1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_confirm_operations,
25 validate_hello_event_declarations, validate_hello_flow_operations,
26 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
27 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
28 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
29 },
30 peer::{
31 encode_route_bind, ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY,
32 OPERATOR_APP_CAPABILITY,
33 },
34 scope::{
35 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
36 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
37 SCOPE_SYNC_OP,
38 },
39 session::{
40 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
41 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
42 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
43 ROLE_VERSIONS_FIELD,
44 },
45 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
46 Priority, RouteTarget, PROTOCOL_VERSION,
47};
48use tokio::time::{timeout_at, Instant};
49use tracing::{debug, info, warn};
50
51use crate::{
52 caller_identity::CallerIdentity,
53 capability_requirements::{
54 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
55 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
56 RequirementStatus, RuntimeModule,
57 },
58 child_roster::DaemonShutdownFlag,
59 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
60 forwarding::{
61 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
62 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
63 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
64 },
65 observability::{
66 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
67 },
68 privileged_prompt::{prompt_text, PrivilegedOperation},
69 provenance::{
70 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
71 },
72 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
73 router::{daemon_error_flags, RouteCtx, RouterError},
74 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
75 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
76 stderr_tail::{CaptureState, TailEntry},
77 supervise::{
78 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
79 SupervisorHandle, SwapHelloAdmission,
80 },
81 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
82};
83
84pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
90
91const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
92const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
93const CAP_PING_PONG: &str = "ping_pong_v1";
94const CAP_SESSION_ATTACH: &str = "session_attach_v1";
95const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
96
97const SUBC_CONTROL_OPS: &[&str] = &[
98 ops::ACTIVITY_RECORD,
99 ops::ACTIVITY_QUERY,
100 ops::ACTIVITY_SUBSCRIBE,
101 ops::SERVER_DESCRIBE,
102 ops::CATALOG_LIST,
103 ops::ROUTE_OPEN,
104 ops::ROUTE_POLL,
105 ops::ROUTE_CLOSING,
106 ops::ROUTE_CLOSED,
107 ops::SUPERVISOR_LIST,
108 ops::SUPERVISOR_RESTART,
109 ops::SUPERVISOR_SWAP,
110 ops::SUPERVISOR_RELOAD,
111 ops::SUPERVISOR_RESCAN,
112 ops::SUPERVISOR_RELEASE_RESERVED,
113 ops::SUPERVISOR_SET_ENABLED,
114 ops::SUPERVISOR_HEALTH_PROBE,
115 ops::SUPERVISOR_HEALTH,
116 ops::SUPERVISOR_STDERR_TAIL,
117 ops::SUPERVISOR_TERMINALS,
118 ops::SUPERVISOR_ROUTES,
119 ops::SUPERVISOR_PROVENANCE,
120 ops::SUPERVISOR_SPAWN_SNAPSHOT,
121 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
122];
123
124const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
131
132mod rescan_checked;
133use rescan_checked::{RescanMode, RescanPlan};
134
135const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
136 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
137 "supervisor.live_roots",
138 SCOPE_SYNC_OP,
139 SCOPE_APPLY_OP,
140 SCOPE_DESCRIBE_OP,
141 "operator.confirm",
142];
143
144const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
148
149const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
150
151pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
159
160pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
175
176pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
194
195const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
196const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
197
198fn reload_verdict(
199 configured: &Path,
200 spawned_from: Option<&Path>,
201 image: subc_control::RunningImageAgreement,
202) -> PendingReloadVerdict {
203 let path = match spawned_from {
204 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
205 ReloadPathAgreement::Match
206 }
207 Some(spawned_from) => ReloadPathAgreement::Mismatch {
208 configured: configured.to_path_buf(),
209 spawned_from: spawned_from.to_path_buf(),
210 },
211 None => ReloadPathAgreement::Unavailable {
212 reason: if matches!(
213 image,
214 subc_control::RunningImageAgreement::Unavailable {
215 reason: subc_control::RunningImageUnavailableReason::NotRunning
216 }
217 ) {
218 ReloadPathUnavailableReason::NotRunning
219 } else {
220 ReloadPathUnavailableReason::SpawnedPathUnavailable
221 },
222 },
223 };
224 PendingReloadVerdict {
225 path,
226 image,
227 spec: None,
228 }
229}
230
231fn launch_spec_agreement(
232 configured: &crate::supervise::ModuleSpec,
233 running: bool,
234 spawned: Option<&crate::supervise::LaunchSpecDigest>,
235) -> subc_control::LaunchSpecAgreement {
236 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
237 if !running {
238 return Agreement::Unavailable {
239 reason: Reason::NotRunning,
240 };
241 }
242 match spawned {
243 None => Agreement::Unavailable {
244 reason: Reason::SpawnedSpecUnavailable,
245 },
246 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
247 Agreement::Match
248 }
249 Some(_) => Agreement::Mismatch,
250 }
251}
252
253fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
254 if configured == spawned_from {
255 return true;
256 }
257 #[cfg(windows)]
258 {
259 match (
265 std::fs::canonicalize(configured),
266 std::fs::canonicalize(spawned_from),
267 ) {
268 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
269 _ => false,
270 }
271 }
272 #[cfg(not(windows))]
273 {
274 false
275 }
276}
277
278#[derive(Clone)]
279struct DaemonProvenanceFacts {
280 build: DaemonBuildProvenance,
281 pid: Option<u32>,
282 started_at_ms: Option<u64>,
283 start_clock: Option<crate::clock::StartClock>,
284 executable_path: Option<PathBuf>,
285 executable_identity: Option<SpawnedFileIdentity>,
286 process_start_time: Option<u64>,
287 probe: ExecutableIdentityProbe,
288}
289
290impl Default for DaemonProvenanceFacts {
291 fn default() -> Self {
292 Self {
293 build: DaemonBuildProvenance {
294 build_git_sha: None,
295 build_lock_digest: None,
296 },
297 pid: None,
298 started_at_ms: None,
299 start_clock: None,
300 executable_path: None,
301 executable_identity: None,
302 process_start_time: None,
303 probe: ExecutableIdentityProbe::default(),
304 }
305 }
306}
307
308#[derive(Debug, Clone)]
309struct SupervisorRescanContext {
310 supervisor: Supervisor,
311 config_path: PathBuf,
312 configured_port: Option<u16>,
313 storage_config: Option<crate::daemon_config::StorageConfig>,
314 admission_facts_carrier_module_id: Option<String>,
315 admission_facts_targets: Option<Vec<String>>,
316 scope_authority_owners: Vec<String>,
317 privileged_operator_prompt: PrivilegedOperatorPrompt,
318 generation: Arc<std::sync::atomic::AtomicU64>,
324}
325
326const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
347 "reloading",
348 "supervisor_not_live",
349 "registration_not_active",
350 "no_forwarding_connection",
351 "relay_send_failed",
352];
353
354#[derive(Clone)]
356pub struct ControlHandler {
357 registry: Arc<Registry>,
358 activity: Option<crate::activity::ActivityLedger>,
359 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
360 forwarding: Arc<ForwardingTable>,
361 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
362 supervisor: SupervisorHandle,
363 subc_capabilities: Arc<[String]>,
364 route_bind_relay_timeout: Duration,
368 route_bind_relay_timeouts: BTreeMap<String, Duration>,
372 route_bind_breakers: RouteBindBreakers,
375 route_bind_concurrency: RouteBindConcurrency,
378 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
383 route_bind_breaker_threshold: u32,
385 route_bind_breaker_cooldown: Duration,
387 health_probe_timeout: Duration,
388 storage_config: Option<crate::daemon_config::StorageConfig>,
391 machine_id: Option<crate::machine_id::MachineId>,
395 admission_facts_carrier_module_id: Option<String>,
396 admission_facts_targets: Option<Vec<String>>,
397 scopes: Arc<RwLock<ScopeTable>>,
401 scope_deadline_added: Arc<tokio::sync::Notify>,
405 scope_authority_owners: Vec<String>,
408 privileged_operator_prompt: PrivilegedOperatorPrompt,
410 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
413 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
414 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
415 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
416 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
417 pub(crate) operator_apps:
418 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
419 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
424 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
430 #[cfg(test)]
432 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
433 #[cfg(test)]
434 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
435 #[allow(dead_code)]
439 #[cfg(windows)]
440 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
441 #[allow(dead_code)]
442 #[cfg(windows)]
443 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
444 #[allow(dead_code)]
448 #[cfg(windows)]
449 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
450 #[cfg(all(test, windows))]
453 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
454 #[cfg(test)]
455 caller_lookup: Option<TestCallerLookup>,
456 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
459 rescan: Option<SupervisorRescanContext>,
460 connected_clients: ConnectedClients,
461 counters: DaemonCounters,
462 local_endpoint_unavailable: Option<String>,
465 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
466 daemon_shutdown: DaemonShutdownFlag,
471 daemon_provenance: DaemonProvenanceFacts,
472 #[cfg(test)]
473 control_dispatch_delay: Option<Duration>,
474 #[cfg(test)]
475 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
476 #[cfg(test)]
480 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
481 #[cfg(test)]
487 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
488}
489
490impl fmt::Debug for ControlHandler {
491 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
492 f.debug_struct("ControlHandler")
493 .field("registry", &self.registry)
494 .field("forwarding", &self.forwarding)
495 .field("process_liveness", &self.process_liveness.is_some())
496 .field("supervisor", &self.supervisor)
497 .field("subc_capabilities", &self.subc_capabilities)
498 .finish()
499 }
500}
501
502struct RouteOpenRequest {
503 target: RouteTarget,
504 identity: BindIdentity,
505 consumer_identity: Option<ConsumerIdentity>,
506 consumer_capabilities: Option<Vec<String>>,
507 role_versions: Option<BTreeMap<String, String>>,
508 admission_facts: Option<serde_json::Value>,
509 scope: Option<ScopeSelector>,
510}
511
512#[derive(Clone, Copy)]
513enum RouteOpenPhase {
514 Admission,
515 ProjectRoot,
516 EgressReservation,
517 BindSend,
518 BindAckWait,
519}
520
521struct RouteOpenTimings {
525 module_id: String,
526 principal: String,
527 admission: Duration,
528 project_root: Duration,
529 egress_reservation: Duration,
530 bind_send: Duration,
531 bind_ack_wait: Duration,
532 phase: Option<RouteOpenPhase>,
533 phase_started_at: Instant,
534}
535
536impl RouteOpenTimings {
537 fn new() -> Self {
538 Self {
539 module_id: String::new(),
540 principal: "unresolved".to_string(),
542 admission: Duration::ZERO,
543 project_root: Duration::ZERO,
544 egress_reservation: Duration::ZERO,
545 bind_send: Duration::ZERO,
546 bind_ack_wait: Duration::ZERO,
547 phase: Some(RouteOpenPhase::Admission),
548 phase_started_at: Instant::now(),
549 }
550 }
551
552 fn finish(&mut self) {
553 let elapsed = self.phase_started_at.elapsed();
554 match self.phase.take() {
555 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
556 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
557 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
558 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
559 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
560 None => {}
561 }
562 }
563
564 fn enter(&mut self, phase: RouteOpenPhase) {
565 self.finish();
566 self.phase = Some(phase);
567 self.phase_started_at = Instant::now();
568 }
569}
570
571struct RouteBindReservationGuard {
572 forwarding: Arc<ForwardingTable>,
573 endpoint: ModuleEndpointId,
574 relay_corr: u64,
575 armed: bool,
576}
577
578struct ModuleControlRpcGuard {
579 forwarding: Arc<ForwardingTable>,
580 endpoint: ModuleEndpointId,
581 corr: u64,
582 armed: bool,
583}
584
585impl ModuleControlRpcGuard {
586 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
587 Self {
588 forwarding,
589 endpoint,
590 corr,
591 armed: true,
592 }
593 }
594
595 fn disarm(&mut self) {
596 self.armed = false;
597 }
598}
599
600impl Drop for ModuleControlRpcGuard {
601 fn drop(&mut self) {
602 if self.armed {
603 let _ = self
604 .forwarding
605 .cancel_module_control_rpc(self.endpoint, self.corr);
606 }
607 }
608}
609
610impl RouteBindReservationGuard {
611 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
612 Self {
613 forwarding,
614 endpoint,
615 relay_corr,
616 armed: true,
617 }
618 }
619
620 fn release_and_disarm(&mut self) {
621 if !self.armed {
622 return;
623 }
624 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
625 self.endpoint,
626 self.relay_corr,
627 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
628 ) {
629 send_goodbye_target_best_effort(
630 &self.forwarding.counters(),
631 &target,
632 "canceled route.bind",
633 );
634 }
635 self.armed = false;
636 }
637
638 fn disarm(&mut self) {
639 self.armed = false;
640 }
641}
642
643impl Drop for RouteBindReservationGuard {
644 fn drop(&mut self) {
645 self.release_and_disarm();
646 }
647}
648
649#[derive(Debug, Clone, Default)]
679pub(crate) struct RouteBindBreakers {
680 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
681}
682
683#[derive(Debug, Clone, Default)]
684pub(crate) struct RouteBindConcurrency {
685 modules: Arc<Mutex<HashMap<String, usize>>>,
686}
687
688struct RouteBindConcurrencyGuard {
689 concurrency: RouteBindConcurrency,
690 module_id: String,
691}
692
693impl RouteBindConcurrency {
694 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
697 let mut modules = self
698 .modules
699 .lock()
700 .expect("route.bind concurrency mutex poisoned");
701 let in_flight = modules.entry(module_id.to_string()).or_default();
702 if *in_flight >= limit {
703 return Err(*in_flight);
704 }
705 *in_flight += 1;
706 Ok(RouteBindConcurrencyGuard {
707 concurrency: self.clone(),
708 module_id: module_id.to_string(),
709 })
710 }
711}
712
713impl Drop for RouteBindConcurrencyGuard {
714 fn drop(&mut self) {
715 let mut modules = self
716 .concurrency
717 .modules
718 .lock()
719 .expect("route.bind concurrency mutex poisoned");
720 let remove = {
721 let in_flight = modules
722 .get_mut(&self.module_id)
723 .expect("admitted route.bind has a concurrency entry");
724 *in_flight -= 1;
725 *in_flight == 0
726 };
727 if remove {
728 modules.remove(&self.module_id);
729 }
730 }
731}
732
733#[derive(Debug, Default)]
734struct ModuleBreakerState {
735 consecutive_timeouts: u32,
737 cooldown_until: Option<Instant>,
740 probe_in_flight: Option<Arc<()>>,
745}
746
747enum RouteBindAdmission<'a> {
749 Admitted {
750 guard: RouteBindBreakerGuard<'a>,
751 probe: bool,
754 },
755 Refused {
756 consecutive_timeouts: u32,
757 retry_in: Duration,
761 probe_in_flight: bool,
762 },
763}
764
765struct RouteBindBreakerGuard<'a> {
772 breakers: RouteBindBreakers,
773 module_id: &'a str,
774 probe_token: Option<Arc<()>>,
775 settled: bool,
776}
777
778impl RouteBindBreakerGuard<'_> {
779 fn record_accepted(&mut self) -> bool {
783 self.settled = true;
784 self.breakers.record_accepted(self.module_id)
785 }
786
787 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
790 self.settled = true;
791 self.breakers.record_timeout(
792 self.module_id,
793 self.probe_token.as_ref(),
794 threshold,
795 cooldown,
796 )
797 }
798
799 fn record_inconclusive(&mut self) {
808 self.settled = true;
809 self.breakers
810 .record_inconclusive(self.module_id, self.probe_token.as_ref());
811 }
812}
813
814impl Drop for RouteBindBreakerGuard<'_> {
815 fn drop(&mut self) {
816 if !self.settled {
817 self.breakers
818 .record_inconclusive(self.module_id, self.probe_token.as_ref());
819 }
820 }
821}
822
823struct BreakerOpened {
827 consecutive_timeouts: u32,
828 reopened_after_probe: bool,
831}
832
833impl RouteBindBreakers {
834 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
835 self.modules
836 .lock()
837 .expect("route.bind breaker mutex poisoned")
838 }
839
840 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
843 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
844 probe: probe_token.is_some(),
845 guard: RouteBindBreakerGuard {
846 breakers: self.clone(),
847 module_id,
848 probe_token,
849 settled: false,
850 },
851 };
852
853 let mut modules = self.lock();
854 let Some(state) = modules.get_mut(module_id) else {
855 return admitted(None);
856 };
857 let Some(cooldown_until) = state.cooldown_until else {
858 return admitted(None);
859 };
860 if state.probe_in_flight.is_some() {
861 return RouteBindAdmission::Refused {
862 consecutive_timeouts: state.consecutive_timeouts,
863 retry_in: Duration::ZERO,
864 probe_in_flight: true,
865 };
866 }
867 let now = Instant::now();
868 if now < cooldown_until {
869 return RouteBindAdmission::Refused {
870 consecutive_timeouts: state.consecutive_timeouts,
871 retry_in: cooldown_until - now,
872 probe_in_flight: false,
873 };
874 }
875 let token = Arc::new(());
876 state.probe_in_flight = Some(Arc::clone(&token));
877 admitted(Some(token))
878 }
879
880 fn record_accepted(&self, module_id: &str) -> bool {
881 self.lock()
882 .remove(module_id)
883 .is_some_and(|state| state.cooldown_until.is_some())
884 }
885
886 fn record_timeout(
887 &self,
888 module_id: &str,
889 probe_token: Option<&Arc<()>>,
890 threshold: u32,
891 cooldown: Duration,
892 ) -> Option<BreakerOpened> {
893 let mut modules = self.lock();
894 let state = modules.entry(module_id.to_string()).or_default();
895 let was_open = state.cooldown_until.is_some();
896 let was_probe = Self::owns_probe(state, probe_token);
897 if was_probe {
898 state.probe_in_flight = None;
899 }
900 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
901 if state.consecutive_timeouts < threshold {
902 return None;
903 }
904 state.cooldown_until = Some(Instant::now() + cooldown);
905 Some(BreakerOpened {
906 consecutive_timeouts: state.consecutive_timeouts,
907 reopened_after_probe: was_open && was_probe,
908 })
909 }
910
911 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
912 state
916 .probe_in_flight
917 .as_ref()
918 .zip(token)
919 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
920 }
921
922 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
923 if let Some(state) = self.lock().get_mut(module_id) {
924 if Self::owns_probe(state, probe_token) {
925 state.probe_in_flight = None;
926 }
927 }
928 }
929
930 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
945 self.lock()
946 .remove(module_id)
947 .map(|state| state.consecutive_timeouts)
948 .filter(|discarded| *discarded > 0)
949 }
950
951 fn open_snapshot(&self) -> Option<serde_json::Value> {
958 let now = Instant::now();
959 let modules = self.lock();
960 let open = modules
961 .iter()
962 .filter_map(|(module_id, state)| {
963 let cooldown_until = state.cooldown_until?;
964 Some((
965 module_id.clone(),
966 serde_json::json!({
967 "consecutive_timeouts": state.consecutive_timeouts,
968 "cooldown_remaining_ms":
969 cooldown_until.saturating_duration_since(now).as_millis() as u64,
970 "probe_in_flight": state.probe_in_flight.is_some(),
971 }),
972 ))
973 })
974 .collect::<serde_json::Map<String, serde_json::Value>>();
975 (!open.is_empty()).then_some(serde_json::Value::Object(open))
976 }
977}
978
979impl ControlHandler {
980 pub fn new(registry: Arc<Registry>) -> Self {
981 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
982 }
983
984 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
985 let counters = forwarding.counters();
986 let route_bind_breakers = forwarding.route_bind_breakers();
990 let route_bind_concurrency = forwarding.route_bind_concurrency();
991 let route_outages = forwarding.route_outages();
992 Self {
993 registry,
994 activity: None,
995 activity_streams: Arc::new(Mutex::new(HashMap::new())),
996 forwarding,
997 process_liveness: None,
998 supervisor: SupervisorHandle::new(),
999 subc_capabilities: Arc::from([
1000 CAP_MANIFEST_REGISTRATION.to_string(),
1001 CAP_CHANNEL_LIFECYCLE.to_string(),
1002 CAP_PING_PONG.to_string(),
1003 CAP_SESSION_ATTACH.to_string(),
1004 CAP_ADMISSION_FACTS_RELAY.to_string(),
1005 CAP_SCOPES_V1.to_string(),
1006 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1007 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1008 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1009 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1010 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1011 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1012 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1013 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1014 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1015 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1016 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1017 ]),
1018 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1019 route_bind_relay_timeouts: BTreeMap::new(),
1020 route_bind_breakers,
1021 route_bind_concurrency,
1022 route_outages,
1023 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1024 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1025 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1026 storage_config: None,
1027 machine_id: None,
1028 admission_facts_carrier_module_id: None,
1029 admission_facts_targets: None,
1030 scopes: Arc::new(RwLock::new(ScopeTable::new(
1031 crate::daemon_config::default_scope_authority_owners(),
1032 ))),
1033 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1034 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1035 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1036 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1037 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1038 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1039 local_connections: Arc::new(Mutex::new(HashSet::new())),
1040 operator_app_policy: Default::default(),
1041 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1042 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1043 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1044 #[cfg(test)]
1045 app_open_test_platform: None,
1046 #[cfg(test)]
1047 app_open_test_launch: None,
1048 #[cfg(windows)]
1049 windows_verify: None,
1050 #[cfg(windows)]
1051 windows_flash: None,
1052 #[cfg(windows)]
1053 windows_track: None,
1054 #[cfg(all(test, windows))]
1055 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1056 #[cfg(test)]
1057 caller_lookup: None,
1058 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1059 rescan: None,
1060 connected_clients: ConnectedClients::new(),
1061 local_endpoint_unavailable: None,
1062 counters,
1063 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1064 daemon_shutdown: DaemonShutdownFlag::default(),
1065 daemon_provenance: DaemonProvenanceFacts::default(),
1066 #[cfg(test)]
1067 control_dispatch_delay: None,
1068 #[cfg(test)]
1069 provenance_probe_override: None,
1070 #[cfg(test)]
1071 route_bind_relay_sent_hold: None,
1072 #[cfg(test)]
1073 hello_registry_to_forwarding_hold: None,
1074 }
1075 }
1076
1077 pub fn with_storage_config(
1080 mut self,
1081 storage_config: Option<crate::daemon_config::StorageConfig>,
1082 ) -> Self {
1083 self.storage_config = storage_config;
1084 self
1085 }
1086
1087 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1090 self.activity = Some(crate::activity::ActivityLedger::open(
1091 run_dir.join("activity.jsonl"),
1092 incarnation,
1093 ));
1094 self.supervisor.set_activity(self.activity.clone());
1095 self
1096 }
1097
1098 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1101 self.machine_id = machine_id;
1102 self
1103 }
1104
1105 pub fn with_admission_facts_config(
1109 mut self,
1110 carrier_module_id: Option<String>,
1111 targets: Option<Vec<String>>,
1112 ) -> Self {
1113 self.admission_facts_carrier_module_id = carrier_module_id;
1114 self.admission_facts_targets = targets;
1115 self
1116 }
1117
1118 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1122 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1123 self.scope_authority_owners = owners;
1124 self
1125 }
1126
1127 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1129 self.privileged_operator_prompt = mode;
1130 self
1131 }
1132
1133 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1136 self.route_bind_relay_timeout = timeout;
1137 self
1138 }
1139
1140 pub fn with_route_bind_relay_timeouts(
1146 mut self,
1147 timeouts: impl IntoIterator<Item = (String, Duration)>,
1148 ) -> Self {
1149 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1150 self
1151 }
1152
1153 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1159 self.route_bind_relay_timeouts
1160 .get(module_id)
1161 .copied()
1162 .unwrap_or(self.route_bind_relay_timeout)
1163 }
1164
1165 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1173 self.route_bind_breaker_threshold = threshold.max(1);
1174 self.route_bind_breaker_cooldown = cooldown;
1175 self
1176 }
1177
1178 #[cfg(test)]
1179 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1180 self.health_probe_timeout = timeout;
1181 self
1182 }
1183
1184 #[cfg(test)]
1185 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1186 self.control_dispatch_delay = Some(delay);
1187 self
1188 }
1189
1190 #[cfg(test)]
1194 pub(crate) fn with_route_bind_relay_sent_hold(
1195 mut self,
1196 hold: Arc<tokio::sync::Barrier>,
1197 ) -> Self {
1198 self.route_bind_relay_sent_hold = Some(hold);
1199 self
1200 }
1201
1202 #[cfg(test)]
1207 pub(crate) fn with_hello_registry_to_forwarding_hold(
1208 mut self,
1209 hold: Arc<std::sync::Barrier>,
1210 ) -> Self {
1211 self.hello_registry_to_forwarding_hold = Some(hold);
1212 self
1213 }
1214
1215 #[cfg(test)]
1216 fn hold_between_registry_and_forwarding(&self) {
1217 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1218 hold.wait();
1219 hold.wait();
1220 }
1221 }
1222
1223 pub fn with_process_liveness(
1224 mut self,
1225 process_liveness: Arc<dyn ModuleProcessLiveness>,
1226 ) -> Self {
1227 self.process_liveness = Some(process_liveness);
1228 self
1229 }
1230
1231 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1232 self.supervisor = supervisor;
1233 self.supervisor.set_activity(self.activity.clone());
1234 self
1235 }
1236
1237 pub fn with_daemon_provenance(
1238 mut self,
1239 pid: u32,
1240 started_at_ms: u64,
1241 executable_path: Option<PathBuf>,
1242 build_git_sha: Option<String>,
1243 build_lock_digest: Option<String>,
1244 ) -> Self {
1245 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1246 let process_start_time = process_start_time(pid);
1247 self.daemon_provenance = DaemonProvenanceFacts {
1248 build: DaemonBuildProvenance {
1249 build_git_sha,
1250 build_lock_digest,
1251 },
1252 pid: Some(pid),
1253 started_at_ms: Some(started_at_ms),
1254 start_clock: None,
1255 executable_path,
1256 executable_identity,
1257 process_start_time,
1258 probe: ExecutableIdentityProbe::default(),
1259 };
1260 self
1261 }
1262
1263 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1264 self.daemon_provenance.start_clock = Some(clock);
1265 self
1266 }
1267
1268 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1270 self.scopes
1271 .write()
1272 .unwrap_or_else(|poisoned| poisoned.into_inner())
1273 .set_wall_clock(Arc::new(clock));
1274 self
1275 }
1276
1277 #[cfg(test)]
1278 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1279 self.provenance_probe_override = Some(result);
1280 self
1281 }
1282
1283 pub fn with_capability_config(
1287 self,
1288 modules: impl IntoIterator<Item = (String, bool)>,
1289 reserved_capabilities: BTreeMap<String, String>,
1290 ) -> Self {
1291 self.capability_evaluator
1292 .configure(modules, reserved_capabilities);
1293 self
1294 }
1295
1296 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1300 self.daemon_shutdown = flag;
1301 self
1302 }
1303
1304 pub fn with_supervisor_rescan(
1305 mut self,
1306 supervisor: Supervisor,
1307 config_path: impl Into<PathBuf>,
1308 configured_port: Option<u16>,
1309 ) -> Self {
1310 self.rescan = Some(SupervisorRescanContext {
1311 supervisor,
1312 config_path: config_path.into(),
1313 configured_port,
1314 storage_config: self.storage_config.clone(),
1315 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1316 admission_facts_targets: self.admission_facts_targets.clone(),
1317 scope_authority_owners: self.scope_authority_owners.clone(),
1318 privileged_operator_prompt: self.privileged_operator_prompt,
1319 generation: Arc::default(),
1320 });
1321 self
1322 }
1323
1324 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1325 self.connected_clients = connected_clients;
1326 self
1327 }
1328
1329 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1332 self.local_endpoint_unavailable = reason;
1333 self
1334 }
1335
1336 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1337 Arc::clone(&self.forwarding)
1338 }
1339
1340 pub(crate) fn counters(&self) -> DaemonCounters {
1341 self.counters.clone()
1342 }
1343
1344 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1347 tokio::spawn(async move {
1348 loop {
1349 self.capability_evaluator
1350 .wait_for_change_or_deadline()
1351 .await;
1352 self.refresh_capability_requirements();
1353 }
1354 });
1355 }
1356
1357 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1363 tokio::spawn(async move {
1364 loop {
1365 let has_deadlines = self
1366 .scopes
1367 .read()
1368 .unwrap_or_else(|poisoned| poisoned.into_inner())
1369 .has_deadlines();
1370 if !has_deadlines {
1371 self.scope_deadline_added.notified().await;
1372 continue;
1373 }
1374 tokio::time::sleep(Duration::from_secs(1)).await;
1375 if let Err(error) = self.sweep_expired_scopes() {
1376 warn!(%error, "scope expiry sweep failed");
1377 }
1378 }
1379 });
1380 }
1381
1382 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1383 let mut table = self
1384 .scopes
1385 .write()
1386 .unwrap_or_else(|poisoned| poisoned.into_inner());
1387 let swept = table.sweep_expired();
1388 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1389 drop(table);
1390 self.close_scope_drained_routes(drained);
1391 Ok(())
1392 }
1393
1394 fn publish_scope_changes(
1395 &self,
1396 changes: &[crate::scopes::ScopeTagChange],
1397 expired: &[crate::scopes::ScopeExpired],
1398 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1399 if changes.is_empty() {
1400 return Ok(Vec::new());
1401 }
1402 let drained = self
1403 .forwarding
1404 .publish_scope_changes(changes)
1405 .map_err(RouterError::Forwarding)?;
1406 let mut counts = HashMap::new();
1407 for route in &drained {
1408 *counts
1409 .entry((
1410 route.scope.owner.as_str(),
1411 route.scope.scope_ref.as_str(),
1412 route.scope.tag.scope_epoch,
1413 ))
1414 .or_insert(0usize) += 1;
1415 }
1416 for scope in expired {
1417 let routes_closed = counts
1418 .get(&(
1419 scope.owner.as_str(),
1420 scope.scope_ref.as_str(),
1421 scope.scope_epoch,
1422 ))
1423 .copied()
1424 .unwrap_or(0);
1425 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1426 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1427 routes_closed, "scope expired");
1428 }
1429 Ok(drained)
1430 }
1431
1432 fn runtime_capability_snapshot(
1433 &self,
1434 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1435 let runtime = self
1436 .supervisor
1437 .list()
1438 .into_iter()
1439 .map(|module| {
1440 let status = module.status().map_err(|err| {
1441 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1442 })?;
1443 Ok(RuntimeModule {
1444 module_id: status.module_id,
1445 state: status.state,
1446 enabled: status.enabled,
1447 })
1448 })
1449 .collect::<Result<Vec<_>, RouterError>>()?;
1450 let registrations = self.registry.list_registered_modules().map_err(|err| {
1456 RouterError::backend(
1457 0,
1458 0,
1459 format!("failed to list capability registrations: {err}"),
1460 )
1461 })?;
1462 let registrations = registrations
1463 .into_iter()
1464 .map(|registration| RegisteredModule {
1465 module_id: registration.manifest.module_id,
1466 module_version: registration.manifest.module_version,
1467 capabilities: registration.manifest.capabilities,
1468 })
1469 .collect();
1470 Ok((runtime, registrations))
1471 }
1472
1473 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1480 let cached_registration = RegisteredModule {
1481 module_id: registration.manifest.module_id.clone(),
1482 module_version: registration.manifest.module_version.clone(),
1483 capabilities: registration.manifest.capabilities.clone(),
1484 };
1485 let change = self.capability_evaluator.record_hello(&cached_registration);
1486 log_manifest_change(&cached_registration.module_id, &change);
1487 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1488 self.enforce_capability_denies();
1489 }
1490 self.refresh_capability_requirements();
1491 }
1492
1493 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1497 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1498 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1499 self.supervisor.set_swap_promotion_observer(observer);
1500 }
1501
1502 pub fn refresh_capability_requirements(&self) {
1503 match self.runtime_capability_snapshot() {
1504 Ok((runtime, registrations)) => {
1505 log_requirement_events(
1506 self.capability_evaluator
1507 .evaluate_now(&runtime, ®istrations),
1508 &self.daemon_shutdown,
1509 );
1510 }
1511 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1512 }
1513 }
1514
1515 fn enforce_capability_denies(&self) {
1519 let registrations = match self.registry.list_registered_modules() {
1520 Ok(snapshot) => snapshot,
1521 Err(err) => {
1522 warn!(error = %err, "failed to read registrations for capability deny census");
1523 return;
1524 }
1525 };
1526 let manifests = registrations
1527 .into_iter()
1528 .map(|registration| {
1529 (
1530 registration.manifest.module_id.clone(),
1531 registration.manifest,
1532 )
1533 })
1534 .collect::<BTreeMap<_, _>>();
1535 let census = match self.forwarding.route_census(None) {
1536 Ok(census) => census,
1537 Err(err) => {
1538 warn!(error = %err, "failed to read route census for capability deny enforcement");
1539 return;
1540 }
1541 };
1542
1543 for (target_module_id, routes) in census {
1544 let Some(target_manifest) = manifests.get(&target_module_id) else {
1545 continue;
1546 };
1547 let mut closed_routes = Vec::new();
1548 let mut goodbyes = Vec::new();
1549 for route in routes {
1550 let Principal::Reserved {
1551 module_id: opening_module_id,
1552 } = &route.principal
1553 else {
1554 continue;
1555 };
1556 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1557 continue;
1558 };
1559 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1560 continue;
1561 };
1562
1563 match self.forwarding.release_client_route(
1564 route.goodbye_target.connection_id,
1565 route.goodbye_target.channel,
1566 route.goodbye_target.epoch,
1567 ) {
1568 Ok(RouteRelease::Removed(module_goodbye)) => {
1569 warn!(
1570 opening_module_id,
1571 target_module_id,
1572 capability,
1573 "force-closing route because an attested capability deny edge now matches"
1574 );
1575 goodbyes.push(module_goodbye);
1576 goodbyes.push(route.goodbye_target.clone());
1577 closed_routes.push(route);
1578 }
1579 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1580 Err(err) => warn!(
1581 opening_module_id,
1582 target_module_id,
1583 capability,
1584 error = %err,
1585 "failed to force-close capability-denied route"
1586 ),
1587 }
1588 }
1589
1590 if closed_routes.is_empty() {
1591 continue;
1592 }
1593 send_route_control_pushes(
1594 &self.forwarding,
1595 closed_routes,
1596 ClientControlPush::RouteClosed {
1597 module_id: target_module_id,
1598 channels: Vec::new(),
1599 reason: RouteCloseReason::CapabilityDenied,
1600 drained: false,
1601 abandoned: 0,
1602 excluded_subscriptions: 0,
1603 terminal: Some(false),
1604 },
1605 );
1606 self.emit_route_goodbyes(goodbyes);
1609 }
1610 }
1611
1612 fn not_ready_reason(
1618 &self,
1619 registration: &crate::registry::ModuleRegistration,
1620 ) -> Option<NotReadyReason> {
1621 if !registration.ready {
1622 return Some(NotReadyReason {
1623 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1624 capability: None,
1625 });
1626 }
1627 self.first_unprovided_required_capability(registration)
1628 .map(|capability| NotReadyReason {
1629 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1630 capability: Some(capability),
1631 })
1632 }
1633
1634 fn first_unprovided_required_capability(
1652 &self,
1653 registration: &crate::registry::ModuleRegistration,
1654 ) -> Option<String> {
1655 let required = registration
1656 .manifest
1657 .capabilities
1658 .iter()
1659 .flat_map(|declarations| declarations.requires.iter())
1660 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1661 .map(|requirement| requirement.capability.as_str())
1662 .collect::<BTreeSet<_>>();
1663 if required.is_empty() {
1664 return None;
1665 }
1666 let module_id = registration.manifest.module_id.as_str();
1667 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1668 if required
1669 .iter()
1670 .any(|capability| verdict(capability).is_none())
1671 {
1672 self.refresh_capability_requirements();
1673 }
1674 required
1675 .into_iter()
1676 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1677 .map(str::to_string)
1678 }
1679
1680 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1681 self.capability_evaluator
1682 .statuses()
1683 .into_iter()
1684 .map(capability_requirement_status)
1685 .collect()
1686 }
1687
1688 fn deregister_connection(
1695 &self,
1696 connection_id: ConnectionId,
1697 reason: RegistrationEndReason,
1698 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1699 self.registry
1700 .deregister_connection_with_reason(connection_id, reason)
1701 }
1702
1703 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1704 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1705 return None;
1706 }
1707 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1708 parse_client_control_request(&frame.body)
1709 else {
1710 return None;
1711 };
1712 Some(target_module_id(&target).to_string())
1713 }
1714
1715 pub(crate) fn route_open_capacity_refusal(
1716 &self,
1717 ctx: &RouteCtx,
1718 frame: &Frame,
1719 target_module_id: &str,
1720 in_flight: usize,
1721 limit: usize,
1722 ) -> Result<Frame, RouterError> {
1723 let caller = self.route_open_caller_from_frame(frame);
1726 self.route_open_admission_refusal_frame(
1727 ctx,
1728 &caller,
1729 frame,
1730 target_module_id,
1731 "open_admission_full",
1732 (in_flight, limit),
1733 format!(
1734 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1735 ),
1736 )
1737 }
1738
1739 fn route_open_target_capacity_refusal(
1740 &self,
1741 ctx: &RouteCtx,
1742 caller: &RouteOpenCaller,
1743 frame: &Frame,
1744 target_module_id: &str,
1745 in_flight: usize,
1746 ) -> Result<Frame, RouterError> {
1747 self.route_open_admission_refusal_frame(
1748 ctx,
1749 caller,
1750 frame,
1751 target_module_id,
1752 "target_binds_full",
1753 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1754 format!(
1755 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1756 ),
1757 )
1758 }
1759
1760 #[allow(clippy::too_many_arguments)]
1768 fn route_open_admission_refusal_frame(
1769 &self,
1770 ctx: &RouteCtx,
1771 caller: &RouteOpenCaller,
1772 frame: &Frame,
1773 target_module_id: &str,
1774 reason: &'static str,
1775 (in_flight, limit): (usize, usize),
1776 message: impl Into<String>,
1777 ) -> Result<Frame, RouterError> {
1778 let code = error_codes::TARGET_UNAVAILABLE;
1779 self.counters.increment_route_open_refused(code);
1780 info!(
1781 target: "control",
1782 code,
1783 reason,
1784 module_id = ?target_module_id,
1785 connection_id = ctx.connection_id.get(),
1786 principal = %caller.principal,
1787 scope = %caller.scope,
1788 harness = %caller.harness,
1789 in_flight,
1790 limit,
1791 "route.open refused"
1792 );
1793 control_error_frame(frame, code, message.into())
1794 }
1795
1796 #[cfg(test)]
1802 pub fn handle_control(
1803 &self,
1804 connection_id: ConnectionId,
1805 frame: Frame,
1806 ) -> Result<Vec<Frame>, RouterError> {
1807 match frame.header.ty {
1808 FrameType::Ping => Ok(vec![pong(&frame)?]),
1809 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1810 FrameType::Goodbye => self.handle_goodbye(connection_id),
1811 ty => Ok(vec![control_error_frame(
1812 &frame,
1813 "unsupported_control_frame",
1814 format!("unsupported channel-0 frame {ty:?}"),
1815 )?]),
1816 }
1817 }
1818
1819 pub async fn handle_control_frame(
1820 &self,
1821 ctx: &RouteCtx,
1822 frame: Frame,
1823 ) -> Result<Vec<Frame>, RouterError> {
1824 self.handle_control_frame_timed(ctx, frame, None).await
1825 }
1826
1827 #[cfg(any(test, feature = "test-support"))]
1828 pub fn with_test_operator_app_policy(
1829 mut self,
1830 policy: crate::operator_apps::TestOperatorAppPolicy,
1831 ) -> Self {
1832 self.operator_app_policy = policy.into();
1833 self
1834 }
1835
1836 #[cfg(all(windows, any(test, feature = "test-support")))]
1840 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1841 self.windows_verify = Some(verify);
1842 self
1843 }
1844
1845 #[cfg(all(windows, any(test, feature = "test-support")))]
1847 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1848 self.windows_flash = Some(flash);
1849 self
1850 }
1851
1852 #[cfg(all(windows, any(test, feature = "test-support")))]
1856 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1857 self.windows_track = Some(track);
1858 self
1859 }
1860
1861 pub(crate) fn record_captured_peer(
1862 &self,
1863 connection: ConnectionId,
1864 peer: subc_os::local_peer::CapturedPeer,
1865 ) {
1866 self.record_observed_peer(connection, peer.observation().clone());
1867 #[cfg(windows)]
1868 if let Some(stamp) = self.stamp_windows_launch(&peer) {
1869 self.operator_apps
1870 .lock()
1871 .unwrap_or_else(|p| p.into_inner())
1872 .insert(connection, stamp);
1873 }
1874 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1875 self.operator_apps
1876 .lock()
1877 .unwrap_or_else(|p| p.into_inner())
1878 .insert(connection, stamp);
1879 }
1880 }
1881
1882 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1883 self.local_connections
1884 .lock()
1885 .unwrap_or_else(|p| p.into_inner())
1886 .insert(connection);
1887 }
1888
1889 fn connection_operator_app(
1892 &self,
1893 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1894 ) -> Option<subc_protocol::peer::OperatorApp> {
1895 stamp
1896 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1897 .map(|stamp| stamp.app.clone())
1898 }
1899
1900 fn handle_connection_describe(
1901 &self,
1902 ctx: &RouteCtx,
1903 frame: Frame,
1904 ) -> Result<Vec<Frame>, RouterError> {
1905 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1906 return Ok(vec![control_error_frame(
1907 &frame,
1908 "invalid_control_body",
1909 "malformed connection.describe body",
1910 )?]);
1911 }
1912 let local = self
1913 .local_connections
1914 .lock()
1915 .unwrap_or_else(|p| p.into_inner())
1916 .contains(&ctx.connection_id);
1917 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1918 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1919 Ok(vec![control_response_body_frame(
1920 &frame,
1921 &subc_control::ConnectionDescribeReply::new(local, operator_app),
1922 ops::CONNECTION_DESCRIBE,
1923 )?])
1924 }
1925
1926 pub(crate) fn record_observed_peer(
1927 &self,
1928 connection: ConnectionId,
1929 peer: subc_os::local_peer::ObservedPeer,
1930 ) {
1931 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1932 let peer = match peer {
1933 OsPeer::Unobserved => ObservedPeer::unobserved(),
1934 OsPeer::Observed(peer) => {
1935 let observed = match peer.user {
1936 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1937 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1938 };
1939 observed
1940 .with_exe(
1941 peer.exe
1942 .and_then(|path| path.into_os_string().into_string().ok()),
1943 )
1944 .with_signing_id(peer.signing_id)
1945 }
1946 };
1947 self.observed_peers
1948 .lock()
1949 .unwrap_or_else(|p| p.into_inner())
1950 .insert(connection, peer);
1951 }
1952
1953 pub(crate) fn record_caller_endpoints(
1954 &self,
1955 connection: ConnectionId,
1956 endpoints: Option<(SocketAddr, SocketAddr)>,
1957 ) {
1958 let mut callers = self
1959 .caller_endpoints
1960 .lock()
1961 .unwrap_or_else(|p| p.into_inner());
1962 callers.remove(&connection);
1963 if let Some(endpoints) = endpoints {
1964 callers.insert(connection, endpoints);
1965 }
1966 }
1967
1968 #[cfg(test)]
1969 pub(crate) fn with_caller_lookup_for_test(
1970 mut self,
1971 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1972 ) -> Self {
1973 self.caller_lookup = Some(Arc::new(lookup));
1974 self
1975 }
1976
1977 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1978 let endpoints = self
1979 .caller_endpoints
1980 .lock()
1981 .unwrap_or_else(|p| p.into_inner())
1982 .get(&connection)
1983 .copied();
1984 #[cfg(test)]
1985 let lookup = self.caller_lookup.clone();
1986 tokio::task::spawn_blocking(move || {
1987 #[cfg(test)]
1988 if let Some(lookup) = lookup {
1989 return lookup(endpoints);
1990 }
1991 endpoints
1992 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1993 .unwrap_or_default()
1994 })
1995 .await
1996 .unwrap_or_default()
1997 }
1998
1999 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2000 frame.header.channel == 0
2001 && frame.header.ty == FrameType::Request
2002 && self
2003 .forwarding
2004 .module_endpoint_for_connection(ctx.connection_id)
2005 .is_ok_and(|endpoint| endpoint.is_none())
2006 && parse_client_control_request(&frame.body)
2007 .ok()
2008 .is_some_and(|request| privileged_operation(&request).is_some())
2009 }
2010
2011 pub(crate) async fn handle_control_frame_timed(
2012 &self,
2013 ctx: &RouteCtx,
2014 frame: Frame,
2015 dispatch_started_at: Option<StdInstant>,
2016 ) -> Result<Vec<Frame>, RouterError> {
2017 match frame.header.ty {
2018 FrameType::Ping => Ok(vec![pong(&frame)?]),
2019 FrameType::Hello => {
2020 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2021 }
2022 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2023 FrameType::Cancel => {
2024 if self
2027 .forwarding
2028 .operator_confirms()
2029 .cancel(ctx.connection_id, frame.header.corr)
2030 || self
2031 .supervisor
2032 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2033 || self.cancel_activity_subscription(ctx, &frame)?
2034 {
2035 Ok(Vec::new())
2036 } else {
2037 Ok(vec![control_error_frame(
2038 &frame,
2039 "unknown_subscription",
2040 "no subscription has this correlation id",
2041 )?])
2042 }
2043 }
2044 FrameType::Request => {
2045 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2048 if op
2049 .as_ref()
2050 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2051 {
2052 return self.handle_activity_record(ctx, frame).await;
2053 }
2054 if op.as_ref().is_some_and(|probe| {
2055 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2056 }) {
2057 return self.handle_activity_read(ctx, frame).await;
2058 }
2059 if op
2060 .as_ref()
2061 .is_some_and(|probe| probe.op == "operator.confirm")
2062 {
2063 return self.handle_operator_confirm(ctx, frame);
2064 }
2065 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2066 return self.handle_app_open(ctx, frame).await;
2067 }
2068 if op
2069 .as_ref()
2070 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2071 {
2072 return self.handle_connection_describe(ctx, frame);
2073 }
2074 if op
2075 .as_ref()
2076 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2077 {
2078 return self.handle_scope_served_by(ctx.connection_id, frame);
2079 }
2080 if self
2081 .forwarding
2082 .module_endpoint_for_connection(ctx.connection_id)
2083 .map_err(RouterError::Forwarding)?
2084 .is_some()
2085 {
2086 if !is_known_module_request_op(&frame.body) {
2087 return Ok(vec![control_error_frame(
2088 &frame,
2089 "unsupported_control_frame",
2090 "module-originated channel-0 REQUEST is not supported",
2091 )?]);
2092 }
2093 let request = match parse_module_control_request_from_module(&frame.body) {
2094 Ok(request) => request,
2095 Err((err, ControlRequestBodyError::UnknownOp)) => {
2096 return Ok(vec![control_error_frame(
2097 &frame,
2098 "unsupported_control_frame",
2099 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2100 )?])
2101 }
2102 Err((err, ControlRequestBodyError::InvalidBody)) => {
2103 return Ok(vec![control_error_frame(
2104 &frame,
2105 "invalid_control_body",
2106 format!("malformed module control body: {err}"),
2107 )?])
2108 }
2109 };
2110 let op = module_control_request_op(&request);
2111 let corr = frame.header.corr;
2112 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2113 let result =
2114 self.handle_module_control_request(ctx.connection_id, frame, request);
2115 log_slow_control_dispatch(
2116 dispatch_started_at,
2117 op,
2118 ctx.connection_id,
2119 corr,
2120 None,
2121 );
2122 return result;
2123 }
2124
2125 if is_known_module_request_op(&frame.body) {
2126 return Ok(vec![control_error_frame(
2127 &frame,
2128 "not_registered",
2129 "catalog.update requires an active module registration owned by this connection",
2130 )?]);
2131 }
2132
2133 let request = match parse_client_control_request(&frame.body) {
2134 Ok(request) => request,
2135 Err((err, ControlRequestBodyError::UnknownOp)) => {
2136 return Ok(vec![control_error_frame(
2137 &frame,
2138 "unknown_control_op",
2139 format!("unknown client control op: {err}"),
2140 )?])
2141 }
2142 Err((err, ControlRequestBodyError::InvalidBody)) => {
2143 return Ok(vec![control_error_frame(
2144 &frame,
2145 "invalid_control_body",
2146 format!("malformed client control body: {err}"),
2147 )?])
2148 }
2149 };
2150 let op = client_control_request_op(&request);
2151 let corr = frame.header.corr;
2152 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2153 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2154 #[cfg(test)]
2155 if let Some(delay) = self.control_dispatch_delay {
2156 tokio::time::sleep(delay).await;
2157 }
2158 let result = self
2159 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2160 .await;
2161 if let Some(timings) = route_open_timings.as_mut() {
2162 timings.finish();
2163 }
2164 log_slow_control_dispatch(
2165 dispatch_started_at,
2166 op,
2167 ctx.connection_id,
2168 corr,
2169 route_open_timings.as_ref(),
2170 );
2171 result
2172 }
2173 FrameType::Push => {
2174 let Some(endpoint) = self
2175 .forwarding
2176 .module_endpoint_for_connection(ctx.connection_id)
2177 .map_err(RouterError::Forwarding)?
2178 else {
2179 return Ok(vec![control_error_frame(
2180 &frame,
2181 "unsupported_control_frame",
2182 "client-originated channel-0 PUSH is not supported",
2183 )?]);
2184 };
2185 self.handle_status_update(endpoint, frame)
2186 }
2187 FrameType::Response | FrameType::Error
2188 if self
2189 .forwarding
2190 .module_endpoint_for_connection(ctx.connection_id)
2191 .map_err(RouterError::Forwarding)?
2192 .is_some() =>
2193 {
2194 self.handle_module_relay_response(ctx.connection_id, frame)
2195 }
2196 ty => Ok(vec![control_error_frame(
2197 &frame,
2198 "unsupported_control_frame",
2199 format!("unsupported channel-0 frame {ty:?}"),
2200 )?]),
2201 }
2202 }
2203
2204 pub fn cleanup_connection(
2205 &self,
2206 connection_id: ConnectionId,
2207 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2208 self.local_connections
2209 .lock()
2210 .unwrap_or_else(|p| p.into_inner())
2211 .remove(&connection_id);
2212 self.caller_endpoints
2213 .lock()
2214 .unwrap_or_else(|p| p.into_inner())
2215 .remove(&connection_id);
2216 self.build_class_warned
2217 .lock()
2218 .unwrap_or_else(|p| p.into_inner())
2219 .remove(&connection_id);
2220 self.observed_peers
2221 .lock()
2222 .unwrap_or_else(|p| p.into_inner())
2223 .remove(&connection_id);
2224 self.operator_apps
2225 .lock()
2226 .unwrap_or_else(|p| p.into_inner())
2227 .remove(&connection_id);
2228 self.forwarding
2229 .operator_confirms()
2230 .connection_closed(connection_id);
2231 self.cleanup_connection_with_end_reason(
2232 connection_id,
2233 RegistrationEndReason::ConnectionClosed,
2234 )
2235 }
2236
2237 fn cleanup_connection_with_end_reason(
2238 &self,
2239 connection_id: ConnectionId,
2240 requested_reason: RegistrationEndReason,
2241 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2242 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2243 self.registry
2244 .get_module_by_connection(connection_id)?
2245 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2246 .and_then(|module| module.registration_end_reason().ok().flatten())
2247 .unwrap_or(requested_reason)
2248 } else {
2249 requested_reason
2250 };
2251 let crash_closed = self
2252 .registry
2253 .get_module_by_connection(connection_id)?
2254 .and_then(|registration| {
2255 self.forwarding
2256 .module_endpoint_for_connection(connection_id)
2257 .ok()
2258 .flatten()
2259 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2260 .map(|routes| (registration.manifest.module_id, routes))
2261 });
2262 let crash_closed = crash_closed.map(|(module_id, routes)| {
2263 let terminal = match self.supervisor.get(&module_id) {
2264 None => false,
2265 Some(module) => match module.will_recover_after_connection_loss() {
2266 Ok(will_recover) => !will_recover,
2267 Err(err) => {
2268 warn!(
2269 %module_id,
2270 error = %err,
2271 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2272 );
2273 false
2274 }
2275 },
2276 };
2277 let (reason, drained) = match self
2281 .forwarding
2282 .daemon_drain_result_for_connection(connection_id)
2283 {
2284 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2285 Ok(None) => (RouteCloseReason::Crash, false),
2286 Err(err) => {
2287 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2288 (RouteCloseReason::Crash, false)
2289 }
2290 };
2291 (module_id, routes, reason, drained, terminal)
2292 });
2293 let registrations = self.deregister_connection(connection_id, end_reason);
2294 let cleanup = if crash_closed.is_some() {
2295 self.forwarding.cleanup_connection_counted(connection_id)
2296 } else {
2297 self.forwarding
2303 .cleanup_connection(connection_id)
2304 .map(|released| crate::forwarding::ConnectionCleanup {
2305 released,
2306 abandoned_relays: 0,
2307 })
2308 };
2309 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2314 let abandoned = cleanup
2315 .as_ref()
2316 .map_or(0, |cleanup| cleanup.abandoned_relays);
2317 send_route_control_pushes(
2318 &self.forwarding,
2319 routes,
2320 ClientControlPush::RouteClosed {
2321 module_id,
2322 channels: Vec::new(),
2323 reason,
2324 drained,
2325 abandoned,
2326 excluded_subscriptions: 0,
2327 terminal: Some(terminal),
2328 },
2329 );
2330 }
2331 if let Ok(cleanup) = cleanup {
2332 self.emit_route_goodbyes(cleanup.released);
2333 }
2334 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2338 crate::supervise::notify_registration_release();
2339 self.capability_evaluator.wake_deadline_loop();
2340 self.refresh_capability_requirements();
2341 }
2342 self.supervisor.remove_spawn_subscribers(connection_id);
2343 self.activity_streams
2344 .lock()
2345 .unwrap_or_else(|p| p.into_inner())
2346 .retain(|(id, _), task| {
2347 if *id == connection_id {
2348 task.abort();
2349 false
2350 } else {
2351 true
2352 }
2353 });
2354 self.hello_launch_nonces
2357 .lock()
2358 .unwrap_or_else(|poisoned| poisoned.into_inner())
2359 .forget(connection_id);
2360 self.scopes
2361 .write()
2362 .unwrap_or_else(|poisoned| poisoned.into_inner())
2363 .release_connection(connection_id);
2364 registrations
2365 }
2366
2367 pub(crate) fn handle_route_goodbye(
2368 &self,
2369 connection_id: ConnectionId,
2370 route_channel: u16,
2371 route_epoch: u32,
2372 ) -> Result<bool, RouterError> {
2373 debug!(
2374 connection_id = connection_id.get(),
2375 route_channel, route_epoch, "handling route GOODBYE"
2376 );
2377 let RouteRelease::Removed(released_route) = self
2378 .forwarding
2379 .release_client_route(connection_id, route_channel, route_epoch)
2380 .map_err(RouterError::Forwarding)?
2381 else {
2382 return Ok(false);
2383 };
2384 self.emit_route_goodbyes(vec![released_route]);
2385 Ok(true)
2386 }
2387
2388 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2389 for released in released_routes {
2390 let frame = match Frame::build_with_version(
2391 released.negotiated_ver,
2392 FrameType::Goodbye,
2393 control_flags(),
2394 released.channel,
2395 released.epoch,
2396 0,
2397 Vec::new(),
2398 ) {
2399 Ok(frame) => frame,
2400 Err(err) => {
2401 warn!(
2402 route_channel = released.channel,
2403 error = %err,
2404 "failed to build route GOODBYE frame"
2405 );
2406 continue;
2407 }
2408 };
2409 if !released.close_on_delivery_failure() {
2410 crate::forwarding::send_module_route_goodbye(
2411 &self.counters,
2412 &released.sink,
2413 frame,
2414 released.module_id.as_deref(),
2415 "client route released",
2416 );
2417 continue;
2418 }
2419 if let Err(err) = released.sink.try_send(frame) {
2420 warn!(
2421 target_connection_id = released.connection_id.get(),
2422 route_channel = released.channel,
2423 error = %err,
2424 "route GOODBYE was not delivered to client; closing target connection"
2425 );
2426 if self
2427 .forwarding
2428 .escalate_client_delivery_failure(
2429 released.connection_id,
2430 released.channel,
2431 released.epoch,
2432 CloseReason::new(
2433 "route_goodbye_delivery_failed",
2434 format!(
2435 "failed to enqueue route GOODBYE for channel {}: {err}",
2436 released.channel
2437 ),
2438 ),
2439 crate::forwarding::UndeliveredFrame {
2440 module_id: released.module_id.as_deref(),
2441 sink: &released.sink,
2442 },
2443 )
2444 .unwrap_or(false)
2445 {
2446 self.counters.increment_goodbye_relay_client_failed();
2447 }
2448 }
2449 }
2450 }
2451
2452 fn send_abandoned_route_bind_goodbye(
2464 &self,
2465 module_sink: &crate::FrameSink,
2466 negotiated_ver: u8,
2467 module_channel: u16,
2468 module_epoch: u32,
2469 ) {
2470 let frame = match Frame::build_with_version(
2471 negotiated_ver,
2472 FrameType::Goodbye,
2473 control_flags(),
2474 module_channel,
2475 module_epoch,
2476 0,
2477 Vec::new(),
2478 ) {
2479 Ok(frame) => frame,
2480 Err(err) => {
2481 warn!(
2482 route_channel = module_channel,
2483 error = %err,
2484 "failed to build GOODBYE for abandoned route.bind"
2485 );
2486 return;
2487 }
2488 };
2489 crate::forwarding::send_module_route_goodbye(
2490 &self.counters,
2491 module_sink,
2492 frame,
2493 None,
2494 "abandoned route.bind",
2495 );
2496 }
2497
2498 fn handle_hello(
2499 &self,
2500 connection_id: ConnectionId,
2501 sink: Option<crate::FrameSink>,
2502 frame: Frame,
2503 ) -> Result<Vec<Frame>, RouterError> {
2504 debug!(
2505 connection_id = connection_id.get(),
2506 corr = frame.header.corr,
2507 "handling HELLO"
2508 );
2509 if self
2513 .registry
2514 .get_module_by_connection(connection_id)
2515 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2516 .is_some()
2517 {
2518 return Ok(vec![control_error_frame(
2519 &frame,
2520 "invalid_hello",
2521 "connection is already registered as a module",
2522 )?]);
2523 }
2524 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2525 Ok(value) => value,
2526 Err(err) => {
2527 return Ok(vec![control_error_frame(
2528 &frame,
2529 "invalid_hello",
2530 format!("malformed HELLO body: {err}"),
2531 )?])
2532 }
2533 };
2534 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2535 return Ok(vec![control_error_frame(
2536 &frame,
2537 "invalid_capability_grammar",
2538 err.to_string(),
2539 )?]);
2540 }
2541 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2542 return Ok(vec![control_error_frame(
2543 &frame,
2544 "invalid_manifest",
2545 err.to_string(),
2546 )?]);
2547 }
2548 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2549 return Ok(vec![control_error_frame(
2550 &frame,
2551 "invalid_event_declaration",
2552 err.to_string(),
2553 )?]);
2554 }
2555 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2556 return Ok(vec![control_error_frame(
2557 &frame,
2558 "invalid_manifest",
2559 err.to_string(),
2560 )?]);
2561 }
2562 if let Some(provenance) = hello_value
2563 .get("manifest")
2564 .and_then(|manifest| manifest.get("provenance"))
2565 {
2566 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2567 return Ok(vec![control_error_frame(
2568 &frame,
2569 "invalid_manifest",
2570 format!("malformed manifest provenance: {err}"),
2571 )?]);
2572 }
2573 }
2574 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2575 let module_id = hello_value["manifest"]["module_id"]
2576 .as_str()
2577 .unwrap_or("<unknown>");
2578 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2579 }
2580 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2581 let module_id = hello_value["manifest"]["module_id"]
2582 .as_str()
2583 .unwrap_or("<unknown>");
2584 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2585 }
2586 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2587 let module_id = hello_value["manifest"]["module_id"]
2588 .as_str()
2589 .unwrap_or("<unknown>");
2590 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2591 }
2592 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2593 Ok(hello) => hello,
2594 Err(err) => {
2595 return Ok(vec![control_error_frame(
2596 &frame,
2597 "invalid_hello",
2598 format!("malformed HELLO body: {err}"),
2599 )?])
2600 }
2601 };
2602
2603 if hello.protocol_ver != hello.manifest.protocol_ver {
2604 return Ok(vec![control_error_frame(
2605 &frame,
2606 "invalid_manifest",
2607 format!(
2608 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2609 hello.protocol_ver, hello.manifest.protocol_ver
2610 ),
2611 )?]);
2612 }
2613
2614 if hello.manifest.module_id.trim().is_empty() {
2615 return Ok(vec![control_error_frame(
2616 &frame,
2617 "invalid_manifest",
2618 "manifest module_id must not be empty",
2619 )?]);
2620 }
2621
2622 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2623 Ok(negotiated_ver) => negotiated_ver,
2624 Err(message) => {
2625 return Ok(vec![control_error_frame(
2626 &frame,
2627 "version_unsupported",
2628 message,
2629 )?])
2630 }
2631 };
2632
2633 let swap_admission = self
2641 .supervisor
2642 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2643 if swap_admission == SwapHelloAdmission::Refused {
2644 warn!(
2645 module_id = %hello.manifest.module_id,
2646 connection_id = connection_id.get(),
2647 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2648 );
2649 return Ok(vec![control_error_frame(
2650 &frame,
2651 "swap_token_invalid",
2652 format!(
2653 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2654 hello.manifest.module_id
2655 ),
2656 )?]);
2657 }
2658 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2659
2660 let nonce_verified = if swap_candidate {
2665 true
2666 } else {
2667 match self.supervisor.reserved_hello_verification(
2668 &hello.manifest.module_id,
2669 hello.launch_nonce.as_deref(),
2670 ) {
2671 Ok(verified) => verified,
2672 Err(rejection) => {
2673 let message = match rejection {
2674 ReservedHelloRejection::Exact { module_id } => format!(
2675 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2676 ),
2677 ReservedHelloRejection::Prefix {
2678 prefix,
2679 owner_module_id,
2680 } => format!(
2681 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2682 hello.manifest.module_id
2683 ),
2684 };
2685 return Ok(vec![control_error_frame(
2686 &frame,
2687 "reserved_module",
2688 message,
2689 )?]);
2690 }
2691 }
2692 };
2693
2694 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2695 &hello.manifest.module_id,
2696 hello.manifest.capabilities.as_ref(),
2697 );
2698 if let Some(refusal) = reserved_capability_refusals.first() {
2699 let capability = refusal.capability.clone();
2700 let bound_module = refusal.claimants[0].clone();
2701 log_duplicate_claim_events(reserved_capability_refusals);
2702 return Ok(vec![control_error_frame(
2703 &frame,
2704 "reserved_capability",
2705 format!(
2706 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2707 capability, bound_module, hello.manifest.module_id
2708 ),
2709 )?]);
2710 }
2711
2712 if self
2715 .forwarding
2716 .connection_has_client_routes(connection_id)
2717 .map_err(RouterError::Forwarding)?
2718 {
2719 return Ok(vec![control_error_frame(
2720 &frame,
2721 "invalid_hello",
2722 "connection has open client routes and cannot also register as a module",
2723 )?]);
2724 }
2725
2726 self.hello_launch_nonces
2731 .lock()
2732 .unwrap_or_else(|poisoned| poisoned.into_inner())
2733 .record(connection_id, hello.launch_nonce.as_deref());
2734 let control_ops = effective_module_control_ops(hello.control_ops);
2735 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2738 if swap_candidate {
2739 return self.register_swap_candidate(
2740 connection_id,
2741 sink,
2742 &frame,
2743 hello.manifest,
2744 negotiated_ver,
2745 control_ops,
2746 hello_ack,
2747 );
2748 }
2749 let registration = match self
2756 .registry
2757 .register_unpublished_verified_with_control_ops(
2758 hello.manifest,
2759 negotiated_ver,
2760 connection_id,
2761 control_ops,
2762 nonce_verified,
2763 ) {
2764 Ok(registration) => registration,
2765 Err(RegistryError::DuplicateModuleId { module_id }) => {
2766 return Ok(vec![control_error_frame(
2767 &frame,
2768 "duplicate_module_id",
2769 format!(
2770 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2771 ),
2772 )?])
2773 }
2774 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2775 return Ok(vec![control_error_frame(
2776 &frame,
2777 "invalid_module_id",
2778 err.to_string(),
2779 )?])
2780 }
2781 Err(err) => {
2782 return Ok(vec![control_error_frame(
2783 &frame,
2784 "registry_error",
2785 err.to_string(),
2786 )?])
2787 }
2788 };
2789 #[cfg(test)]
2790 self.hold_between_registry_and_forwarding();
2791
2792 let reply = if let Some(sink) = sink {
2793 let concurrency = manifest_concurrency(®istration.manifest);
2809 if let Err(err) = self.forwarding.register_module_connection_acked(
2810 connection_id,
2811 registration.manifest.module_id.clone(),
2812 negotiated_ver,
2813 concurrency,
2814 sink,
2815 hello_ack,
2816 ) {
2817 if matches!(
2822 self.deregister_connection(
2823 connection_id,
2824 RegistrationEndReason::RegistrationFailed,
2825 ),
2826 Ok(r) if !r.is_empty()
2827 ) {
2828 crate::supervise::notify_registration_release();
2829 }
2830 return Ok(vec![control_error_frame(
2831 &frame,
2832 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2833 "invalid_hello"
2834 } else {
2835 forwarding_error_code(&err)
2836 },
2837 err.to_string(),
2838 )?]);
2839 }
2840 Vec::new()
2841 } else {
2842 vec![hello_ack]
2845 };
2846
2847 if !self
2852 .registry
2853 .publish_registration(connection_id)
2854 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2855 {
2856 debug!(
2857 module_id = %registration.manifest.module_id,
2858 connection_id = connection_id.get(),
2859 "registration was removed before it could be published"
2860 );
2861 }
2862
2863 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2872 info!(
2873 module_id = %registration.manifest.module_id,
2874 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2875 );
2876 }
2877
2878 self.apply_registration_capabilities(®istration);
2879
2880 self.supervisor
2881 .record_activity_registration(®istration.manifest.module_id);
2882
2883 info!(
2884 module_id = %registration.manifest.module_id,
2885 module_version = %registration.manifest.module_version,
2886 negotiated_ver,
2887 routable_provider = manifest_provides_routable_role(®istration.manifest),
2888 connection_id = connection_id.get(),
2889 "module registered"
2890 );
2891
2892 Ok(reply)
2893 }
2894
2895 #[allow(clippy::too_many_arguments)]
2912 fn register_swap_candidate(
2913 &self,
2914 connection_id: ConnectionId,
2915 sink: Option<crate::FrameSink>,
2916 frame: &Frame,
2917 manifest: ModuleManifest,
2918 negotiated_ver: u8,
2919 control_ops: Vec<String>,
2920 hello_ack: Frame,
2921 ) -> Result<Vec<Frame>, RouterError> {
2922 let module_id = manifest.module_id.clone();
2923 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2924 manifest,
2925 negotiated_ver,
2926 connection_id,
2927 control_ops,
2928 ) {
2929 Ok(registration) => registration,
2930 Err(RegistryError::DuplicateModuleId { module_id }) => {
2931 return Ok(vec![control_error_frame(
2932 frame,
2933 "duplicate_module_id",
2934 format!(
2935 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2936 ),
2937 )?])
2938 }
2939 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2940 return Ok(vec![control_error_frame(
2941 frame,
2942 "invalid_module_id",
2943 err.to_string(),
2944 )?])
2945 }
2946 Err(err) => {
2947 return Ok(vec![control_error_frame(
2948 frame,
2949 "registry_error",
2950 err.to_string(),
2951 )?])
2952 }
2953 };
2954 #[cfg(test)]
2955 self.hold_between_registry_and_forwarding();
2956 let reply = if let Some(sink) = sink {
2957 let concurrency = manifest_concurrency(®istration.manifest);
2961 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2962 connection_id,
2963 module_id.clone(),
2964 negotiated_ver,
2965 concurrency,
2966 sink,
2967 hello_ack,
2968 ) {
2969 if matches!(
2970 self.deregister_connection(
2971 connection_id,
2972 RegistrationEndReason::RegistrationFailed,
2973 ),
2974 Ok(r) if !r.is_empty()
2975 ) {
2976 crate::supervise::notify_registration_release();
2977 }
2978 return Ok(vec![control_error_frame(
2979 frame,
2980 forwarding_error_code(&err),
2981 err.to_string(),
2982 )?]);
2983 }
2984 Vec::new()
2985 } else {
2986 vec![hello_ack]
2987 };
2988 if !self
2989 .registry
2990 .publish_registration(connection_id)
2991 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2992 {
2993 debug!(
2994 module_id = %module_id,
2995 connection_id = connection_id.get(),
2996 "swap candidate registration was removed before it could be published"
2997 );
2998 }
2999 self.supervisor.mark_swap_candidate_admitted(&module_id);
3000 info!(
3001 module_id = %module_id,
3002 module_version = %registration.manifest.module_version,
3003 negotiated_ver,
3004 ready = registration.ready,
3005 connection_id = connection_id.get(),
3006 "swap candidate registered; not routable until cutover"
3007 );
3008 Ok(reply)
3009 }
3010
3011 fn build_hello_ack(
3012 &self,
3013 frame: &Frame,
3014 negotiated_ver: u8,
3015 module_id: &str,
3016 ) -> Result<Frame, RouterError> {
3017 let ack = ModuleHelloAckBody {
3018 negotiated_ver,
3019 subc_ops: module_subc_ops(),
3020 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3021 storage: self
3022 .storage_config
3023 .as_ref()
3024 .map(|cfg| cfg.descriptor_for(module_id)),
3025 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3026 };
3027 let body = serde_json::to_vec(&ack).map_err(|err| {
3028 RouterError::backend(
3029 0,
3030 frame.header.corr,
3031 format!("failed to encode HELLO_ACK: {err}"),
3032 )
3033 })?;
3034
3035 Frame::build_with_version(
3036 negotiated_ver,
3037 FrameType::HelloAck,
3038 control_flags(),
3039 0,
3040 0,
3041 frame.header.corr,
3042 body,
3043 )
3044 .map_err(RouterError::FrameBuild)
3045 }
3046
3047 async fn handle_client_control_request(
3048 &self,
3049 ctx: &RouteCtx,
3050 frame: Frame,
3051 request: ClientControlRequest,
3052 route_open_timings: &mut Option<RouteOpenTimings>,
3053 ) -> Result<Vec<Frame>, RouterError> {
3054 if let Some(operation) = privileged_operation(&request) {
3055 let caller = self.resolve_caller(ctx.connection_id).await;
3056 let op = client_control_request_op(&request);
3057 let text = prompt_text(operation, &caller);
3058 match self.privileged_operator_prompt {
3059 PrivilegedOperatorPrompt::Log => {
3060 info!(target: "subc_daemon::privileged_prompt", operation = op,
3061 target = ?operation, caller = %caller.description(), summary = %text,
3062 "privileged request would prompt the operator");
3063 }
3064 PrivilegedOperatorPrompt::Enforce => {
3065 let key =
3066 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3067 let stamp = self
3068 .operator_apps
3069 .lock()
3070 .unwrap_or_else(|p| p.into_inner())
3071 .get(&ctx.connection_id)
3072 .cloned();
3073 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3074 &self.operator_app_policy,
3075 &self.launch_table,
3076 stamp.as_ref(),
3077 )
3078 .then(|| self.shortcut_context(stamp, None, None));
3079 let outcome = self
3080 .forwarding
3081 .operator_confirms()
3082 .approve_privileged_with_shortcut(
3083 ctx,
3084 &frame,
3085 op,
3086 text,
3087 caller.description(),
3088 key,
3089 shortcut,
3090 )
3091 .await;
3092 if outcome.code != "confirmed" {
3093 return Ok(vec![outcome.frame(&frame)]);
3094 }
3095 }
3096 }
3097 }
3098 match request {
3099 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3100 ClientControlRequest::CatalogList { module_id } => {
3101 self.handle_catalog_list(frame, module_id)
3102 }
3103 ClientControlRequest::RouteOpen {
3104 target,
3105 identity,
3106 consumer_identity,
3107 consumer_capabilities,
3108 role_versions,
3109 admission_facts,
3110 scope,
3111 } => {
3112 self.handle_route_open(
3113 ctx,
3114 frame,
3115 RouteOpenRequest {
3116 target,
3117 identity,
3118 consumer_identity,
3119 consumer_capabilities,
3120 role_versions,
3121 admission_facts,
3122 scope,
3123 },
3124 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3125 )
3126 .await
3127 }
3128 ClientControlRequest::RoutePoll {
3129 route_channel,
3130 route_epoch,
3131 kind,
3132 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3133 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
3134 ClientControlRequest::SupervisorSpawnSnapshot {} => {
3135 self.handle_supervisor_spawn_snapshot(frame)
3136 }
3137 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3138 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3139 }
3140 ClientControlRequest::SupervisorRestart {
3141 module_id,
3142 drain_timeout_ms,
3143 } => {
3144 self.log_supervisor_request_received(
3145 ctx,
3146 frame.header.corr,
3147 ops::SUPERVISOR_RESTART,
3148 Some(&module_id),
3149 None,
3150 )?;
3151 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3152 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3153 .await
3154 }
3155 ClientControlRequest::SupervisorSwap {
3156 module_id,
3157 ready_timeout_ms,
3158 } => {
3159 self.log_supervisor_request_received(
3160 ctx,
3161 frame.header.corr,
3162 ops::SUPERVISOR_SWAP,
3163 Some(&module_id),
3164 None,
3165 )?;
3166 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3167 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3168 .await
3169 }
3170 ClientControlRequest::SupervisorReload { module_id } => {
3171 self.log_supervisor_request_received(
3172 ctx,
3173 frame.header.corr,
3174 ops::SUPERVISOR_RELOAD,
3175 Some(&module_id),
3176 None,
3177 )?;
3178 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3179 self.handle_supervisor_reload(frame, module_id, requester)
3180 .await
3181 }
3182 ClientControlRequest::SupervisorRescan { preview } => {
3183 if !preview {
3184 self.log_supervisor_request_received(
3185 ctx,
3186 frame.header.corr,
3187 ops::SUPERVISOR_RESCAN,
3188 None,
3189 None,
3190 )?;
3191 }
3192 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3193 self.handle_supervisor_rescan(
3194 frame,
3195 if preview {
3196 RescanMode::Preview
3197 } else {
3198 RescanMode::Execute
3199 },
3200 requester,
3201 )
3202 .await
3203 }
3204 ClientControlRequest::SupervisorRescanChecked {
3205 preview,
3206 candidate,
3207 expect,
3208 } => {
3209 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
3210 .await
3211 }
3212 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3213 self.log_supervisor_request_received(
3214 ctx,
3215 frame.header.corr,
3216 ops::SUPERVISOR_RELEASE_RESERVED,
3217 Some(&module_id),
3218 None,
3219 )?;
3220 self.handle_supervisor_release_reserved(frame, module_id)
3221 .await
3222 }
3223 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3224 self.log_supervisor_request_received(
3225 ctx,
3226 frame.header.corr,
3227 ops::SUPERVISOR_SET_ENABLED,
3228 Some(&module_id),
3229 Some(enabled),
3230 )?;
3231 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3232 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3233 .await
3234 }
3235 ClientControlRequest::SupervisorHealthProbe { module_id } => {
3236 self.handle_supervisor_health_probe(frame, module_id).await
3237 }
3238 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3239 ClientControlRequest::SupervisorRoutes { module_id } => {
3240 self.handle_supervisor_routes(frame, module_id)
3241 }
3242 ClientControlRequest::SupervisorProvenance { module_id } => {
3243 self.handle_supervisor_provenance(frame, module_id).await
3244 }
3245 ClientControlRequest::SupervisorStderrTail {
3246 module_id,
3247 max_lines,
3248 max_bytes,
3249 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3250 ClientControlRequest::SupervisorTerminals { module_id } => {
3251 self.handle_supervisor_terminals(frame, module_id).await
3252 }
3253 }
3254 }
3255
3256 fn log_supervisor_request_received(
3257 &self,
3258 ctx: &RouteCtx,
3259 corr: u64,
3260 op: &'static str,
3261 module_id: Option<&str>,
3262 enabled: Option<bool>,
3263 ) -> Result<(), RouterError> {
3264 let caller = self.supervisor_requester(ctx, corr)?;
3265 let caller = principal_label(&caller);
3266 info!(
3269 target: "control",
3270 op,
3271 module_id,
3272 enabled,
3273 connection_id = ctx.connection_id.get(),
3274 caller = %caller,
3275 "supervisor request received"
3276 );
3277 Ok(())
3278 }
3279
3280 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3281 Ok(self
3282 .registry
3283 .get_module_by_connection(ctx.connection_id)
3284 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3285 .map(|registration| Principal::Reserved {
3286 module_id: registration.manifest.module_id,
3287 })
3288 .unwrap_or(Principal::Direct))
3289 }
3290
3291 fn handle_module_control_request(
3292 &self,
3293 connection_id: ConnectionId,
3294 frame: Frame,
3295 request: ModuleControlRequestFromModule,
3296 ) -> Result<Vec<Frame>, RouterError> {
3297 match request {
3298 ModuleControlRequestFromModule::CatalogUpdate {
3299 provides,
3300 capabilities,
3301 ready,
3302 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3303 ModuleControlRequestFromModule::LiveRoots {} => {
3304 let registered = self
3305 .registry
3306 .get_module_by_connection(connection_id)
3307 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3308 let Some(registration) = registered else {
3309 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3310 };
3311 let response = self
3312 .forwarding
3313 .live_roots(®istration.manifest.module_id)
3314 .map_err(RouterError::Forwarding)?;
3315 Ok(vec![control_response_body_frame(
3316 &frame,
3317 &response,
3318 "ModuleControlResponseToModule::LiveRoots",
3319 )?])
3320 }
3321 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3322 self.handle_scope_sync(connection_id, frame, generation, scopes)
3323 }
3324 ModuleControlRequestFromModule::ScopeApply {
3325 generation,
3326 upsert,
3327 end,
3328 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3329 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3330 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3331 }
3332 }
3333 }
3334
3335 fn handle_scope_served_by(
3336 &self,
3337 connection_id: ConnectionId,
3338 frame: Frame,
3339 ) -> Result<Vec<Frame>, RouterError> {
3340 let registered = self
3341 .registry
3342 .get_module_by_connection(connection_id)
3343 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3344 let Some(registered) = registered else {
3345 return Ok(vec![control_error_frame(
3346 &frame,
3347 "not_registered",
3348 "scope.served_by requires an active module registration owned by this connection",
3349 )?]);
3350 };
3351 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3352 {
3353 Ok(request) => request,
3354 Err(err) => {
3355 return Ok(vec![control_error_frame(
3356 &frame,
3357 "invalid_control_body",
3358 format!("malformed scope.served_by body: {err}"),
3359 )?])
3360 }
3361 };
3362 let answer = self
3363 .scopes
3364 .read()
3365 .unwrap_or_else(|poisoned| poisoned.into_inner())
3366 .served_by(
3367 ®istered.manifest.module_id,
3368 &request.scope_ref,
3369 request.scope_epoch,
3370 &request.module_id,
3371 );
3372 let reply = subc_control::ScopeServedByReply::new(
3373 answer,
3374 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3375 );
3376 Ok(vec![control_response_body_frame(
3377 &frame,
3378 &reply,
3379 "ScopeServedByReply",
3380 )?])
3381 }
3382
3383 fn handle_operator_confirm(
3384 &self,
3385 ctx: &RouteCtx,
3386 frame: Frame,
3387 ) -> Result<Vec<Frame>, RouterError> {
3388 use crate::operator_confirm::{audit, Outcome};
3389 let registration = self
3390 .registry
3391 .get_module_by_connection(ctx.connection_id)
3392 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3393 let Some(registration) = registration else {
3394 let outcome = Outcome::refusal("not_registered");
3395 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3396 return Ok(vec![outcome.frame(&frame)]);
3397 };
3398 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3399 Ok(request) => request,
3400 Err(_) => {
3401 let outcome = Outcome::refusal("invalid_control_body");
3402 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3403 return Ok(vec![outcome.frame(&frame)]);
3404 }
3405 };
3406 let declared = matches!(
3407 request.kind,
3408 Some(subc_protocol::session::OperatorConfirmKind::Operation)
3409 ) && request.operation.as_ref().is_some_and(|operation| {
3410 registration
3411 .manifest
3412 .confirm_operations
3413 .as_ref()
3414 .is_some_and(|operations| operations.contains(operation))
3415 });
3416 let module_id = registration.manifest.module_id;
3417 let nonce = self
3421 .hello_launch_nonces
3422 .lock()
3423 .unwrap_or_else(|p| p.into_inner())
3424 .nonce(ctx.connection_id)
3425 .map(str::to_owned);
3426 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3427 self.supervisor
3428 .spawned_consumer_authorized(&module_id, nonce)
3429 });
3430 let confirms = self.forwarding.operator_confirms();
3431 self.forwarding
3432 .with_operator_route(
3433 ctx.connection_id,
3434 request.route_channel,
3435 request.route_epoch,
3436 |binding| {
3437 let shortcut = binding
3438 .filter(|binding| {
3439 nonce_proven
3440 && crate::operator_confirm::valid_summary(&request.summary)
3441 && declared
3442 && crate::operator_confirm::ShortcutContext::route_eligible(
3443 &self.operator_app_policy,
3444 &self.launch_table,
3445 binding,
3446 )
3447 })
3448 .map(|binding| {
3449 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3450 &self.operator_app_policy,
3451 binding,
3452 );
3453 let stamp = if phone.is_some() {
3454 None
3455 } else {
3456 binding.operator_app.clone()
3457 };
3458 self.shortcut_context(
3459 stamp,
3460 phone,
3461 Some((request.route_channel, request.route_epoch)),
3462 )
3463 });
3464 confirms.admit_with_shortcut(
3465 ctx,
3466 frame,
3467 module_id,
3468 nonce_proven,
3469 request,
3470 binding,
3471 shortcut,
3472 )
3473 },
3474 )
3475 .map_err(RouterError::Forwarding)
3476 }
3477
3478 fn shortcut_context(
3479 &self,
3480 stamp: Option<crate::operator_apps::OperatorAppStamp>,
3481 phone_fingerprint: Option<String>,
3482 route: Option<(u16, u32)>,
3483 ) -> crate::operator_confirm::ShortcutContext {
3484 crate::operator_confirm::ShortcutContext {
3485 ledger: self.activity.clone(),
3486 forwarding: self.forwarding.clone(),
3487 launches: self.launch_table.clone(),
3488 stamp,
3489 phone_fingerprint,
3490 route,
3491 connections: self.operator_apps.clone(),
3492 #[cfg(test)]
3493 hooks: Default::default(),
3494 }
3495 }
3496
3497 #[cfg(any(test, feature = "test-support"))]
3499 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3500 self.operator_app_policy.phone_enabled = enabled;
3501 self
3502 }
3503
3504 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3507 let mut scopes = ScopeRecords::from(scopes);
3508 for (index, record) in scopes.records.iter_mut().enumerate() {
3509 if let Some(refusal) = Self::canonicalize_workdir(record) {
3510 scopes.refusals.insert(index, refusal);
3511 }
3512 }
3513 scopes
3514 }
3515
3516 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3517 let workdir = record.workdir.as_mut()?;
3518 let path = std::path::Path::new(workdir);
3519 if !path.is_absolute() {
3520 return Some((
3521 error_codes::INVALID_WORKDIR,
3522 "workdir must be absolute".to_string(),
3523 ));
3524 }
3525 match ProjectRootId::from_path_allowing_missing(path) {
3526 Ok(root) => match root.as_path().to_str() {
3527 Some(canonical) => {
3528 *workdir = canonical.to_string();
3529 None
3530 }
3531 None => Some((
3532 error_codes::INVALID_WORKDIR,
3533 "canonical workdir is not UTF-8".to_string(),
3534 )),
3535 },
3536 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3537 }
3538 }
3539
3540 fn handle_scope_sync(
3544 &self,
3545 connection_id: ConnectionId,
3546 frame: Frame,
3547 generation: u64,
3548 scopes: Vec<ScopeRecord>,
3549 ) -> Result<Vec<Frame>, RouterError> {
3550 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3551 }
3552
3553 fn handle_scope_change(
3554 &self,
3555 connection_id: ConnectionId,
3556 frame: Frame,
3557 generation: u64,
3558 scopes: Vec<ScopeRecord>,
3559 end: Option<Vec<ScopeEnd>>,
3560 ) -> Result<Vec<Frame>, RouterError> {
3561 let is_apply = end.is_some();
3562 let Some(registration) = self
3563 .registry
3564 .get_module_by_connection(connection_id)
3565 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3566 else {
3567 return Ok(vec![control_error_frame(
3568 &frame,
3569 "not_registered",
3570 "scope.sync and scope.apply require an active module registration owned by this connection",
3571 )?]);
3572 };
3573 let owner = registration.manifest.module_id;
3574 let scopes = Self::prepare_scope_records(scopes);
3575 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3576 let is_current_launch = |connection: ConnectionId| {
3577 self.hello_launch_nonces
3578 .lock()
3579 .unwrap_or_else(|poisoned| poisoned.into_inner())
3580 .presented(connection, current_nonce.as_deref())
3581 };
3582 let mut table = self
3587 .scopes
3588 .write()
3589 .unwrap_or_else(|poisoned| poisoned.into_inner());
3590 let outcome = match end {
3591 Some(end) => table.apply(
3592 &owner,
3593 connection_id,
3594 is_current_launch,
3595 generation,
3596 scopes,
3597 end,
3598 ),
3599 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3600 };
3601 let drained = match &outcome {
3602 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3603 Err(_) => Vec::new(),
3604 };
3605 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3606 drop(table);
3607 if has_deadlines {
3608 self.scope_deadline_added.notify_one();
3609 }
3610 match outcome {
3611 Ok(applied) => {
3612 let counts = ScopeOutcomeCounts::of(&applied.results);
3613 info!(
3614 owner = %owner,
3615 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3616 generation,
3617 records = applied.results.len(),
3618 created = counts.created,
3619 replaced = counts.replaced,
3620 updated = counts.updated,
3621 unchanged = counts.unchanged,
3622 refused = counts.refused,
3623 ended = applied.ended.len(),
3624 tag_changes = applied.tag_changes.len(),
3625 routes_closed = drained.len(),
3626 "scope change accepted"
3627 );
3628 for refused in applied
3634 .results
3635 .iter()
3636 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3637 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3638 {
3639 warn!(
3640 owner = %owner,
3641 generation,
3642 scope_ref = %refused.scope_ref,
3643 scope_epoch = refused.scope_epoch,
3644 code = refused.code.as_deref().unwrap_or(""),
3645 "scope record refused"
3646 );
3647 }
3648 self.close_scope_drained_routes(drained);
3649 let response = if is_apply {
3650 ModuleControlResponseToModule::ScopeApply {
3651 generation,
3652 results: applied.results,
3653 end_results: applied.end_results,
3654 ended: applied.ended,
3655 }
3656 } else {
3657 ModuleControlResponseToModule::ScopeSync {
3658 generation,
3659 results: applied.results,
3660 ended: applied.ended,
3661 }
3662 };
3663 Ok(vec![control_response_body_frame(
3664 &frame,
3665 &response,
3666 "scope change response",
3667 )?])
3668 }
3669 Err(refusal) => {
3670 info!(
3671 owner = %owner,
3672 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3673 generation,
3674 code = refusal.code,
3675 "scope change refused"
3676 );
3677 Ok(vec![control_error_frame(
3678 &frame,
3679 refusal.code,
3680 refusal.message,
3681 )?])
3682 }
3683 }
3684 }
3685
3686 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3692 if drained.is_empty() {
3693 return;
3694 }
3695 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3696 BTreeMap::new();
3697 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3698 for route in drained {
3699 warn!(
3700 module_id = %route.module_id,
3701 reason = ?route.reason,
3702 client_connection_id = route.client.connection_id.get(),
3703 route_channel = route.client.channel,
3704 "closing route because its scope changed"
3705 );
3706 pushes
3707 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3708 .or_insert_with(|| (route.reason, Vec::new()))
3709 .1
3710 .push(EndpointRoute {
3711 goodbye_target: route.client.clone(),
3712 principal: Principal::Unverified,
3713 bound_at: Instant::now(),
3714 draining: false,
3715 drain_reason: None,
3716 });
3717 goodbyes.push(route.module);
3718 goodbyes.push(route.client);
3719 }
3720 for ((module_id, _), (reason, routes)) in pushes {
3721 send_route_control_pushes(
3722 &self.forwarding,
3723 routes,
3724 ClientControlPush::RouteClosed {
3725 module_id,
3726 channels: Vec::new(),
3727 reason,
3728 drained: false,
3729 abandoned: 0,
3730 excluded_subscriptions: 0,
3731 terminal: Some(false),
3732 },
3733 );
3734 }
3735 self.emit_route_goodbyes(goodbyes);
3736 }
3737
3738 fn handle_scope_describe(
3741 &self,
3742 connection_id: ConnectionId,
3743 frame: Frame,
3744 owner: Principal,
3745 scope_ref: String,
3746 ) -> Result<Vec<Frame>, RouterError> {
3747 let registered = self
3748 .registry
3749 .get_module_by_connection(connection_id)
3750 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3751 let Some(registered) = registered else {
3752 return Ok(vec![control_error_frame(
3753 &frame,
3754 "not_registered",
3755 "scope.describe requires an active module registration owned by this connection",
3756 )?]);
3757 };
3758 let mut description = self
3759 .scopes
3760 .read()
3761 .unwrap_or_else(|poisoned| poisoned.into_inner())
3762 .describe(&owner, &scope_ref);
3763 if !registered
3766 .manifest
3767 .capabilities
3768 .as_ref()
3769 .is_some_and(|caps| {
3770 caps.provides
3771 .iter()
3772 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3773 })
3774 {
3775 if let Some(stamp) = description.stamp.as_mut() {
3776 stamp.attributes.on_behalf_of_agent = None;
3777 }
3778 }
3779 if !registered
3780 .manifest
3781 .capabilities
3782 .as_ref()
3783 .is_some_and(|caps| {
3784 caps.provides
3785 .iter()
3786 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3787 })
3788 {
3789 if let Some(stamp) = description.stamp.as_mut() {
3790 stamp.attributes.flow_version = None;
3791 stamp.attributes.flow_code_hash = None;
3792 stamp.attributes.flow_install_id = None;
3793 }
3794 }
3795 let owner_configured = match &owner {
3796 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3801 _ => false,
3802 };
3803 let response = ModuleControlResponseToModule::ScopeDescribe {
3804 status: description.status,
3805 scope_epoch: description.scope_epoch,
3806 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3807 owner_synced: description.owner_synced,
3808 owner_configured,
3809 scope: description.stamp,
3810 };
3811 Ok(vec![control_response_body_frame(
3812 &frame,
3813 &response,
3814 "ModuleControlResponseToModule::ScopeDescribe",
3815 )?])
3816 }
3817
3818 fn handle_catalog_update(
3819 &self,
3820 connection_id: ConnectionId,
3821 frame: Frame,
3822 provides: Vec<ProviderRole>,
3823 capabilities: Option<CapabilityDeclarations>,
3824 ready: Option<bool>,
3825 ) -> Result<Vec<Frame>, RouterError> {
3826 self.refresh_capability_requirements();
3827 let Some(registration) = self
3828 .registry
3829 .get_module_by_connection(connection_id)
3830 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3831 else {
3832 return Ok(vec![control_error_frame(
3833 &frame,
3834 "not_registered",
3835 "catalog.update requires an active module registration owned by this connection",
3836 )?]);
3837 };
3838
3839 if let Some(message) =
3840 catalog_update_frozen_field_message(®istration.manifest, &provides)
3841 {
3842 return Ok(vec![control_error_frame(
3843 &frame,
3844 "catalog_update_frozen_field",
3845 message,
3846 )?]);
3847 }
3848
3849 let mut candidate = registration.manifest.clone();
3850 candidate.provides = provides.clone();
3851 candidate.capabilities = capabilities
3852 .clone()
3853 .or_else(|| registration.manifest.capabilities.clone());
3854 if let Some(rich_ui) = &candidate.rich_ui {
3855 if let Err(err) = rich_ui.validate() {
3856 return Ok(vec![control_error_frame(
3857 &frame,
3858 "invalid_manifest",
3859 err.to_string(),
3860 )?]);
3861 }
3862 }
3863 if let Err(err) = candidate.validate_capability_grammar() {
3864 return Ok(vec![control_error_frame(
3865 &frame,
3866 "invalid_capability_grammar",
3867 err.to_string(),
3868 )?]);
3869 }
3870
3871 let mut conflicts = self
3874 .capability_evaluator
3875 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3876 if let Some(conflict) = conflicts.first() {
3877 let message = format!(
3878 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3879 conflict.capability, conflict.claimants[0], candidate.module_id
3880 );
3881 for conflict in &mut conflicts {
3882 conflict.source = DuplicateClaimSource::CatalogUpdate;
3883 }
3884 log_duplicate_claim_events(conflicts);
3885 return Ok(vec![control_error_frame(
3886 &frame,
3887 "reserved_capability",
3888 message,
3889 )?]);
3890 }
3891
3892 let updated = self
3893 .registry
3894 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3895 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3896 if updated.is_none() {
3897 return Ok(vec![control_error_frame(
3898 &frame,
3899 "not_registered",
3900 "catalog.update requires an active module registration owned by this connection",
3901 )?]);
3902 }
3903 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3904 log_duplicate_claim_events(
3905 self.capability_evaluator
3906 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3907 );
3908 }
3909 if capability_census_trigger(
3910 registration.manifest.capabilities.as_ref(),
3911 updated
3912 .as_ref()
3913 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3914 ) {
3915 self.enforce_capability_denies();
3916 }
3917 self.refresh_capability_requirements();
3918
3919 let response = ModuleControlResponseToModule::CatalogUpdate {};
3920 control_response_body_frame(
3921 &frame,
3922 &response,
3923 "ModuleControlResponseToModule::CatalogUpdate",
3924 )
3925 .map(|frame| vec![frame])
3926 }
3927
3928 pub(crate) fn with_activity_ledger_config(
3929 mut self,
3930 run_dir: &Path,
3931 incarnation: String,
3932 config: crate::activity::ActivityConfig,
3933 ) -> Self {
3934 self.activity = Some(crate::activity::ActivityLedger::open_config(
3935 run_dir.join("activity.jsonl"),
3936 incarnation,
3937 config,
3938 ));
3939 self.supervisor.set_activity(self.activity.clone());
3940 self
3941 }
3942
3943 pub(crate) async fn configure_activity(
3944 &self,
3945 config: crate::activity::ActivityConfig,
3946 ) -> Result<(), &'static str> {
3947 match &self.activity {
3948 Some(ledger) => ledger.configure(config).await,
3949 None => Ok(()),
3950 }
3951 }
3952
3953 fn cancel_activity_subscription(
3954 &self,
3955 ctx: &RouteCtx,
3956 frame: &Frame,
3957 ) -> Result<bool, RouterError> {
3958 let task = self
3959 .activity_streams
3960 .lock()
3961 .unwrap_or_else(|p| p.into_inner())
3962 .remove(&(ctx.connection_id, frame.header.corr));
3963 let Some(task) = task else {
3964 return Ok(false);
3965 };
3966 task.abort();
3967 let end = Frame::build_with_version(
3968 response_version(frame),
3969 FrameType::StreamEnd,
3970 control_flags(),
3971 0,
3972 0,
3973 frame.header.corr,
3974 Vec::new(),
3975 )
3976 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3977 let sink = ctx.egress.clone();
3978 tokio::spawn(async move {
3979 let _ = sink.send(end).await;
3980 });
3981 Ok(true)
3982 }
3983
3984 async fn handle_activity_read(
3985 &self,
3986 ctx: &RouteCtx,
3987 frame: Frame,
3988 ) -> Result<Vec<Frame>, RouterError> {
3989 use subc_control::{
3990 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3991 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3992 };
3993 #[derive(Deserialize)]
3994 struct Identity {
3995 #[serde(default)]
3996 consumer_identity: Option<ConsumerIdentity>,
3997 }
3998 let principal = match self
4002 .forwarding
4003 .module_id_for_connection(ctx.connection_id)
4004 .map_err(RouterError::Forwarding)?
4005 {
4006 Some(module_id) => Ok(Principal::Reserved { module_id }),
4007 None => {
4008 let identity: Identity = match serde_json::from_slice(&frame.body) {
4009 Ok(identity) => identity,
4010 Err(_) => {
4011 return Ok(vec![control_error_frame(
4012 &frame,
4013 ACTIVITY_MALFORMED,
4014 "malformed activity read",
4015 )?])
4016 }
4017 };
4018 self.route_open_principal(&frame, identity.consumer_identity)?
4019 }
4020 };
4021 let admitted = matches!(principal, Ok(Principal::Direct))
4022 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4023 if !admitted {
4024 return Ok(vec![control_error_frame(
4025 &frame,
4026 ACTIVITY_READ_FORBIDDEN,
4027 "activity reads require an operator or trusted mirror principal",
4028 )?]);
4029 }
4030 let Some(ledger) = &self.activity else {
4031 return Ok(vec![control_error_frame(
4032 &frame,
4033 ACTIVITY_UNAVAILABLE,
4034 "activity ledger unavailable",
4035 )?]);
4036 };
4037 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4038 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4039 {
4040 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4041 Ok(request) => ledger.query(request).await,
4042 Err(_) => Err(ACTIVITY_MALFORMED),
4043 };
4044 return match result {
4045 Ok(reply) => Ok(vec![control_response_body_frame(
4046 &frame,
4047 &reply,
4048 ops::ACTIVITY_QUERY,
4049 )?]),
4050 Err(code) => Ok(vec![control_error_frame(
4051 &frame,
4052 code,
4053 "activity query refused",
4054 )?]),
4055 };
4056 }
4057 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4058 Ok(request) => request,
4059 Err(_) => {
4060 return Ok(vec![control_error_frame(
4061 &frame,
4062 ACTIVITY_MALFORMED,
4063 "malformed activity subscription",
4064 )?])
4065 }
4066 };
4067 let mut subscription = match ledger.subscribe(request.since).await {
4068 Ok(subscription) => subscription,
4069 Err(code) => {
4070 return Ok(vec![control_error_frame(
4071 &frame,
4072 code,
4073 "activity subscription refused",
4074 )?])
4075 }
4076 };
4077 let key = (ctx.connection_id, frame.header.corr);
4078 let sink = ctx.egress.clone();
4079 let mut streams = self
4080 .activity_streams
4081 .lock()
4082 .unwrap_or_else(|p| p.into_inner());
4083 streams.retain(|_, task| !task.is_finished());
4084 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4085 return Ok(vec![control_error_frame(
4086 &frame,
4087 "duplicate_subscription",
4088 "activity subscription correlation id already in use",
4089 )?]);
4090 }
4091 let task = tokio::spawn(async move {
4092 loop {
4093 let entry = if let Some(entry) = subscription.replay.pop_front() {
4094 Some(entry)
4095 } else {
4096 subscription.receiver.recv().await
4097 };
4098 let Some(entry) = entry else {
4099 break;
4100 };
4101 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4102 return;
4103 };
4104 let Ok(next) = Frame::build_with_version(
4105 response_version(&frame),
4106 FrameType::StreamData,
4107 control_flags(),
4108 0,
4109 0,
4110 frame.header.corr,
4111 body,
4112 ) else {
4113 return;
4114 };
4115 if sink.send(next).await.is_err() {
4116 return;
4117 }
4118 }
4119 if let Some(code) = subscription.terminal_code() {
4122 if let Ok(end) =
4123 control_error_frame(&frame, code, "activity subscriber fell behind")
4124 {
4125 let _ = sink.send(end).await;
4126 }
4127 }
4128 });
4129 streams.insert(key, task.abort_handle());
4130 Ok(Vec::new())
4131 }
4132
4133 async fn handle_activity_record(
4134 &self,
4135 ctx: &RouteCtx,
4136 frame: Frame,
4137 ) -> Result<Vec<Frame>, RouterError> {
4138 use subc_control::{
4139 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4140 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4141 };
4142 let result = async {
4143 let module = self
4144 .forwarding
4145 .module_id_for_connection(ctx.connection_id)
4146 .ok()
4147 .flatten()
4148 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4149 let request: ActivityRecordRequest =
4150 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4151 crate::activity::validate(&module, &request)?;
4152 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4153 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4154 let at_ms = std::time::SystemTime::now()
4155 .duration_since(std::time::UNIX_EPOCH)
4156 .unwrap_or_default()
4157 .as_millis() as u64;
4158 ledger.record(module, request, actor, at_ms).await
4159 }
4160 .await;
4161 match result {
4162 Ok(entry) => Ok(vec![control_response_body_frame(
4163 &frame,
4164 &ActivityRecordResponse { seq: entry.seq },
4165 "activity.record",
4166 )?]),
4167 Err(code) => Ok(vec![control_error_frame(
4168 &frame,
4169 code,
4170 "activity recording refused",
4171 )?]),
4172 }
4173 }
4174
4175 fn activity_actor(
4176 &self,
4177 connection: ConnectionId,
4178 module: &str,
4179 on: Option<&subc_control::ActivityOn>,
4180 ) -> Result<subc_control::ActivityActor, &'static str> {
4181 use crate::forwarding::{DataRoute, DataRouteState};
4182 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4183 let principal = |id: &str| Principal::Reserved {
4184 module_id: id.to_owned(),
4185 };
4186 let mut actor = ActivityActor {
4187 principal: principal(module),
4188 scope_owner: None,
4189 agent_id: None,
4190 run_id: None,
4191 flow_id: None,
4192 project_root: None,
4193 workdir: None,
4194 project_id: None,
4195 };
4196 let scope = match on {
4197 None => return Ok(actor),
4198 Some(ActivityOn::Scope(scope)) => {
4199 let selector = subc_protocol::scope::ScopeSelector::new(
4200 principal(module),
4201 scope.scope_ref.clone(),
4202 Some(scope.epoch),
4203 );
4204 self.scopes
4205 .read()
4206 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4207 .attribute(&principal(module), module, &selector, true)
4208 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4209 }
4210 Some(ActivityOn::Route(route)) => {
4211 let DataRoute::Module(DataRouteState::Bound(binding)) = self
4212 .forwarding
4213 .lookup_data_route(connection, route.channel, route.epoch)
4214 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4215 else {
4216 return Err(ACTIVITY_ACTOR_UNKNOWN);
4217 };
4218 if binding.module_id != module {
4219 return Err(ACTIVITY_ACTOR_UNKNOWN);
4220 }
4221 actor.principal = binding.principal.clone();
4222 actor.project_root = binding
4223 .project_root
4224 .as_ref()
4225 .map(|root| root.as_path().to_path_buf());
4226 let Some(bound) = &binding.scope else {
4227 return Ok(actor);
4228 };
4229 let owner = principal(&bound.owner);
4230 let selector = subc_protocol::scope::ScopeSelector::new(
4231 owner.clone(),
4232 bound.scope_ref.clone(),
4233 Some(bound.tag.scope_epoch),
4234 );
4235 let admitted = self
4236 .scopes
4237 .read()
4238 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4239 .attribute(&owner, module, &selector, true)
4240 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4241 if admitted.tag != bound.tag {
4242 return Err(ACTIVITY_ACTOR_UNKNOWN);
4243 }
4244 admitted
4245 }
4246 };
4247 actor.scope_owner = Some(scope.stamp.owner);
4248 actor.agent_id = scope.stamp.attributes.agent_id;
4249 actor.run_id = scope.stamp.attributes.run_id;
4250 actor.flow_id = scope.stamp.attributes.flow_id;
4251 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4252 actor.project_id = scope.stamp.project_id;
4253 Ok(actor)
4254 }
4255
4256 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4257 self.refresh_capability_requirements();
4258 let mut counters = self.counters.snapshot();
4264 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4265 obj.insert(
4266 "activity_unparseable_lines".into(),
4267 ledger.read_errors().into(),
4268 );
4269 obj.insert(
4270 "activity_lifecycle_append_failures".into(),
4271 ledger.lifecycle_failures().into(),
4272 );
4273 obj.insert(
4274 "activity_started_digest_skips".into(),
4275 ledger.lifecycle_digest_skips().into(),
4276 );
4277 }
4278 if let (Ok((connections_with_routes, max)), Some(obj)) = (
4279 self.forwarding.client_route_concentration(),
4280 counters.as_object_mut(),
4281 ) {
4282 obj.insert(
4283 "client_connections_with_routes".into(),
4284 connections_with_routes.into(),
4285 );
4286 obj.insert("max_routes_on_one_connection".into(), max.into());
4287 }
4288 if let (Some(open_breakers), Some(obj)) = (
4294 self.route_bind_breakers.open_snapshot(),
4295 counters.as_object_mut(),
4296 ) {
4297 obj.insert("route_bind_breakers_open".into(), open_breakers);
4298 }
4299 if let (Some(reason), Some(obj)) = (
4304 self.local_endpoint_unavailable.clone(),
4305 counters.as_object_mut(),
4306 ) {
4307 obj.insert("local_endpoint_unavailable".into(), reason.into());
4308 }
4309 let response = ClientControlResponse::ServerDescribe {
4310 protocol_ver: PROTOCOL_VERSION,
4311 subc_ops: subc_ops(),
4312 capabilities: self.describe_capabilities(),
4313 connected_clients: self.connected_clients.count(),
4314 counters: Some(counters),
4315 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4316 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4317 capability_requirements: self.capability_requirement_statuses(),
4318 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4319 config_path: self
4320 .rescan
4321 .as_ref()
4322 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4323 };
4324 Ok(vec![control_response_body_frame(
4325 &frame,
4326 &response,
4327 "ClientControlResponse::ServerDescribe",
4328 )?])
4329 }
4330
4331 fn handle_catalog_list(
4332 &self,
4333 frame: Frame,
4334 module_id: Option<String>,
4335 ) -> Result<Vec<Frame>, RouterError> {
4336 let (generation, modules) = self
4337 .registry
4338 .list_modules_with_nonce_verification()
4339 .map_err(|err| {
4340 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4341 })?;
4342 let entries = modules
4343 .into_iter()
4344 .filter(|(registration, _)| {
4345 module_id
4346 .as_deref()
4347 .map(|wanted| registration.manifest.module_id == wanted)
4348 .unwrap_or(true)
4349 })
4350 .map(|(registration, nonce_verified)| {
4351 let not_ready = self.not_ready_reason(®istration);
4352 let roles = registration.manifest.provides;
4353 CatalogEntry::new(
4354 registration.manifest.module_id,
4355 roles,
4356 registration.control_ops,
4357 )
4358 .with_ready(not_ready.is_none())
4359 .with_not_ready(not_ready)
4360 .with_module_version(Some(registration.manifest.module_version))
4361 .with_capabilities(registration.manifest.capabilities)
4362 .with_self_signals(registration.manifest.self_signals)
4363 .with_events(registration.manifest.events)
4364 .with_operation_delivery(registration.manifest.operation_delivery)
4365 .with_operation_payload(registration.manifest.operation_payload)
4366 .with_flow_operations(registration.manifest.flow_operations)
4367 .with_confirm_operations(registration.manifest.confirm_operations)
4368 .with_rich_ui(registration.manifest.rich_ui)
4369 .with_role_declarations(registration.manifest.role_declarations)
4370 .with_nonce_verified(nonce_verified)
4371 })
4372 .collect();
4373 let response = ClientControlResponse::CatalogList {
4374 generation,
4375 modules: entries,
4376 subc_ops: subc_ops(),
4377 };
4378 Ok(vec![control_response_body_frame(
4379 &frame,
4380 &response,
4381 "ClientControlResponse::CatalogList",
4382 )?])
4383 }
4384
4385 fn attest_route_open_consumer(
4390 &self,
4391 consumer_identity: Option<&ConsumerIdentity>,
4392 ) -> Option<Principal> {
4393 let Some(consumer_identity) = consumer_identity else {
4394 return Some(Principal::Direct);
4395 };
4396 self.supervisor
4397 .spawned_consumer_authorized(
4398 &consumer_identity.module_id,
4399 &consumer_identity.launch_nonce,
4400 )
4401 .then(|| Principal::Reserved {
4402 module_id: consumer_identity.module_id.clone(),
4403 })
4404 }
4405
4406 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4410 let Ok(ClientControlRequest::RouteOpen {
4411 identity,
4412 consumer_identity,
4413 scope,
4414 ..
4415 }) = parse_client_control_request(&frame.body)
4416 else {
4417 return RouteOpenCaller::unknown();
4418 };
4419 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4420 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4421 }
4422
4423 fn route_open_principal(
4424 &self,
4425 frame: &Frame,
4426 consumer_identity: Option<ConsumerIdentity>,
4427 ) -> Result<Result<Principal, Frame>, RouterError> {
4428 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4429 self.route_open_principal_attested(frame, consumer_identity, attested)
4430 }
4431
4432 fn route_open_principal_attested(
4436 &self,
4437 frame: &Frame,
4438 consumer_identity: Option<ConsumerIdentity>,
4439 attested: Option<Principal>,
4440 ) -> Result<Result<Principal, Frame>, RouterError> {
4441 if let Some(principal) = attested {
4442 return Ok(Ok(principal));
4443 }
4444 let Some(consumer_identity) = consumer_identity else {
4445 return Ok(Ok(Principal::Direct));
4446 };
4447
4448 Ok(Err(control_error_frame(
4449 frame,
4450 "bad_consumer_identity",
4451 format!(
4452 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4453 consumer_identity.module_id
4454 ),
4455 )?))
4456 }
4457
4458 #[allow(clippy::too_many_arguments)]
4465 fn route_open_refusal_frame(
4466 &self,
4467 ctx: &RouteCtx,
4468 caller: &RouteOpenCaller,
4469 frame: &Frame,
4470 module_id: &str,
4471 reason: &'static str,
4472 code: &'static str,
4473 message: impl Into<String>,
4474 ) -> Result<Frame, RouterError> {
4475 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4476 control_error_frame(frame, code, message.into())
4477 }
4478
4479 #[allow(clippy::too_many_arguments)]
4488 fn route_bind_rejection(
4489 &self,
4490 ctx: &RouteCtx,
4491 caller: &RouteOpenCaller,
4492 frame: &Frame,
4493 target_module_id: &str,
4494 body: ErrorBody,
4495 daemon_authored: bool,
4496 ) -> Result<Vec<Frame>, RouterError> {
4497 if daemon_authored {
4503 let scope_code = match body.code.as_str() {
4504 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4505 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4506 _ => None,
4507 };
4508 if let Some(code) = scope_code {
4509 self.observe_route_open_refusal(
4510 ctx,
4511 caller,
4512 target_module_id,
4513 "scope_changed_before_commit",
4514 code,
4515 );
4516 return Ok(vec![control_error_body_frame(frame, body)?]);
4517 }
4518 }
4519 self.counters
4520 .increment_route_open_refused("module_rejected");
4521 info!(
4522 target: "control",
4523 code = "module_rejected",
4524 module_code = ?body.code,
4525 module_id = ?target_module_id,
4526 connection_id = ctx.connection_id.get(),
4527 principal = %caller.principal,
4528 scope = %caller.scope,
4529 harness = %caller.harness,
4530 "route.open refused"
4531 );
4532 let refusal = if daemon_authored {
4533 control_error_body_frame(frame, body)?
4534 } else {
4535 control_relayed_error_body_frame(frame, body)?
4536 };
4537 Ok(vec![refusal])
4538 }
4539
4540 #[allow(clippy::too_many_arguments)]
4558 fn route_open_breaker_refusal_frame(
4559 &self,
4560 ctx: &RouteCtx,
4561 caller: &RouteOpenCaller,
4562 frame: &Frame,
4563 module_id: &str,
4564 consecutive_timeouts: u32,
4565 retry_in: Duration,
4566 probe_in_flight: bool,
4567 ) -> Result<Frame, RouterError> {
4568 self.counters
4569 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4570 debug!(
4571 target: "control",
4572 code = "module_timeout",
4573 module_id = ?module_id,
4574 connection_id = ctx.connection_id.get(),
4575 principal = %caller.principal,
4576 scope = %caller.scope,
4577 harness = %caller.harness,
4578 consecutive_timeouts,
4579 retry_in_ms = retry_in.as_millis() as u64,
4580 probe_in_flight,
4581 "route.open refused by open bind-relay breaker"
4582 );
4583 let detail = if probe_in_flight {
4590 "one test route is already being tried; retry once it settles".to_string()
4591 } else {
4592 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4593 };
4594 control_error_frame(
4595 frame,
4596 "module_timeout",
4597 format!(
4598 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4599 timed out in a row); {detail}; its established routes are unaffected"
4600 ),
4601 )
4602 }
4603
4604 fn observe_route_open_refusal(
4617 &self,
4618 ctx: &RouteCtx,
4619 caller: &RouteOpenCaller,
4620 module_id: &str,
4621 reason: &'static str,
4622 code: &'static str,
4623 ) {
4624 self.counters.increment_route_open_refused(code);
4625 info!(
4626 target: "control",
4627 code,
4628 reason,
4629 module_id = ?module_id,
4630 connection_id = ctx.connection_id.get(),
4631 principal = %caller.principal,
4632 scope = %caller.scope,
4633 harness = %caller.harness,
4634 "route.open refused"
4635 );
4636 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4637 self.route_outages.record_not_serving(module_id, reason);
4638 }
4639 }
4640
4641 fn observe_route_open_accept(
4686 &self,
4687 ctx: &RouteCtx,
4688 module_id: &str,
4689 principal: &str,
4690 scope: &str,
4691 ) {
4692 self.route_outages.record_accepted(module_id);
4693 self.counters.increment_route_open_accepted(principal);
4694 info!(
4695 target: "control",
4696 principal,
4697 module_id,
4698 connection_id = ctx.connection_id.get(),
4699 scope = %scope,
4700 "route.open accepted"
4701 );
4702 }
4703
4704 fn supervised_absent_route_open_refusal_frame(
4705 &self,
4706 ctx: &RouteCtx,
4707 caller: &RouteOpenCaller,
4708 frame: &Frame,
4709 module_id: &str,
4710 code: &'static str,
4711 status: &crate::supervise::ModuleStatus,
4712 ) -> Result<Frame, RouterError> {
4713 self.counters.increment_route_open_refused(code);
4714 info!(
4715 target: "control",
4716 code,
4717 reason = "supervised_not_registered",
4718 module_id = ?module_id,
4719 connection_id = ctx.connection_id.get(),
4720 principal = %caller.principal,
4721 scope = %caller.scope,
4722 harness = %caller.harness,
4723 state = %status.state,
4724 enabled = status.enabled,
4725 live = status.live,
4726 "route.open refused"
4727 );
4728 self.route_outages
4732 .record_not_serving(module_id, "supervised_not_registered");
4733 let message = if let Some(pid) = status.waiting_for_previous_pid {
4734 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4735 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4736 })
4737 } else {
4738 format!(
4739 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4740 status.state, status.enabled, status.live
4741 )
4742 };
4743 control_error_frame(frame, code, message)
4744 }
4745
4746 async fn handle_route_open(
4747 &self,
4748 ctx: &RouteCtx,
4749 frame: Frame,
4750 request: RouteOpenRequest,
4751 timings: &mut RouteOpenTimings,
4752 ) -> Result<Vec<Frame>, RouterError> {
4753 let RouteOpenRequest {
4754 target,
4755 mut identity,
4756 consumer_identity,
4757 consumer_capabilities,
4758 role_versions,
4759 admission_facts,
4760 scope,
4761 } = request;
4762 let target_module_id = target_module_id(&target).to_string();
4763 timings.module_id.clone_from(&target_module_id);
4764 if self
4765 .registry
4766 .get_module_by_connection(ctx.connection_id)
4767 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4768 .is_some()
4769 {
4770 return Ok(vec![control_error_frame(
4771 &frame,
4772 "invalid_request",
4773 "module connections cannot open client routes",
4774 )?]);
4775 }
4776 debug!(
4777 connection_id = ctx.connection_id.get(),
4778 corr = frame.header.corr,
4779 module_id = %target_module_id,
4780 "handling route.open"
4781 );
4782
4783 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4788 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4789
4790 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4796 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4797 self.observe_route_open_refusal(
4798 ctx,
4799 &caller,
4800 &target_module_id,
4801 "invalid_role_versions",
4802 error_codes::INVALID_REQUEST,
4803 );
4804 return Ok(vec![control_error_body_frame(
4805 &frame,
4806 ErrorBody {
4807 code: error_codes::INVALID_REQUEST.to_string(),
4808 message: error.to_string(),
4809 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4810 },
4811 )?]);
4812 }
4813
4814 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4815 if status.waiting_for_previous_pid.is_some() {
4816 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4817 ctx,
4818 &caller,
4819 &frame,
4820 &target_module_id,
4821 "module_warming",
4822 &status,
4823 )?]);
4824 }
4825 }
4826
4827 let Some(registration) = self
4846 .registry
4847 .get_module(&target_module_id)
4848 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4849 else {
4850 if let Some((status, warming)) =
4851 self.supervisor_status(&target_module_id, frame.header.corr)?
4852 {
4853 if status.protocol == ModuleProtocol::None {
4860 return Ok(vec![self.route_open_refusal_frame(
4861 ctx,
4862 &caller,
4863 &frame,
4864 &target_module_id,
4865 "protocol_none",
4866 error_codes::MODULE_NO_PROTOCOL,
4867 format!(
4868 "module_id '{target_module_id}' is declared protocol: none; \
4869 it speaks no subc wire and serves no routes"
4870 ),
4871 )?]);
4872 }
4873 let code = if warming {
4874 "module_warming"
4875 } else {
4876 "target_unavailable"
4877 };
4878 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4879 ctx,
4880 &caller,
4881 &frame,
4882 &target_module_id,
4883 code,
4884 &status,
4885 )?]);
4886 }
4887 if let Some(removed_ago_ms) =
4888 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4889 {
4890 return Ok(vec![self.route_open_refusal_frame(
4891 ctx,
4892 &caller,
4893 &frame,
4894 &target_module_id,
4895 "removed",
4896 error_codes::MODULE_REMOVED,
4897 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4898 )?]);
4899 }
4900 return Ok(vec![self.route_open_refusal_frame(
4901 ctx,
4902 &caller,
4903 &frame,
4904 &target_module_id,
4905 "not_registered",
4906 error_codes::UNKNOWN_MODULE,
4907 format!("module_id '{target_module_id}' is not registered"),
4908 )?]);
4909 };
4910
4911 if !registration.ready {
4915 self.counters
4916 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4917 info!(
4918 target: "control",
4919 code = error_codes::MODULE_WARMING,
4920 module_id = ?target_module_id,
4921 connection_id = ctx.connection_id.get(),
4922 principal = %caller.principal,
4923 scope = %caller.scope,
4924 harness = %caller.harness,
4925 reason = "declared_not_ready",
4926 "route.open refused"
4927 );
4928 self.route_outages
4931 .record_not_serving(&target_module_id, "declared_not_ready");
4932 return Ok(vec![control_error_body_frame(
4933 &frame,
4934 ErrorBody {
4935 code: error_codes::MODULE_WARMING.to_string(),
4936 message: format!(
4937 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4938 ),
4939 detail: Some(serde_json::json!({
4940 "reason": "declared_not_ready"
4941 })),
4942 },
4943 )?]);
4944 }
4945
4946 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4965 self.counters
4966 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4967 info!(
4968 target: "control",
4969 code = error_codes::MODULE_WARMING,
4970 module_id = ?target_module_id,
4971 connection_id = ctx.connection_id.get(),
4972 principal = %caller.principal,
4973 scope = %caller.scope,
4974 harness = %caller.harness,
4975 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4976 capability = %capability,
4977 "route.open refused"
4978 );
4979 return Ok(vec![control_error_body_frame(
4980 &frame,
4981 ErrorBody {
4982 code: error_codes::MODULE_WARMING.to_string(),
4983 message: format!(
4984 "module_id '{target_module_id}' requires capability '{capability}', \
4985 which no registered module provides; retry"
4986 ),
4987 detail: Some(serde_json::json!({
4988 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4989 "capability": capability,
4990 })),
4991 },
4992 )?]);
4993 }
4994
4995 if !target_has_required_role(&target, ®istration.manifest.provides) {
4996 return Ok(vec![self.route_open_refusal_frame(
4997 ctx,
4998 &caller,
4999 &frame,
5000 &target_module_id,
5001 "role_not_provided",
5002 "target_unavailable",
5003 format!("module_id '{target_module_id}' does not provide the requested target"),
5004 )?]);
5005 }
5006
5007 if registration.state != ChannelState::Active {
5008 return Ok(vec![self.route_open_refusal_frame(
5009 ctx,
5010 &caller,
5011 &frame,
5012 &target_module_id,
5013 "registration_not_active",
5014 "target_unavailable",
5015 format!("module_id '{target_module_id}' is not active"),
5016 )?]);
5017 }
5018
5019 if self
5020 .forwarding
5021 .module_is_draining(&target_module_id)
5022 .map_err(RouterError::Forwarding)?
5023 {
5024 return Ok(vec![self.route_open_refusal_frame(
5025 ctx,
5026 &caller,
5027 &frame,
5028 &target_module_id,
5029 "reloading",
5030 "module_reloading",
5031 format!("module_id '{target_module_id}' is reloading"),
5032 )?]);
5033 }
5034
5035 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5036 process_liveness.process_live(&target_module_id) == Some(false)
5037 }) {
5038 if process_liveness.process_replacing(&target_module_id) {
5045 return Ok(vec![self.route_open_refusal_frame(
5046 ctx,
5047 &caller,
5048 &frame,
5049 &target_module_id,
5050 "reloading",
5051 "module_reloading",
5052 format!("module_id '{target_module_id}' is reloading"),
5053 )?]);
5054 }
5055 return Ok(vec![self.route_open_refusal_frame(
5056 ctx,
5057 &caller,
5058 &frame,
5059 &target_module_id,
5060 "supervisor_not_live",
5061 "target_unavailable",
5062 format!("module_id '{target_module_id}' is not live"),
5063 )?]);
5064 }
5065
5066 if !self
5067 .forwarding
5068 .has_live_module_connection(&target_module_id)
5069 .map_err(RouterError::Forwarding)?
5070 {
5071 return Ok(vec![self.route_open_refusal_frame(
5072 ctx,
5073 &caller,
5074 &frame,
5075 &target_module_id,
5076 "no_forwarding_connection",
5077 "target_unavailable",
5078 format!("module_id '{target_module_id}' has no live forwarding connection"),
5079 )?]);
5080 }
5081
5082 if let Some(error) =
5083 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5084 {
5085 self.observe_route_open_refusal(
5086 ctx,
5087 &caller,
5088 &target_module_id,
5089 "op_not_allowed",
5090 "op_not_allowed",
5091 );
5092 return Ok(vec![error]);
5093 }
5094
5095 let principal =
5096 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5097 Ok(principal) => principal,
5098 Err(error) => {
5099 self.observe_route_open_refusal(
5100 ctx,
5101 &caller,
5102 &target_module_id,
5103 "bad_consumer_identity",
5104 "bad_consumer_identity",
5105 );
5106 return Ok(vec![error]);
5107 }
5108 };
5109 timings.principal = principal_label(&principal);
5110
5111 let build_class = identity.build_class.take().filter(|build_class| {
5114 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5115 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5116 } else {
5117 if self
5118 .build_class_warned
5119 .lock()
5120 .unwrap_or_else(|p| p.into_inner())
5121 .insert(ctx.connection_id)
5122 {
5123 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5124 "ignoring build_class from a non-Callosum opener");
5125 }
5126 false
5127 }
5128 });
5129
5130 if let Principal::Reserved {
5134 module_id: opening_module_id,
5135 } = &principal
5136 {
5137 if let Some(opening_registration) = self
5138 .registry
5139 .get_module(opening_module_id)
5140 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5141 {
5142 if let Some(capability) =
5143 denied_capability(&opening_registration.manifest, ®istration.manifest)
5144 {
5145 warn!(
5146 opening_module_id,
5147 target_module_id,
5148 capability,
5149 "refusing route.open because an attested capability deny edge matches"
5150 );
5151 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5152 &frame,
5153 &target_module_id,
5154 "capability_deny_edge",
5155 "capability_forbidden",
5156 format!(
5157 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5158 ),
5159 )?]);
5160 }
5161 }
5162 }
5163
5164 if admission_facts.is_some() {
5165 let carrier_matches = matches!(
5166 &principal,
5167 Principal::Reserved { module_id }
5168 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5169 );
5170 if !carrier_matches {
5171 return Ok(vec![self.route_open_refusal_frame(
5172 ctx,
5173 &caller,
5174 &frame,
5175 &target_module_id,
5176 "admission_facts_carrier_not_permitted",
5177 "admission_facts_not_permitted",
5178 "admission facts may only be carried by the configured reserved module",
5179 )?]);
5180 }
5181
5182 let target_allowed = self
5183 .admission_facts_targets
5184 .as_ref()
5185 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5186 if !target_allowed {
5187 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5188 &frame,
5189 &target_module_id,
5190 "admission_facts_target_not_listed",
5191 "admission_facts_target_not_allowed",
5192 format!(
5193 "admission facts are not permitted for target module_id '{target_module_id}'"
5194 ),
5195 )?]);
5196 }
5197
5198 }
5202
5203 let (bound_scope, mut scope_stamp) = match &scope {
5209 None => (None, None),
5210 Some(selector) => {
5211 let owner_configured = match &selector.owner {
5212 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5218 _ => false,
5219 };
5220 let mut scopes = self
5221 .scopes
5222 .write()
5223 .unwrap_or_else(|poisoned| poisoned.into_inner());
5224 let admitted = scopes
5225 .admit(&principal, &target_module_id, selector, owner_configured)
5226 .and_then(|admission| {
5227 crate::scopes::check_target_flow_support(
5228 &admission.stamp,
5229 &target_module_id,
5230 registration.manifest.capabilities.as_ref(),
5231 )?;
5232 crate::scopes::check_target_agent_run_support(
5233 &admission.stamp,
5234 &target_module_id,
5235 registration.manifest.capabilities.as_ref(),
5236 )?;
5237 let bound = scopes.reserve_bind(
5238 &admission,
5239 &principal,
5240 ctx.connection_id,
5241 selector.hold_presence,
5242 );
5243 Ok((admission, bound))
5244 });
5245 match admitted {
5246 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5247 Err(refusal) => {
5248 return Ok(vec![self.route_open_refusal_frame(
5249 ctx,
5250 &caller,
5251 &frame,
5252 &target_module_id,
5253 refusal.code,
5254 refusal.code,
5255 refusal.message,
5256 )?]);
5257 }
5258 }
5259 }
5260 };
5261 if let Some(stamp) = &scope_stamp {
5264 caller.scope = route_open_scope_label(stamp);
5265 }
5266
5267 timings.enter(RouteOpenPhase::ProjectRoot);
5289 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
5290 Ok(project_root) => project_root,
5291 Err(err) => {
5292 return Ok(vec![control_error_frame(
5293 &frame,
5294 "invalid_project_root",
5295 err.to_string(),
5296 )?])
5297 }
5298 };
5299 timings.enter(RouteOpenPhase::Admission);
5300 identity.project_root = project_root.as_path().to_path_buf();
5301
5302 let _concurrency_guard = match self
5313 .route_bind_concurrency
5314 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5315 {
5316 Ok(guard) => guard,
5317 Err(in_flight) => {
5318 return Ok(vec![self.route_open_target_capacity_refusal(
5319 ctx,
5320 &caller,
5321 &frame,
5322 &target_module_id,
5323 in_flight,
5324 )?]);
5325 }
5326 };
5327
5328 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5331 RouteBindAdmission::Admitted { guard, probe } => {
5332 if probe {
5333 info!(
5334 module_id = %target_module_id,
5335 connection_id = ctx.connection_id.get(),
5336 "route.bind breaker half-open: admitting one probe"
5337 );
5338 }
5339 guard
5340 }
5341 RouteBindAdmission::Refused {
5342 consecutive_timeouts,
5343 retry_in,
5344 probe_in_flight,
5345 } => {
5346 return Ok(vec![self.route_open_breaker_refusal_frame(
5347 ctx,
5348 &caller,
5349 &frame,
5350 &target_module_id,
5351 consecutive_timeouts,
5352 retry_in,
5353 probe_in_flight,
5354 )?]);
5355 }
5356 };
5357
5358 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5363 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5364 let operator_app = self
5365 .operator_apps
5366 .lock()
5367 .unwrap_or_else(|p| p.into_inner())
5368 .get(&ctx.connection_id)
5369 .cloned();
5370 timings.enter(RouteOpenPhase::EgressReservation);
5371 let pending = match self
5372 .forwarding
5373 .begin_route_bind_relay_for(
5374 ctx.connection_id,
5375 ctx.egress.clone(),
5376 response_version(&frame),
5377 frame.header.corr,
5378 &target_module_id,
5379 principal.clone(),
5380 bound_scope,
5381 Some(project_root),
5382 identity.harness.clone(),
5383 build_class,
5384 operator_app.clone(),
5385 relay_deadline,
5386 )
5387 .await
5388 {
5389 Ok(pending) => pending,
5390 Err(err) => {
5391 return Ok(vec![self.route_open_refusal_frame(
5392 ctx,
5393 &caller,
5394 &frame,
5395 &target_module_id,
5396 "relay_reservation_failed",
5397 forwarding_error_code(&err),
5398 err.to_string(),
5399 )?])
5400 }
5401 };
5402 timings.enter(RouteOpenPhase::Admission);
5403 let crate::forwarding::PendingRouteBindRelay {
5404 endpoint,
5405 module_sink,
5406 negotiated_ver,
5407 client_channel,
5408 client_epoch,
5409 module_channel,
5410 module_epoch,
5411 corr: relay_corr,
5412 receiver,
5413 } = pending;
5414 let mut reservation =
5415 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5416
5417 let relay_registration = self
5422 .registry
5423 .get_module_by_connection(endpoint.connection_id)
5424 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5425 let relay_capabilities = relay_registration
5426 .as_ref()
5427 .and_then(|registration| registration.manifest.capabilities.as_ref());
5428 if let Some(selector) = &scope {
5429 if let Err(refusal) = self
5432 .scopes
5433 .read()
5434 .unwrap_or_else(|poisoned| poisoned.into_inner())
5435 .check_bind_proof(&principal, selector)
5436 {
5437 reservation.release_and_disarm();
5438 return Ok(vec![self.route_open_refusal_frame(
5439 ctx,
5440 &caller,
5441 &frame,
5442 &target_module_id,
5443 refusal.code,
5444 refusal.code,
5445 refusal.message,
5446 )?]);
5447 }
5448 }
5449 if let Some(stamp) = scope_stamp
5450 .as_ref()
5451 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5452 {
5453 if let Err(refusal) = crate::scopes::check_target_flow_support(
5454 stamp,
5455 &target_module_id,
5456 relay_capabilities,
5457 )
5458 .and_then(|()| {
5459 crate::scopes::check_target_agent_run_support(
5460 stamp,
5461 &target_module_id,
5462 relay_capabilities,
5463 )
5464 }) {
5465 reservation.release_and_disarm();
5466 return Ok(vec![self.route_open_refusal_frame(
5467 ctx,
5468 &caller,
5469 &frame,
5470 &target_module_id,
5471 refusal.code,
5472 refusal.code,
5473 refusal.message,
5474 )?]);
5475 }
5476 }
5477
5478 if !relay_capabilities.is_some_and(|caps| {
5481 caps.provides
5482 .iter()
5483 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5484 }) {
5485 if let Some(stamp) = scope_stamp.as_mut() {
5486 stamp.workdir = None;
5487 stamp.project_id = None;
5488 }
5489 }
5490 let worker_run = scope_stamp
5495 .as_ref()
5496 .is_some_and(crate::scopes::is_worker_run)
5497 && relay_capabilities.is_some_and(|caps| {
5498 caps.provides
5499 .iter()
5500 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5501 });
5502 if !worker_run
5503 && !relay_capabilities.is_some_and(|caps| {
5504 caps.provides
5505 .iter()
5506 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5507 })
5508 {
5509 if let Some(stamp) = scope_stamp.as_mut() {
5510 stamp.attributes.on_behalf_of_agent = None;
5511 }
5512 }
5513 if !relay_capabilities.is_some_and(|caps| {
5514 caps.provides
5515 .iter()
5516 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5517 }) {
5518 if let Some(stamp) = scope_stamp.as_mut() {
5519 stamp.attributes.flow_version = None;
5520 stamp.attributes.flow_code_hash = None;
5521 stamp.attributes.flow_install_id = None;
5522 }
5523 }
5524 let call_mode = if worker_run
5525 || relay_capabilities
5526 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5527 {
5528 Some(derive_call_mode(scope_stamp.as_ref()))
5529 } else {
5530 if let Some(stamp) = scope_stamp.as_mut() {
5531 stamp.expires_at_ms = None;
5532 }
5533 None
5534 };
5535
5536 debug!(
5537 connection_id = ctx.connection_id.get(),
5538 client_channel,
5539 client_epoch,
5540 module_channel,
5541 module_epoch,
5542 "reserved route handle pair"
5543 );
5544 let route_principal_label = principal_label(&principal);
5547 let relay = ModuleControlRequest::RouteBind {
5548 route_channel: module_channel,
5549 epoch: module_epoch,
5550 target,
5551 identity,
5552 principal: Some(principal),
5553 consumer_capabilities,
5554 role_versions,
5555 admission_facts,
5556 scope: scope_stamp,
5557 call_mode,
5558 };
5559 let peer = RouteBindPeer::new()
5560 .with_observed_peer(
5561 relay_capabilities
5562 .is_some_and(|caps| {
5563 caps.provides
5564 .iter()
5565 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5566 })
5567 .then(|| {
5568 self.observed_peers
5569 .lock()
5570 .unwrap_or_else(|p| p.into_inner())
5571 .get(&ctx.connection_id)
5572 .cloned()
5573 .unwrap_or_else(ObservedPeer::unobserved)
5574 }),
5575 )
5576 .with_operator_app(
5577 self.connection_operator_app(operator_app.as_ref())
5578 .filter(|_| {
5579 relay_capabilities.is_some_and(|caps| {
5580 caps.provides
5581 .iter()
5582 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5583 })
5584 }),
5585 );
5586 let relay_body = encode_route_bind(&relay, &peer).map_err(|err| {
5587 RouterError::backend(
5588 0,
5589 frame.header.corr,
5590 format!("failed to encode route.bind request: {err}"),
5591 )
5592 })?;
5593 let relay_frame = Frame::build_with_version(
5594 negotiated_ver,
5595 FrameType::Request,
5596 control_flags(),
5597 0,
5598 0,
5599 relay_corr,
5600 relay_body,
5601 )
5602 .map_err(RouterError::FrameBuild)?;
5603
5604 timings.enter(RouteOpenPhase::BindSend);
5605 let sent = module_sink.send(relay_frame).await;
5606 timings.enter(RouteOpenPhase::Admission);
5607 if let Err(err) = sent {
5608 reservation.release_and_disarm();
5609 return Ok(vec![self.route_open_refusal_frame(
5610 ctx,
5611 &caller,
5612 &frame,
5613 &target_module_id,
5614 "relay_send_failed",
5615 "target_unavailable",
5616 err.to_string(),
5617 )?]);
5618 }
5619
5620 #[cfg(test)]
5621 if let Some(hold) = &self.route_bind_relay_sent_hold {
5622 hold.wait().await;
5626 hold.wait().await;
5627 }
5628
5629 if self
5634 .forwarding
5635 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5636 .map_err(RouterError::Forwarding)?
5637 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5638 {
5639 self.send_abandoned_route_bind_goodbye(
5640 &module_sink,
5641 negotiated_ver,
5642 module_channel,
5643 module_epoch,
5644 );
5645 }
5646
5647 timings.enter(RouteOpenPhase::BindAckWait);
5648 let outcome = timeout_at(relay_deadline, receiver).await;
5649 timings.enter(RouteOpenPhase::Admission);
5650 match outcome {
5651 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5652 reservation.disarm();
5653 if breaker.record_accepted() {
5654 info!(
5655 module_id = %target_module_id,
5656 "route.bind breaker closed: the probe was accepted"
5657 );
5658 }
5659 self.observe_route_open_accept(
5660 ctx,
5661 &target_module_id,
5662 &route_principal_label,
5663 &caller.scope,
5664 );
5665 Ok(Vec::new())
5666 }
5667 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5668 reservation.release_and_disarm();
5669 breaker.record_inconclusive();
5673 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5674 }
5675 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5676 reservation.release_and_disarm();
5677 breaker.record_inconclusive();
5678 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5679 }
5680 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5681 reservation.release_and_disarm();
5682 breaker.record_inconclusive();
5683 tracing::warn!(
5690 module_id = %target_module_id,
5691 "route.bind relay abandoned: {message}"
5692 );
5693 Ok(vec![self.route_open_refusal_frame(
5694 ctx,
5695 &caller,
5696 &frame,
5697 &target_module_id,
5698 "relay_abandoned",
5699 "target_unavailable",
5700 message,
5701 )?])
5702 }
5703 Ok(Err(_)) => {
5704 reservation.release_and_disarm();
5705 breaker.record_inconclusive();
5706 Ok(vec![self.route_open_refusal_frame(
5707 ctx,
5708 &caller,
5709 &frame,
5710 &target_module_id,
5711 "relay_waiter_canceled",
5712 "target_unavailable",
5713 "route.bind relay waiter was canceled before the module responded",
5714 )?])
5715 }
5716 Err(_) => {
5717 reservation.release_and_disarm();
5718 if let Some(opened) = breaker.record_timeout(
5722 self.route_bind_breaker_threshold,
5723 self.route_bind_breaker_cooldown,
5724 ) {
5725 warn!(
5726 module_id = %target_module_id,
5727 consecutive_timeouts = opened.consecutive_timeouts,
5728 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5729 reopened_after_probe = opened.reopened_after_probe,
5730 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5731 );
5732 }
5733 tracing::warn!(
5741 module_id = %target_module_id,
5742 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5743 "route.bind relay timed out: module did not ack within budget"
5744 );
5745 Ok(vec![self.route_open_refusal_frame(
5746 ctx,
5747 &caller,
5748 &frame,
5749 &target_module_id,
5750 "relay_timed_out",
5751 "module_timeout",
5752 format!(
5753 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5754 route_bind_relay_timeout
5755 ),
5756 )?])
5757 }
5758 }
5759 }
5760
5761 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5762 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5763 snapshot: self.supervisor.spawn_snapshot(),
5764 };
5765 Ok(vec![control_response_body_frame(
5766 &frame,
5767 &response,
5768 "ClientControlResponse::SupervisorSpawnSnapshot",
5769 )?])
5770 }
5771
5772 fn handle_supervisor_spawn_subscribe(
5773 &self,
5774 ctx: &RouteCtx,
5775 frame: Frame,
5776 since: Option<SpawnCursor>,
5777 ) -> Result<Vec<Frame>, RouterError> {
5778 match self.supervisor.subscribe_spawns(
5779 ctx.connection_id,
5780 frame.header.corr,
5781 response_version(&frame),
5782 since,
5783 ctx.egress.clone(),
5784 ) {
5785 Ok(()) => Ok(Vec::new()),
5786 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5787 Ok(vec![control_error_body_frame(
5788 &frame,
5789 ErrorBody {
5790 code: "spawn_cursor_incarnation_mismatch".to_string(),
5791 message: "spawn cursor belongs to a different daemon incarnation"
5792 .to_string(),
5793 detail: Some(serde_json::json!({
5794 "current_daemon_incarnation": current
5795 })),
5796 },
5797 )?])
5798 }
5799 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5800 &frame,
5801 ErrorBody {
5802 code: "spawn_cursor_too_old".to_string(),
5803 message: "spawn cursor predates the retained event ring".to_string(),
5804 detail: Some(serde_json::json!({
5805 "oldest_retained_cursor": oldest
5806 })),
5807 },
5808 )?]),
5809 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5810 0,
5811 frame.header.corr,
5812 format!("failed to open supervisor spawn subscription: {error}"),
5813 )),
5814 }
5815 }
5816
5817 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5818 let generation = self
5819 .registry
5820 .generation()
5821 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5822 let mut modules = Vec::new();
5823 for module in self.supervisor.list() {
5824 let status = module.status_for_control("list").map_err(|err| {
5825 RouterError::backend(
5826 0,
5827 frame.header.corr,
5828 format!("failed to read supervisor status: {err}"),
5829 )
5830 })?;
5831 let (configured, _) = module.configuration().map_err(|err| {
5832 RouterError::backend(
5833 0,
5834 frame.header.corr,
5835 format!("failed to read module configuration: {err}"),
5836 )
5837 })?;
5838 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5839 RouterError::backend(
5840 0,
5841 frame.header.corr,
5842 format!("failed to read retry time: {err}"),
5843 )
5844 })?;
5845 let image = module.running_image_agreement().await;
5847 let resources = Some(module.child_resource_usage());
5850 let mut verdict =
5851 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5852 verdict.spec = Some(launch_spec_agreement(
5853 &configured,
5854 status.process_alive,
5855 status.launch_spec_digest.as_ref(),
5856 ));
5857 let pending_reload = Some(verdict);
5858 let configured_enabled = self
5859 .capability_evaluator
5860 .configured_enabled(&status.module_id);
5861 modules.push(
5864 SupervisorEntry::new(
5865 status.module_id,
5866 status.state.to_string(),
5867 status.enabled,
5868 status.live,
5869 status.health.status,
5870 )
5871 .with_configured_enabled(configured_enabled)
5872 .with_next_retry_at_ms(next_retry_at_ms)
5873 .with_launch_nonce_env(Some(!cfg!(unix)))
5874 .with_launch_nonce_source(status.launch_nonce_source)
5875 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5876 .with_protocol(status.protocol)
5877 .with_pending_reload(pending_reload)
5878 .with_last_probe_ms(status.health.last_probe_ms)
5879 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5880 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5881 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5882 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5883 .with_restart_count(Some(status.restart_count))
5884 .with_max_restarts(Some(status.max_restarts))
5885 .with_lifetime_restarts(Some(status.lifetime_restarts))
5886 .with_spawn_generation(Some(status.spawn_generation))
5887 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5888 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5889 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5890 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5891 .with_resources(resources),
5892 );
5893 }
5894 let response = ClientControlResponse::SupervisorList {
5895 generation,
5896 modules,
5897 };
5898 Ok(vec![control_response_body_frame(
5899 &frame,
5900 &response,
5901 "ClientControlResponse::SupervisorList",
5902 )?])
5903 }
5904
5905 fn handle_supervisor_stderr_tail(
5906 &self,
5907 frame: Frame,
5908 module_id: String,
5909 max_lines: Option<u32>,
5910 max_bytes: Option<u32>,
5911 ) -> Result<Vec<Frame>, RouterError> {
5912 let Some(module) = self.supervisor.get(&module_id) else {
5913 return Ok(vec![control_error_frame(
5914 &frame,
5915 "unknown_module",
5916 format!("module_id '{module_id}' is not supervised"),
5917 )?]);
5918 };
5919
5920 let snapshot = module.stderr_tail(
5921 max_lines.map(|value| value as usize),
5922 max_bytes.map(|value| value as usize),
5923 );
5924
5925 let response = ClientControlResponse::SupervisorStderrTail {
5926 module_id,
5927 tail: StderrTail {
5928 capture: match snapshot.capture {
5929 CaptureState::Captured => StderrCaptureState::Captured,
5930 CaptureState::Incomplete { reason } => {
5931 StderrCaptureState::Incomplete { reason }
5932 }
5933 CaptureState::NotCaptured { reason } => {
5934 StderrCaptureState::NotCaptured { reason }
5935 }
5936 },
5937 entries: snapshot
5938 .entries
5939 .into_iter()
5940 .map(|entry| match entry {
5941 TailEntry::Line {
5942 text,
5943 truncated,
5944 at_ms,
5945 } => StderrTailEntry::Line {
5946 text,
5947 truncated,
5948 at_ms,
5949 },
5950 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5951 })
5952 .collect(),
5953 dropped_lines: snapshot.dropped_lines,
5954 },
5955 };
5956 Ok(vec![control_response_body_frame(
5957 &frame,
5958 &response,
5959 "ClientControlResponse::SupervisorStderrTail",
5960 )?])
5961 }
5962
5963 async fn handle_supervisor_terminals(
5964 &self,
5965 frame: Frame,
5966 module_id: String,
5967 ) -> Result<Vec<Frame>, RouterError> {
5968 let Some(module) = self.supervisor.get(&module_id) else {
5969 return Ok(vec![control_error_frame(
5970 &frame,
5971 "unknown_module",
5972 format!("module_id '{module_id}' is not supervised"),
5973 )?]);
5974 };
5975
5976 let terminals = module
5979 .read_durable_terminal_history()
5980 .await
5981 .map_err(|error| {
5982 RouterError::backend(
5983 0,
5984 frame.header.corr,
5985 format!("failed to read terminal history: {error}"),
5986 )
5987 })?;
5988 let response = ClientControlResponse::SupervisorTerminals {
5989 module_id,
5990 terminals,
5991 };
5992 Ok(vec![control_response_body_frame(
5993 &frame,
5994 &response,
5995 "ClientControlResponse::SupervisorTerminals",
5996 )?])
5997 }
5998
5999 fn handle_supervisor_routes(
6000 &self,
6001 frame: Frame,
6002 module_id: Option<String>,
6003 ) -> Result<Vec<Frame>, RouterError> {
6004 let modules = self
6005 .forwarding
6006 .route_census(module_id.as_deref())
6007 .map_err(RouterError::Forwarding)?
6008 .into_iter()
6009 .map(|(module_id, routes)| SupervisorRouteModule {
6010 module_id,
6011 routes: routes
6012 .into_iter()
6013 .map(|route| SupervisorRoute {
6014 consumer: match route.principal {
6015 Principal::Reserved { module_id } => {
6016 SupervisorRouteConsumer::Reserved { module_id }
6017 }
6018 Principal::Direct | Principal::Unverified => {
6019 SupervisorRouteConsumer::Direct {
6020 connection_id: route.goodbye_target.connection_id.get(),
6021 }
6022 }
6023 },
6024 age_ms: Instant::now()
6025 .saturating_duration_since(route.bound_at)
6026 .as_millis()
6027 .try_into()
6028 .unwrap_or(u64::MAX),
6029 draining: route.draining,
6030 drain_reason: route.drain_reason,
6031 })
6032 .collect(),
6033 })
6034 .collect();
6035 let response = ClientControlResponse::SupervisorRoutes { modules };
6036 Ok(vec![control_response_body_frame(
6037 &frame,
6038 &response,
6039 "ClientControlResponse::SupervisorRoutes",
6040 )?])
6041 }
6042
6043 async fn handle_supervisor_provenance(
6044 &self,
6045 frame: Frame,
6046 module_id: Option<String>,
6047 ) -> Result<Vec<Frame>, RouterError> {
6048 let mut selected = if let Some(module_id) = module_id {
6049 let Some(module) = self.supervisor.get(&module_id) else {
6050 return Ok(vec![control_error_frame(
6051 &frame,
6052 "unknown_module",
6053 format!("module_id '{module_id}' is not supervised"),
6054 )?]);
6055 };
6056 vec![module]
6057 } else {
6058 self.supervisor.list()
6059 };
6060
6061 let mut modules = Vec::with_capacity(selected.len());
6062 for module in selected.drain(..) {
6063 let (status, observed_image) = module
6064 .status_and_running_image_agreement()
6065 .await
6066 .map_err(|err| {
6067 RouterError::backend(
6068 0,
6069 frame.header.corr,
6070 format!("failed to read supervisor status: {err}"),
6071 )
6072 })?;
6073 let module_declared = self
6074 .registry
6075 .get_module(&status.module_id)
6076 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6077 .and_then(|registration| registration.manifest.provenance)
6078 .map(|build| ModuleDeclaredProvenance::Reported { build })
6079 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6080 #[cfg(test)]
6081 let running_image = match &self.provenance_probe_override {
6082 Some(result) => result.clone(),
6083 None => observed_image,
6084 };
6085 #[cfg(not(test))]
6086 let running_image = observed_image;
6087 modules.push(SupervisorModuleProvenance {
6088 module_id: status.module_id,
6089 module_declared,
6090 daemon_observed: SupervisorObservedProcess {
6091 launch_nonce_source: status.launch_nonce_source,
6092 pid: status.pid,
6093 spawned_at_ms: status.spawned_at_ms,
6094 spawned_from: status.spawned_from,
6095 running_image,
6096 },
6097 });
6098 }
6099 let daemon = SupervisorDaemonProvenance {
6100 daemon_build: self.daemon_provenance.build.clone(),
6101 daemon_observed: DaemonObservedProcess {
6102 pid: self.daemon_provenance.pid,
6103 started_at_ms: self
6104 .daemon_provenance
6105 .start_clock
6106 .map(|clock| clock.started_at_ms())
6107 .or(self.daemon_provenance.started_at_ms),
6108 running_image: self
6109 .daemon_provenance
6110 .probe
6111 .observe(
6112 self.daemon_provenance.pid,
6113 self.daemon_provenance.executable_path.as_deref(),
6114 self.daemon_provenance.executable_identity.clone(),
6115 self.daemon_provenance.process_start_time,
6116 )
6117 .await,
6118 },
6119 };
6120 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6121 Ok(vec![control_response_body_frame(
6122 &frame,
6123 &response,
6124 "ClientControlResponse::SupervisorProvenance",
6125 )?])
6126 }
6127
6128 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6129 self.refresh_capability_requirements();
6130 let generation = self
6131 .registry
6132 .generation()
6133 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6134 let modules = self
6135 .supervisor
6136 .list()
6137 .into_iter()
6138 .map(|module| {
6139 let status = module.status_for_control("health").map_err(|err| {
6140 RouterError::backend(
6141 0,
6142 frame.header.corr,
6143 format!("failed to read supervisor health: {err}"),
6144 )
6145 })?;
6146 let module_id = status.module_id;
6147 let capability_detail = self
6148 .capability_evaluator
6149 .required_problem_detail(&module_id);
6150 Ok(SupervisorHealthEntry {
6151 module_id,
6152 status: status.health.status,
6153 detail: append_capability_problem_detail(
6154 status.health.detail,
6155 capability_detail,
6156 ),
6157 metrics: status.health.metrics,
6158 consecutive_failures: status.health.consecutive_failures,
6159 late_answer_count: status.health.late_answer_count,
6160 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6161 last_action: status.health.last_action,
6162 last_action_ms: status.health.last_action_ms,
6163 last_probe_ms: status.health.last_probe_ms,
6164 })
6165 })
6166 .collect::<Result<Vec<_>, RouterError>>()?;
6167 let response = ClientControlResponse::SupervisorHealth {
6168 generation,
6169 modules,
6170 };
6171 Ok(vec![control_response_body_frame(
6172 &frame,
6173 &response,
6174 "ClientControlResponse::SupervisorHealth",
6175 )?])
6176 }
6177
6178 async fn handle_supervisor_restart(
6179 &self,
6180 frame: Frame,
6181 module_id: String,
6182 drain_timeout_ms: Option<u64>,
6183 requester: Principal,
6184 ) -> Result<Vec<Frame>, RouterError> {
6185 let operation_lock = self.supervisor.operation_lock();
6186 let _operation_guard = operation_lock.lock().await;
6187 let Some(module) = self.supervisor.get(&module_id) else {
6188 return Ok(vec![control_error_frame(
6189 &frame,
6190 "unknown_module",
6191 format!("module_id '{module_id}' is not supervised"),
6192 )?]);
6193 };
6194
6195 self.route_outages.mark_operator_action(&module_id);
6196 if let Err(err) = module
6197 .restart_requested(drain_timeout_ms, Some(requester))
6198 .await
6199 {
6200 self.route_outages
6201 .operator_action_ended_unrefused(&module_id);
6202 let (code, message) = match err {
6203 crate::supervise::SuperviseError::Disabled { .. } => {
6204 ("module_disabled", err.to_string())
6205 }
6206 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6207 ("swap_in_progress", err.to_string())
6208 }
6209 _ => (
6210 "target_unavailable",
6211 format!("failed to restart module_id '{module_id}': {err}"),
6212 ),
6213 };
6214 return Ok(vec![control_error_frame(&frame, code, message)?]);
6215 }
6216
6217 let response = ClientControlResponse::SupervisorAck {
6218 module_id,
6219 applied: true,
6220 };
6221 Ok(vec![control_response_body_frame(
6222 &frame,
6223 &response,
6224 "ClientControlResponse::SupervisorAck",
6225 )?])
6226 }
6227
6228 async fn handle_supervisor_swap(
6232 &self,
6233 frame: Frame,
6234 module_id: String,
6235 ready_timeout_ms: Option<u64>,
6236 requester: Principal,
6237 ) -> Result<Vec<Frame>, RouterError> {
6238 let module = {
6245 let operation_lock = self.supervisor.operation_lock();
6246 let _operation_guard = operation_lock.lock().await;
6247 self.supervisor.get(&module_id)
6248 };
6249 let Some(module) = module else {
6250 return Ok(vec![control_error_frame(
6251 &frame,
6252 "unknown_module",
6253 format!("module_id '{module_id}' is not supervised"),
6254 )?]);
6255 };
6256
6257 self.route_outages.mark_operator_action(&module_id);
6258 if let Err(err) = module
6259 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6260 .await
6261 {
6262 self.route_outages
6263 .operator_action_ended_unrefused(&module_id);
6264 use crate::supervise::SuperviseError;
6265 let message = err.to_string();
6266 let error = match err {
6267 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6268 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6269 code: "swap_refused".to_string(),
6270 message,
6271 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6272 },
6273 SuperviseError::SwapFailed {
6274 arm,
6275 candidate_exit,
6276 ..
6277 } => ErrorBody {
6278 code: "swap_failed".to_string(),
6279 message,
6280 detail: Some(serde_json::json!({
6281 "arm": arm.as_str(),
6282 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6283 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6284 })),
6285 },
6286 _ => ErrorBody::new(
6287 "target_unavailable",
6288 format!("failed to swap module_id '{module_id}': {message}"),
6289 ),
6290 };
6291 return Ok(vec![control_error_body_frame(&frame, error)?]);
6292 }
6293 self.route_outages
6297 .operator_action_ended_unrefused(&module_id);
6298
6299 let response = ClientControlResponse::SupervisorAck {
6300 module_id,
6301 applied: true,
6302 };
6303 Ok(vec![control_response_body_frame(
6304 &frame,
6305 &response,
6306 "ClientControlResponse::SupervisorAck",
6307 )?])
6308 }
6309
6310 async fn handle_supervisor_reload(
6311 &self,
6312 frame: Frame,
6313 module_id: String,
6314 requester: Principal,
6315 ) -> Result<Vec<Frame>, RouterError> {
6316 let operation_lock = self.supervisor.operation_lock();
6317 let _operation_guard = operation_lock.lock().await;
6318 let Some(module) = self.supervisor.get(&module_id) else {
6319 return Ok(vec![control_error_frame(
6320 &frame,
6321 "unknown_module",
6322 format!("module_id '{module_id}' is not supervised"),
6323 )?]);
6324 };
6325
6326 self.route_outages.mark_operator_action(&module_id);
6327 if let Err(err) = module.reload_requested(Some(requester)).await {
6328 self.route_outages
6329 .operator_action_ended_unrefused(&module_id);
6330 let (code, message) = match err {
6331 crate::supervise::SuperviseError::Disabled { .. } => {
6332 ("module_disabled", err.to_string())
6333 }
6334 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6335 ("swap_in_progress", err.to_string())
6336 }
6337 _ => (
6338 "reload_failed",
6339 format!("failed to reload module_id '{module_id}': {err}"),
6340 ),
6341 };
6342 return Ok(vec![control_error_frame(&frame, code, message)?]);
6343 }
6344
6345 let response = ClientControlResponse::SupervisorAck {
6346 module_id,
6347 applied: true,
6348 };
6349 Ok(vec![control_response_body_frame(
6350 &frame,
6351 &response,
6352 "ClientControlResponse::SupervisorAck",
6353 )?])
6354 }
6355
6356 async fn handle_supervisor_rescan(
6360 &self,
6361 frame: Frame,
6362 mode: RescanMode,
6363 requester: Principal,
6364 ) -> Result<Vec<Frame>, RouterError> {
6365 let preview = mode.previews();
6366 let Some(context) = self.rescan.clone() else {
6367 return Ok(vec![control_error_frame(
6368 &frame,
6369 "rescan_unavailable",
6370 "the daemon was not started with a reloadable config path".to_string(),
6371 )?]);
6372 };
6373
6374 let operation_lock = self.supervisor.operation_lock();
6375 let _operation_guard = operation_lock.lock().await;
6376 let loaded = match self.load_rescan_config(&mode, &context) {
6380 Ok(config) => config,
6381 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6382 };
6383 let Some((config, activity_config)) = loaded else {
6391 return Ok(vec![control_error_frame(
6392 &frame,
6393 "invalid_daemon_config",
6394 format!(
6395 "daemon config not found at {}; refusing to rescan (an absent config would \
6396 retire every supervised module)",
6397 context.config_path.display()
6398 ),
6399 )?]);
6400 };
6401 let (
6402 configured_port,
6403 storage_config,
6404 admission_facts_carrier_module_id,
6405 admission_facts_targets,
6406 scope_authority_owners,
6407 privileged_operator_prompt,
6408 modules,
6409 reserved_capabilities,
6410 ) = (
6411 config.port,
6412 config.storage,
6413 config.admission_facts_carrier_module_id,
6414 config.admission_facts_targets,
6415 config.scope_authority_owners,
6416 config.privileged_operator_prompt,
6417 config.modules,
6418 config.reserved_capabilities,
6419 );
6420
6421 let mut restart_required = Vec::new();
6430 for section in RestartRequiredSection::ALL {
6431 let changed = match section {
6432 RestartRequiredSection::Port => configured_port != context.configured_port,
6433 RestartRequiredSection::Storage => storage_config != context.storage_config,
6434 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6435 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6436 }
6437 RestartRequiredSection::AdmissionFactsTargets => {
6438 admission_facts_targets != context.admission_facts_targets
6439 }
6440 RestartRequiredSection::ScopeAuthorityOwners => {
6441 scope_authority_owners != context.scope_authority_owners
6442 }
6443 RestartRequiredSection::PrivilegedOperatorPrompt => {
6444 privileged_operator_prompt != context.privileged_operator_prompt
6445 }
6446 };
6447 if changed {
6448 restart_required.push(section.label().to_string());
6449 }
6450 }
6451 if !restart_required.is_empty() {
6452 warn!(
6453 config_path = %context.config_path.display(),
6454 sections = %restart_required.join(", "),
6455 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6456 );
6457 }
6458
6459 for configured in &modules {
6460 if let Err(err) = validate_spec(&configured.module_spec()) {
6461 return Ok(vec![control_error_frame(
6462 &frame,
6463 "invalid_daemon_config",
6464 format!("supervisor rescan rejected daemon config: {err}"),
6465 )?]);
6466 }
6467 }
6468
6469 let configured_capabilities = modules
6470 .iter()
6471 .map(|module| (module.module_id.clone(), module.enabled))
6472 .collect::<Vec<_>>();
6473 let preview_capability_warnings = if preview {
6474 let (_, registrations) = self.runtime_capability_snapshot()?;
6475 let current_modules = self
6476 .supervisor
6477 .list()
6478 .into_iter()
6479 .map(|module| module.module_id().to_string())
6480 .collect::<BTreeSet<_>>();
6481 let resulting_modules = configured_capabilities.clone();
6482 let removed = current_modules
6483 .into_iter()
6484 .filter(|module_id| {
6485 !resulting_modules
6486 .iter()
6487 .any(|(configured_id, _)| configured_id == module_id)
6488 })
6489 .collect::<Vec<_>>();
6490 let disabled = match &mode {
6493 RescanMode::Candidate(_) => resulting_modules
6494 .iter()
6495 .filter(|(_, enabled)| !enabled)
6496 .map(|(module_id, _)| module_id.clone())
6497 .collect::<Vec<_>>(),
6498 _ => Vec::new(),
6499 };
6500 self.capability_evaluator.preview_warnings(
6501 resulting_modules,
6502 &removed,
6503 &disabled,
6504 ®istrations,
6505 )
6506 } else {
6507 Vec::new()
6508 };
6509 let reconciled = match &mode {
6510 RescanMode::Candidate(candidate) => self
6511 .preview_candidate(&context, modules, candidate)
6512 .map(|result| (result, Vec::new())),
6513 _ => {
6514 self.reconcile_supervised_modules(
6515 &context.supervisor,
6516 modules,
6517 preview,
6518 requester.clone(),
6519 )
6520 .await
6521 }
6522 };
6523 let (result, changed) = match reconciled {
6524 Ok(result) => result,
6525 Err(message) => {
6526 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6527 }
6528 };
6529 if !preview {
6530 if let Some(ledger) = &self.activity {
6531 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6532 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6533 }
6534 if let Err(code) = self.configure_activity(activity_config).await {
6535 warn!(
6536 code,
6537 "activity retention could not be applied during rescan"
6538 );
6539 }
6540 self.capability_evaluator
6541 .configure(configured_capabilities, reserved_capabilities);
6542 self.capability_evaluator.wake_deadline_loop();
6543 self.refresh_capability_requirements();
6544 }
6545 let mut result = result;
6546 result.restart_required = restart_required;
6547 result.capability_warnings = preview_capability_warnings;
6548 let response = ClientControlResponse::SupervisorRescan { result };
6549 Ok(vec![control_response_body_frame(
6550 &frame,
6551 &response,
6552 "ClientControlResponse::SupervisorRescan",
6553 )?])
6554 }
6555
6556 async fn handle_supervisor_release_reserved(
6557 &self,
6558 frame: Frame,
6559 module_id: String,
6560 ) -> Result<Vec<Frame>, RouterError> {
6561 let Some(context) = self.rescan.clone() else {
6562 return Ok(vec![control_error_frame(
6563 &frame,
6564 "release_unavailable",
6565 "reserved-id release requires a daemon started with a reloadable config path",
6566 )?]);
6567 };
6568 let operation_lock = self.supervisor.operation_lock();
6569 let _operation_guard = operation_lock.lock().await;
6570 let loaded = match crate::daemon_config::load(&context.config_path) {
6571 Ok(Some(config)) => config,
6572 Ok(None) => {
6573 return Ok(vec![control_error_frame(
6574 &frame,
6575 "invalid_daemon_config",
6576 format!(
6577 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6578 context.config_path.display()
6579 ),
6580 )?])
6581 }
6582 Err(err) => {
6583 return Ok(vec![control_error_frame(
6584 &frame,
6585 "invalid_daemon_config",
6586 format!("unable to verify reserved-id release against daemon config: {err}"),
6587 )?])
6588 }
6589 };
6590 if loaded
6591 .modules
6592 .iter()
6593 .any(|configured| configured.module_id == module_id)
6594 {
6595 return Ok(vec![control_error_frame(
6596 &frame,
6597 "reserved_module_configured",
6598 format!(
6599 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6600 ),
6601 )?]);
6602 }
6603 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6604 return Ok(vec![control_error_frame(
6605 &frame,
6606 "reserved_gate_not_retained",
6607 format!(
6608 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6609 ),
6610 )?]);
6611 }
6612
6613 let response = ClientControlResponse::SupervisorAck {
6614 module_id,
6615 applied: true,
6616 };
6617 Ok(vec![control_response_body_frame(
6618 &frame,
6619 &response,
6620 "ClientControlResponse::SupervisorAck",
6621 )?])
6622 }
6623
6624 fn plan_rescan(
6628 &self,
6629 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6630 ) -> Result<RescanPlan, String> {
6631 let mut current = BTreeMap::new();
6632 for module in self.supervisor.list() {
6633 let (spec, health) = module.configuration().map_err(|err| {
6634 format!(
6635 "failed to read configuration for module_id '{}': {err}",
6636 module.module_id()
6637 )
6638 })?;
6639 let enabled = module
6640 .status()
6641 .map_err(|err| {
6642 format!(
6643 "failed to read status for module_id '{}': {err}",
6644 module.module_id()
6645 )
6646 })?
6647 .enabled;
6648 current.insert(
6649 module.module_id().to_string(),
6650 (module, spec, health, enabled),
6651 );
6652 }
6653 let configured = configured_modules
6654 .into_iter()
6655 .map(|module| (module.module_id.clone(), module))
6656 .collect::<BTreeMap<_, _>>();
6657
6658 let added = configured
6659 .keys()
6660 .filter(|module_id| !current.contains_key(*module_id))
6661 .cloned()
6662 .collect::<Vec<_>>();
6663 let removed = current
6664 .keys()
6665 .filter(|module_id| !configured.contains_key(*module_id))
6666 .cloned()
6667 .collect::<Vec<_>>();
6668 let mut changed_pending_reload = Vec::new();
6669 let mut configuration_changes = BTreeSet::new();
6670 let mut enabled_changes = BTreeSet::new();
6671 let mut unchanged = 0_u32;
6672
6673 for (module_id, configured_module) in &configured {
6674 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6675 else {
6676 continue;
6677 };
6678 let launch_changed = *current_spec != configured_module.module_spec();
6682 let configuration_changed =
6683 launch_changed || *current_health != configured_module.health;
6684 let enabled_changed = *current_enabled != configured_module.enabled;
6685 if configuration_changed {
6686 configuration_changes.insert(module_id.clone());
6687 }
6688 if launch_changed {
6689 changed_pending_reload.push(module_id.clone());
6690 }
6691 if enabled_changed {
6692 enabled_changes.insert(module_id.clone());
6693 }
6694 if !configuration_changed && !enabled_changed {
6695 unchanged = unchanged.saturating_add(1);
6696 }
6697 }
6698 Ok(RescanPlan {
6699 current,
6700 configured,
6701 added,
6702 removed,
6703 changed_pending_reload,
6704 configuration_changes,
6705 enabled_changes,
6706 unchanged,
6707 })
6708 }
6709
6710 async fn reconcile_supervised_modules(
6719 &self,
6720 supervisor: &Supervisor,
6721 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6722 preview: bool,
6723 requester: Principal,
6724 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6725 let plan = self.plan_rescan(configured_modules)?;
6726
6727 if preview {
6733 return Ok((plan.preview_result(), Vec::new()));
6734 }
6735 let changed = plan
6736 .configuration_changes
6737 .union(&plan.enabled_changes)
6738 .cloned()
6739 .collect::<Vec<_>>();
6740 let RescanPlan {
6741 current,
6742 configured,
6743 added,
6744 removed,
6745 changed_pending_reload,
6746 configuration_changes,
6747 enabled_changes,
6748 unchanged,
6749 } = plan;
6750 if !(added.is_empty()
6755 && removed.is_empty()
6756 && configuration_changes.is_empty()
6757 && enabled_changes.is_empty())
6758 {
6759 if let Some(context) = &self.rescan {
6760 context
6761 .generation
6762 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6763 }
6764 }
6765
6766 for module_id in &removed {
6767 let module = ¤t
6768 .get(module_id)
6769 .expect("removed module came from current supervisor state")
6770 .0;
6771 module
6772 .retire_requested(Some(requester.clone()))
6773 .await
6774 .map_err(|err| {
6775 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6776 })?;
6777 self.supervisor.record_rescan_removal(module_id);
6811 self.supervisor.retire(module_id);
6812 self.route_outages.forget(module_id);
6813 }
6814
6815 for module_id in configured.keys() {
6816 let Some((module, _, _, _)) = current.get(module_id) else {
6817 continue;
6818 };
6819 let configured_module = configured
6820 .get(module_id)
6821 .expect("configured module id came from configured map");
6822 if configuration_changes.contains(module_id) {
6823 module
6824 .update_configuration(
6825 configured_module.module_spec(),
6826 configured_module.health.clone(),
6827 configured_module.drain_timeout_ms,
6828 )
6829 .await
6830 .map_err(|err| {
6831 format!(
6832 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6833 )
6834 })?;
6835 }
6836 if enabled_changes.contains(module_id) {
6837 self.route_outages.mark_operator_action(module_id);
6840 module
6841 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6842 .await
6843 .map_err(|err| {
6844 self.route_outages.operator_action_ended_unrefused(module_id);
6845 format!(
6846 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6847 configured_module.enabled
6848 )
6849 })?;
6850 }
6851 }
6852
6853 for module_id in &added {
6854 let configured_module = configured
6855 .get(module_id)
6856 .expect("added module id came from configured map");
6857 supervisor
6858 .supervise_configured_requested(
6859 configured_module.module_spec(),
6860 configured_module.enabled,
6861 configured_module.health.clone(),
6862 configured_module.drain_timeout_ms,
6863 configured_module.restart,
6864 configured_module.register_timeout,
6865 Some(requester.clone()),
6866 )
6867 .map_err(|err| {
6868 format!("failed to add module_id '{module_id}' during rescan: {err}")
6869 })?;
6870 }
6871
6872 Ok((
6873 SupervisorRescanResult {
6874 added,
6875 removed,
6876 changed_pending_reload,
6877 enabled_changes: enabled_changes.iter().cloned().collect(),
6878 unchanged,
6879 preview: false,
6880 restart_required: Vec::new(),
6883 capability_warnings: Vec::new(),
6884 module_actions: Vec::new(),
6885 earlier_pending: Vec::new(),
6886 expect: None,
6887 },
6888 changed,
6889 ))
6890 }
6891
6892 async fn handle_supervisor_set_enabled(
6893 &self,
6894 frame: Frame,
6895 module_id: String,
6896 enabled: bool,
6897 requester: Principal,
6898 ) -> Result<Vec<Frame>, RouterError> {
6899 let operation_lock = self.supervisor.operation_lock();
6900 let _operation_guard = operation_lock.lock().await;
6901 let Some(module) = self.supervisor.get(&module_id) else {
6902 return Ok(vec![control_error_frame(
6903 &frame,
6904 "unknown_module",
6905 format!("module_id '{module_id}' is not supervised"),
6906 )?]);
6907 };
6908
6909 self.route_outages.mark_operator_action(&module_id);
6912 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6913 Ok(applied) => applied,
6914 Err(err) => {
6915 self.route_outages
6916 .operator_action_ended_unrefused(&module_id);
6917 return Ok(vec![control_error_frame(
6918 &frame,
6919 "target_unavailable",
6920 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6921 )?]);
6922 }
6923 };
6924 if !applied {
6925 self.route_outages
6928 .operator_action_ended_unrefused(&module_id);
6929 }
6930
6931 self.capability_evaluator.wake_deadline_loop();
6932 self.refresh_capability_requirements();
6933 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6934 Ok(vec![control_response_body_frame(
6935 &frame,
6936 &response,
6937 "ClientControlResponse::SupervisorAck",
6938 )?])
6939 }
6940
6941 async fn handle_supervisor_health_probe(
6942 &self,
6943 frame: Frame,
6944 module_id: String,
6945 ) -> Result<Vec<Frame>, RouterError> {
6946 self.refresh_capability_requirements();
6947 let Some(registration) = self
6948 .registry
6949 .get_module(&module_id)
6950 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6951 else {
6952 return Ok(vec![control_error_frame(
6953 &frame,
6954 "unknown_module",
6955 format!("module_id '{module_id}' is not registered"),
6956 )?]);
6957 };
6958
6959 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6972 {
6973 return Ok(vec![control_error_frame(
6974 &frame,
6975 "health_not_advertised",
6976 format!("module_id '{module_id}' did not advertise health.check"),
6977 )?]);
6978 }
6979
6980 let deadline = Instant::now() + self.health_probe_timeout;
6981 let pending = match self.forwarding.begin_module_control_rpc_for(
6982 &module_id,
6983 MODULE_CONTROL_OP_HEALTH_CHECK,
6984 deadline,
6985 ) {
6986 Ok(pending) => pending,
6987 Err(err) => {
6988 return Ok(vec![control_error_frame(
6989 &frame,
6990 forwarding_error_code(&err),
6991 err.to_string(),
6992 )?])
6993 }
6994 };
6995
6996 let PendingModuleControlRpc {
6997 endpoint,
6998 module_sink,
6999 negotiated_ver,
7000 corr: probe_corr,
7001 receiver,
7002 } = pending;
7003 let mut guard =
7004 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7005 let probe_body =
7006 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
7007 RouterError::backend(
7008 0,
7009 frame.header.corr,
7010 format!("failed to encode health.check request: {err}"),
7011 )
7012 })?;
7013 let probe_frame = Frame::build_with_version(
7014 negotiated_ver,
7015 FrameType::Request,
7016 control_flags(),
7017 0,
7018 0,
7019 probe_corr,
7020 probe_body,
7021 )
7022 .map_err(RouterError::FrameBuild)?;
7023
7024 match timeout_at(deadline, module_sink.send(probe_frame)).await {
7027 Ok(Ok(())) => {}
7028 Ok(Err(err)) => {
7029 return Ok(vec![control_error_frame(
7030 &frame,
7031 "target_unavailable",
7032 err.to_string(),
7033 )?]);
7034 }
7035 Err(_) => {
7036 return Ok(vec![control_error_frame(
7037 &frame,
7038 "module_timeout",
7039 format!(
7040 "module_id '{module_id}' did not answer health.check within {:?}",
7041 self.health_probe_timeout
7042 ),
7043 )?]);
7044 }
7045 }
7046
7047 match timeout_at(deadline, receiver).await {
7048 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7049 guard.disarm();
7050 let Some(report) = response.health_report() else {
7051 return Ok(vec![control_error_frame(
7052 &frame,
7053 "invalid_control_body",
7054 "health.check RPC returned a non-health response",
7055 )?]);
7056 };
7057 let HealthReport {
7062 status,
7063 detail,
7064 metrics,
7065 } = report;
7066 let capability_detail = self
7067 .capability_evaluator
7068 .required_problem_detail(&module_id);
7069 let response = ClientControlResponse::SupervisorHealthProbe {
7070 module_id,
7071 status,
7072 detail: append_capability_problem_detail(detail, capability_detail),
7073 metrics,
7074 };
7075 Ok(vec![control_response_body_frame(
7076 &frame,
7077 &response,
7078 "ClientControlResponse::SupervisorHealthProbe",
7079 )?])
7080 }
7081 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7082 guard.disarm();
7083 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7086 }
7087 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7088 guard.disarm();
7089 Ok(vec![control_error_frame(
7090 &frame,
7091 "target_unavailable",
7092 message,
7093 )?])
7094 }
7095 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7096 guard.disarm();
7097 Ok(vec![control_error_frame(
7098 &frame,
7099 "invalid_control_body",
7100 message,
7101 )?])
7102 }
7103 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7104 guard.disarm();
7105 Ok(vec![control_error_frame(
7106 &frame,
7107 "invalid_control_body",
7108 format!("expected module-control op '{expected}', got '{actual}'"),
7109 )?])
7110 }
7111 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7112 guard.disarm();
7113 Ok(vec![control_error_frame(
7114 &frame,
7115 "module_timeout",
7116 format!(
7117 "module_id '{module_id}' answered health.check after {:?}",
7118 self.health_probe_timeout
7119 ),
7120 )?])
7121 }
7122 Ok(Err(_)) => Ok(vec![control_error_frame(
7123 &frame,
7124 "target_unavailable",
7125 "health.check waiter was canceled before the module responded",
7126 )?]),
7127 Err(_) => Ok(vec![control_error_frame(
7128 &frame,
7129 "module_timeout",
7130 format!(
7131 "module_id '{module_id}' did not answer health.check within {:?}",
7132 self.health_probe_timeout
7133 ),
7134 )?]),
7135 }
7136 }
7137
7138 fn supervisor_status(
7139 &self,
7140 module_id: &str,
7141 corr: u64,
7142 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7143 self.supervisor
7144 .get(module_id)
7145 .map(|module| {
7146 let warming = module.is_warming_for_control("status").map_err(|err| {
7147 RouterError::backend(
7148 0,
7149 corr,
7150 format!(
7151 "failed to read supervisor warming state for module_id '{module_id}': {err}"
7152 ),
7153 )
7154 })?;
7155 module.status_for_control("status").map_err(|err| {
7156 RouterError::backend(
7157 0,
7158 corr,
7159 format!(
7160 "failed to read supervisor status for module_id '{module_id}': {err}"
7161 ),
7162 )
7163 }).map(|status| (status, warming))
7164 })
7165 .transpose()
7166 }
7167
7168 fn guard_module_control_op(
7169 &self,
7170 frame: &Frame,
7171 module_id: &str,
7172 op: &str,
7173 ) -> Result<Option<Frame>, RouterError> {
7174 if self.module_grants_op(module_id, op, frame.header.corr)? {
7175 return Ok(None);
7176 }
7177
7178 Ok(Some(control_error_frame(
7179 frame,
7180 "op_not_allowed",
7181 format!("module_id '{module_id}' did not grant control op '{op}'"),
7182 )?))
7183 }
7184
7185 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7186 let Some(registration) = self
7187 .registry
7188 .get_module(module_id)
7189 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7190 else {
7191 return Ok(false);
7192 };
7193 Ok(module_registration_grants_op(®istration.control_ops, op))
7194 }
7195
7196 fn handle_status_update(
7197 &self,
7198 endpoint: ModuleEndpointId,
7199 frame: Frame,
7200 ) -> Result<Vec<Frame>, RouterError> {
7201 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7202 Ok(update) => update,
7203 Err(err) => {
7204 if is_known_module_push_op(&frame.body) {
7209 return Ok(vec![control_error_frame(
7210 &frame,
7211 "invalid_control_body",
7212 format!("malformed module control push body: {err}"),
7213 )?]);
7214 }
7215 return Ok(Vec::new());
7216 }
7217 };
7218
7219 match update {
7220 ModuleControlPush::RouteStatus {
7221 route_channel,
7222 route_epoch,
7223 status,
7224 } => {
7225 self.forwarding
7226 .cache_status(endpoint, route_channel, route_epoch, status)
7227 .map_err(RouterError::Forwarding)?;
7228 }
7229 }
7230 Ok(Vec::new())
7231 }
7232
7233 fn handle_route_poll(
7234 &self,
7235 ctx: &RouteCtx,
7236 frame: Frame,
7237 route_channel: u16,
7238 route_epoch: u32,
7239 kind: PollKind,
7240 ) -> Result<Vec<Frame>, RouterError> {
7241 let snapshot = self
7242 .forwarding
7243 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7244 .map_err(RouterError::Forwarding)?;
7245 let response = match (kind, snapshot) {
7246 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7247 ClientControlResponse::RoutePoll {
7248 route_channel,
7249 route_epoch,
7250 status,
7251 live: None,
7252 }
7253 }
7254 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7255 route_channel,
7256 route_epoch,
7257 status: None,
7258 live: None,
7259 },
7260 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7261 let live = self
7287 .process_liveness
7288 .as_ref()
7289 .and_then(|source| source.process_live(&module_id))
7290 .unwrap_or(true);
7291 ClientControlResponse::RoutePoll {
7292 route_channel,
7293 route_epoch,
7294 status: None,
7295 live: Some(live),
7296 }
7297 }
7298 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7299 route_channel,
7300 route_epoch,
7301 status: None,
7302 live: Some(false),
7303 },
7304 };
7305
7306 Ok(vec![control_response_body_frame(
7307 &frame,
7308 &response,
7309 "ClientControlResponse::RoutePoll",
7310 )?])
7311 }
7312
7313 pub(crate) fn observe_module_control_completion(
7314 &self,
7315 completion: ModuleControlRpcCompletion,
7316 ) -> bool {
7317 match completion {
7318 ModuleControlRpcCompletion::Unknown => false,
7319 ModuleControlRpcCompletion::Settled => true,
7320 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7321 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7322 info!(
7323 module_id = %module_id,
7324 latency_ms,
7325 "late health.check answer proves the module is alive"
7326 );
7327 match self
7328 .supervisor
7329 .record_late_health_answer(&module_id, latency_ms)
7330 {
7331 Ok(true) => {}
7332 Ok(false) => debug!(
7333 module_id = %module_id,
7334 latency_ms,
7335 "late health.check answer has no active supervisor snapshot"
7336 ),
7337 Err(err) => warn!(
7338 module_id = %module_id,
7339 latency_ms,
7340 error = %err,
7341 "failed to record late health.check answer"
7342 ),
7343 }
7344 true
7345 }
7346 }
7347 }
7348
7349 fn refuse_to_end_module_connection_for_a_client(
7370 &self,
7371 module_connection_id: ConnectionId,
7372 corr: u64,
7373 err: ForwardingError,
7374 ) -> Result<(), RouterError> {
7375 if let ForwardingError::ConnectionClosing { connection_id } = err {
7376 if connection_id != module_connection_id {
7377 warn!(
7378 module_connection_id = module_connection_id.get(),
7379 client_connection_id = connection_id.get(),
7380 corr,
7381 "dropping a route.bind response for a closing client; the module connection keeps serving"
7382 );
7383 return Ok(());
7384 }
7385 }
7386 Err(RouterError::Forwarding(err))
7387 }
7388
7389 fn handle_module_relay_response(
7390 &self,
7391 connection_id: ConnectionId,
7392 frame: Frame,
7393 ) -> Result<Vec<Frame>, RouterError> {
7394 let mut secondary_error = None;
7395 let outcome = match frame.header.ty {
7396 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7397 Ok(probe) if probe.op == "route.bind" => {
7398 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7399 Ok(ModuleControlResponse::RouteBindAck {}) => {
7400 RouteBindRelayOutcome::Accepted
7401 }
7402 Ok(other) => {
7403 let message =
7404 format!("route.bind response carried unexpected body: {other:?}");
7405 secondary_error = Some(control_error_frame(
7406 &frame,
7407 "invalid_control_body",
7408 message.clone(),
7409 )?);
7410 RouteBindRelayOutcome::ModuleGone(message)
7411 }
7412 Err(err) => {
7413 let message = format!("malformed route.bind response body: {err}");
7414 secondary_error = Some(control_error_frame(
7415 &frame,
7416 "invalid_control_body",
7417 message.clone(),
7418 )?);
7419 RouteBindRelayOutcome::ModuleGone(message)
7420 }
7421 }
7422 }
7423 Ok(probe) => {
7424 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7425 {
7426 Ok(response) => ModuleControlRpcOutcome::Response(response),
7427 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7428 "malformed {} response body: {err}",
7429 probe.op
7430 )),
7431 };
7432 let completion = self
7433 .forwarding
7434 .complete_module_control_rpc(
7435 connection_id,
7436 frame.header.corr,
7437 Some(&probe.op),
7438 outcome,
7439 )
7440 .map_err(RouterError::Forwarding)?;
7441 if !self.observe_module_control_completion(completion) {
7442 debug!(
7443 connection_id = connection_id.get(),
7444 corr = frame.header.corr,
7445 op = %probe.op,
7446 "dropping late or unknown module-control RPC response"
7447 );
7448 }
7449 return Ok(Vec::new());
7450 }
7451 Err(err) => {
7452 if let Some(expected_op) = self
7453 .forwarding
7454 .pending_module_control_op(connection_id, frame.header.corr)
7455 .map_err(RouterError::Forwarding)?
7456 {
7457 let completion = self
7458 .forwarding
7459 .complete_module_control_rpc(
7460 connection_id,
7461 frame.header.corr,
7462 None,
7463 ModuleControlRpcOutcome::MalformedResponse(format!(
7464 "malformed {expected_op} response body: {err}"
7465 )),
7466 )
7467 .map_err(RouterError::Forwarding)?;
7468 if !self.observe_module_control_completion(completion) {
7469 debug!(
7470 connection_id = connection_id.get(),
7471 corr = frame.header.corr,
7472 "dropping late malformed module-control RPC response"
7473 );
7474 }
7475 return Ok(Vec::new());
7476 }
7477 let message = format!("malformed route.bind response body: {err}");
7478 secondary_error = Some(control_error_frame(
7479 &frame,
7480 "invalid_control_body",
7481 message.clone(),
7482 )?);
7483 RouteBindRelayOutcome::ModuleGone(message)
7484 }
7485 },
7486 FrameType::Error => {
7487 if self
7488 .forwarding
7489 .pending_module_control_op(connection_id, frame.header.corr)
7490 .map_err(RouterError::Forwarding)?
7491 .is_some()
7492 {
7493 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7494 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7495 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7496 "malformed module-control ERROR body: {err}"
7497 )),
7498 };
7499 let completion = self
7500 .forwarding
7501 .complete_module_control_rpc(
7502 connection_id,
7503 frame.header.corr,
7504 None,
7505 outcome,
7506 )
7507 .map_err(RouterError::Forwarding)?;
7508 if !self.observe_module_control_completion(completion) {
7509 debug!(
7510 connection_id = connection_id.get(),
7511 corr = frame.header.corr,
7512 "dropping late or unknown module-control RPC error"
7513 );
7514 }
7515 return Ok(Vec::new());
7516 }
7517 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7518 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7519 Err(err) => {
7520 let message = format!("malformed route.bind ERROR body: {err}");
7521 secondary_error = Some(control_error_frame(
7522 &frame,
7523 "invalid_control_body",
7524 message.clone(),
7525 )?);
7526 RouteBindRelayOutcome::ModuleGone(message)
7527 }
7528 }
7529 }
7530 ty => {
7531 return Ok(vec![control_error_frame(
7532 &frame,
7533 "unsupported_control_frame",
7534 format!("unsupported module channel-0 frame {ty:?}"),
7535 )?])
7536 }
7537 };
7538
7539 let settled =
7540 self.forwarding
7541 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7542 let completion = match settled {
7543 Ok(completion) => completion,
7544 Err(err) => {
7545 self.refuse_to_end_module_connection_for_a_client(
7546 connection_id,
7547 frame.header.corr,
7548 err,
7549 )?;
7550 return Ok(secondary_error.into_iter().collect());
7551 }
7552 };
7553 if let Some(target) = completion.abandoned.as_ref() {
7554 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7555 }
7556 if !completion.settled {
7557 debug!(
7558 connection_id = connection_id.get(),
7559 corr = frame.header.corr,
7560 frame_type = ?frame.header.ty,
7561 "dropping late or unknown route.bind relay response"
7562 );
7563 }
7564 Ok(secondary_error.into_iter().collect())
7565 }
7566
7567 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7568 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7569 self.cleanup_connection_with_end_reason(
7573 connection_id,
7574 RegistrationEndReason::ExplicitGoodbye,
7575 )
7576 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7577 Ok(Vec::new())
7578 }
7579}
7580
7581impl Default for ControlHandler {
7582 fn default() -> Self {
7583 Self::new(Arc::new(Registry::default()))
7584 }
7585}
7586
7587impl crate::supervise::SwapPromotionObserver for ControlHandler {
7588 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7589 self.apply_registration_capabilities(registration);
7590 }
7591}
7592
7593fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7594 CapabilityRequirementStatus {
7595 consumer: status.consumer,
7596 capability: status.capability,
7597 need: match status.need {
7598 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7599 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7600 },
7601 verdict: status.verdict.as_str().to_string(),
7602 episode_seq: status.episode_seq,
7603 config_satisfiable: status.config_satisfiable,
7604 runtime_available: status.runtime_available,
7605 detail: status.detail,
7606 }
7607}
7608
7609fn append_capability_problem_detail(
7610 detail: Option<String>,
7611 capability_detail: Option<String>,
7612) -> Option<String> {
7613 match (detail, capability_detail) {
7614 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7615 (Some(detail), None) => Some(detail),
7616 (None, Some(capability_detail)) => Some(capability_detail),
7617 (None, None) => None,
7618 }
7619}
7620
7621fn subc_ops() -> Vec<String> {
7622 SUBC_CONTROL_OPS
7623 .iter()
7624 .map(|op| (*op).to_string())
7625 .collect()
7626}
7627
7628fn module_subc_ops() -> Vec<String> {
7629 SUBC_CONTROL_OPS
7630 .iter()
7631 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7632 .map(|op| (*op).to_string())
7633 .collect()
7634}
7635
7636#[cfg(test)]
7637fn module_baseline_control_ops() -> Vec<String> {
7638 MODULE_BASELINE_CONTROL_OPS
7639 .iter()
7640 .map(|op| (*op).to_string())
7641 .collect()
7642}
7643
7644fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7645 let mut seen = HashSet::new();
7646 let mut effective = Vec::new();
7647 for op in MODULE_BASELINE_CONTROL_OPS {
7648 if seen.insert((*op).to_string()) {
7649 effective.push((*op).to_string());
7650 }
7651 }
7652 for op in declared.unwrap_or_default() {
7653 if seen.insert(op.clone()) {
7654 effective.push(op);
7655 }
7656 }
7657 effective
7658}
7659
7660fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7661 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7662}
7663
7664fn target_module_id(target: &RouteTarget) -> &str {
7665 match target {
7666 RouteTarget::ToolProvider { module_id }
7667 | RouteTarget::ManagementSurface { module_id }
7668 | RouteTarget::InternalService { module_id, .. } => module_id,
7669 }
7670}
7671
7672fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7673 roles.iter().any(|role| match (target, role) {
7674 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7675 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7676 (
7677 RouteTarget::InternalService { service_id, .. },
7678 ProviderRole::InternalService {
7679 service_id: provided,
7680 ..
7681 },
7682 ) => service_id == provided,
7683 _ => false,
7684 })
7685}
7686
7687fn is_routable_role(role: &ProviderRole) -> bool {
7688 matches!(
7689 role,
7690 ProviderRole::ToolProvider { .. }
7691 | ProviderRole::ManagementSurface { .. }
7692 | ProviderRole::InternalService { .. }
7693 )
7694}
7695
7696#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7697enum ControlRequestBodyError {
7698 UnknownOp,
7699 InvalidBody,
7700}
7701
7702#[derive(Debug, Deserialize)]
7703struct ControlOpProbe {
7704 op: String,
7705}
7706
7707const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7710
7711fn is_known_module_push_op(body: &[u8]) -> bool {
7712 serde_json::from_slice::<ControlOpProbe>(body)
7713 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7714 .unwrap_or(false)
7715}
7716
7717fn is_known_module_request_op(body: &[u8]) -> bool {
7718 serde_json::from_slice::<ControlOpProbe>(body)
7719 .map(|probe| is_module_to_subc_op(&probe.op))
7720 .unwrap_or(false)
7721}
7722
7723fn is_module_to_subc_op(op: &str) -> bool {
7724 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7725}
7726
7727fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7728 debug!(
7729 op = %op,
7730 connection_id = connection_id.get(),
7731 corr,
7732 "control dispatch"
7733 );
7734}
7735
7736fn log_slow_control_dispatch(
7737 dispatch_started_at: Option<StdInstant>,
7738 op: &'static str,
7739 connection_id: ConnectionId,
7740 corr: u64,
7741 route_open_timings: Option<&RouteOpenTimings>,
7742) {
7743 let Some(dispatch_started_at) = dispatch_started_at else {
7744 return;
7745 };
7746 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7748 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7749 if let Some(timings) = route_open_timings {
7750 warn!(
7751 op = %op,
7752 connection_id = connection_id.get(),
7753 corr,
7754 elapsed_ms = elapsed.as_millis() as u64,
7755 module_id = ?timings.module_id,
7756 principal = %timings.principal,
7757 admission_ms = timings.admission.as_millis() as u64,
7758 project_root_ms = timings.project_root.as_millis() as u64,
7759 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7760 bind_send_ms = timings.bind_send.as_millis() as u64,
7761 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7762 "slow control dispatch"
7763 );
7764 return;
7765 }
7766 warn!(
7767 op = %op,
7768 connection_id = connection_id.get(),
7769 corr,
7770 elapsed_ms = elapsed.as_millis() as u64,
7771 "slow control dispatch"
7772 );
7773 }
7774}
7775
7776#[cfg(test)]
7778type TestCallerLookup =
7779 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7780
7781fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7783 match request {
7784 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7785 Some(PrivilegedOperation::Restart { module_id })
7786 }
7787 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7788 Some(PrivilegedOperation::Swap { module_id })
7789 }
7790 ClientControlRequest::SupervisorReload { module_id } => {
7791 Some(PrivilegedOperation::Reload { module_id })
7792 }
7793 ClientControlRequest::SupervisorRescan { preview } => {
7794 if *preview {
7795 None
7796 } else {
7797 Some(PrivilegedOperation::Rescan)
7798 }
7799 }
7800 ClientControlRequest::SupervisorRescanChecked {
7805 preview,
7806 candidate,
7807 expect,
7808 } => (!*preview && candidate.is_none() && expect.is_some())
7809 .then_some(PrivilegedOperation::Rescan),
7810 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7811 Some(PrivilegedOperation::SetEnabled {
7812 module_id,
7813 enabled: *enabled,
7814 })
7815 }
7816 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7817 Some(PrivilegedOperation::ReleaseReserved { module_id })
7818 }
7819 ClientControlRequest::ServerDescribe {}
7820 | ClientControlRequest::CatalogList { .. }
7821 | ClientControlRequest::RouteOpen { .. }
7822 | ClientControlRequest::RoutePoll { .. }
7823 | ClientControlRequest::SupervisorList {}
7824 | ClientControlRequest::SupervisorHealth {}
7825 | ClientControlRequest::SupervisorHealthProbe { .. }
7826 | ClientControlRequest::SupervisorRoutes { .. }
7827 | ClientControlRequest::SupervisorProvenance { .. }
7828 | ClientControlRequest::SupervisorStderrTail { .. }
7829 | ClientControlRequest::SupervisorTerminals { .. }
7830 | ClientControlRequest::SupervisorSpawnSnapshot {}
7831 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7832 }
7833}
7834
7835fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7836 match request {
7837 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7838 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7839 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7840 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7841 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7842 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7843 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7844 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7845 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7846 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7847 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7848 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7849 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7850 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7851 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7852 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7853 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7854 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7855 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7856 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7857 }
7858}
7859
7860struct RouteOpenCaller {
7868 principal: String,
7872 scope: String,
7875 harness: String,
7878}
7879
7880impl RouteOpenCaller {
7881 fn new(
7882 principal: Option<&Principal>,
7883 scope: Option<&ScopeSelector>,
7884 identity: &BindIdentity,
7885 ) -> Self {
7886 Self {
7887 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7888 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7889 harness: bounded_log_text(&identity.harness),
7890 }
7891 }
7892
7893 fn unknown() -> Self {
7896 Self {
7897 principal: "unknown".to_string(),
7898 scope: "unknown".to_string(),
7899 harness: "unknown".to_string(),
7900 }
7901 }
7902}
7903
7904const MAX_LOGGED_TEXT_CHARS: usize = 64;
7907
7908fn bounded_log_text(value: &str) -> String {
7912 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7913 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7914 bounded.push('…');
7915 }
7916 format!("{bounded:?}")
7917}
7918
7919fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7925 use subc_protocol::scope::ScopeKind;
7926 let kind = match stamp.kind {
7927 ScopeKind::Head => "head",
7928 ScopeKind::Worker => "worker",
7929 ScopeKind::Ephemeral => "ephemeral",
7930 };
7931 format!(
7932 "{kind}/{}/{}@{}",
7933 principal_label(&stamp.owner),
7934 bounded_log_text(&stamp.scope_ref),
7935 stamp.scope_epoch
7936 )
7937}
7938
7939fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7946 let epoch = selector
7947 .scope_epoch
7948 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7949 format!(
7950 "requested/{}/{}@{epoch}",
7951 bounded_log_text(&principal_label(&selector.owner)),
7952 bounded_log_text(&selector.scope_ref),
7953 )
7954}
7955
7956fn principal_label(principal: &Principal) -> String {
7957 match principal {
7958 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7959 Principal::Direct => "direct".to_string(),
7960 other => format!("{other:?}"),
7961 }
7962}
7963
7964fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7965 match request {
7966 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7967 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7968 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7969 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7970 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7971 }
7972}
7973
7974fn parse_client_control_request(
7975 body: &[u8],
7976) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7977 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7978 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7979 Ok(probe)
7980 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7981 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7982 {
7983 ControlRequestBodyError::InvalidBody
7984 }
7985 Ok(_) => ControlRequestBodyError::UnknownOp,
7986 Err(_) => ControlRequestBodyError::InvalidBody,
7987 };
7988 (err, classification)
7989 })
7990}
7991
7992fn parse_module_control_request_from_module(
7993 body: &[u8],
7994) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7995 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7996 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7997 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7998 Ok(_) => ControlRequestBodyError::UnknownOp,
7999 Err(_) => ControlRequestBodyError::InvalidBody,
8000 };
8001 (err, classification)
8002 })
8003}
8004
8005#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8006enum ProviderRoleKind {
8007 ToolProvider,
8008 PipelineStage,
8009 ManagementSurface,
8010 InternalService,
8011}
8012
8013fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8014 match role {
8015 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8016 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8017 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8018 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8019 }
8020}
8021
8022fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8023 roles.iter().map(provider_role_kind).collect()
8024}
8025
8026const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8029
8030#[derive(Debug, Default, PartialEq, Eq)]
8032struct ScopeOutcomeCounts {
8033 created: usize,
8034 replaced: usize,
8035 updated: usize,
8036 unchanged: usize,
8037 refused: usize,
8038}
8039
8040impl ScopeOutcomeCounts {
8041 fn of(results: &[ScopeRecordResult]) -> Self {
8042 let mut counts = Self::default();
8043 for result in results {
8044 let slot = match result.outcome {
8045 ScopeRecordOutcome::Created => &mut counts.created,
8046 ScopeRecordOutcome::Replaced => &mut counts.replaced,
8047 ScopeRecordOutcome::Updated => &mut counts.updated,
8048 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8049 ScopeRecordOutcome::Refused => &mut counts.refused,
8050 };
8051 *slot += 1;
8052 }
8053 counts
8054 }
8055}
8056
8057#[cfg(test)]
8058mod scope_outcome_count_tests {
8059 use super::*;
8060
8061 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8062 ScopeRecordResult {
8063 scope_ref: "r".to_string(),
8064 scope_epoch: 1,
8065 outcome,
8066 code: None,
8067 message: None,
8068 version: None,
8069 parent_state: None,
8070 }
8071 }
8072
8073 #[test]
8076 fn every_outcome_is_counted_in_its_own_field() {
8077 let results = [
8078 result(ScopeRecordOutcome::Created),
8079 result(ScopeRecordOutcome::Created),
8080 result(ScopeRecordOutcome::Replaced),
8081 result(ScopeRecordOutcome::Updated),
8082 result(ScopeRecordOutcome::Unchanged),
8083 result(ScopeRecordOutcome::Refused),
8084 result(ScopeRecordOutcome::Refused),
8085 result(ScopeRecordOutcome::Refused),
8086 ];
8087 assert_eq!(
8088 ScopeOutcomeCounts::of(&results),
8089 ScopeOutcomeCounts {
8090 created: 2,
8091 replaced: 1,
8092 updated: 1,
8093 unchanged: 1,
8094 refused: 3,
8095 }
8096 );
8097 }
8098}
8099
8100fn capability_census_trigger(
8104 old: Option<&CapabilityDeclarations>,
8105 new: Option<&CapabilityDeclarations>,
8106) -> bool {
8107 let old_provides = old
8108 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8109 .unwrap_or_default();
8110 let old_denies = old
8111 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8112 .unwrap_or_default();
8113 let new = new.cloned().unwrap_or(CapabilityDeclarations {
8114 provides: Vec::new(),
8115 requires: Vec::new(),
8116 must_never_reach: Vec::new(),
8117 });
8118
8119 new.provides
8120 .iter()
8121 .any(|capability| !old_provides.contains(capability))
8122 || new
8123 .must_never_reach
8124 .iter()
8125 .any(|capability| !old_denies.contains(capability))
8126}
8127
8128fn denied_capability<'a>(
8131 opening_manifest: &'a ModuleManifest,
8132 target_manifest: &ModuleManifest,
8133) -> Option<&'a str> {
8134 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
8135 let target_capabilities = target_manifest.capabilities.as_ref()?;
8136 opening_capabilities
8137 .must_never_reach
8138 .iter()
8139 .find(|denied| {
8140 target_capabilities
8141 .provides
8142 .iter()
8143 .any(|provided| provided == *denied)
8144 })
8145 .map(String::as_str)
8146}
8147
8148fn catalog_update_frozen_field_message(
8149 registered: &ModuleManifest,
8150 provides: &[ProviderRole],
8151) -> Option<String> {
8152 let old_has_provides = !registered.provides.is_empty();
8153 let new_has_provides = !provides.is_empty();
8154 if old_has_provides != new_has_provides {
8155 return Some(format!(
8156 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8157 registered.module_id
8158 ));
8159 }
8160
8161 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
8162 return Some(format!(
8163 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8164 registered.module_id
8165 ));
8166 }
8167
8168 let registered_concurrency = manifest_concurrency(registered);
8169 let mut candidate = registered.clone();
8170 candidate.provides = provides.to_vec();
8171 let candidate_concurrency = manifest_concurrency(&candidate);
8172 if candidate_concurrency != registered_concurrency {
8173 return Some(format!(
8174 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8175 registered.module_id, registered_concurrency, candidate_concurrency
8176 ));
8177 }
8178
8179 None
8182}
8183
8184fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8185 manifest.provides.iter().any(is_routable_role)
8186}
8187
8188fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8194 manifest
8195 .provides
8196 .iter()
8197 .find_map(|provider| match provider {
8198 ProviderRole::ToolProvider { concurrency, .. }
8199 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8200 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8201 })
8202 .unwrap_or(Concurrency::ModuleManaged)
8203}
8204
8205fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8212 let has_management_surface = manifest
8213 .provides
8214 .iter()
8215 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8216 if !has_management_surface {
8217 return false;
8218 }
8219 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8220 return false;
8221 };
8222 let Some(provides) = raw
8223 .get("manifest")
8224 .and_then(|manifest| manifest.get("provides"))
8225 .and_then(serde_json::Value::as_array)
8226 else {
8227 return false;
8228 };
8229 provides.iter().any(|role| {
8234 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8235 && role.get("concurrency").is_none()
8236 })
8237}
8238
8239fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8240 if peer_version != PROTOCOL_VERSION {
8241 return Err(format!(
8242 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8243 ));
8244 }
8245 Ok(PROTOCOL_VERSION)
8246}
8247
8248fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8249 Frame::build_with_version(
8250 response_version(frame),
8251 FrameType::Pong,
8252 frame.header.flags,
8253 0,
8254 0,
8255 frame.header.corr,
8256 Vec::new(),
8257 )
8258 .map_err(RouterError::FrameBuild)
8259}
8260
8261pub(crate) fn control_error_frame(
8262 frame: &Frame,
8263 code: &'static str,
8264 message: impl Into<String>,
8265) -> Result<Frame, RouterError> {
8266 control_error_body_frame(
8267 frame,
8268 ErrorBody {
8269 code: code.to_string(),
8270 message: message.into(),
8271 detail: None,
8272 },
8273 )
8274}
8275
8276fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8284 control_error_frame_with_flags(frame, error, daemon_error_flags())
8285}
8286
8287fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8293 control_error_frame_with_flags(frame, error, control_flags())
8294}
8295
8296fn control_error_frame_with_flags(
8297 frame: &Frame,
8298 error: ErrorBody,
8299 flags: Flags,
8300) -> Result<Frame, RouterError> {
8301 let body = serde_json::to_vec(&error).map_err(|err| {
8302 RouterError::backend(
8303 0,
8304 frame.header.corr,
8305 format!("failed to encode control ERROR: {err}"),
8306 )
8307 })?;
8308
8309 Frame::build_with_version(
8310 response_version(frame),
8311 FrameType::Error,
8312 flags,
8313 0,
8314 0,
8315 frame.header.corr,
8316 body,
8317 )
8318 .map_err(RouterError::FrameBuild)
8319}
8320
8321pub(crate) fn control_response_body_frame<T: Serialize>(
8322 frame: &Frame,
8323 reply: &T,
8324 label: &'static str,
8325) -> Result<Frame, RouterError> {
8326 let body = serde_json::to_vec(reply).map_err(|err| {
8327 RouterError::backend(
8328 0,
8329 frame.header.corr,
8330 format!("failed to encode {label}: {err}"),
8331 )
8332 })?;
8333
8334 Frame::build_with_version(
8335 response_version(frame),
8336 FrameType::Response,
8337 control_flags(),
8338 0,
8339 0,
8340 frame.header.corr,
8341 body,
8342 )
8343 .map_err(RouterError::FrameBuild)
8344}
8345
8346fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8376 match err {
8377 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8378 ForwardingError::NoModuleConnection => "target_unavailable",
8379 ForwardingError::ModuleReloading { .. } => "module_reloading",
8380 ForwardingError::ClientRouteChannelExhausted { .. }
8381 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8382 ForwardingError::StaleModuleEndpoint
8383 | ForwardingError::UnknownReservation { .. }
8384 | ForwardingError::ConnectionClosing { .. }
8385 | ForwardingError::ClientEgressClosed { .. }
8386 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8387 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8390 ForwardingError::RelayCorrelationExhausted
8391 | ForwardingError::RouteOpenBuild(_)
8392 | ForwardingError::Poisoned => "forwarding_error",
8393 }
8394}
8395
8396fn response_version(frame: &Frame) -> u8 {
8397 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8398 frame.header.ver
8399 } else {
8400 PROTOCOL_VERSION
8401 }
8402}
8403
8404fn control_flags() -> Flags {
8405 Flags::new(false, Priority::Passive, false)
8406}
8407
8408fn send_goodbye_target_best_effort(
8413 counters: &DaemonCounters,
8414 target: &GoodbyeTarget,
8415 context: &'static str,
8416) {
8417 let Ok(frame) = Frame::build_with_version(
8418 target.negotiated_ver,
8419 FrameType::Goodbye,
8420 control_flags(),
8421 target.channel,
8422 target.epoch,
8423 0,
8424 Vec::new(),
8425 ) else {
8426 return;
8427 };
8428 crate::forwarding::send_module_route_goodbye(
8429 counters,
8430 &target.sink,
8431 frame,
8432 target.module_id.as_deref(),
8433 context,
8434 );
8435}
8436
8437pub(crate) fn send_route_control_pushes(
8438 forwarding: &ForwardingTable,
8439 routes: Vec<EndpointRoute>,
8440 push: ClientControlPush,
8441) {
8442 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8443 for route in routes {
8444 let target = route.goodbye_target;
8445 if let Some((existing, channels)) = targets
8446 .iter_mut()
8447 .find(|(existing, _)| existing.connection_id == target.connection_id)
8448 {
8449 debug_assert_eq!(
8450 existing.negotiated_ver, target.negotiated_ver,
8451 "one connection cannot negotiate multiple frame versions"
8452 );
8453 if !channels.contains(&target.channel) {
8454 channels.push(target.channel);
8455 }
8456 continue;
8457 }
8458 let channel = target.channel;
8459 targets.push((target, vec![channel]));
8460 }
8461 for (target, mut channels) in targets {
8462 channels.sort_unstable();
8463 let mut push = push.clone();
8464 match &mut push {
8465 ClientControlPush::RouteClosing {
8466 channels: covered, ..
8467 }
8468 | ClientControlPush::RouteClosed {
8469 channels: covered, ..
8470 } => *covered = channels,
8471 }
8472 let body = match serde_json::to_vec(&push) {
8473 Ok(body) => body,
8474 Err(err) => {
8475 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8476 continue;
8477 }
8478 };
8479 let frame = match Frame::build_with_version(
8480 target.negotiated_ver,
8481 FrameType::Push,
8482 control_flags(),
8483 0,
8484 0,
8485 0,
8486 body.clone(),
8487 ) {
8488 Ok(frame) => frame,
8489 Err(err) => {
8490 warn!(
8491 route_channel = target.channel,
8492 error = %err,
8493 "failed to build route lifecycle control PUSH frame"
8494 );
8495 continue;
8496 }
8497 };
8498 if let Err(err) = target.sink.try_send(frame) {
8499 if target.close_on_delivery_failure() {
8500 warn!(
8501 target_connection_id = target.connection_id.get(),
8502 route_channel = target.channel,
8503 error = %err,
8504 "route lifecycle control PUSH was not delivered to client; closing target connection"
8505 );
8506 let _ = forwarding.escalate_client_delivery_failure(
8507 target.connection_id,
8508 target.channel,
8509 target.epoch,
8510 CloseReason::new(
8511 "route_lifecycle_push_delivery_failed",
8512 format!(
8513 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8514 target.channel
8515 ),
8516 ),
8517 crate::forwarding::UndeliveredFrame {
8518 module_id: target.module_id.as_deref(),
8519 sink: &target.sink,
8520 },
8521 );
8522 }
8523 }
8524 }
8525}
8526
8527#[cfg(test)]
8528mod tests {
8529 mod peer_manifest {
8530 include!("peer_manifest_tests.rs");
8531 }
8532 mod activity_read {
8533 include!("activity_read_tests.rs");
8534 }
8535
8536 mod lifecycle_activity {
8537 include!("control_lifecycle_tests.rs");
8538 }
8539 #[cfg(unix)]
8540 #[tokio::test]
8541 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8542 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8543 let path = dir.join("subc.jsonc");
8544 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8545 "program":"/bin/sleep","args":["60"],"protocol":"none",
8546 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8547 }}}).to_string()).unwrap();
8548 let mut configured = crate::daemon_config::load(&path)
8549 .unwrap()
8550 .unwrap()
8551 .modules
8552 .pop()
8553 .unwrap();
8554 let registry = std::sync::Arc::new(crate::Registry::default());
8555 let handle = crate::SupervisorHandle::new();
8556 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8557 .with_handle(handle.clone());
8558 let module = supervisor
8559 .supervise_configured_with_health(
8560 configured.module_spec(),
8561 true,
8562 configured.health.clone(),
8563 None,
8564 configured.restart,
8565 )
8566 .unwrap();
8567 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8568 let before = module.status().unwrap().pid;
8569 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8570 configured.health.cadence = std::time::Duration::from_secs(3600);
8571 let health_only = handler
8572 .reconcile_supervised_modules(
8573 &supervisor,
8574 vec![configured.clone()],
8575 false,
8576 Principal::Direct,
8577 )
8578 .await
8579 .unwrap();
8580 assert!(
8581 health_only.0.changed_pending_reload.is_empty(),
8582 "health policy is already applied live"
8583 );
8584 assert_eq!(module.status().unwrap().pid, before);
8585 assert_eq!(
8586 module.configuration().unwrap().1.http,
8587 configured.health.http
8588 );
8589 configured.args = vec!["61".into()];
8590 let launch = handler
8591 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8592 .await
8593 .unwrap();
8594 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8595 assert_eq!(
8596 module.status().unwrap().pid,
8597 before,
8598 "a launch edit is stored until reload"
8599 );
8600 module.drain().await.unwrap();
8601 }
8602 use cortexkit_test_support::ScratchDir;
8603 use std::{
8604 collections::BTreeMap,
8605 fmt,
8606 path::PathBuf,
8607 sync::{Arc, Mutex},
8608 time::Duration,
8609 };
8610
8611 use serde_json::{json, Value};
8612 use subc_protocol::{
8613 manifest::{
8614 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8615 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8616 },
8617 session::HealthStatus,
8618 FrameType,
8619 };
8620
8621 use super::*;
8622 use crate::{
8623 forwarding::{DataRoute, DataRouteState},
8624 registry::ChannelState,
8625 router::FrameSink,
8626 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8627 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8628 RouteCtx, Router,
8629 };
8630 use tokio::{
8631 sync::mpsc,
8632 time::{sleep, Instant},
8633 };
8634 use tracing::{
8635 field::{Field, Visit},
8636 Event, Subscriber,
8637 };
8638 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8639
8640 fn fake_aft_stub_path() -> PathBuf {
8659 let mut path = std::env::current_exe().expect("current_exe available in tests");
8660 path.pop(); path.pop(); path.push(if cfg!(windows) {
8663 "fake-aft-stub.exe"
8664 } else {
8665 "fake-aft-stub"
8666 });
8667 assert!(
8668 path.exists(),
8669 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8670 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8671 path.display()
8672 );
8673 path
8674 }
8675
8676 fn client_retries(code: &str) -> bool {
8681 subc_protocol::error_codes::is_retryable_route_open(code)
8682 }
8683
8684 #[test]
8695 fn retryability_of_forwarding_codes_matches_the_failure() {
8696 let transient = [
8699 ForwardingError::NoModuleConnection,
8700 ForwardingError::ModuleReloading {
8701 module_id: "m".into(),
8702 },
8703 ForwardingError::StaleModuleEndpoint,
8704 ForwardingError::UnknownReservation {
8705 client_channel: 1,
8706 module_channel: 1,
8707 },
8708 ForwardingError::ConnectionClosing {
8709 connection_id: ConnectionId::new(1),
8710 },
8711 ForwardingError::ClientEgressClosed {
8712 connection_id: ConnectionId::new(1),
8713 },
8714 ForwardingError::ModuleEgressUnavailable {
8715 connection_id: ConnectionId::new(1),
8716 },
8717 ];
8718 for err in transient {
8719 let code = forwarding_error_code(&err);
8720 assert!(
8721 client_retries(code),
8722 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8723 );
8724 }
8725
8726 let permanent = [
8731 ForwardingError::ConnectionRoleConflict {
8732 connection_id: ConnectionId::new(1),
8733 },
8734 ForwardingError::ClientRouteChannelExhausted {
8735 connection_id: ConnectionId::new(1),
8736 },
8737 ForwardingError::ModuleRouteChannelExhausted {
8738 endpoint: ModuleEndpointId {
8739 connection_id: ConnectionId::new(1),
8740 generation: 1,
8741 },
8742 },
8743 ForwardingError::RelayCorrelationExhausted,
8744 ForwardingError::RouteOpenBuild("x".into()),
8745 ForwardingError::Poisoned,
8746 ];
8747 for err in permanent {
8748 let code = forwarding_error_code(&err);
8749 assert!(
8750 !client_retries(code),
8751 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8752 );
8753 }
8754 }
8755
8756 #[tokio::test]
8784 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8785 let handler = ControlHandler::default();
8786 let frame =
8787 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8788
8789 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8793 assert_eq!(
8794 stamped,
8795 Principal::Direct,
8796 "a caller that proved nothing must not be stamped as a supervised module"
8797 );
8798
8799 let forged = handler
8804 .route_open_principal(
8805 &frame,
8806 Some(ConsumerIdentity {
8807 module_id: "aft".to_string(),
8808 launch_nonce: "not-a-real-nonce".to_string(),
8809 }),
8810 )
8811 .unwrap();
8812 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8813 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8814 }
8815
8816 #[test]
8833 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8834 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8835 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8836 let ClientControlRequest::RouteOpen {
8837 consumer_identity, ..
8838 } = parsed
8839 else {
8840 panic!("route.open body must parse as RouteOpen");
8841 };
8842 assert_eq!(
8843 consumer_identity,
8844 Some(ConsumerIdentity {
8845 module_id: "aft".to_string(),
8846 launch_nonce: "n".to_string(),
8847 }),
8848 "the wire field name must reach the value route_open_principal reads"
8849 );
8850 }
8851
8852 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8853 ModuleManifest::builder(module_id, "0.1.0")
8854 .protocol_ver(protocol_ver)
8855 .provides(vec![ProviderRole::ToolProvider {
8856 tools: vec![Tool {
8857 name: "read".to_string(),
8858 description: None,
8859 execution_mode: ExecutionMode::Pure,
8860 schema: json!({"type": "object"}),
8861 }],
8862 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8863 concurrency: Concurrency::ModuleManaged,
8864 emits_push: true,
8865 sub_supervises: true,
8866 }])
8867 .build()
8868 }
8869
8870 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8871 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8872 }
8873
8874 fn hello_frame_with_control_ops(
8875 module_id: &str,
8876 protocol_ver: u8,
8877 corr: u64,
8878 control_ops: Option<Vec<String>>,
8879 ) -> Frame {
8880 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8881 }
8882
8883 fn hello_frame_with_nonce(
8884 module_id: &str,
8885 protocol_ver: u8,
8886 corr: u64,
8887 launch_nonce: Option<&str>,
8888 ) -> Frame {
8889 hello_frame_full(
8890 module_id,
8891 protocol_ver,
8892 corr,
8893 None,
8894 launch_nonce.map(ToOwned::to_owned),
8895 )
8896 }
8897
8898 fn hello_frame_full(
8899 module_id: &str,
8900 protocol_ver: u8,
8901 corr: u64,
8902 control_ops: Option<Vec<String>>,
8903 launch_nonce: Option<String>,
8904 ) -> Frame {
8905 let body = serde_json::to_vec(&ModuleHelloBody {
8906 manifest: manifest(module_id, protocol_ver),
8907 protocol_ver,
8908 control_ops,
8909 launch_nonce,
8910 })
8911 .unwrap();
8912 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8913 }
8914
8915 fn hello_frame_with_rich_ui(
8916 module_id: &str,
8917 protocol_ver: u8,
8918 corr: u64,
8919 launch_nonce: Option<&str>,
8920 rich_ui: Value,
8921 ) -> Frame {
8922 let mut body = serde_json::to_value(ModuleHelloBody {
8923 manifest: manifest(module_id, protocol_ver),
8924 protocol_ver,
8925 control_ops: None,
8926 launch_nonce: launch_nonce.map(str::to_string),
8927 })
8928 .unwrap();
8929 body["manifest"]["rich_ui"] = rich_ui;
8930 Frame::build(
8931 FrameType::Hello,
8932 control_flags(),
8933 0,
8934 0,
8935 corr,
8936 serde_json::to_vec(&body).unwrap(),
8937 )
8938 .unwrap()
8939 }
8940
8941 fn non_routable_hello_frame_with_control_ops(
8942 module_id: &str,
8943 corr: u64,
8944 control_ops: Option<Vec<String>>,
8945 ) -> Frame {
8946 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8947 manifest.provides.clear();
8948 let body = serde_json::to_vec(&ModuleHelloBody {
8949 manifest,
8950 protocol_ver: PROTOCOL_VERSION,
8951 control_ops,
8952 launch_nonce: None,
8953 })
8954 .unwrap();
8955 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8956 }
8957
8958 fn capability_grammar_hello_frame(
8959 capabilities: Value,
8960 runtime_computed: Option<Value>,
8961 corr: u64,
8962 ) -> Frame {
8963 let mut body = serde_json::to_value(ModuleHelloBody {
8964 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8965 protocol_ver: PROTOCOL_VERSION,
8966 control_ops: None,
8967 launch_nonce: None,
8968 })
8969 .expect("HELLO body serializes");
8970 body["manifest"]["capabilities"] = capabilities;
8971 if let Some(runtime_computed) = runtime_computed {
8972 body["runtime_computed"] = runtime_computed;
8973 }
8974 Frame::build(
8975 FrameType::Hello,
8976 control_flags(),
8977 0,
8978 0,
8979 corr,
8980 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8981 )
8982 .expect("HELLO frame builds")
8983 }
8984
8985 fn channel_request(channel: u16, corr: u64) -> Frame {
8986 Frame::build(
8987 FrameType::Request,
8988 Flags::new(true, Priority::Interactive, false),
8989 channel,
8990 0,
8991 corr,
8992 b"opaque".to_vec(),
8993 )
8994 .unwrap()
8995 }
8996
8997 fn route_ctx(
8998 connection_id: ConnectionId,
8999 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9000 let (tx, rx) = mpsc::channel(8);
9001 (
9002 RouteCtx {
9003 connection_id,
9004 egress: FrameSink::new(tx),
9005 },
9006 rx,
9007 )
9008 }
9009
9010 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9011 serde_json::from_slice(&frame.body).unwrap()
9012 }
9013
9014 async fn hello_via_sink(
9019 handler: &ControlHandler,
9020 ctx: &RouteCtx,
9021 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9022 hello: Frame,
9023 ) -> Frame {
9024 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9025 assert!(
9026 replies.is_empty(),
9027 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9028 );
9029 let ack = rx
9030 .try_recv()
9031 .expect("HELLO_ACK is queued on the module sink")
9032 .frame;
9033 assert_eq!(ack.header.ty, FrameType::HelloAck);
9034 ack
9035 }
9036
9037 fn parse_error(frame: &Frame) -> Value {
9038 serde_json::from_slice(&frame.body).unwrap()
9039 }
9040
9041 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9042 serde_json::from_slice(&frame.body).unwrap()
9043 }
9044
9045 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9046 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9047 route_channel,
9048 route_epoch: 0,
9049 kind,
9050 })
9051 .unwrap();
9052 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9053 }
9054
9055 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9056 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9057 module_id: module_id.to_string(),
9058 })
9059 .unwrap();
9060 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9061 }
9062
9063 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9064 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9065 }
9066
9067 fn route_open_frame_with_consumer_capabilities(
9068 corr: u64,
9069 module_id: &str,
9070 project_root: ScratchDir,
9071 consumer_capabilities: Option<Vec<String>>,
9072 ) -> Frame {
9073 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9074 target: RouteTarget::ToolProvider {
9075 module_id: module_id.to_string(),
9076 },
9077 identity: BindIdentity::new(
9078 project_root.path().to_path_buf(),
9079 "unit".to_string(),
9080 "session".to_string(),
9081 ),
9082 consumer_identity: None,
9083 consumer_capabilities,
9084 role_versions: None,
9085 admission_facts: None,
9086 scope: None,
9087 })
9088 .unwrap();
9089 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9090 }
9091
9092 fn route_open_frame_with_role_versions(
9093 corr: u64,
9094 module_id: &str,
9095 project_root: ScratchDir,
9096 role_versions: Option<BTreeMap<String, String>>,
9097 ) -> Frame {
9098 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9099 target: RouteTarget::ToolProvider {
9100 module_id: module_id.to_string(),
9101 },
9102 identity: BindIdentity::new(
9103 project_root.path().to_path_buf(),
9104 "unit".to_string(),
9105 format!("session-{corr}"),
9106 ),
9107 consumer_identity: None,
9108 consumer_capabilities: None,
9109 role_versions,
9110 admission_facts: None,
9111 scope: None,
9112 })
9113 .unwrap();
9114 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9115 }
9116
9117 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9118 entries
9119 .iter()
9120 .map(|(role, version)| (role.to_string(), version.to_string()))
9121 .collect()
9122 }
9123
9124 fn route_open_frame_with_admission_facts(
9125 corr: u64,
9126 module_id: &str,
9127 project_root: ScratchDir,
9128 consumer_identity: Option<subc_control::ConsumerIdentity>,
9129 facts: Option<Value>,
9130 ) -> Frame {
9131 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9132 target: RouteTarget::ToolProvider {
9133 module_id: module_id.to_string(),
9134 },
9135 identity: BindIdentity::new(
9136 project_root.path().to_path_buf(),
9137 "unit".to_string(),
9138 format!("session-{corr}"),
9139 ),
9140 consumer_identity,
9141 consumer_capabilities: None,
9142 role_versions: None,
9143 admission_facts: facts,
9144 scope: None,
9145 })
9146 .unwrap();
9147 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9148 }
9149
9150 #[derive(Clone, Default)]
9151 struct EventCapture {
9152 events: Arc<Mutex<Vec<CapturedEvent>>>,
9153 }
9154
9155 #[derive(Clone, Debug)]
9156 struct CapturedEvent {
9157 target: String,
9158 level: tracing::Level,
9159 fields: BTreeMap<String, String>,
9160 }
9161
9162 impl EventCapture {
9163 fn events(&self) -> Vec<CapturedEvent> {
9164 self.events.lock().unwrap().clone()
9165 }
9166 }
9167
9168 impl<S> Layer<S> for EventCapture
9169 where
9170 S: Subscriber,
9171 {
9172 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9173 let mut visitor = EventFieldVisitor::default();
9174 event.record(&mut visitor);
9175 self.events.lock().unwrap().push(CapturedEvent {
9176 target: event.metadata().target().to_string(),
9177 level: *event.metadata().level(),
9178 fields: visitor.fields,
9179 });
9180 }
9181 }
9182
9183 #[derive(Default)]
9184 struct EventFieldVisitor {
9185 fields: BTreeMap<String, String>,
9186 }
9187
9188 impl Visit for EventFieldVisitor {
9189 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9190 self.fields
9191 .insert(field.name().to_string(), format!("{value:?}"));
9192 }
9193 }
9194
9195 fn health_response(corr: u64, status: HealthStatus) -> Frame {
9196 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9197 status,
9198 detail: Some("warming".to_string()),
9199 metrics: Some(json!({"queue_depth": 3})),
9200 })
9201 .unwrap();
9202 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9203 }
9204
9205 fn route_bind_ack(corr: u64) -> Frame {
9206 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9207 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9208 }
9209
9210 fn unique_project_root(label: &str) -> ScratchDir {
9211 ScratchDir::new(label)
9212 }
9213
9214 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9215 match parse_route_poll(frame) {
9216 ClientControlResponse::RoutePoll {
9217 status: None,
9218 live: Some(live),
9219 ..
9220 } => assert_eq!(live, expected_live),
9221 other => panic!("unexpected route.poll response: {other:?}"),
9222 }
9223 }
9224
9225 fn bind_liveness_route(
9226 registry: &Registry,
9227 forwarding: &ForwardingTable,
9228 module_id: &str,
9229 ) -> (RouteCtx, u16, u32) {
9230 let module_connection = ConnectionId::new(101);
9231 let client_connection = ConnectionId::new(202);
9232 let registration = registry
9233 .register_with_control_ops(
9234 manifest(module_id, PROTOCOL_VERSION),
9235 PROTOCOL_VERSION,
9236 module_connection,
9237 module_baseline_control_ops(),
9238 )
9239 .unwrap();
9240 let (module_tx, _module_rx) = mpsc::channel(8);
9241 let endpoint = forwarding
9242 .register_module_connection(
9243 module_connection,
9244 module_id.to_string(),
9245 PROTOCOL_VERSION,
9246 manifest_concurrency(®istration.manifest),
9247 FrameSink::new(module_tx),
9248 )
9249 .unwrap();
9250 let (client_ctx, _client_rx) = route_ctx(client_connection);
9251 let pending = forwarding
9252 .begin_route_bind_relay_for_test(
9253 client_connection,
9254 client_ctx.egress.clone(),
9255 1,
9256 module_id,
9257 )
9258 .unwrap();
9259 assert_eq!(pending.endpoint, endpoint);
9260 let route_channel = pending.client_channel;
9261 let route_epoch = pending.client_epoch;
9262 forwarding
9263 .complete_pending_relay(
9264 module_connection,
9265 pending.corr,
9266 RouteBindRelayOutcome::Accepted,
9267 )
9268 .unwrap();
9269 (client_ctx, route_channel, route_epoch)
9270 }
9271
9272 struct FakeProcessLiveness {
9273 live: Option<bool>,
9274 }
9275
9276 impl ModuleProcessLiveness for FakeProcessLiveness {
9277 fn process_live(&self, _module_id: &str) -> Option<bool> {
9278 self.live
9279 }
9280 }
9281
9282 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9283 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9284 {
9285 let registry = Arc::new(Registry::default());
9286 let supervisor_handle = SupervisorHandle::new();
9287 let supervisor = Supervisor::new_for_test(
9288 Arc::clone(®istry),
9289 RestartPolicy::new(1, Duration::from_millis(10)),
9290 )
9291 .with_handle(supervisor_handle.clone());
9292 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9293 let module = supervisor
9294 .spawn(ModuleSpec {
9295 module_id: "stderr-tail-wire".to_string(),
9296 program: fake_aft_stub_path(),
9297 args: Vec::new(),
9298 env: vec![
9299 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9300 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9301 ],
9302 reserved: false,
9303 reserved_prefixes: Vec::new(),
9304 protocol: ModuleProtocol::Subc,
9305 overlap: Default::default(),
9306 })
9307 .unwrap();
9308
9309 let deadline = Instant::now() + Duration::from_secs(5);
9310 loop {
9311 let tail = module.stderr_tail(None, None);
9312 if tail
9313 .entries
9314 .iter()
9315 .any(|entry| matches!(entry, TailEntry::ProcessStart))
9316 && tail.entries.iter().any(|entry| {
9317 matches!(
9318 entry,
9319 TailEntry::Line {
9320 truncated: true,
9321 ..
9322 }
9323 )
9324 })
9325 {
9326 break;
9327 }
9328 assert!(
9329 Instant::now() < deadline,
9330 "module did not produce a truncated line and restart boundary: {tail:?}"
9331 );
9332 sleep(Duration::from_millis(10)).await;
9333 }
9334
9335 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9336 let request = ClientControlRequest::SupervisorStderrTail {
9337 module_id: "stderr-tail-wire".to_string(),
9338 max_lines: None,
9339 max_bytes: None,
9340 };
9341 let frame = Frame::build(
9342 FrameType::Request,
9343 control_flags(),
9344 0,
9345 0,
9346 1,
9347 serde_json::to_vec(&request).unwrap(),
9348 )
9349 .unwrap();
9350 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9351 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9352 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9353 serde_json::from_slice(&responses[0].body).unwrap()
9354 else {
9355 panic!("expected supervisor.stderr_tail response");
9356 };
9357
9358 assert!(
9359 tail.entries
9360 .iter()
9361 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9362 "the control response lost the restart boundary"
9363 );
9364 let Some(StderrTailEntry::Line {
9365 text,
9366 truncated,
9367 at_ms,
9368 }) = tail.entries.iter().find(|entry| {
9369 matches!(
9370 entry,
9371 StderrTailEntry::Line {
9372 truncated: true,
9373 ..
9374 }
9375 )
9376 })
9377 else {
9378 panic!("the control response lost the truncated line");
9379 };
9380 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9381 assert!(*truncated);
9382 assert!(
9383 at_ms.is_some(),
9384 "the control response lost the line's capture time"
9385 );
9386 }
9387
9388 #[tokio::test(flavor = "current_thread")]
9393 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9394 let dir = ScratchDir::new("terminals-off-worker");
9395 let journal_path = dir.join("terminals.jsonl");
9396 let registry = Arc::new(Registry::default());
9397 let supervisor_handle = SupervisorHandle::new();
9398 let supervisor =
9399 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9400 .with_handle(supervisor_handle.clone())
9401 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9402 let module = supervisor
9403 .spawn(ModuleSpec {
9404 module_id: "terminal-off-worker".to_string(),
9405 program: fake_aft_stub_path(),
9406 args: Vec::new(),
9407 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9408 reserved: false,
9409 reserved_prefixes: Vec::new(),
9410 protocol: ModuleProtocol::Subc,
9411 overlap: Default::default(),
9412 })
9413 .unwrap();
9414 let deadline = Instant::now() + Duration::from_secs(5);
9415 while module.terminal_history().entries.len() != 2 {
9416 assert!(Instant::now() < deadline, "module did not record two exits");
9417 sleep(Duration::from_millis(10)).await;
9418 }
9419
9420 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9421 let handler =
9422 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
9423 let frame = Frame::build(
9424 FrameType::Request,
9425 control_flags(),
9426 0,
9427 0,
9428 1,
9429 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9430 module_id: "terminal-off-worker".to_string(),
9431 })
9432 .unwrap(),
9433 )
9434 .unwrap();
9435 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9436 let spawned_at = std::time::Instant::now();
9437 let read = tokio::spawn({
9438 let handler = Arc::clone(&handler);
9439 async move { handler.handle_control_frame(&ctx, frame).await }
9440 });
9441 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9444 .await
9445 .unwrap()
9446 .expect("the history read reached its pause");
9447 let elapsed = spawned_at.elapsed();
9448 assert!(
9449 elapsed < Duration::from_secs(2) && !read.is_finished(),
9450 "this task could not run while the history read was paused \
9451 (resumed after {elapsed:?}, read finished: {})",
9452 read.is_finished()
9453 );
9454
9455 drop(release);
9456 let responses = read.await.unwrap().unwrap();
9457 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9458 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9459 panic!("expected supervisor.terminals response");
9460 };
9461 assert_eq!(terminals.entries.len(), 2);
9462 assert_eq!(terminals.journal_skipped_lines, 0);
9463 assert_eq!(terminals.journal_read_errors, 0);
9464 }
9465
9466 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9467 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9468 let registry = Arc::new(Registry::default());
9469 let supervisor_handle = SupervisorHandle::new();
9470 let supervisor =
9471 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9472 .with_handle(supervisor_handle.clone());
9473 let module = supervisor
9474 .spawn(ModuleSpec {
9475 module_id: "terminal-golden".to_string(),
9476 program: fake_aft_stub_path(),
9477 args: Vec::new(),
9478 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9479 reserved: false,
9480 reserved_prefixes: Vec::new(),
9481 protocol: ModuleProtocol::Subc,
9482 overlap: Default::default(),
9483 })
9484 .unwrap();
9485
9486 let deadline = Instant::now() + Duration::from_secs(5);
9487 while module.terminal_history().entries.len() != 2 {
9488 assert!(
9489 Instant::now() < deadline,
9490 "module did not retain two terminal exits: {:?}",
9491 module.terminal_history()
9492 );
9493 sleep(Duration::from_millis(10)).await;
9494 }
9495
9496 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9497 let request = ClientControlRequest::SupervisorTerminals {
9498 module_id: "terminal-golden".to_string(),
9499 };
9500 let frame = Frame::build(
9501 FrameType::Request,
9502 control_flags(),
9503 0,
9504 0,
9505 1,
9506 serde_json::to_vec(&request).unwrap(),
9507 )
9508 .unwrap();
9509 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9510 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9511 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9512 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9513 panic!("expected supervisor.terminals response");
9514 };
9515 assert_eq!(terminals.entries.len(), 2);
9516 assert_eq!(terminals.dropped, 0);
9517
9518 let mut rendered = serde_json::to_value(response).unwrap();
9519 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9522 for (index, entry) in rendered["entries"]
9523 .as_array_mut()
9524 .expect("terminal response entries array")
9525 .iter_mut()
9526 .enumerate()
9527 {
9528 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9529 }
9530
9531 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9532 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9533 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9534 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9535 std::fs::write(&golden_path, &serialized).unwrap();
9536 }
9537 let expected: Value =
9538 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9539 assert_eq!(rendered, expected);
9540 }
9541
9542 #[test]
9543 fn hello_registers_manifest_and_returns_ack() {
9544 let registry = Arc::new(Registry::default());
9545 let handler = ControlHandler::new(Arc::clone(®istry));
9546 let conn = ConnectionId::new(1);
9547
9548 let responses = handler
9549 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9550 .unwrap();
9551
9552 assert_eq!(responses.len(), 1);
9553 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9554 assert_eq!(responses[0].header.channel, 0);
9555 assert_eq!(responses[0].header.corr, 7);
9556 let ack = parse_ack(&responses[0]);
9557 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9558 assert!(ack
9559 .subc_capabilities
9560 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9561 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9562 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9563 assert!(ack
9564 .subc_ops
9565 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9566 assert!(ack
9567 .subc_ops
9568 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9569
9570 let registration = registry.get_module("aft").unwrap().unwrap();
9571 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9572 assert_eq!(registration.state, ChannelState::Active);
9573 assert_eq!(registration.connection_id, conn);
9574 assert_eq!(registration.control_ops, module_baseline_control_ops());
9575 }
9576
9577 #[test]
9578 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9579 let invalid_identifiers = [
9580 ("case_change", "credentials-Provider/v1"),
9581 ("leading_zero", "credentials-provider/v01"),
9582 ("trailing_hyphen", "credentials-provider-/v1"),
9583 ("consecutive_hyphens", "credentials--provider/v1"),
9584 ("uppercase", "Credentials-provider/v1"),
9585 ("missing_v", "credentials-provider/1"),
9586 ("whitespace", "credentials provider/v1"),
9587 ("zero_version", "credentials-provider/v0"),
9588 ("out_of_range_version", "credentials-provider/v4294967296"),
9589 (
9590 "overlength_name",
9591 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9592 ),
9593 ];
9594 let mut cases = invalid_identifiers
9595 .into_iter()
9596 .map(|(name, identifier)| {
9597 (
9598 format!("identifier_{name}"),
9599 "capabilities.provides[0]".to_string(),
9600 identifier.to_string(),
9601 json!({ "provides": [identifier] }),
9602 None,
9603 )
9604 })
9605 .collect::<Vec<_>>();
9606 cases.extend([
9607 (
9608 "unknown_need".to_string(),
9609 "capabilities.requires[0].need".to_string(),
9610 "deferred".to_string(),
9611 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9612 None,
9613 ),
9614 (
9615 "duplicate_provides".to_string(),
9616 "capabilities.provides[1]".to_string(),
9617 "credentials-provider/v1".to_string(),
9618 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9619 None,
9620 ),
9621 (
9622 "duplicate_must_never_reach".to_string(),
9623 "capabilities.must_never_reach[1]".to_string(),
9624 "credentials-provider/v1".to_string(),
9625 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9626 None,
9627 ),
9628 (
9629 "duplicate_requires_same_need".to_string(),
9630 "capabilities.requires[1]".to_string(),
9631 "credentials-provider/v1".to_string(),
9632 json!({ "requires": [
9633 { "capability": "credentials-provider/v1", "need": "required" },
9634 { "capability": "credentials-provider/v1", "need": "required" }
9635 ] }),
9636 None,
9637 ),
9638 (
9639 "duplicate_requires_conflicting_need".to_string(),
9640 "capabilities.requires[1]".to_string(),
9641 "credentials-provider/v1".to_string(),
9642 json!({ "requires": [
9643 { "capability": "credentials-provider/v1", "need": "required" },
9644 { "capability": "credentials-provider/v1", "need": "optional" }
9645 ] }),
9646 None,
9647 ),
9648 (
9649 "capabilities_root_pointer".to_string(),
9650 "runtime_computed[0]".to_string(),
9651 "/capabilities".to_string(),
9652 json!({}),
9653 Some(json!(["/capabilities"])),
9654 ),
9655 (
9656 "capabilities_descendant_pointer".to_string(),
9657 "runtime_computed[0]".to_string(),
9658 "/capabilities/provides".to_string(),
9659 json!({}),
9660 Some(json!(["/capabilities/provides"])),
9661 ),
9662 (
9663 "malformed_pointer_without_leading_slash".to_string(),
9664 "runtime_computed[0]".to_string(),
9665 "capabilities".to_string(),
9666 json!({}),
9667 Some(json!(["capabilities"])),
9668 ),
9669 (
9670 "malformed_pointer_escape".to_string(),
9671 "runtime_computed[0]".to_string(),
9672 "/roles/~2/tools".to_string(),
9673 json!({}),
9674 Some(json!(["/roles/~2/tools"])),
9675 ),
9676 (
9677 "unknown_capabilities_field".to_string(),
9678 "capabilities.future".to_string(),
9679 "<array>".to_string(),
9680 json!({ "future": [] }),
9681 None,
9682 ),
9683 ]);
9684
9685 for (index, (name, field, value, capabilities, runtime_computed)) in
9686 cases.into_iter().enumerate()
9687 {
9688 let registry = Arc::new(Registry::default());
9689 let handler = ControlHandler::new(Arc::clone(®istry));
9690 let response = handler
9691 .handle_control(
9692 ConnectionId::new((index + 1) as u64),
9693 capability_grammar_hello_frame(
9694 capabilities,
9695 runtime_computed,
9696 index as u64 + 1,
9697 ),
9698 )
9699 .expect("invalid HELLO returns a refusal");
9700
9701 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9702 let error = parse_error(&response[0]);
9703 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9704 let message = error["message"]
9705 .as_str()
9706 .expect("error message is a string");
9707 assert!(
9708 message.contains(&field),
9709 "{name}: field missing from {message}"
9710 );
9711 assert!(
9712 message.contains(&value),
9713 "{name}: value missing from {message}"
9714 );
9715 assert_eq!(
9716 registry
9717 .active_registration_count()
9718 .expect("registry reads"),
9719 0,
9720 "{name}: refused HELLO must not create a catalog entry"
9721 );
9722 }
9723 }
9724
9725 #[test]
9726 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9727 let registry = Arc::new(Registry::default());
9728 let handler = ControlHandler::new(Arc::clone(®istry));
9729 let capabilities = json!({
9730 "provides": ["credentials-provider/v1"],
9731 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9732 "must_never_reach": ["federation-transport/v1"]
9733 });
9734 let response = handler
9735 .handle_control(
9736 ConnectionId::new(99),
9737 capability_grammar_hello_frame(
9738 capabilities.clone(),
9739 Some(json!(["/roles/0/tools"])),
9740 99,
9741 ),
9742 )
9743 .expect("valid HELLO registers");
9744 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9745
9746 let request = Frame::build(
9747 FrameType::Request,
9748 control_flags(),
9749 0,
9750 0,
9751 100,
9752 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9753 .expect("catalog request serializes"),
9754 )
9755 .expect("catalog request frame builds");
9756 let response = handler
9757 .handle_catalog_list(request, None)
9758 .expect("catalog list succeeds");
9759 let ClientControlResponse::CatalogList { modules, .. } =
9760 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9761 else {
9762 panic!("catalog request must return catalog.list");
9763 };
9764 assert_eq!(modules.len(), 1);
9765 assert_eq!(
9766 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9767 capabilities
9768 );
9769 }
9770
9771 #[test]
9772 fn catalog_list_mirrors_management_operation_description() {
9773 let registry = Arc::new(Registry::default());
9774 let handler = ControlHandler::new(Arc::clone(®istry));
9775 let description = "List managed records and return their identifiers and metadata.";
9776 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9777 manifest.provides = vec![ProviderRole::ManagementSurface {
9778 operations: vec![ManagementOperation {
9779 name: "records.list".to_string(),
9780 kind: ManagementOperationKind::Query,
9781 description: Some(description.to_string()),
9782 }],
9783 config_schema: json!({"type": "object"}),
9784 observability: vec![ObservabilitySurface {
9785 name: "records.stats".to_string(),
9786 kind: ObservabilityKind::Snapshot,
9787 }],
9788 identity_scope: vec![IdentityScope::Project],
9789 concurrency: Concurrency::ModuleManaged,
9790 }];
9791 registry
9792 .register_with_control_ops(
9793 manifest,
9794 PROTOCOL_VERSION,
9795 ConnectionId::new(99),
9796 Vec::new(),
9797 )
9798 .expect("described management manifest registers");
9799
9800 let request = Frame::build(
9801 FrameType::Request,
9802 control_flags(),
9803 0,
9804 0,
9805 100,
9806 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9807 .expect("catalog request serializes"),
9808 )
9809 .expect("catalog request frame builds");
9810 let response = handler
9811 .handle_catalog_list(request, None)
9812 .expect("catalog list succeeds");
9813 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9814 assert_eq!(
9815 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9816 "catalog.list must preserve the declared operation description verbatim"
9817 );
9818 }
9819
9820 #[test]
9821 fn catalog_list_mirrors_registered_operation_delivery() {
9822 let registry = Arc::new(Registry::default());
9823 let handler = ControlHandler::new(Arc::clone(®istry));
9824 let delivery =
9825 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9826 let mut body = serde_json::to_value(ModuleHelloBody {
9827 manifest: manifest("watch-management", PROTOCOL_VERSION),
9828 protocol_ver: PROTOCOL_VERSION,
9829 control_ops: None,
9830 launch_nonce: None,
9831 })
9832 .unwrap();
9833 body["manifest"]["provides"] = json!([{
9834 "role": "management_surface",
9835 "operations": [
9836 {"name": "status", "kind": "query"},
9837 {"name": "future", "kind": "query"},
9838 {"name": "reset", "kind": "mutate"}
9839 ],
9840 "config_schema": {}, "observability": [], "identity_scope": []
9841 }]);
9842 body["manifest"]["operation_delivery"] = delivery.clone();
9843 let hello = Frame::build(
9844 FrameType::Hello,
9845 control_flags(),
9846 0,
9847 0,
9848 99,
9849 serde_json::to_vec(&body).unwrap(),
9850 )
9851 .unwrap();
9852 let response = handler
9853 .handle_control(ConnectionId::new(99), hello)
9854 .unwrap();
9855 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9856
9857 let (_, registrations) = registry.list_modules().unwrap();
9858 let registered = ®istrations[0].manifest;
9859 assert!(registered.is_watch_operation("status"));
9860 assert!(!registered.is_watch_operation("future"));
9861 assert!(!registered.is_watch_operation("reset"));
9862 assert!(!registered.is_watch_operation("missing"));
9863
9864 let request = Frame::build(
9865 FrameType::Request,
9866 control_flags(),
9867 0,
9868 0,
9869 100,
9870 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9871 )
9872 .unwrap();
9873 let response = handler.handle_catalog_list(request, None).unwrap();
9874 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9875 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9876 let ClientControlResponse::CatalogList { modules, .. } =
9877 serde_json::from_slice(&response[0].body).unwrap()
9878 else {
9879 panic!("expected catalog.list");
9880 };
9881 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9882 }
9883
9884 #[test]
9885 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9886 let registry = Arc::new(Registry::default());
9887 let supervisor = SupervisorHandle::new();
9888 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9889 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9890 let descriptor = json!({
9891 "schema": "ckios.richui.manifest/1",
9892 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9893 "bytes": 4096,
9894 "chunks": 4
9895 });
9896 let connection = ConnectionId::new(201);
9897 let hello = handler
9898 .handle_control(
9899 connection,
9900 hello_frame_with_rich_ui(
9901 "vault",
9902 PROTOCOL_VERSION,
9903 201,
9904 Some("the-real-nonce"),
9905 descriptor.clone(),
9906 ),
9907 )
9908 .unwrap();
9909 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9910
9911 let request =
9912 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9913 let provides = registry
9914 .get_module_by_connection(connection)
9915 .unwrap()
9916 .unwrap()
9917 .manifest
9918 .provides;
9919 handler
9920 .handle_catalog_update(connection, request, provides, None, Some(false))
9921 .unwrap();
9922
9923 let request = Frame::build(
9924 FrameType::Request,
9925 control_flags(),
9926 0,
9927 0,
9928 203,
9929 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9930 )
9931 .unwrap();
9932 let response = handler.handle_catalog_list(request, None).unwrap();
9933 let ClientControlResponse::CatalogList { modules, .. } =
9934 serde_json::from_slice(&response[0].body).unwrap()
9935 else {
9936 panic!("expected catalog.list");
9937 };
9938 assert_eq!(
9939 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9940 descriptor
9941 );
9942 assert!(modules[0].nonce_verified);
9943 }
9944
9945 fn hello_with_role_declarations(declarations: Value) -> Frame {
9946 let mut body = serde_json::to_value(ModuleHelloBody {
9947 manifest: manifest("role-test", PROTOCOL_VERSION),
9948 protocol_ver: PROTOCOL_VERSION,
9949 control_ops: None,
9950 launch_nonce: None,
9951 })
9952 .unwrap();
9953 body["manifest"]["role_declarations"] = declarations;
9954 Frame::build(
9955 FrameType::Hello,
9956 control_flags(),
9957 0,
9958 0,
9959 301,
9960 serde_json::to_vec(&body).unwrap(),
9961 )
9962 .unwrap()
9963 }
9964
9965 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9966 let request = Frame::build(
9967 FrameType::Request,
9968 control_flags(),
9969 0,
9970 0,
9971 302,
9972 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9973 )
9974 .unwrap();
9975 let response = handler.handle_catalog_list(request, None).unwrap();
9976 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9977 let ClientControlResponse::CatalogList { mut modules, .. } =
9978 serde_json::from_slice(&response[0].body).unwrap()
9979 else {
9980 panic!("expected catalog.list");
9981 };
9982 (raw["modules"][0].clone(), modules.remove(0))
9983 }
9984
9985 #[test]
9986 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9987 let registry = Arc::new(Registry::default());
9988 let handler = ControlHandler::new(Arc::clone(®istry));
9989 let declarations = json!({
9990 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9991 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9992 });
9993 let connection = ConnectionId::new(301);
9994 let response = handler
9995 .handle_control(
9996 connection,
9997 hello_with_role_declarations(declarations.clone()),
9998 )
9999 .unwrap();
10000 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10001 let registered = registry
10002 .get_module_by_connection(connection)
10003 .unwrap()
10004 .unwrap()
10005 .manifest;
10006 assert_eq!(
10010 serde_json::to_value(®istered.role_declarations).unwrap(),
10011 declarations
10012 );
10013 let (raw, entry) = role_declarations_catalog(&handler);
10014 assert_eq!(raw["role_declarations"], declarations);
10015 assert_eq!(entry.role_declarations, registered.role_declarations);
10016
10017 let request =
10018 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10019 handler
10020 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10021 .unwrap();
10022 let (raw, entry) = role_declarations_catalog(&handler);
10023 assert_eq!(raw["role_declarations"], declarations);
10024 assert_eq!(entry.role_declarations, registered.role_declarations);
10025 }
10026
10027 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10028 let registry = Arc::new(Registry::default());
10029 let handler = ControlHandler::new(Arc::clone(®istry));
10030 let capture = EventCapture::default();
10031 let _subscriber =
10032 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10033 let response = handler
10034 .handle_control(
10035 ConnectionId::new(301),
10036 hello_with_role_declarations(declarations),
10037 )
10038 .unwrap();
10039 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10040 assert!(registry
10041 .get_module_by_connection(ConnectionId::new(301))
10042 .unwrap()
10043 .unwrap()
10044 .manifest
10045 .role_declarations
10046 .is_empty());
10047 let (raw, entry) = role_declarations_catalog(&handler);
10048 assert!(raw.get("role_declarations").is_none());
10049 assert!(entry.role_declarations.is_empty());
10050 let warnings: Vec<_> = capture
10051 .events()
10052 .into_iter()
10053 .filter(|event| {
10054 event.fields.get("message").map(String::as_str)
10055 == Some("dropping whole role_declarations map from HELLO")
10056 })
10057 .collect();
10058 assert_eq!(warnings.len(), 1);
10059 assert_eq!(warnings[0].level, tracing::Level::WARN);
10060 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10061 assert_eq!(warnings[0].fields["reason"], reason);
10062 }
10063
10064 #[test]
10065 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10066 for value in [
10067 json!(null),
10068 json!(false),
10069 json!(0),
10070 json!("opaque"),
10071 json!([]),
10072 ] {
10073 assert_hello_drops_role_declarations_with_warning(
10074 json!({"routing/v1": {}, "broken/v1": value}),
10075 "role_declarations values must be objects",
10076 );
10077 }
10078 assert_hello_drops_role_declarations_with_warning(
10079 json!([]),
10080 "role_declarations must be an object",
10081 );
10082 }
10083
10084 #[test]
10085 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10086 assert_hello_drops_role_declarations_with_warning(
10087 json!({"routing/v1": {}, "bad key": {}}),
10088 "role_declarations key must use capability-name grammar",
10089 );
10090 }
10091
10092 #[test]
10093 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10094 assert_hello_drops_role_declarations_with_warning(
10095 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10096 "role_declarations exceeds 16384 compact JSON bytes",
10097 );
10098 }
10099
10100 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10101 let mut body = serde_json::to_value(ModuleHelloBody {
10102 manifest: manifest("flow-ops", PROTOCOL_VERSION),
10103 protocol_ver: PROTOCOL_VERSION,
10104 control_ops: None,
10105 launch_nonce: None,
10106 })
10107 .unwrap();
10108 body["manifest"]["provides"] = json!([{
10109 "role": "management_surface",
10110 "operations": [
10111 {"name": "diagnostics.findings", "kind": "query"},
10112 {"name": "reset", "kind": "mutate"}
10113 ],
10114 "config_schema": {}, "observability": [], "identity_scope": []
10115 }]);
10116 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10117 if let Some(flow_operations) = flow_operations {
10118 body["manifest"]["flow_operations"] = flow_operations;
10119 }
10120 Frame::build(
10121 FrameType::Hello,
10122 control_flags(),
10123 0,
10124 0,
10125 401,
10126 serde_json::to_vec(&body).unwrap(),
10127 )
10128 .unwrap()
10129 }
10130
10131 #[test]
10132 fn catalog_list_mirrors_flow_operations_unchanged() {
10133 for (capability, flow_operations) in [
10136 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10137 ("call-modes/v1", json!(["diagnostics.findings"])),
10138 ("flow-scopes/v1", json!([])),
10139 ] {
10140 let registry = Arc::new(Registry::default());
10141 let handler = ControlHandler::new(Arc::clone(®istry));
10142 let connection = ConnectionId::new(401);
10143 let response = handler
10144 .handle_control(
10145 connection,
10146 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10147 )
10148 .unwrap();
10149 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10150 let registered = registry
10151 .get_module_by_connection(connection)
10152 .unwrap()
10153 .unwrap()
10154 .manifest;
10155 assert_eq!(
10156 serde_json::to_value(®istered.flow_operations).unwrap(),
10157 flow_operations
10158 );
10159 let (raw, entry) = role_declarations_catalog(&handler);
10160 assert_eq!(raw["flow_operations"], flow_operations);
10161 assert_eq!(entry.flow_operations, registered.flow_operations);
10162 }
10163
10164 let registry = Arc::new(Registry::default());
10165 let handler = ControlHandler::new(Arc::clone(®istry));
10166 let response = handler
10167 .handle_control(
10168 ConnectionId::new(401),
10169 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10170 )
10171 .unwrap();
10172 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10173 let (raw, entry) = role_declarations_catalog(&handler);
10174 assert!(raw.get("flow_operations").is_none());
10175 assert_eq!(entry.flow_operations, None);
10176 }
10177
10178 fn assert_hello_drops_flow_operations_with_warning(
10179 capabilities: Value,
10180 flow_operations: Value,
10181 reason: &str,
10182 ) {
10183 let registry = Arc::new(Registry::default());
10184 let handler = ControlHandler::new(Arc::clone(®istry));
10185 let capture = EventCapture::default();
10186 let _subscriber =
10187 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10188 let response = handler
10189 .handle_control(
10190 ConnectionId::new(401),
10191 hello_with_flow_operations(capabilities, Some(flow_operations)),
10192 )
10193 .unwrap();
10194 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10195 let registered = registry
10196 .get_module_by_connection(ConnectionId::new(401))
10197 .unwrap()
10198 .expect("the module still registers")
10199 .manifest;
10200 assert_eq!(registered.module_id, "flow-ops");
10201 assert_eq!(registered.flow_operations, None);
10202 let (raw, entry) = role_declarations_catalog(&handler);
10203 assert_eq!(raw["module_id"], "flow-ops");
10204 assert!(raw.get("flow_operations").is_none());
10205 assert_eq!(entry.flow_operations, None);
10206 let warnings: Vec<_> = capture
10207 .events()
10208 .into_iter()
10209 .filter(|event| {
10210 event.fields.get("message").map(String::as_str)
10211 == Some("dropping flow_operations from HELLO")
10212 })
10213 .collect();
10214 assert_eq!(warnings.len(), 1);
10215 assert_eq!(warnings[0].level, tracing::Level::WARN);
10216 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10217 assert_eq!(warnings[0].fields["reason"], reason);
10218 }
10219
10220 #[test]
10221 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10222 assert_hello_drops_flow_operations_with_warning(
10223 json!(["flow-scopes/v1"]),
10224 json!(["diagnostics.findings", "missing"]),
10225 "flow_operations names 'missing', which is not a declared management operation",
10226 );
10227 }
10228
10229 #[test]
10230 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10231 assert_hello_drops_flow_operations_with_warning(
10232 json!([]),
10233 json!(["diagnostics.findings"]),
10234 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10235 );
10236 assert_hello_drops_flow_operations_with_warning(
10238 json!(["agent-run-scopes/v1"]),
10239 json!([]),
10240 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10241 );
10242 }
10243
10244 #[test]
10245 fn hello_malformed_flow_operations_is_dropped_with_warning() {
10246 assert_hello_drops_flow_operations_with_warning(
10247 json!(["flow-scopes/v1"]),
10248 json!(["reset", "reset"]),
10249 "flow_operations names 'reset' more than once",
10250 );
10251 assert_hello_drops_flow_operations_with_warning(
10252 json!(["flow-scopes/v1"]),
10253 json!(vec!["reset"; 257]),
10254 "flow_operations lists 257 names, more than 256",
10255 );
10256 assert_hello_drops_flow_operations_with_warning(
10257 json!(["flow-scopes/v1"]),
10258 json!("reset"),
10259 "flow_operations must be an array of operation names",
10260 );
10261 }
10262
10263 #[test]
10264 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10265 let registry = Arc::new(Registry::default());
10266 let handler = ControlHandler::new(registry.clone());
10267 let descriptor = json!({
10268 "schema": "ckios.richui.manifest/1",
10269 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10270 "bytes": 128,
10271 "chunks": 2
10272 });
10273 let malformed = [
10274 ("schema", json!("")),
10275 ("schema", json!("a".repeat(65))),
10276 ("schema", json!("Bad schema")),
10277 (
10278 "manifest_digest",
10279 json!(format!("sha256:{}", "A".repeat(64))),
10280 ),
10281 ("manifest_digest", json!("sha256:abc")),
10282 ("bytes", json!(0)),
10283 ("bytes", json!(16_777_217)),
10284 ("bytes", json!("128")),
10285 ("chunks", json!(0)),
10286 ("chunks", json!(1025)),
10287 ("chunks", json!("2")),
10288 ];
10289 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10290 let mut invalid = descriptor.clone();
10291 invalid[field] = invalid_value;
10292 let response = handler
10293 .handle_control(
10294 ConnectionId::new(210 + index as u64),
10295 hello_frame_with_rich_ui(
10296 &format!("bad-rich-ui-{index}"),
10297 PROTOCOL_VERSION,
10298 210 + index as u64,
10299 None,
10300 invalid,
10301 ),
10302 )
10303 .unwrap();
10304 assert_eq!(response[0].header.ty, FrameType::Error);
10305 let error = parse_error(&response[0]);
10306 assert_eq!(error["code"], "invalid_manifest");
10307 assert!(error["message"]
10308 .as_str()
10309 .unwrap()
10310 .contains(&format!("rich_ui.{field}")));
10311 assert!(registry
10312 .get_module(&format!("bad-rich-ui-{index}"))
10313 .unwrap()
10314 .is_none());
10315 }
10316 }
10317
10318 #[test]
10319 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10320 let registry = Arc::new(Registry::default());
10321 let handler = ControlHandler::new(Arc::clone(®istry));
10322 let connection = ConnectionId::new(219);
10323 registry
10324 .register_with_control_ops(
10325 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10326 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10327 "bad schema",
10328 format!("sha256:{}", "a".repeat(64)),
10329 128,
10330 2,
10331 )))
10332 .build(),
10333 PROTOCOL_VERSION,
10334 connection,
10335 Vec::new(),
10336 )
10337 .unwrap();
10338 let request =
10339 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10340 let response = handler
10341 .handle_catalog_update(connection, request, Vec::new(), None, None)
10342 .unwrap();
10343 assert_eq!(response[0].header.ty, FrameType::Error);
10344 let error = parse_error(&response[0]);
10345 assert_eq!(error["code"], "invalid_manifest");
10346 assert!(error["message"]
10347 .as_str()
10348 .unwrap()
10349 .contains("rich_ui.schema"));
10350 }
10351
10352 #[test]
10353 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10354 let registry = Arc::new(Registry::default());
10355 let supervisor = SupervisorHandle::new();
10356 supervisor.open_swap("provider", "candidate-nonce".to_string());
10357 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10358 let connection = ConnectionId::new(218);
10359 let hello = handler
10360 .handle_control(
10361 connection,
10362 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10363 )
10364 .unwrap();
10365 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10366 assert!(registry.list_modules().unwrap().1.is_empty());
10367
10368 registry.promote_candidate("provider").unwrap().unwrap();
10369 let request = Frame::build(
10370 FrameType::Request,
10371 control_flags(),
10372 0,
10373 0,
10374 219,
10375 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10376 )
10377 .unwrap();
10378 let response = handler.handle_catalog_list(request, None).unwrap();
10379 let ClientControlResponse::CatalogList { modules, .. } =
10380 serde_json::from_slice(&response[0].body).unwrap()
10381 else {
10382 panic!("expected catalog.list");
10383 };
10384 assert_eq!(modules.len(), 1);
10385 assert!(modules[0].nonce_verified);
10386 }
10387
10388 #[test]
10389 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10390 let registry = Arc::new(Registry::default());
10391 let handler = ControlHandler::new(Arc::clone(®istry));
10392 let hello = Frame::build(
10393 FrameType::Hello,
10394 control_flags(),
10395 0,
10396 0,
10397 220,
10398 serde_json::to_vec(&ModuleHelloBody {
10399 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10400 protocol_ver: PROTOCOL_VERSION,
10401 control_ops: None,
10402 launch_nonce: Some("unverified".to_string()),
10403 })
10404 .unwrap(),
10405 )
10406 .unwrap();
10407 assert_eq!(
10408 handler
10409 .handle_control(ConnectionId::new(220), hello)
10410 .unwrap()[0]
10411 .header
10412 .ty,
10413 FrameType::HelloAck
10414 );
10415 let request = Frame::build(
10416 FrameType::Request,
10417 control_flags(),
10418 0,
10419 0,
10420 221,
10421 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10422 )
10423 .unwrap();
10424 let response = handler.handle_catalog_list(request, None).unwrap();
10425 let ClientControlResponse::CatalogList { modules, .. } =
10426 serde_json::from_slice(&response[0].body).unwrap()
10427 else {
10428 panic!("expected catalog.list");
10429 };
10430 assert_eq!(
10431 serde_json::to_vec(&modules[0]).unwrap(),
10432 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10433 );
10434 assert!(!modules[0].nonce_verified);
10435 }
10436
10437 #[test]
10438 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10439 let registry = Arc::new(Registry::default());
10440 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10441 [("vault".to_string(), true), ("squatter".to_string(), true)],
10442 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10443 );
10444 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10445 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10446 provides: vec!["credentials-provider/v1".to_string()],
10447 requires: Vec::new(),
10448 must_never_reach: Vec::new(),
10449 });
10450 let frame = Frame::build(
10451 FrameType::Hello,
10452 control_flags(),
10453 0,
10454 0,
10455 77,
10456 serde_json::to_vec(&ModuleHelloBody {
10457 manifest: squatter,
10458 protocol_ver: PROTOCOL_VERSION,
10459 control_ops: None,
10460 launch_nonce: None,
10461 })
10462 .expect("HELLO serializes"),
10463 )
10464 .expect("HELLO frame builds");
10465 let response = handler
10466 .handle_control(ConnectionId::new(77), frame)
10467 .expect("reserved claim receives a typed refusal");
10468 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10469 assert_eq!(
10470 registry
10471 .active_registration_count()
10472 .expect("registry reads"),
10473 0,
10474 "a reserved capability refusal must not leave a catalog entry"
10475 );
10476 }
10477
10478 #[test]
10479 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10480 for settlement in ["timeout", "inconclusive", "drop"] {
10481 let breakers = RouteBindBreakers::default();
10482 let RouteBindAdmission::Admitted {
10483 guard: mut old,
10484 probe: false,
10485 } = breakers.admit("prov")
10486 else {
10487 panic!("ordinary relay admitted")
10488 };
10489 let RouteBindAdmission::Admitted {
10490 guard: mut opener, ..
10491 } = breakers.admit("prov")
10492 else {
10493 panic!("second relay admitted")
10494 };
10495 assert!(
10496 !opener
10497 .record_timeout(1, Duration::ZERO)
10498 .unwrap()
10499 .reopened_after_probe
10500 );
10501 let RouteBindAdmission::Admitted {
10502 guard: mut probe,
10503 probe: true,
10504 } = breakers.admit("prov")
10505 else {
10506 panic!("one cooldown probe admitted")
10507 };
10508 match settlement {
10509 "timeout" => assert!(
10510 !old.record_timeout(1, Duration::ZERO)
10511 .unwrap()
10512 .reopened_after_probe
10513 ),
10514 "inconclusive" => old.record_inconclusive(),
10515 "drop" => drop(old),
10516 _ => unreachable!(),
10517 }
10518 assert!(
10519 matches!(
10520 breakers.admit("prov"),
10521 RouteBindAdmission::Refused {
10522 probe_in_flight: true,
10523 ..
10524 }
10525 ),
10526 "{settlement} of a pre-open relay cannot release the real probe"
10527 );
10528 assert!(
10529 probe
10530 .record_timeout(1, Duration::ZERO)
10531 .unwrap()
10532 .reopened_after_probe
10533 );
10534 assert!(matches!(
10535 breakers.admit("prov"),
10536 RouteBindAdmission::Admitted { probe: true, .. }
10537 ));
10538 }
10539 let breakers = RouteBindBreakers::default();
10540 let admit = || match breakers.admit("prov") {
10541 RouteBindAdmission::Admitted { guard, .. } => guard,
10542 _ => panic!("relay admitted"),
10543 };
10544 admit().record_timeout(1, Duration::ZERO);
10545 let mut old_probe = admit();
10546 breakers.reset_for_new_module_connection("prov");
10547 admit().record_timeout(1, Duration::ZERO);
10548 let _new_probe = admit();
10549 old_probe.record_inconclusive();
10550 assert!(matches!(
10551 breakers.admit("prov"),
10552 RouteBindAdmission::Refused {
10553 probe_in_flight: true,
10554 ..
10555 }
10556 ));
10557 }
10558
10559 #[tokio::test]
10560 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10561 for candidate in [false, true] {
10562 let registry = Arc::new(Registry::default());
10563 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10564 [("vault".to_string(), true), ("squatter".to_string(), true)],
10565 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10566 );
10567 let conn = ConnectionId::new(77);
10568 let (ctx, mut rx) = route_ctx(conn);
10569 let initial = capability_manifest("squatter", &[], &[]);
10570 if candidate {
10571 registry
10572 .register_candidate_with_control_ops(
10573 initial.clone(),
10574 PROTOCOL_VERSION,
10575 conn,
10576 module_baseline_control_ops(),
10577 )
10578 .unwrap();
10579 handler
10580 .forwarding
10581 .register_candidate_module_connection(
10582 conn,
10583 "squatter".to_string(),
10584 PROTOCOL_VERSION,
10585 manifest_concurrency(&initial),
10586 ctx.egress.clone(),
10587 )
10588 .unwrap();
10589 } else {
10590 hello_via_sink(
10591 &handler,
10592 &ctx,
10593 &mut rx,
10594 hello_frame_with_manifest(initial.clone(), 1),
10595 )
10596 .await;
10597 }
10598 let response = handler
10599 .handle_control_frame(
10600 &ctx,
10601 catalog_update_with_capabilities_frame(
10602 2,
10603 capability_manifest("squatter", &["credentials-provider/v1"], &[])
10604 .capabilities
10605 .unwrap(),
10606 ),
10607 )
10608 .await
10609 .unwrap();
10610 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10611 assert_eq!(
10612 registry
10613 .get_module_by_connection(conn)
10614 .unwrap()
10615 .unwrap()
10616 .manifest,
10617 initial
10618 );
10619 }
10620 }
10621
10622 #[test]
10623 fn server_describe_surfaces_required_capability_verdict_fields() {
10624 let registry = Arc::new(Registry::default());
10625 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10626 [
10627 ("consumer".to_string(), true),
10628 ("provider".to_string(), false),
10629 ],
10630 BTreeMap::new(),
10631 );
10632 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10633 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10634 provides: Vec::new(),
10635 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10636 capability: "credentials-provider/v1".to_string(),
10637 need: subc_protocol::manifest::CapabilityNeed::Required,
10638 }],
10639 must_never_reach: Vec::new(),
10640 });
10641 let hello = Frame::build(
10642 FrameType::Hello,
10643 control_flags(),
10644 0,
10645 0,
10646 78,
10647 serde_json::to_vec(&ModuleHelloBody {
10648 manifest: consumer,
10649 protocol_ver: PROTOCOL_VERSION,
10650 control_ops: None,
10651 launch_nonce: None,
10652 })
10653 .expect("HELLO serializes"),
10654 )
10655 .expect("HELLO frame builds");
10656 handler
10657 .handle_control(ConnectionId::new(78), hello)
10658 .expect("consumer registers");
10659 let describe = Frame::build(
10660 FrameType::Request,
10661 control_flags(),
10662 0,
10663 0,
10664 79,
10665 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10666 .expect("request serializes"),
10667 )
10668 .expect("describe frame builds");
10669 let response = handler
10670 .handle_server_describe(describe)
10671 .expect("server.describe succeeds");
10672 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10673 let requirement = &rendered["capability_requirements"][0];
10674 assert_eq!(requirement["consumer"], "consumer");
10675 assert_eq!(requirement["verdict"], "never_provided");
10676 assert_eq!(requirement["episode_seq"], 1);
10677 assert_eq!(requirement["config_satisfiable"], false);
10678 assert_eq!(requirement["runtime_available"], false);
10679 assert!(requirement["detail"]
10680 .as_str()
10681 .expect("detail string")
10682 .contains("credentials-provider/v1"));
10683 }
10684
10685 fn deregister_required_provider(
10691 daemon_shutdown_begun: bool,
10692 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10693 const CAPABILITY: &str = "project-identity/v1";
10694 let registry = Arc::new(Registry::default());
10695 let daemon_shutdown = DaemonShutdownFlag::default();
10696 let handler = ControlHandler::new(Arc::clone(®istry))
10697 .with_capability_config(
10698 [
10699 ("consumer".to_string(), true),
10700 ("provider".to_string(), true),
10701 ],
10702 BTreeMap::new(),
10703 )
10704 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10705 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10706 consumer.capabilities = Some(CapabilityDeclarations {
10707 provides: Vec::new(),
10708 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10709 capability: CAPABILITY.to_string(),
10710 need: subc_protocol::manifest::CapabilityNeed::Required,
10711 }],
10712 must_never_reach: Vec::new(),
10713 });
10714 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10715 let provider_conn = ConnectionId::new(182);
10716 for (conn, manifest, corr) in [
10717 (ConnectionId::new(181), consumer, 181),
10718 (provider_conn, provider, 182),
10719 ] {
10720 let hello = handler
10721 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10722 .expect("module registers");
10723 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10724 }
10725 assert_eq!(
10726 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10727 Some(CapabilityVerdict::Provided),
10728 "the registered provider must satisfy the requirement first"
10729 );
10730 if daemon_shutdown_begun {
10731 daemon_shutdown.set_for_test();
10732 }
10733
10734 let capture = EventCapture::default();
10735 let _subscriber =
10736 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10737 handler
10738 .cleanup_connection(provider_conn)
10739 .expect("provider deregisters");
10740 let events = capture
10741 .events()
10742 .into_iter()
10743 .filter(|event| {
10744 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10745 })
10746 .collect();
10747 (
10748 events,
10749 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10750 )
10751 }
10752
10753 #[test]
10754 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10755 let (events, verdict) = deregister_required_provider(true);
10756 assert_eq!(
10757 verdict,
10758 Some(CapabilityVerdict::NeverProvided),
10759 "shutdown must not change the verdict route.open refusals read"
10760 );
10761 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10762 let event = &events[0];
10763 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10764 assert_eq!(
10765 event.fields.get("cause").map(String::as_str),
10766 Some("\"daemon_shutdown\""),
10767 "{event:?}"
10768 );
10769 assert_eq!(
10770 event.fields.get("verdict").map(String::as_str),
10771 Some("\"never_provided\""),
10772 "{event:?}"
10773 );
10774 }
10775
10776 #[test]
10777 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10778 let (events, verdict) = deregister_required_provider(false);
10779 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10780 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10781 let event = &events[0];
10782 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10783 assert_eq!(event.fields.get("cause"), None, "{event:?}");
10784 assert_eq!(
10785 event.fields.get("verdict").map(String::as_str),
10786 Some("\"never_provided\""),
10787 "{event:?}"
10788 );
10789 }
10790
10791 #[test]
10792 fn catalog_list_omits_capabilities_for_legacy_manifest() {
10793 let registry = Arc::new(Registry::default());
10794 let handler = ControlHandler::new(Arc::clone(®istry));
10795 let hello = handler
10796 .handle_control(
10797 ConnectionId::new(101),
10798 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10799 )
10800 .expect("legacy HELLO registers");
10801 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10802
10803 let request = Frame::build(
10804 FrameType::Request,
10805 control_flags(),
10806 0,
10807 0,
10808 102,
10809 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10810 .expect("catalog request serializes"),
10811 )
10812 .expect("catalog request frame builds");
10813 let response = handler
10814 .handle_catalog_list(request, None)
10815 .expect("catalog list succeeds");
10816 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10817 assert!(
10818 body["modules"][0].get("capabilities").is_none(),
10819 "legacy manifest must retain an absent capabilities field on catalog.list"
10820 );
10821 }
10822
10823 #[test]
10824 fn hello_ack_omits_storage_when_no_storage_config() {
10825 let registry = Arc::new(Registry::default());
10826 let handler = ControlHandler::new(Arc::clone(®istry));
10827 let responses = handler
10828 .handle_control(
10829 ConnectionId::new(1),
10830 hello_frame("aft", PROTOCOL_VERSION, 7),
10831 )
10832 .unwrap();
10833 let ack = parse_ack(&responses[0]);
10834 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10835 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10836 }
10837
10838 #[tokio::test]
10839 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10840 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10841 let registry = Arc::new(Registry::default());
10842 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
10843 let responses = handler
10844 .handle_control(
10845 ConnectionId::new(1),
10846 hello_frame("aft", PROTOCOL_VERSION, 7),
10847 )
10848 .unwrap();
10849 let ack = parse_ack(&responses[0]);
10850 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10851
10852 let described = handler
10853 .handle_control_frame(
10854 &route_ctx(ConnectionId::new(2)).0,
10855 Frame::build(
10856 FrameType::Request,
10857 control_flags(),
10858 0,
10859 0,
10860 9,
10861 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10862 )
10863 .unwrap(),
10864 )
10865 .await
10866 .unwrap();
10867 let ClientControlResponse::ServerDescribe { machine_id, .. } =
10868 serde_json::from_slice(&described[0].body).unwrap()
10869 else {
10870 panic!("server.describe answered with another shape");
10871 };
10872 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10873 }
10874
10875 #[test]
10876 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10877 let registry = Arc::new(Registry::default());
10880 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
10881 crate::daemon_config::StorageConfig::Sqlite {
10882 data_home: std::path::PathBuf::from("/data"),
10883 },
10884 ));
10885
10886 let responses = handler
10887 .handle_control(
10888 ConnectionId::new(1),
10889 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10890 )
10891 .unwrap();
10892 let ack = parse_ack(&responses[0]);
10893 assert_eq!(
10894 ack.storage,
10895 Some(serde_json::json!({
10896 "module_id": "alfonso-routing",
10897 "storage_namespace": "default",
10898 "isolation": { "kind": "module" },
10899 "backend": {
10900 "backend": "sqlite",
10901 "path": "/data/cortexkit/alfonso-routing/store.db"
10902 }
10903 })),
10904 "the delivered descriptor is the module's own sqlite store path"
10905 );
10906 }
10907
10908 #[test]
10909 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10910 let registry = Arc::new(Registry::default());
10911 let handler = ControlHandler::new(Arc::clone(®istry));
10912 let conn = ConnectionId::new(1);
10913 let responses = handler
10914 .handle_control(
10915 conn,
10916 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10917 )
10918 .unwrap();
10919 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10920 let registration = registry.get_module("aft").unwrap().unwrap();
10921 assert_eq!(registration.control_ops, module_baseline_control_ops());
10922
10923 let frame =
10924 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10925 assert!(handler
10926 .guard_module_control_op(&frame, "aft", "route.bind")
10927 .unwrap()
10928 .is_none());
10929 let error = handler
10930 .guard_module_control_op(&frame, "aft", "test.synthetic")
10931 .unwrap()
10932 .expect("synthetic ungranted op should be rejected");
10933 assert_eq!(error.header.ty, FrameType::Error);
10934 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10935 }
10936
10937 #[test]
10938 fn hello_control_ops_some_adds_optional_grants() {
10939 let registry = Arc::new(Registry::default());
10940 let handler = ControlHandler::new(Arc::clone(®istry));
10941 handler
10942 .handle_control(
10943 ConnectionId::new(1),
10944 hello_frame_with_control_ops(
10945 "aft",
10946 PROTOCOL_VERSION,
10947 7,
10948 Some(vec![
10949 "future.synthetic".to_string(),
10950 "route.bind".to_string(),
10951 ]),
10952 ),
10953 )
10954 .unwrap();
10955 let registration = registry.get_module("aft").unwrap().unwrap();
10956 assert_eq!(
10957 registration.control_ops,
10958 vec![
10959 "route.bind".to_string(),
10960 "route.status".to_string(),
10961 "future.synthetic".to_string(),
10962 ]
10963 );
10964 let frame =
10965 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10966 assert!(handler
10967 .guard_module_control_op(&frame, "aft", "future.synthetic")
10968 .unwrap()
10969 .is_none());
10970 }
10971
10972 #[tokio::test]
10973 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10974 let registry = Arc::new(Registry::default());
10975 let forwarding = Arc::new(ForwardingTable::default());
10976 let handler =
10977 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10978 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10979 hello_via_sink(
10980 &handler,
10981 &module_ctx,
10982 &mut module_rx,
10983 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10984 )
10985 .await;
10986
10987 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10988 let responses = handler
10989 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10990 .await
10991 .unwrap();
10992 assert_eq!(responses.len(), 1);
10993 assert_eq!(responses[0].header.ty, FrameType::Error);
10994 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10995 assert!(module_rx.try_recv().is_err());
10996 }
10997
10998 #[tokio::test]
10999 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11000 let registry = Arc::new(Registry::default());
11001 let forwarding = Arc::new(ForwardingTable::default());
11002 let handler =
11003 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11004 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11005 hello_via_sink(
11006 &handler,
11007 &module_ctx,
11008 &mut module_rx,
11009 hello_frame_with_control_ops(
11010 "aft",
11011 PROTOCOL_VERSION,
11012 7,
11013 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11014 ),
11015 )
11016 .await;
11017
11018 let project_root = unique_project_root("demux");
11019 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11020 let route_handler = handler.clone();
11021 let route_task = tokio::spawn(async move {
11022 route_handler
11023 .handle_control_frame(
11024 &route_client_ctx,
11025 route_open_frame(100, "aft", project_root),
11026 )
11027 .await
11028 .unwrap()
11029 });
11030 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11031 .await
11032 .unwrap()
11033 .unwrap();
11034 assert!(matches!(
11035 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11036 ModuleControlRequest::RouteBind { .. }
11037 ));
11038
11039 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11040 let health_handler = handler.clone();
11041 let health_task = tokio::spawn(async move {
11042 health_handler
11043 .handle_control_frame(
11044 &health_client_ctx,
11045 supervisor_health_probe_frame(101, "aft"),
11046 )
11047 .await
11048 .unwrap()
11049 });
11050 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11051 .await
11052 .unwrap()
11053 .unwrap();
11054 assert_eq!(
11055 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11056 ModuleControlRequest::HealthCheck {}
11057 );
11058
11059 handler
11060 .handle_control_frame(
11061 &module_ctx,
11062 health_response(health_frame.header.corr, HealthStatus::Degraded),
11063 )
11064 .await
11065 .unwrap();
11066 let health_response = health_task.await.unwrap();
11067 assert_eq!(health_response.len(), 1);
11068 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11069 ClientControlResponse::SupervisorHealthProbe {
11070 module_id,
11071 status,
11072 detail,
11073 metrics,
11074 } => {
11075 assert_eq!(module_id, "aft");
11076 assert_eq!(status, HealthStatus::Degraded);
11077 assert_eq!(detail.as_deref(), Some("warming"));
11078 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11079 }
11080 other => panic!("unexpected health response: {other:?}"),
11081 }
11082
11083 handler
11084 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11085 .await
11086 .unwrap();
11087 let route_response = route_task.await.unwrap();
11088 assert!(route_response.is_empty());
11089 let published = route_client_rx.recv().await.unwrap();
11090 assert!(matches!(
11091 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11092 ClientControlResponse::RouteOpen { .. }
11093 ));
11094 }
11095
11096 async fn relay_route_open(
11101 handler: &ControlHandler,
11102 client_connection: ConnectionId,
11103 client_egress: &FrameSink,
11104 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11105 corr: u64,
11106 module_id: &str,
11107 project_root_label: &str,
11108 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11109 let ctx = RouteCtx {
11110 connection_id: client_connection,
11111 egress: client_egress.clone(),
11112 };
11113 let handler = handler.clone();
11114 let project_root = unique_project_root(project_root_label);
11115 let module_id = module_id.to_string();
11116 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11117 let task = tokio::spawn(async move {
11118 let _guard = tracing::dispatcher::set_default(&dispatch);
11119 handler
11120 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11121 .await
11122 .unwrap()
11123 });
11124 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11125 .await
11126 .expect("module receives the relayed route.bind")
11127 .expect("module egress is open");
11128 (task, bind.frame)
11129 }
11130
11131 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11132 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11133 ModuleControlRequest::RouteBind {
11134 route_channel,
11135 epoch,
11136 ..
11137 } => (route_channel, epoch),
11138 other => panic!("expected a route.bind request, got {other:?}"),
11139 }
11140 }
11141
11142 fn published_route(frame: &Frame) -> (u16, u32) {
11143 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11144 ClientControlResponse::RouteOpen {
11145 route_channel,
11146 route_epoch,
11147 } => (route_channel, route_epoch),
11148 other => panic!("expected a route.open response, got {other:?}"),
11149 }
11150 }
11151
11152 #[tokio::test]
11172 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11173 let registry = Arc::new(Registry::default());
11174 let forwarding = Arc::new(ForwardingTable::default());
11175 let handler =
11176 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11177
11178 let module_connection = ConnectionId::new(30);
11179 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11180 hello_via_sink(
11181 &handler,
11182 &module_ctx,
11183 &mut module_rx,
11184 hello_frame("aft", PROTOCOL_VERSION, 7),
11185 )
11186 .await;
11187
11188 let dying_client = ConnectionId::new(31);
11189 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11190
11191 let (first_task, first_bind) = relay_route_open(
11194 &handler,
11195 dying_client,
11196 &dying_ctx.egress,
11197 &mut module_rx,
11198 100,
11199 "aft",
11200 "closing-first",
11201 )
11202 .await;
11203 handler
11204 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11205 .await
11206 .unwrap();
11207 assert!(first_task.await.unwrap().is_empty());
11208 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11209
11210 let (second_task, second_bind) = relay_route_open(
11212 &handler,
11213 dying_client,
11214 &dying_ctx.egress,
11215 &mut module_rx,
11216 101,
11217 "aft",
11218 "closing-second",
11219 )
11220 .await;
11221 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11222
11223 assert!(forwarding
11226 .escalate_client_delivery_failure(
11227 dying_client,
11228 first_channel,
11229 first_epoch,
11230 CloseReason::new(
11231 "module_to_client_delivery_failed",
11232 "client egress refused a module frame",
11233 ),
11234 crate::forwarding::UndeliveredFrame {
11235 module_id: None,
11236 sink: &dying_ctx.egress,
11237 },
11238 )
11239 .unwrap());
11240 assert!(!dying_ctx.egress.is_closed());
11241
11242 let ack = handler
11244 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11245 .await;
11246 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11247 if module_loop_error.is_some() {
11248 handler.cleanup_connection(module_connection).unwrap();
11252 }
11253 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11257 .await
11258 .ok()
11259 .flatten();
11260
11261 assert!(
11263 registry
11264 .get_module_by_connection(module_connection)
11265 .unwrap()
11266 .is_some(),
11267 "one client's closing connection ended the shared module connection: \
11268 {module_loop_error:?}"
11269 );
11270 let cotenant = ConnectionId::new(32);
11272 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11273 let (cotenant_task, cotenant_bind) = relay_route_open(
11274 &handler,
11275 cotenant,
11276 &cotenant_ctx.egress,
11277 &mut module_rx,
11278 102,
11279 "aft",
11280 "closing-cotenant",
11281 )
11282 .await;
11283 handler
11284 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11285 .await
11286 .unwrap();
11287 assert!(cotenant_task.await.unwrap().is_empty());
11288 let (cotenant_channel, cotenant_epoch) =
11289 published_route(&cotenant_rx.recv().await.unwrap());
11290 assert!(matches!(
11291 forwarding
11292 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11293 .unwrap(),
11294 DataRoute::Client(DataRouteState::Bound(_))
11295 ));
11296
11297 let goodbye = post_ack_module_frame
11300 .expect("module receives a GOODBYE for the abandoned route channel");
11301 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11302 assert_eq!(goodbye.header.channel, abandoned_channel);
11303 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11304
11305 assert!(dying_rx.try_recv().is_err());
11309 let second_response = second_task.await.unwrap();
11310 assert_eq!(second_response.len(), 1);
11311 assert_eq!(
11312 parse_error(&second_response[0])["code"],
11313 "target_unavailable"
11314 );
11315 }
11316
11317 async fn catalog_snapshot(
11320 handler: &ControlHandler,
11321 client_ctx: &RouteCtx,
11322 corr: u64,
11323 ) -> (u64, Vec<(String, bool)>) {
11324 let body =
11325 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11326 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11327 let reply = handler
11328 .handle_control_frame(client_ctx, request)
11329 .await
11330 .unwrap()
11331 .pop()
11332 .unwrap();
11333 let ClientControlResponse::CatalogList {
11334 generation,
11335 modules,
11336 ..
11337 } = serde_json::from_slice(&reply.body).unwrap()
11338 else {
11339 panic!("not a catalog.list reply");
11340 };
11341 (
11342 generation,
11343 modules
11344 .into_iter()
11345 .map(|entry| (entry.module_id, entry.ready))
11346 .collect(),
11347 )
11348 }
11349
11350 fn hello_held_before_forwarding() -> (
11356 ControlHandler,
11357 Arc<std::sync::Barrier>,
11358 std::thread::JoinHandle<Vec<Frame>>,
11359 RouteCtx,
11360 mpsc::Receiver<crate::router::OutboundFrame>,
11361 ) {
11362 let hold = Arc::new(std::sync::Barrier::new(2));
11363 let handler = ControlHandler::with_forwarding(
11364 Arc::new(Registry::default()),
11365 Arc::new(ForwardingTable::default()),
11366 )
11367 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11368 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11369 let hello = {
11370 let handler = handler.clone();
11371 let module_ctx = module_ctx.clone();
11372 std::thread::spawn(move || {
11373 handler
11374 .handle_hello(
11375 module_ctx.connection_id,
11376 Some(module_ctx.egress.clone()),
11377 hello_frame("aft", PROTOCOL_VERSION, 7),
11378 )
11379 .unwrap()
11380 })
11381 };
11382 (handler, hold, hello, module_ctx, module_rx)
11383 }
11384
11385 #[tokio::test]
11394 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11395 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11396 let client = ConnectionId::new(41);
11397 let (client_ctx, mut client_rx) = route_ctx(client);
11398
11399 hold.wait();
11400 assert!(handler.registry.get_module("aft").unwrap().is_some());
11402 assert!(!handler
11403 .forwarding
11404 .has_live_module_connection("aft")
11405 .unwrap());
11406 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11407 assert!(
11408 listed.is_empty(),
11409 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11410 );
11411 let refused = handler
11412 .handle_control_frame(
11413 &client_ctx,
11414 route_open_frame(301, "aft", unique_project_root("listed-window")),
11415 )
11416 .await
11417 .unwrap();
11418 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11419 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11420 let duplicate = handler
11421 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11422 .await
11423 .unwrap();
11424 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11425
11426 hold.wait();
11427 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11428 let ack = module_rx.try_recv().unwrap().frame;
11429 assert_eq!(ack.header.ty, FrameType::HelloAck);
11430 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11431 assert_eq!(listed, vec![("aft".to_string(), true)]);
11432
11433 let (open_task, bind) = relay_route_open(
11434 &handler,
11435 client,
11436 &client_ctx.egress,
11437 &mut module_rx,
11438 303,
11439 "aft",
11440 "listed-after",
11441 )
11442 .await;
11443 assert!(handler
11444 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11445 .await
11446 .unwrap()
11447 .is_empty());
11448 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11449 published_route(&client_rx.try_recv().unwrap().frame);
11450 }
11451
11452 #[tokio::test]
11459 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11460 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11461 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11462 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11463
11464 hold.wait();
11465 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11466 assert!(listed.is_empty());
11467 assert_eq!(
11468 during, before,
11469 "the generation moved while the module was not listed"
11470 );
11471
11472 hold.wait();
11473 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11474 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11475 assert_eq!(listed, vec![("aft".to_string(), true)]);
11476 assert_eq!(
11477 after,
11478 before + 1,
11479 "the module appeared without exactly one generation bump"
11480 );
11481 }
11482
11483 #[tokio::test]
11488 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11489 let registry = Arc::new(Registry::default());
11490 let handler = ControlHandler::with_forwarding(
11491 Arc::clone(®istry),
11492 Arc::new(ForwardingTable::default()),
11493 );
11494 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11495 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11496 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11497 drop(module_rx);
11498
11499 let replies = handler
11500 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11501 .await
11502 .unwrap();
11503 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11504 assert!(registry.get_module("aft").unwrap().is_none());
11505 assert!(registry
11506 .get_module_by_connection(module_ctx.connection_id)
11507 .unwrap()
11508 .is_none());
11509 assert_eq!(
11510 catalog_snapshot(&handler, &client_ctx, 301).await,
11511 (before, Vec::new())
11512 );
11513 }
11514
11515 async fn route_open_held_after_bind_send() -> (
11519 ControlHandler,
11520 Arc<tokio::sync::Barrier>,
11521 RouteCtx,
11522 mpsc::Receiver<crate::router::OutboundFrame>,
11523 ) {
11524 let hold = Arc::new(tokio::sync::Barrier::new(2));
11525 let handler = ControlHandler::with_forwarding(
11526 Arc::new(Registry::default()),
11527 Arc::new(ForwardingTable::default()),
11528 )
11529 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11530 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11531 hello_via_sink(
11532 &handler,
11533 &module_ctx,
11534 &mut module_rx,
11535 hello_frame("aft", PROTOCOL_VERSION, 7),
11536 )
11537 .await;
11538 (handler, hold, module_ctx, module_rx)
11539 }
11540
11541 #[tokio::test]
11548 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11549 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11550 let client = ConnectionId::new(41);
11551 let (client_ctx, mut client_rx) = route_ctx(client);
11552
11553 let (open_task, bind) = relay_route_open(
11554 &handler,
11555 client,
11556 &client_ctx.egress,
11557 &mut module_rx,
11558 200,
11559 "aft",
11560 "acked-before-mark",
11561 )
11562 .await;
11563 let (module_channel, module_epoch) = route_bind_channel(&bind);
11564 hold.wait().await;
11566
11567 assert!(handler
11569 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11570 .await
11571 .unwrap()
11572 .is_empty());
11573 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11574
11575 hold.wait().await;
11576 assert!(
11577 open_task.await.unwrap().is_empty(),
11578 "route.open answers accepted"
11579 );
11580
11581 match module_rx.try_recv() {
11584 Err(mpsc::error::TryRecvError::Empty) => {}
11585 Ok(outbound) => panic!(
11586 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11587 outbound.frame.header.ty,
11588 outbound.frame.header.channel,
11589 outbound.frame.header.epoch
11590 ),
11591 Err(err) => panic!("module egress closed: {err}"),
11592 }
11593
11594 let router = Router::with_control_handler(Arc::new(handler.clone()));
11597 let request = Frame::build(
11598 FrameType::Request,
11599 Flags::new(true, Priority::Interactive, false),
11600 client_channel,
11601 client_epoch,
11602 201,
11603 b"first request".to_vec(),
11604 )
11605 .unwrap();
11606 router
11607 .route_for_connection(&client_ctx, request)
11608 .await
11609 .unwrap();
11610 let delivered = module_rx.try_recv().unwrap().frame;
11611 assert_eq!(delivered.header.ty, FrameType::Request);
11612 assert_eq!(
11613 (delivered.header.channel, delivered.header.epoch),
11614 (module_channel, module_epoch)
11615 );
11616 assert_eq!(delivered.body, b"first request");
11617 }
11618
11619 #[tokio::test]
11625 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11626 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11627 let client = ConnectionId::new(42);
11628 let (client_ctx, _client_rx) = route_ctx(client);
11629
11630 let (open_task, bind) = relay_route_open(
11631 &handler,
11632 client,
11633 &client_ctx.egress,
11634 &mut module_rx,
11635 300,
11636 "aft",
11637 "released-before-mark",
11638 )
11639 .await;
11640 let (module_channel, module_epoch) = route_bind_channel(&bind);
11641 hold.wait().await;
11642
11643 handler.cleanup_connection(client).unwrap();
11644 assert!(
11645 module_rx.try_recv().is_err(),
11646 "client teardown must not GOODBYE a bind route.open is still queueing"
11647 );
11648
11649 hold.wait().await;
11650 let response = open_task.await.unwrap();
11651 assert_eq!(response.len(), 1);
11652 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11653
11654 let goodbye = module_rx
11655 .try_recv()
11656 .expect("module is told to drop the binding the daemon abandoned")
11657 .frame;
11658 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11659 assert_eq!(
11660 (goodbye.header.channel, goodbye.header.epoch),
11661 (module_channel, module_epoch)
11662 );
11663 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11664 }
11665
11666 #[tokio::test]
11671 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11672 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11673 let client = ConnectionId::new(43);
11674 let (client_ctx, client_rx) = route_ctx(client);
11675
11676 let (open_task, bind) = relay_route_open(
11677 &handler,
11678 client,
11679 &client_ctx.egress,
11680 &mut module_rx,
11681 400,
11682 "aft",
11683 "gone-client-before-mark",
11684 )
11685 .await;
11686 let (module_channel, module_epoch) = route_bind_channel(&bind);
11687 hold.wait().await;
11688
11689 drop(client_rx);
11690 assert!(client_ctx.egress.is_closed());
11691 handler
11692 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11693 .await
11694 .unwrap();
11695
11696 hold.wait().await;
11697 let response = open_task.await.unwrap();
11698 assert_eq!(response.len(), 1);
11699 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11700
11701 let goodbye = module_rx
11702 .try_recv()
11703 .expect("module is told to drop the binding it acked")
11704 .frame;
11705 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11706 assert_eq!(
11707 (goodbye.header.channel, goodbye.header.epoch),
11708 (module_channel, module_epoch)
11709 );
11710 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11711 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11712 }
11713
11714 #[test]
11721 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11722 let handler = ControlHandler::default();
11723 let module_connection = ConnectionId::new(30);
11724 let client_connection = ConnectionId::new(31);
11725
11726 handler
11727 .refuse_to_end_module_connection_for_a_client(
11728 module_connection,
11729 77,
11730 ForwardingError::ConnectionClosing {
11731 connection_id: client_connection,
11732 },
11733 )
11734 .expect("a closing client must never end the module connection");
11735
11736 assert!(matches!(
11737 handler.refuse_to_end_module_connection_for_a_client(
11738 module_connection,
11739 78,
11740 ForwardingError::ConnectionClosing {
11741 connection_id: module_connection,
11742 },
11743 ),
11744 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11745 connection_id
11746 })) if connection_id == module_connection
11747 ));
11748 assert!(matches!(
11749 handler.refuse_to_end_module_connection_for_a_client(
11750 module_connection,
11751 79,
11752 ForwardingError::Poisoned,
11753 ),
11754 Err(RouterError::Forwarding(ForwardingError::Poisoned))
11755 ));
11756 assert!(matches!(
11757 handler.refuse_to_end_module_connection_for_a_client(
11758 module_connection,
11759 80,
11760 ForwardingError::StaleModuleEndpoint,
11761 ),
11762 Err(RouterError::Forwarding(
11763 ForwardingError::StaleModuleEndpoint
11764 ))
11765 ));
11766 }
11767
11768 #[tokio::test]
11781 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11782 let registry = Arc::new(Registry::default());
11783 let forwarding = Arc::new(ForwardingTable::default());
11784 let supervisor = SupervisorHandle::new();
11785 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11786 let handler =
11787 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11788 .with_supervisor(supervisor);
11789
11790 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11791 hello_via_sink(
11792 &handler,
11793 &target_ctx,
11794 &mut target_rx,
11795 hello_frame("target", PROTOCOL_VERSION, 1),
11796 )
11797 .await;
11798
11799 let wrong_nonce = handler
11803 .handle_control_frame(
11804 &route_ctx(ConnectionId::new(91)).0,
11805 route_open_frame_with_admission_facts(
11806 20,
11807 "target",
11808 unique_project_root("admission-facts"),
11809 Some(subc_control::ConsumerIdentity {
11810 module_id: "fed".to_string(),
11811 launch_nonce: "not-the-real-nonce".to_string(),
11812 }),
11813 None,
11814 ),
11815 )
11816 .await
11817 .unwrap();
11818 assert_eq!(
11819 parse_error(&wrong_nonce[0])["code"],
11820 "bad_consumer_identity",
11821 "a mismatched launch nonce must be refused, not stamped Reserved"
11822 );
11823
11824 let never_spawned = handler
11828 .handle_control_frame(
11829 &route_ctx(ConnectionId::new(92)).0,
11830 route_open_frame_with_admission_facts(
11831 21,
11832 "target",
11833 unique_project_root("admission-facts"),
11834 Some(subc_control::ConsumerIdentity {
11835 module_id: "never-spawned".to_string(),
11836 launch_nonce: "any-nonce".to_string(),
11837 }),
11838 None,
11839 ),
11840 )
11841 .await
11842 .unwrap();
11843 assert_eq!(
11844 parse_error(&never_spawned[0])["code"],
11845 "bad_consumer_identity",
11846 "an unspawned module_id must be refused rather than accepted for lack of a record"
11847 );
11848 }
11849
11850 #[tokio::test]
11873 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11874 let registry = Arc::new(Registry::default());
11875 let forwarding = Arc::new(ForwardingTable::default());
11876 let supervisor = SupervisorHandle::new();
11877 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11878 let handler =
11879 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11880 .with_supervisor(supervisor);
11881
11882 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11883 hello_via_sink(
11884 &handler,
11885 &target_ctx,
11886 &mut target_rx,
11887 hello_frame("target", PROTOCOL_VERSION, 1),
11888 )
11889 .await;
11890
11891 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11892 let route_handler = handler.clone();
11893 let route_task = tokio::spawn(async move {
11894 route_handler
11895 .handle_control_frame(
11896 &client_ctx,
11897 route_open_frame_with_admission_facts(
11898 30,
11899 "target",
11900 unique_project_root("admission-facts"),
11901 Some(subc_control::ConsumerIdentity {
11902 module_id: "fed".to_string(),
11903 launch_nonce: "fed-nonce".to_string(),
11904 }),
11905 None,
11906 ),
11907 )
11908 .await
11909 .unwrap()
11910 });
11911
11912 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11919 .await
11920 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11921 .expect("module control channel closed before route.bind");
11922 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11923 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11924 panic!("expected route.bind")
11925 };
11926 assert_eq!(
11927 principal,
11928 Some(Principal::Reserved {
11929 module_id: "fed".to_string()
11930 }),
11931 "a correctly attested consumer must be stamped Reserved for its own id"
11932 );
11933
11934 handler
11935 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11936 .await
11937 .unwrap();
11938 assert!(route_task.await.unwrap().is_empty());
11939 assert!(
11940 matches!(
11941 serde_json::from_slice::<ClientControlResponse>(
11942 &client_rx.recv().await.unwrap().body
11943 )
11944 .unwrap(),
11945 ClientControlResponse::RouteOpen { .. }
11946 ),
11947 "the route must actually open, not merely avoid an error"
11948 );
11949 }
11950
11951 #[tokio::test(start_paused = true)]
11952 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11953 let registry = Arc::new(Registry::default());
11954 let forwarding = Arc::new(ForwardingTable::default());
11955 let supervisor = SupervisorHandle::new();
11956 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11957 let handler =
11958 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11959 .with_supervisor(supervisor);
11960
11961 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11962 hello_via_sink(
11963 &handler,
11964 &target_ctx,
11965 &mut target_rx,
11966 hello_frame("target", PROTOCOL_VERSION, 1),
11967 )
11968 .await;
11969
11970 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11971 let direct_handler = handler.clone();
11972 let direct_open = tokio::spawn(async move {
11973 direct_handler
11974 .handle_control_frame(
11975 &direct_ctx,
11976 route_open_frame(2, "target", unique_project_root("route-census-direct")),
11977 )
11978 .await
11979 .unwrap()
11980 });
11981 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11982 .await
11983 .expect("no direct route.bind within 5s")
11984 .expect("target control channel closed before direct route.bind");
11985 handler
11986 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11987 .await
11988 .unwrap();
11989 assert!(direct_open.await.unwrap().is_empty());
11990 let _ = direct_rx.recv().await.unwrap();
11991
11992 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11993 let reserved_handler = handler.clone();
11994 let reserved_open = tokio::spawn(async move {
11995 reserved_handler
11996 .handle_control_frame(
11997 &reserved_ctx,
11998 route_open_frame_with_admission_facts(
11999 3,
12000 "target",
12001 unique_project_root("admission-facts"),
12002 Some(ConsumerIdentity {
12003 module_id: "fed".to_string(),
12004 launch_nonce: "fed-nonce".to_string(),
12005 }),
12006 None,
12007 ),
12008 )
12009 .await
12010 .unwrap()
12011 });
12012 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12013 .await
12014 .expect("no reserved route.bind within 5s")
12015 .expect("target control channel closed before reserved route.bind");
12016 handler
12017 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12018 .await
12019 .unwrap();
12020 assert!(reserved_open.await.unwrap().is_empty());
12021 let _ = reserved_rx.recv().await.unwrap();
12022
12023 forwarding
12024 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12025 .unwrap();
12026 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12027 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12028 module_id: Some("target".to_string()),
12029 })
12030 .unwrap();
12031 let census_frame =
12032 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12033 let response = handler
12034 .handle_control_frame(&census_ctx, census_frame)
12035 .await
12036 .unwrap()
12037 .pop()
12038 .unwrap();
12039 let actual: Value = serde_json::from_slice(&response.body).unwrap();
12040 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12041 assert!(matches!(
12042 decoded,
12043 ClientControlResponse::SupervisorRoutes { .. }
12044 ));
12045 let routes = actual["modules"][0]["routes"].as_array().unwrap();
12046 assert_eq!(routes.len(), 2);
12047 assert!(routes.iter().all(|route| route["draining"] == true));
12048 assert!(
12051 routes.iter().all(|route| route["drain_reason"] == "reload"),
12052 "draining routes must name the drain's reason: {routes:?}"
12053 );
12054 assert!(routes.iter().any(|route| {
12055 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12056 }));
12057 assert!(routes.iter().any(|route| {
12058 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12059 }));
12060
12061 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12062 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12063 if std::env::var_os("UPDATE_GOLDEN").is_some() {
12064 std::fs::write(
12065 &golden_path,
12066 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12067 )
12068 .unwrap();
12069 }
12070 let expected: Value =
12071 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12072 assert_eq!(actual, expected);
12073 }
12074
12075 async fn query_live_roots(
12076 handler: &ControlHandler,
12077 module_ctx: &RouteCtx,
12078 ) -> ModuleControlResponseToModule {
12079 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12080 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12081 let response = handler
12082 .handle_control_frame(module_ctx, frame)
12083 .await
12084 .unwrap()
12085 .pop()
12086 .unwrap();
12087 serde_json::from_slice(&response.body).unwrap()
12088 }
12089
12090 #[tokio::test(start_paused = true)]
12091 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12092 let registry = Arc::new(Registry::default());
12093 let forwarding = Arc::new(ForwardingTable::default());
12094 let handler = ControlHandler::with_forwarding(registry, forwarding);
12095 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12096 hello_via_sink(
12097 &handler,
12098 &target_ctx,
12099 &mut target_rx,
12100 hello_frame("target", PROTOCOL_VERSION, 1),
12101 )
12102 .await;
12103 let root = unique_project_root("live-roots-known");
12104 let path = ProjectRootId::from_path_allowing_missing(root.path())
12105 .unwrap()
12106 .as_path()
12107 .to_path_buf();
12108 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12109 let open_handler = handler.clone();
12110 let opened = tokio::spawn(async move {
12111 open_handler
12112 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
12113 .await
12114 .unwrap()
12115 });
12116 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12117 .await
12118 .unwrap()
12119 .unwrap();
12120 handler
12121 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12122 .await
12123 .unwrap();
12124 assert!(opened.await.unwrap().is_empty());
12125 let _ = client_rx.recv().await.unwrap();
12126
12127 let root = unique_project_root("live-roots-pending");
12128 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12129 .unwrap()
12130 .as_path()
12131 .to_path_buf();
12132 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12133 let open_handler = handler.clone();
12134 let pending = tokio::spawn(async move {
12135 open_handler
12136 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
12137 .await
12138 .unwrap()
12139 });
12140 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12141 .await
12142 .unwrap()
12143 .unwrap();
12144 let actual = query_live_roots(&handler, &target_ctx).await;
12145 let ModuleControlResponseToModule::LiveRoots {
12146 roots,
12147 unknown_root_bindings,
12148 total_bindings,
12149 } = actual
12150 else {
12151 panic!("expected live roots")
12152 };
12153 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12154 assert_eq!(unknown_root_bindings, 0);
12155 assert_eq!(
12156 roots.len(),
12157 2,
12158 "root-known arm must retain each canonical root"
12159 );
12160 assert_eq!(
12161 total_bindings,
12162 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12163 );
12164 let counts = roots
12165 .iter()
12166 .map(|root| (root.project_root.clone(), root.bound, root.pending))
12167 .collect::<Vec<_>>();
12168 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12169 expected.sort_by(|a, b| a.0.cmp(&b.0));
12170 assert_eq!(
12171 counts, expected,
12172 "roots must sort by path and count pending separately"
12173 );
12174 handler
12175 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12176 .await
12177 .unwrap();
12178 assert!(pending.await.unwrap().is_empty());
12179 }
12180
12181 #[tokio::test(start_paused = true)]
12182 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12183 let forwarding = Arc::new(ForwardingTable::default());
12184 let handler =
12185 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12186 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12187 hello_via_sink(
12188 &handler,
12189 &target_ctx,
12190 &mut target_rx,
12191 hello_frame("target", PROTOCOL_VERSION, 1),
12192 )
12193 .await;
12194 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12195 let pending = forwarding
12196 .begin_route_bind_relay_for_test(
12197 client_ctx.connection_id,
12198 client_ctx.egress.clone(),
12199 2,
12200 "target",
12201 )
12202 .unwrap();
12203 forwarding
12204 .complete_pending_relay(
12205 target_ctx.connection_id,
12206 pending.corr,
12207 RouteBindRelayOutcome::Accepted,
12208 )
12209 .unwrap();
12210 let actual = query_live_roots(&handler, &target_ctx).await;
12211 let ModuleControlResponseToModule::LiveRoots {
12212 roots,
12213 unknown_root_bindings,
12214 total_bindings,
12215 } = actual
12216 else {
12217 panic!("expected live roots")
12218 };
12219 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12220 assert_eq!(
12221 unknown_root_bindings, 1,
12222 "unknown-root arm must not read as no bindings"
12223 );
12224 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12225 assert_eq!(
12226 total_bindings,
12227 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12228 );
12229 }
12230
12231 #[tokio::test(start_paused = true)]
12239 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12240 let forwarding = Arc::new(ForwardingTable::default());
12241 let handler =
12242 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12243 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12244 let replies = handler
12245 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12246 .await
12247 .unwrap();
12248 let queued_by_hello = module_rx.len();
12249
12250 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12251 let open_handler = handler.clone();
12252 let open = tokio::spawn(async move {
12253 open_handler
12254 .handle_control_frame(
12255 &client_ctx,
12256 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12257 )
12258 .await
12259 .unwrap()
12260 });
12261 let mut spins = 0;
12263 while module_rx.len() == queued_by_hello {
12264 spins += 1;
12265 assert!(spins < 10_000, "route.open never queued a route.bind");
12266 tokio::task::yield_now().await;
12267 }
12268
12269 for reply in replies {
12271 module_ctx.egress.send(reply).await.unwrap();
12272 }
12273
12274 let first = module_rx.recv().await.unwrap().frame;
12275 assert_eq!(
12276 first.header.ty,
12277 FrameType::HelloAck,
12278 "the first frame a registering module reads must be its HELLO_ACK"
12279 );
12280 assert_eq!(first.header.corr, 7);
12281 let second = module_rx.recv().await.unwrap().frame;
12282 assert_eq!(second.header.ty, FrameType::Request);
12283 assert!(
12284 matches!(
12285 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12286 ModuleControlRequest::RouteBind { .. }
12287 ),
12288 "the route.bind follows the ack"
12289 );
12290 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12291
12292 handler
12293 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12294 .await
12295 .unwrap();
12296 assert!(open.await.unwrap().is_empty());
12297 let _ = client_rx.recv().await.unwrap();
12298 }
12299
12300 #[tokio::test(start_paused = true)]
12301 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12302 let handler = ControlHandler::with_forwarding(
12303 Arc::new(Registry::default()),
12304 Arc::new(ForwardingTable::default()),
12305 );
12306 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12307 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12308 hello_via_sink(
12309 &handler,
12310 &first_ctx,
12311 &mut first_rx,
12312 hello_frame("first", PROTOCOL_VERSION, 1),
12313 )
12314 .await;
12315 hello_via_sink(
12316 &handler,
12317 &second_ctx,
12318 &mut second_rx,
12319 hello_frame("second", PROTOCOL_VERSION, 2),
12320 )
12321 .await;
12322 let root = unique_project_root("second-only");
12323 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12324 let cloned = handler.clone();
12325 let open = tokio::spawn(async move {
12326 cloned
12327 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12328 .await
12329 .unwrap()
12330 });
12331 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12332 .await
12333 .unwrap()
12334 .unwrap();
12335 let first = query_live_roots(&handler, &first_ctx).await;
12336 let second = query_live_roots(&handler, &second_ctx).await;
12337 assert!(
12338 matches!(
12339 first,
12340 ModuleControlResponseToModule::LiveRoots {
12341 total_bindings: 0,
12342 ..
12343 }
12344 ),
12345 "cross-module scope must not expose another module's roots"
12346 );
12347 assert!(
12348 matches!(
12349 second,
12350 ModuleControlResponseToModule::LiveRoots {
12351 total_bindings: 1,
12352 ..
12353 }
12354 ),
12355 "second module must see its pending route"
12356 );
12357 handler
12358 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12359 .await
12360 .unwrap();
12361 assert!(open.await.unwrap().is_empty());
12362 }
12363
12364 #[tokio::test(start_paused = true)]
12365 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12366 let handler = ControlHandler::with_forwarding(
12367 Arc::new(Registry::default()),
12368 Arc::new(ForwardingTable::default()),
12369 );
12370 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12371 hello_via_sink(
12372 &handler,
12373 &target_ctx,
12374 &mut target_rx,
12375 hello_frame("target", PROTOCOL_VERSION, 1),
12376 )
12377 .await;
12378 let actual = query_live_roots(&handler, &target_ctx).await;
12379 let ModuleControlResponseToModule::LiveRoots {
12380 roots,
12381 unknown_root_bindings,
12382 total_bindings,
12383 } = actual
12384 else {
12385 panic!("expected live roots")
12386 };
12387 assert!(roots.is_empty());
12388 assert_eq!(unknown_root_bindings, 0);
12389 assert_eq!(total_bindings, 0);
12390 assert_eq!(
12391 total_bindings,
12392 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12393 );
12394 }
12395
12396 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12403 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12404 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12405 let text = std::fs::read_to_string(&path)
12406 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12407 let vectors: Vec<(String, Value)> = text
12408 .lines()
12409 .filter(|line| !line.trim().is_empty())
12410 .map(|line| {
12411 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12412 let id = entry["corpus_id"]
12413 .as_str()
12414 .expect("every vector carries a corpus_id")
12415 .to_string();
12416 (id, entry["package"].clone())
12417 })
12418 .collect();
12419 assert_eq!(
12423 vectors.len(),
12424 3,
12425 "vendored fed corpus changed size; re-sync from subc-federation"
12426 );
12427
12428 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12443 let richest = vectors
12444 .iter()
12445 .filter_map(|(_, package)| package.as_object())
12446 .map(|object| {
12447 object
12448 .keys()
12449 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12450 .count()
12451 })
12452 .max()
12453 .unwrap_or(0);
12454 assert!(
12455 richest >= 2,
12456 "vendored corpus no longer carries a package with unmodelled fields, \
12457 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12458 );
12459
12460 vectors
12461 }
12462
12463 #[tokio::test]
12471 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12472 for (corpus_id, package) in fed_admission_facts_vectors() {
12473 let registry = Arc::new(Registry::default());
12474 let forwarding = Arc::new(ForwardingTable::default());
12475 let supervisor = SupervisorHandle::new();
12476 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12477 let handler =
12478 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12479 .with_supervisor(supervisor)
12480 .with_admission_facts_config(
12481 Some("fed".to_string()),
12482 Some(vec!["target".to_string()]),
12483 );
12484
12485 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12486 hello_via_sink(
12487 &handler,
12488 &target_ctx,
12489 &mut target_rx,
12490 hello_frame("target", PROTOCOL_VERSION, 1),
12491 )
12492 .await;
12493
12494 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12495 let route_handler = handler.clone();
12496 let expected = package.clone();
12497 let route_task = tokio::spawn(async move {
12498 route_handler
12499 .handle_control_frame(
12500 &client_ctx,
12501 route_open_frame_with_admission_facts(
12502 20,
12503 "target",
12504 unique_project_root("admission-facts"),
12505 Some(subc_control::ConsumerIdentity {
12506 module_id: "fed".to_string(),
12507 launch_nonce: "fed-nonce".to_string(),
12508 }),
12509 Some(package),
12510 ),
12511 )
12512 .await
12513 .unwrap()
12514 });
12515
12516 let bind_frame = target_rx.recv().await.unwrap();
12517 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12518 let ModuleControlRequest::RouteBind {
12519 admission_facts, ..
12520 } = bind
12521 else {
12522 panic!("{corpus_id}: expected route.bind")
12523 };
12524 assert_eq!(
12525 admission_facts,
12526 Some(expected),
12527 "{corpus_id}: relay must not add, drop or reshape any field"
12528 );
12529
12530 handler
12531 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12532 .await
12533 .unwrap();
12534 route_task.await.unwrap();
12535 }
12536 }
12537
12538 #[tokio::test]
12539 async fn route_bind_never_relays_the_openers_build_class() {
12540 let registry = Arc::new(Registry::default());
12541 let forwarding = Arc::new(ForwardingTable::default());
12542 let handler =
12543 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12544 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12545 hello_via_sink(
12546 &handler,
12547 &target_ctx,
12548 &mut target_rx,
12549 hello_frame("target", PROTOCOL_VERSION, 1),
12550 )
12551 .await;
12552
12553 let root = unique_project_root("build-class");
12554 let mut identity = BindIdentity::new(
12555 root.path().to_path_buf(),
12556 "unit".to_string(),
12557 "s".to_string(),
12558 );
12559 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12560 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12561 target: RouteTarget::ToolProvider {
12562 module_id: "target".to_string(),
12563 },
12564 identity,
12565 consumer_identity: None,
12566 consumer_capabilities: None,
12567 role_versions: None,
12568 admission_facts: None,
12569 scope: None,
12570 })
12571 .unwrap();
12572 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12573
12574 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12575 let route_handler = handler.clone();
12576 let route_task = tokio::spawn(async move {
12577 route_handler
12578 .handle_control_frame(&client_ctx, open)
12579 .await
12580 .unwrap()
12581 });
12582 let bind_frame = target_rx.recv().await.unwrap();
12583 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12584 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12585 panic!("expected route.bind")
12586 };
12587 assert_eq!(
12588 identity.build_class, None,
12589 "route.bind carried the build class"
12590 );
12591 assert!(
12592 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12593 "the route.bind bytes still name build_class"
12594 );
12595 handler
12596 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12597 .await
12598 .unwrap();
12599 route_task.await.unwrap();
12600 }
12601
12602 #[tokio::test]
12603 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12604 let registry = Arc::new(Registry::default());
12605 let forwarding = Arc::new(ForwardingTable::default());
12606 let supervisor = SupervisorHandle::new();
12607 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12608 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12609 let handler =
12610 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12611 .with_supervisor(supervisor)
12612 .with_admission_facts_config(
12613 Some("fed".to_string()),
12614 Some(vec!["target".to_string()]),
12615 );
12616
12617 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12618 hello_via_sink(
12619 &handler,
12620 &target_ctx,
12621 &mut target_rx,
12622 hello_frame("target", PROTOCOL_VERSION, 1),
12623 )
12624 .await;
12625 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12626 hello_via_sink(
12627 &handler,
12628 &other_ctx,
12629 &mut other_rx,
12630 hello_frame("other", PROTOCOL_VERSION, 2),
12631 )
12632 .await;
12633
12634 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12635 let expected_facts = facts.clone();
12636 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12637 let route_handler = handler.clone();
12638 let route_task = tokio::spawn(async move {
12639 route_handler
12640 .handle_control_frame(
12641 &client_ctx,
12642 route_open_frame_with_admission_facts(
12643 10,
12644 "target",
12645 unique_project_root("admission-facts"),
12646 Some(subc_control::ConsumerIdentity {
12647 module_id: "fed".to_string(),
12648 launch_nonce: "fed-nonce".to_string(),
12649 }),
12650 Some(facts.clone()),
12651 ),
12652 )
12653 .await
12654 .unwrap()
12655 });
12656 let bind_frame = target_rx.recv().await.unwrap();
12657 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12658 let ModuleControlRequest::RouteBind {
12659 admission_facts, ..
12660 } = bind
12661 else {
12662 panic!("expected route.bind")
12663 };
12664 assert_eq!(admission_facts, Some(expected_facts));
12665 handler
12666 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12667 .await
12668 .unwrap();
12669 assert!(route_task.await.unwrap().is_empty());
12670 assert!(matches!(
12671 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12672 .unwrap(),
12673 ClientControlResponse::RouteOpen { .. }
12674 ));
12675
12676 let direct = handler
12677 .handle_control_frame(
12678 &route_ctx(ConnectionId::new(73)).0,
12679 route_open_frame_with_admission_facts(
12680 11,
12681 "target",
12682 unique_project_root("admission-facts"),
12683 None,
12684 Some(json!({"x": 1})),
12685 ),
12686 )
12687 .await
12688 .unwrap();
12689 assert_eq!(
12690 parse_error(&direct[0])["code"],
12691 "admission_facts_not_permitted"
12692 );
12693
12694 let different_reserved = handler
12695 .handle_control_frame(
12696 &route_ctx(ConnectionId::new(77)).0,
12697 route_open_frame_with_admission_facts(
12698 15,
12699 "target",
12700 unique_project_root("admission-facts"),
12701 Some(subc_control::ConsumerIdentity {
12702 module_id: "other".to_string(),
12703 launch_nonce: "other-nonce".to_string(),
12704 }),
12705 Some(json!({"x": 1})),
12706 ),
12707 )
12708 .await
12709 .unwrap();
12710 assert_eq!(
12711 parse_error(&different_reserved[0])["code"],
12712 "admission_facts_not_permitted"
12713 );
12714
12715 let other_target = handler
12716 .handle_control_frame(
12717 &route_ctx(ConnectionId::new(74)).0,
12718 route_open_frame_with_admission_facts(
12719 12,
12720 "other",
12721 unique_project_root("admission-facts"),
12722 Some(subc_control::ConsumerIdentity {
12723 module_id: "fed".to_string(),
12724 launch_nonce: "fed-nonce".to_string(),
12725 }),
12726 Some(json!({"x": 1})),
12727 ),
12728 )
12729 .await
12730 .unwrap();
12731 assert_eq!(
12732 parse_error(&other_target[0])["code"],
12733 "admission_facts_target_not_allowed"
12734 );
12735
12736 let nonexistent = handler
12737 .handle_control_frame(
12738 &route_ctx(ConnectionId::new(75)).0,
12739 route_open_frame_with_admission_facts(
12740 13,
12741 "missing",
12742 unique_project_root("admission-facts"),
12743 None,
12744 Some(json!({"x": 1})),
12745 ),
12746 )
12747 .await
12748 .unwrap();
12749 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12750
12751 let described = handler
12752 .handle_control_frame(
12753 &route_ctx(ConnectionId::new(76)).0,
12754 Frame::build(
12755 FrameType::Request,
12756 control_flags(),
12757 0,
12758 0,
12759 14,
12760 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12761 )
12762 .unwrap(),
12763 )
12764 .await
12765 .unwrap();
12766 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12767 serde_json::from_slice(&described[0].body).unwrap()
12768 else {
12769 panic!("expected server.describe response")
12770 };
12771 assert!(capabilities
12772 .iter()
12773 .any(|cap| cap == "admission_facts_relay_v1"));
12774 }
12775
12776 #[tokio::test]
12777 async fn admission_facts_without_configured_carrier_are_rejected() {
12778 let registry = Arc::new(Registry::default());
12779 let forwarding = Arc::new(ForwardingTable::default());
12780 let handler = ControlHandler::with_forwarding(registry, forwarding);
12781 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12782 hello_via_sink(
12783 &handler,
12784 &target_ctx,
12785 &mut target_rx,
12786 hello_frame("target", PROTOCOL_VERSION, 1),
12787 )
12788 .await;
12789
12790 let responses = handler
12791 .handle_control_frame(
12792 &route_ctx(ConnectionId::new(79)).0,
12793 route_open_frame_with_admission_facts(
12794 16,
12795 "target",
12796 unique_project_root("admission-facts"),
12797 None,
12798 Some(json!({"x": 1})),
12799 ),
12800 )
12801 .await
12802 .unwrap();
12803 assert_eq!(
12804 parse_error(&responses[0])["code"],
12805 "admission_facts_not_permitted"
12806 );
12807 }
12808
12809 #[tokio::test]
12810 async fn route_open_relays_consumer_capabilities_verbatim() {
12811 let registry = Arc::new(Registry::default());
12812 let forwarding = Arc::new(ForwardingTable::default());
12813 let handler =
12814 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12815 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12816 hello_via_sink(
12817 &handler,
12818 &module_ctx,
12819 &mut module_rx,
12820 hello_frame("aft", PROTOCOL_VERSION, 7),
12821 )
12822 .await;
12823
12824 let expected = vec!["elicitation".to_string(), "roots".to_string()];
12825 let expected_for_request = expected.clone();
12826 let project_root = unique_project_root("consumer-capabilities-present");
12827 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12828 let route_handler = handler.clone();
12829 let route_task = tokio::spawn(async move {
12830 route_handler
12831 .handle_control_frame(
12832 &client_ctx,
12833 route_open_frame_with_consumer_capabilities(
12834 401,
12835 "aft",
12836 project_root,
12837 Some(expected_for_request),
12838 ),
12839 )
12840 .await
12841 .unwrap()
12842 });
12843 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12844 .await
12845 .unwrap()
12846 .unwrap();
12847 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12848 let ModuleControlRequest::RouteBind {
12849 consumer_capabilities,
12850 ..
12851 } = bind
12852 else {
12853 panic!("expected route.bind request, got {bind:?}");
12854 };
12855 assert_eq!(consumer_capabilities, Some(expected.clone()));
12856
12857 handler
12858 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12859 .await
12860 .unwrap();
12861 let route_response = route_task.await.unwrap();
12862 assert!(route_response.is_empty());
12863 let published = client_rx.recv().await.unwrap();
12864 assert!(matches!(
12865 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12866 ClientControlResponse::RouteOpen { .. }
12867 ));
12868 }
12869
12870 #[tokio::test]
12871 async fn route_open_without_consumer_capabilities_relays_none() {
12872 let registry = Arc::new(Registry::default());
12873 let forwarding = Arc::new(ForwardingTable::default());
12874 let handler =
12875 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12876 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12877 hello_via_sink(
12878 &handler,
12879 &module_ctx,
12880 &mut module_rx,
12881 hello_frame("aft", PROTOCOL_VERSION, 7),
12882 )
12883 .await;
12884
12885 let project_root = unique_project_root("consumer-capabilities-absent");
12886 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12887 let route_handler = handler.clone();
12888 let route_task = tokio::spawn(async move {
12889 route_handler
12890 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12891 .await
12892 .unwrap()
12893 });
12894 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12895 .await
12896 .unwrap()
12897 .unwrap();
12898 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12899 let ModuleControlRequest::RouteBind {
12900 consumer_capabilities,
12901 ..
12902 } = bind
12903 else {
12904 panic!("expected route.bind request, got {bind:?}");
12905 };
12906 assert_eq!(consumer_capabilities, None);
12907
12908 handler
12909 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12910 .await
12911 .unwrap();
12912 let route_response = route_task.await.unwrap();
12913 assert!(route_response.is_empty());
12914 let published = client_rx.recv().await.unwrap();
12915 assert!(matches!(
12916 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12917 ClientControlResponse::RouteOpen { .. }
12918 ));
12919 }
12920
12921 async fn bind_role_versions_for(
12925 sent: Option<BTreeMap<String, String>>,
12926 connection: u64,
12927 ) -> Option<BTreeMap<String, String>> {
12928 let registry = Arc::new(Registry::default());
12929 let forwarding = Arc::new(ForwardingTable::default());
12930 let handler =
12931 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12932 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12933 hello_via_sink(
12934 &handler,
12935 &module_ctx,
12936 &mut module_rx,
12937 hello_frame("aft", PROTOCOL_VERSION, 7),
12938 )
12939 .await;
12940 let project_root = unique_project_root("role-versions");
12941 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12942 let route_handler = handler.clone();
12943 let route_task = tokio::spawn(async move {
12944 route_handler
12945 .handle_control_frame(
12946 &client_ctx,
12947 route_open_frame_with_role_versions(403, "aft", project_root, sent),
12948 )
12949 .await
12950 .unwrap()
12951 });
12952 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12953 .await
12954 .expect("a well-formed route.open reaches the module as a bind")
12955 .unwrap();
12956 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12957 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12958 panic!("expected route.bind request, got {bind:?}");
12959 };
12960 handler
12961 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12962 .await
12963 .unwrap();
12964 assert!(route_task.await.unwrap().is_empty());
12965 let published = client_rx.recv().await.unwrap();
12966 assert!(matches!(
12967 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12968 ClientControlResponse::RouteOpen { .. }
12969 ));
12970 role_versions
12971 }
12972
12973 #[tokio::test]
12974 async fn route_open_relays_role_versions_verbatim() {
12975 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12976 assert_eq!(
12977 bind_role_versions_for(Some(sent.clone()), 141).await,
12978 Some(sent)
12979 );
12980 }
12981
12982 #[tokio::test]
12985 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12986 assert_eq!(bind_role_versions_for(None, 143).await, None);
12987 assert_eq!(
12988 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12989 None
12990 );
12991 }
12992
12993 #[tokio::test]
12994 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12995 let registry = Arc::new(Registry::default());
12996 let forwarding = Arc::new(ForwardingTable::default());
12997 let handler =
12998 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12999 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13000 hello_via_sink(
13001 &handler,
13002 &module_ctx,
13003 &mut module_rx,
13004 hello_frame("aft", PROTOCOL_VERSION, 7),
13005 )
13006 .await;
13007
13008 let nine: BTreeMap<String, String> = (0..9)
13009 .map(|index| (format!("role-{index}"), "v1".to_string()))
13010 .collect();
13011 for (label, malformed) in [
13012 (
13013 "invalid role name",
13014 role_versions(&[("Tool_Provider", "v1")]),
13015 ),
13016 ("invalid version", role_versions(&[("tool-provider", "v0")])),
13017 ("nine entries", nine),
13018 ] {
13019 let responses = tokio::time::timeout(
13022 Duration::from_secs(1),
13023 handler.handle_control_frame(
13024 &route_ctx(ConnectionId::new(148)).0,
13025 route_open_frame_with_role_versions(
13026 404,
13027 "aft",
13028 unique_project_root("role-versions-malformed"),
13029 Some(malformed),
13030 ),
13031 ),
13032 )
13033 .await
13034 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13035 .unwrap();
13036 assert_eq!(responses.len(), 1, "{label}");
13037 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13038 let error = parse_error(&responses[0]);
13039 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13040 assert_eq!(
13041 error["detail"]["field"], "role_versions",
13042 "{label}: {error}"
13043 );
13044 assert!(
13045 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13046 "{label}: a malformed declaration is terminal"
13047 );
13048 assert!(
13049 module_rx.try_recv().is_err(),
13050 "{label}: the module must never see a bind"
13051 );
13052 }
13053 }
13054
13055 #[tokio::test]
13059 async fn route_role_versions_capability_is_advertised() {
13060 let handler = ControlHandler::new(Arc::new(Registry::default()));
13061 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13062 let ack = hello_via_sink(
13063 &handler,
13064 &ctx,
13065 &mut rx,
13066 hello_frame("m", PROTOCOL_VERSION, 1),
13067 )
13068 .await;
13069 let ack = parse_ack(&ack);
13070 assert!(
13071 ack.subc_capabilities
13072 .iter()
13073 .any(|c| c == "route-role-versions/v1"),
13074 "{:?}",
13075 ack.subc_capabilities
13076 );
13077
13078 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13079 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13080 let reply = handler
13081 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13082 .await
13083 .unwrap()
13084 .pop()
13085 .unwrap();
13086 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13087 serde_json::from_slice(&reply.body).unwrap()
13088 else {
13089 panic!("not a server.describe reply");
13090 };
13091 assert!(
13092 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13093 "{capabilities:?}"
13094 );
13095 }
13096
13097 #[tokio::test]
13101 async fn catalog_events_capability_is_advertised() {
13102 let handler = ControlHandler::new(Arc::new(Registry::default()));
13103 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13104 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13105 let reply = handler
13106 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13107 .await
13108 .unwrap()
13109 .pop()
13110 .unwrap();
13111 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13112 serde_json::from_slice(&reply.body).unwrap()
13113 else {
13114 panic!("not a server.describe reply");
13115 };
13116 assert!(
13117 capabilities
13118 .iter()
13119 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13120 "{capabilities:?}"
13121 );
13122 }
13123
13124 #[tokio::test]
13125 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13126 let handler = ControlHandler::new(Arc::new(Registry::default()));
13127 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13128 let ack = hello_via_sink(
13129 &handler,
13130 &module,
13131 &mut module_rx,
13132 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13133 )
13134 .await;
13135 let ack = parse_ack(&ack);
13136 assert!(
13137 ack.subc_capabilities
13138 .iter()
13139 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13140 "{:?}",
13141 ack.subc_capabilities
13142 );
13143 assert!(ack
13144 .subc_capabilities
13145 .iter()
13146 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13147
13148 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13149 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13150 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13151 let reply = handler
13152 .handle_control_frame(&client, frame)
13153 .await
13154 .unwrap()
13155 .pop()
13156 .unwrap();
13157 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13158 serde_json::from_slice(&reply.body).unwrap()
13159 else {
13160 panic!("not a server.describe reply");
13161 };
13162 assert!(
13163 capabilities
13164 .iter()
13165 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13166 "{capabilities:?}"
13167 );
13168 assert!(capabilities
13169 .iter()
13170 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13171 }
13172
13173 #[tokio::test]
13174 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13175 let handler = ControlHandler::new(Arc::new(Registry::default()));
13176 let (module, mut rx) = route_ctx(ConnectionId::new(156));
13177 let ack = hello_via_sink(
13178 &handler,
13179 &module,
13180 &mut rx,
13181 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13182 )
13183 .await;
13184 assert!(parse_ack(&ack)
13185 .subc_capabilities
13186 .iter()
13187 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13188 assert!(parse_ack(&ack)
13189 .subc_capabilities
13190 .iter()
13191 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13192 let (client, _rx) = route_ctx(ConnectionId::new(157));
13193 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13194 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13195 let reply = handler
13196 .handle_control_frame(&client, frame)
13197 .await
13198 .unwrap()
13199 .pop()
13200 .unwrap();
13201 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13202 serde_json::from_slice(&reply.body).unwrap()
13203 else {
13204 panic!("server.describe reply")
13205 };
13206 assert!(capabilities
13207 .iter()
13208 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13209 assert!(capabilities
13210 .iter()
13211 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13212 }
13213
13214 #[tokio::test]
13215 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13216 let handler = ControlHandler::new(Arc::new(Registry::default()));
13217 let (module, mut rx) = route_ctx(ConnectionId::new(158));
13218 let ack = hello_via_sink(
13219 &handler,
13220 &module,
13221 &mut rx,
13222 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13223 )
13224 .await;
13225 assert!(parse_ack(&ack)
13226 .subc_capabilities
13227 .iter()
13228 .any(|cap| cap == "scope-workdir/v1"));
13229 let (client, _rx) = route_ctx(ConnectionId::new(159));
13230 let frame = Frame::build(
13231 FrameType::Request,
13232 control_flags(),
13233 0,
13234 0,
13235 5,
13236 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13237 )
13238 .unwrap();
13239 let reply = handler
13240 .handle_control_frame(&client, frame)
13241 .await
13242 .unwrap()
13243 .remove(0);
13244 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13245 serde_json::from_slice(&reply.body).unwrap()
13246 else {
13247 panic!("server.describe reply")
13248 };
13249 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13250 }
13251
13252 #[tokio::test]
13253 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13254 let handler = ControlHandler::new(Arc::new(Registry::default()));
13255 let (module, mut rx) = route_ctx(ConnectionId::new(160));
13256 let ack = hello_via_sink(
13257 &handler,
13258 &module,
13259 &mut rx,
13260 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13261 )
13262 .await;
13263 assert!(parse_ack(&ack)
13264 .subc_capabilities
13265 .iter()
13266 .any(|cap| cap == "worker-runs/v1"));
13267 let (client, _rx) = route_ctx(ConnectionId::new(161));
13268 let frame = Frame::build(
13269 FrameType::Request,
13270 control_flags(),
13271 0,
13272 0,
13273 5,
13274 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13275 )
13276 .unwrap();
13277 let reply = handler
13278 .handle_control_frame(&client, frame)
13279 .await
13280 .unwrap()
13281 .remove(0);
13282 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13283 serde_json::from_slice(&reply.body).unwrap()
13284 else {
13285 panic!("server.describe reply")
13286 };
13287 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13288 }
13289
13290 #[tokio::test]
13291 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13292 let handler = ControlHandler::new(Arc::new(Registry::default()));
13293 let (module, mut rx) = route_ctx(ConnectionId::new(154));
13294 let ack = hello_via_sink(
13295 &handler,
13296 &module,
13297 &mut rx,
13298 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13299 )
13300 .await;
13301 assert!(parse_ack(&ack)
13302 .subc_capabilities
13303 .iter()
13304 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13305 let (client, _rx) = route_ctx(ConnectionId::new(155));
13306 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13307 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13308 let reply = handler
13309 .handle_control_frame(&client, frame)
13310 .await
13311 .unwrap()
13312 .pop()
13313 .unwrap();
13314 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13315 serde_json::from_slice(&reply.body).unwrap()
13316 else {
13317 panic!("not a server.describe reply")
13318 };
13319 assert!(capabilities
13320 .iter()
13321 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13322 }
13323
13324 #[tokio::test]
13325 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13326 let registry = Arc::new(Registry::default());
13327 let forwarding = Arc::new(ForwardingTable::default());
13328 let handler =
13329 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13330 .with_health_probe_timeout(Duration::from_secs(5));
13331 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13332 hello_via_sink(
13333 &handler,
13334 &module_ctx,
13335 &mut module_rx,
13336 non_routable_hello_frame_with_control_ops(
13337 "mcp",
13338 300,
13339 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13340 ),
13341 )
13342 .await;
13343 assert!(registry
13344 .get_module("mcp")
13345 .unwrap()
13346 .unwrap()
13347 .manifest
13348 .provides
13349 .is_empty());
13350
13351 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13352 let route_response = handler
13353 .handle_control_frame(
13354 &route_client_ctx,
13355 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13356 )
13357 .await
13358 .unwrap();
13359 assert_eq!(route_response[0].header.ty, FrameType::Error);
13360 assert_eq!(
13361 parse_error(&route_response[0])["code"],
13362 "target_unavailable"
13363 );
13364 assert!(parse_error(&route_response[0])["message"]
13365 .as_str()
13366 .unwrap()
13367 .contains("does not provide the requested target"));
13368 assert!(module_rx.try_recv().is_err());
13369
13370 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13371 let health_handler = handler.clone();
13372 let health_task = tokio::spawn(async move {
13373 health_handler
13374 .handle_control_frame(
13375 &health_client_ctx,
13376 supervisor_health_probe_frame(302, "mcp"),
13377 )
13378 .await
13379 .unwrap()
13380 });
13381 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13382 .await
13383 .unwrap()
13384 .unwrap();
13385 assert_eq!(
13386 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13387 ModuleControlRequest::HealthCheck {}
13388 );
13389 handler
13390 .handle_control_frame(
13391 &module_ctx,
13392 health_response(health_frame.header.corr, HealthStatus::Ok),
13393 )
13394 .await
13395 .unwrap();
13396 let health_response = health_task.await.unwrap();
13397 assert_eq!(health_response[0].header.ty, FrameType::Response);
13398 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13399 ClientControlResponse::SupervisorHealthProbe {
13400 module_id, status, ..
13401 } => {
13402 assert_eq!(module_id, "mcp");
13403 assert_eq!(status, HealthStatus::Ok);
13404 }
13405 other => panic!("unexpected health response: {other:?}"),
13406 }
13407
13408 forwarding
13413 .cleanup_connection(module_ctx.connection_id)
13414 .unwrap();
13415 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13416 let cleanup_response = tokio::time::timeout(
13417 Duration::from_millis(200),
13418 handler.handle_control_frame(
13419 &cleanup_probe_ctx,
13420 supervisor_health_probe_frame(303, "mcp"),
13421 ),
13422 )
13423 .await
13424 .expect("probe should fail immediately when the control lane is gone")
13425 .unwrap();
13426 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13427 assert_eq!(
13428 parse_error(&cleanup_response[0])["code"],
13429 "target_unavailable"
13430 );
13431 assert!(parse_error(&cleanup_response[0])["message"]
13432 .as_str()
13433 .unwrap()
13434 .contains("no module connection"));
13435
13436 handler
13437 .cleanup_connection(module_ctx.connection_id)
13438 .unwrap();
13439 }
13440
13441 #[tokio::test]
13442 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13443 let registry = Arc::new(Registry::default());
13444 let supervisor_handle = SupervisorHandle::new();
13445 let supervisor =
13446 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13447 .with_handle(supervisor_handle.clone())
13448 .with_connection_file_path(
13449 std::env::temp_dir()
13450 .join(format!("subc-route-open-warming-{}", std::process::id())),
13451 );
13452 let module = supervisor
13453 .supervise_configured(
13454 ModuleSpec {
13455 module_id: "warming".to_string(),
13456 program: fake_aft_stub_path(),
13457 args: Vec::new(),
13458 env: Vec::new(),
13459 reserved: false,
13460 reserved_prefixes: Vec::new(),
13461 protocol: ModuleProtocol::Subc,
13462 overlap: Default::default(),
13463 },
13464 true,
13465 )
13466 .unwrap();
13467 assert_eq!(module.state().unwrap(), ModuleState::Running);
13468
13469 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13470 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13471 let response = handler
13472 .handle_control_frame(
13473 &ctx,
13474 route_open_frame(304, "warming", unique_project_root("warming")),
13475 )
13476 .await
13477 .unwrap();
13478 module.stop().await.unwrap();
13479
13480 assert_eq!(response[0].header.ty, FrameType::Error);
13481 let error = parse_error(&response[0]);
13482 assert_eq!(error["code"], "module_warming");
13483 assert!(error["message"]
13484 .as_str()
13485 .unwrap()
13486 .contains("state=running, enabled=true, live=false"));
13487 }
13488
13489 #[test]
13490 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13491 let handler = ControlHandler::new(Arc::new(Registry::default()));
13492 let capture = EventCapture::default();
13493 let _subscriber =
13494 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13495 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13496 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13497 let pending = (0..limit).collect::<Vec<_>>();
13498 let response = handler
13499 .route_open_capacity_refusal(
13500 &ctx,
13501 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13502 "busy",
13503 pending.len(),
13504 limit,
13505 )
13506 .unwrap();
13507 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13508 let event = capture
13509 .events()
13510 .into_iter()
13511 .find(|event| {
13512 event.target == "control"
13513 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13514 })
13515 .expect("connection admission refusal event");
13516 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13517 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13518 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13521 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13522 }
13523
13524 #[test]
13525 fn route_open_target_cap_logs_admission_reason_and_capacity() {
13526 let handler = ControlHandler::new(Arc::new(Registry::default()));
13527 let capture = EventCapture::default();
13528 let _subscriber =
13529 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13530 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13531 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13532 let guards = (0..limit)
13533 .map(|_| {
13534 handler
13535 .route_bind_concurrency
13536 .try_admit("busy", limit)
13537 .unwrap()
13538 })
13539 .collect::<Vec<_>>();
13540 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13541 Err(in_flight) => in_flight,
13542 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13543 };
13544 let response = handler
13545 .route_open_target_capacity_refusal(
13546 &ctx,
13547 &RouteOpenCaller::unknown(),
13548 &route_open_frame(397, "busy", unique_project_root("target-cap")),
13549 "busy",
13550 in_flight,
13551 )
13552 .unwrap();
13553 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13554 let event = capture
13555 .events()
13556 .into_iter()
13557 .find(|event| {
13558 event.target == "control"
13559 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13560 })
13561 .expect("target admission refusal event");
13562 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13563 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13564 drop(guards);
13565 }
13566
13567 #[cfg(unix)]
13568 #[tokio::test(start_paused = true)]
13569 async fn supervisor_list_shows_a_failed_modules_next_retry() {
13570 let registry = Arc::new(Registry::default());
13571 let handle = SupervisorHandle::new();
13572 let supervisor =
13573 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13574 .with_handle(handle.clone());
13575 let module = supervisor
13576 .spawn(crate::ModuleSpec {
13577 module_id: "retry-list".into(),
13578 program: PathBuf::from("/bin/sh"),
13579 args: vec!["-c".into(), "exit 17".into()],
13580 env: vec![],
13581 reserved: false,
13582 reserved_prefixes: vec![],
13583 protocol: ModuleProtocol::None,
13584 overlap: Default::default(),
13585 })
13586 .unwrap();
13587 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13588 while module.state().unwrap() != crate::ModuleState::Failed {
13589 assert!(
13590 std::time::Instant::now() < deadline,
13591 "waiting for budget failure"
13592 );
13593 tokio::task::yield_now().await;
13594 }
13595 let handler = ControlHandler::new(registry).with_supervisor(handle);
13596 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13597 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13598 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13599 assert_eq!(body["modules"][0]["state"], "failed");
13600 let retry = body["modules"][0]["next_retry_at_ms"]
13601 .as_u64()
13602 .expect("scheduled retry on the wire");
13603 let now = std::time::SystemTime::now()
13604 .duration_since(std::time::UNIX_EPOCH)
13605 .unwrap()
13606 .as_millis() as u64;
13607 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13608 module.set_enabled(false).await.unwrap();
13609 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13610 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13611 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13612 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13613 }
13614
13615 #[tokio::test]
13616 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
13617 let registry = Arc::new(Registry::default());
13618 let handle = SupervisorHandle::new();
13619 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13620 .with_handle(handle.clone());
13621 supervisor
13622 .supervise_configured(
13623 crate::ModuleSpec {
13624 module_id: "drift".into(),
13625 program: PathBuf::from("disabled-module"),
13626 args: Vec::new(),
13627 env: Vec::new(),
13628 reserved: false,
13629 reserved_prefixes: Vec::new(),
13630 protocol: ModuleProtocol::Subc,
13631 overlap: Default::default(),
13632 },
13633 false,
13634 )
13635 .unwrap();
13636 let handler = ControlHandler::new(registry).with_supervisor(handle);
13637 for configured in [None, Some(false), Some(true)] {
13638 handler.capability_evaluator.configure(
13639 configured.map(|enabled| ("drift".to_string(), enabled)),
13640 BTreeMap::new(),
13641 );
13642 let frame =
13643 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13644 let replies = handler
13645 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
13646 .await
13647 .unwrap();
13648 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13649 assert!(body.get("configured_enabled").is_none());
13650 assert_eq!(body["op"], "supervisor.ack");
13651 let frame =
13652 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13653 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13654 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13655 assert_eq!(
13656 body["modules"][0].get("configured_enabled"),
13657 configured.map(Value::Bool).as_ref()
13658 );
13659 }
13660 }
13661
13662 #[tokio::test(flavor = "current_thread")]
13663 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
13664 let handler = ControlHandler::new(Arc::new(Registry::default()));
13665 let capture = EventCapture::default();
13666 let _subscriber =
13667 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13668 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
13669 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13670
13671 let _ = handler
13672 .handle_client_control_request(
13673 &ctx,
13674 frame,
13675 ClientControlRequest::SupervisorSetEnabled {
13676 module_id: "broca".to_string(),
13677 enabled: false,
13678 },
13679 &mut None,
13680 )
13681 .await
13682 .unwrap();
13683
13684 let events = capture
13685 .events()
13686 .into_iter()
13687 .filter(|event| {
13688 event.target == "control"
13689 && event.fields.get("message").map(String::as_str)
13690 == Some("supervisor request received")
13691 })
13692 .collect::<Vec<_>>();
13693 assert_eq!(events.len(), 1, "one supervisor request log line");
13694 let event = &events[0];
13695 assert_eq!(event.level, tracing::Level::INFO);
13696 assert_eq!(
13697 event.fields.get("op"),
13698 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
13699 );
13700 assert_eq!(
13701 event.fields.get("module_id"),
13702 Some(&"\"broca\"".to_string())
13703 );
13704 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
13705 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
13706 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
13707 }
13708
13709 #[tokio::test(flavor = "current_thread")]
13710 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
13711 let registry = Arc::new(Registry::default());
13712 let module_connection = ConnectionId::new(303);
13713 registry
13714 .register_with_control_ops(
13715 manifest("aft", PROTOCOL_VERSION),
13716 PROTOCOL_VERSION,
13717 module_connection,
13718 module_baseline_control_ops(),
13719 )
13720 .unwrap();
13721 let handler = ControlHandler::new(registry);
13722 let capture = EventCapture::default();
13723 let _subscriber =
13724 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13725 let (ctx, _rx) = route_ctx(module_connection);
13726 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13727
13728 let _ = handler
13729 .handle_client_control_request(
13730 &ctx,
13731 frame,
13732 ClientControlRequest::SupervisorSetEnabled {
13733 module_id: "broca".to_string(),
13734 enabled: false,
13735 },
13736 &mut None,
13737 )
13738 .await
13739 .unwrap();
13740
13741 let callers = capture
13742 .events()
13743 .into_iter()
13744 .filter(|event| {
13745 event.fields.get("message").map(String::as_str)
13746 == Some("supervisor request received")
13747 })
13748 .map(|event| event.fields.get("caller").cloned())
13749 .collect::<Vec<_>>();
13750 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13751 }
13752
13753 #[tokio::test(flavor = "current_thread")]
13754 async fn supervisor_list_does_not_log_request_received() {
13755 let handler = ControlHandler::new(Arc::new(Registry::default()));
13756 let capture = EventCapture::default();
13757 let _subscriber =
13758 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13759 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13760 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13761
13762 handler
13763 .handle_client_control_request(
13764 &ctx,
13765 frame,
13766 ClientControlRequest::SupervisorList {},
13767 &mut None,
13768 )
13769 .await
13770 .unwrap();
13771
13772 assert!(capture.events().into_iter().all(|event| {
13773 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13774 }));
13775 }
13776
13777 #[tokio::test(flavor = "current_thread")]
13778 async fn supervisor_rescan_preview_does_not_log_request_received() {
13779 let handler = ControlHandler::new(Arc::new(Registry::default()));
13780 let capture = EventCapture::default();
13781 let _subscriber =
13782 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13783 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13784 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13785
13786 handler
13787 .handle_client_control_request(
13788 &ctx,
13789 frame,
13790 ClientControlRequest::SupervisorRescan { preview: true },
13791 &mut None,
13792 )
13793 .await
13794 .unwrap();
13795
13796 assert!(capture.events().into_iter().all(|event| {
13797 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13798 }));
13799 }
13800
13801 struct ApproveOperator;
13802 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13803 fn prompt(
13804 &self,
13805 _: &str,
13806 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13807 ) -> crate::operator_confirm::ProviderResult {
13808 crate::operator_confirm::ProviderResult::Approved
13809 }
13810 }
13811
13812 #[tokio::test]
13813 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13814 assert_eq!(
13815 ControlHandler::default().privileged_operator_prompt,
13816 PrivilegedOperatorPrompt::Log
13817 );
13818 for (initial, changed, running) in [
13819 ("log", "enforce", PrivilegedOperatorPrompt::Log),
13820 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13821 ] {
13822 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13823 let path = dir.join("subc.jsonc");
13824 std::fs::write(
13825 &path,
13826 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13827 )
13828 .unwrap();
13829 let config = crate::daemon_config::load(&path).unwrap().unwrap();
13830 let registry = Arc::new(Registry::default());
13831 let handle = SupervisorHandle::new();
13832 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13833 .with_handle(handle.clone());
13834 let handler = ControlHandler::new(registry)
13835 .with_supervisor(handle)
13836 .with_privileged_operator_prompt(config.privileged_operator_prompt)
13837 .with_supervisor_rescan(supervisor, &path, config.port);
13838 handler.forwarding.operator_confirms().configure(
13839 Arc::new(ApproveOperator),
13840 crate::operator_confirm::OperatorLimits::default(),
13841 );
13842 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13843
13844 for (disk_mode, needs_restart) in [
13845 (Some(initial), false),
13846 (Some(changed), true),
13847 (Some(changed), true),
13848 (None, running == PrivilegedOperatorPrompt::Enforce),
13849 (Some(initial), false),
13850 ] {
13851 let mut doc = json!({"version":1});
13852 if let Some(mode) = disk_mode {
13853 doc["privileged_operator_prompt"] = json!(mode);
13854 }
13855 std::fs::write(&path, doc.to_string()).unwrap();
13856 for preview in [true, false] {
13857 let request = ClientControlRequest::SupervisorRescan { preview };
13858 let frame = Frame::build(
13859 FrameType::Request,
13860 control_flags(),
13861 0,
13862 0,
13863 1,
13864 serde_json::to_vec(&request).unwrap(),
13865 )
13866 .unwrap();
13867 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13868 assert_eq!(frames.len(), 1);
13869 let ClientControlResponse::SupervisorRescan { result } =
13870 serde_json::from_slice(&frames[0].body).unwrap()
13871 else {
13872 panic!("expected a rescan response");
13873 };
13874 let expected = if needs_restart {
13875 vec!["privileged_operator_prompt"]
13876 } else {
13877 Vec::new()
13878 };
13879 assert_eq!(result.restart_required, expected);
13880 assert_eq!(handler.privileged_operator_prompt, running);
13881 assert_eq!(
13882 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13883 running
13884 );
13885 }
13886 }
13887 }
13888 }
13889
13890 #[tokio::test]
13891 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13892 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13893 let path = dir.join("subc.jsonc");
13894 let registry = Arc::new(Registry::default());
13895 let handle = SupervisorHandle::new();
13896 let supervisor =
13897 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13898 let handler = ControlHandler::new(registry)
13899 .with_supervisor(handle)
13900 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13901 .with_supervisor_rescan(supervisor, &path, None);
13902 handler.forwarding.operator_confirms().configure(
13903 Arc::new(ApproveOperator),
13904 crate::operator_confirm::OperatorLimits::default(),
13905 );
13906 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13907 for preview in [true, false] {
13908 std::fs::write(
13909 &path,
13910 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13911 )
13912 .unwrap();
13913 let request = ClientControlRequest::SupervisorRescan { preview };
13914 let frame = Frame::build(
13915 FrameType::Request,
13916 control_flags(),
13917 0,
13918 0,
13919 1,
13920 serde_json::to_vec(&request).unwrap(),
13921 )
13922 .unwrap();
13923 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13924 assert_eq!(frames.len(), 1);
13925 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13926 assert_eq!(
13927 handler.privileged_operator_prompt,
13928 PrivilegedOperatorPrompt::Enforce
13929 );
13930 }
13931 }
13932
13933 #[tokio::test]
13938 async fn route_open_refusal_names_the_check_that_refused() {
13939 let handler = ControlHandler::new(Arc::new(Registry::default()));
13940 let capture = EventCapture::default();
13941 let _subscriber =
13942 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13943 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13944 let response = handler
13945 .handle_control_frame(
13946 &ctx,
13947 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13948 )
13949 .await
13950 .unwrap();
13951
13952 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13953 let event = capture
13954 .events()
13955 .into_iter()
13956 .find(|event| {
13957 event.target == "control"
13958 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13959 })
13960 .expect("route.open refusal event");
13961 assert_eq!(
13962 event.fields.get("reason"),
13963 Some(&"\"not_registered\"".to_string())
13964 );
13965 }
13966
13967 #[tokio::test]
13968 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13969 let registry = Arc::new(Registry::default());
13970 let supervisor_handle = SupervisorHandle::new();
13971 let supervisor =
13972 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13973 .with_handle(supervisor_handle.clone())
13974 .with_connection_file_path(std::env::temp_dir().join(format!(
13975 "subc-route-open-refusal-info-{}",
13976 std::process::id()
13977 )));
13978 let module = supervisor
13979 .supervise_configured(
13980 ModuleSpec {
13981 module_id: "warming".to_string(),
13982 program: fake_aft_stub_path(),
13983 args: Vec::new(),
13984 env: Vec::new(),
13985 reserved: false,
13986 reserved_prefixes: Vec::new(),
13987 protocol: ModuleProtocol::Subc,
13988 overlap: Default::default(),
13989 },
13990 true,
13991 )
13992 .unwrap();
13993 assert_eq!(module.state().unwrap(), ModuleState::Running);
13994
13995 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13996 assert!(handler
13997 .counters()
13998 .snapshot()
13999 .get("route_open_refused_by_code")
14000 .is_none());
14001 let capture = EventCapture::default();
14002 let _subscriber =
14003 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14004 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
14005 let response = handler
14006 .handle_control_frame(
14007 &ctx,
14008 route_open_frame(394, "warming", unique_project_root("refusal-info")),
14009 )
14010 .await
14011 .unwrap();
14012 module.stop().await.unwrap();
14013
14014 assert_eq!(parse_error(&response[0])["code"], "module_warming");
14015 let event = capture
14016 .events()
14017 .into_iter()
14018 .find(|event| {
14019 event.target == "control"
14020 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
14021 })
14022 .expect("route.open refusal event");
14023 assert_eq!(
14024 event.fields.get("module_id"),
14025 Some(&"\"warming\"".to_string())
14026 );
14027 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
14028 assert_eq!(
14029 event.fields.get("reason"),
14030 Some(&"\"supervised_not_registered\"".to_string())
14031 );
14032 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
14033 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
14034 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
14035 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14036 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14037 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
14038 assert_eq!(
14039 handler.counters().snapshot()["route_open_refused_by_code"],
14040 json!({ "module_warming": 1 })
14041 );
14042 }
14043
14044 const OUTAGE_START: &str = "route.open refusing module: not serving";
14045 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
14046
14047 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
14048 capture
14049 .events()
14050 .into_iter()
14051 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
14052 .collect()
14053 }
14054
14055 fn supervise_stub(
14056 registry: &Arc<Registry>,
14057 module_id: &str,
14058 enabled: bool,
14059 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
14060 let supervisor_handle = SupervisorHandle::new();
14061 let supervisor =
14062 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
14063 .with_handle(supervisor_handle.clone())
14064 .with_connection_file_path(std::env::temp_dir().join(format!(
14065 "subc-route-outage-{module_id}-{}",
14066 std::process::id()
14067 )));
14068 let module = supervisor
14069 .supervise_configured(
14070 ModuleSpec {
14071 module_id: module_id.to_string(),
14072 program: fake_aft_stub_path(),
14073 args: Vec::new(),
14074 env: Vec::new(),
14075 reserved: false,
14076 reserved_prefixes: Vec::new(),
14077 protocol: ModuleProtocol::Subc,
14078 overlap: Default::default(),
14079 },
14080 enabled,
14081 )
14082 .unwrap();
14083 (supervisor_handle, module)
14084 }
14085
14086 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
14087 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
14088 module_id: module_id.to_string(),
14089 drain_timeout_ms: Some(50),
14090 })
14091 .unwrap();
14092 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
14093 }
14094
14095 #[test]
14099 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
14100 let registry = Arc::new(Registry::default());
14101 let forwarding = Arc::new(ForwardingTable::default());
14102 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14103 let second = ControlHandler::with_forwarding(registry, forwarding);
14104 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
14105 }
14106
14107 #[tokio::test(flavor = "current_thread")]
14111 async fn route_open_unknown_module_refusals_add_no_outage_state() {
14112 let handler = ControlHandler::new(Arc::new(Registry::default()));
14113 let capture = EventCapture::default();
14114 let _subscriber =
14115 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14116 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
14117 for corr in 0..8 {
14118 let response = handler
14119 .handle_control_frame(
14120 &ctx,
14121 route_open_frame(
14122 380 + corr,
14123 &format!("nobody-{corr}"),
14124 unique_project_root("outage-unknown"),
14125 ),
14126 )
14127 .await
14128 .unwrap();
14129 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14130 }
14131
14132 assert_eq!(handler.route_outages.tracked_module_count(), 0);
14133 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
14134 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
14135 }
14136
14137 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14142 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
14143 let registry = Arc::new(Registry::default());
14144 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
14145 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14146 let capture = EventCapture::default();
14147 let _subscriber =
14148 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14149 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
14150 handler.route_outages.record_accepted("outage-restart");
14153
14154 let response = handler
14155 .handle_control_frame(
14156 &ctx,
14157 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
14158 )
14159 .await
14160 .unwrap();
14161 assert_eq!(response[0].header.ty, FrameType::Error);
14162 let starts = outage_lines(&capture, OUTAGE_START);
14163 assert_eq!(starts.len(), 1, "{starts:?}");
14164 assert_eq!(starts[0].level, tracing::Level::WARN);
14165 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
14166 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
14167 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
14168 handler.route_outages.record_accepted("outage-restart");
14169 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
14170
14171 let restart = handler
14172 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
14173 .await
14174 .unwrap();
14175 assert_eq!(
14176 restart[0].header.ty,
14177 FrameType::Response,
14178 "{:?}",
14179 parse_error(&restart[0])
14180 );
14181 handler
14182 .handle_control_frame(
14183 &ctx,
14184 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
14185 )
14186 .await
14187 .unwrap();
14188 module.stop().await.unwrap();
14189
14190 let starts = outage_lines(&capture, OUTAGE_START);
14191 assert_eq!(starts.len(), 2, "{starts:?}");
14192 assert_eq!(starts[1].level, tracing::Level::INFO);
14193 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
14194 }
14195
14196 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14200 async fn failed_operator_restart_leaves_no_operator_mark() {
14201 let registry = Arc::new(Registry::default());
14202 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
14203 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14204 let capture = EventCapture::default();
14205 let _subscriber =
14206 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14207 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
14208 handler.route_outages.record_accepted("outage-disabled");
14209
14210 let restart = handler
14211 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
14212 .await
14213 .unwrap();
14214 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
14215 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
14216
14217 handler
14218 .handle_control_frame(
14219 &ctx,
14220 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
14221 )
14222 .await
14223 .unwrap();
14224 let starts = outage_lines(&capture, OUTAGE_START);
14225 assert_eq!(starts.len(), 1, "{starts:?}");
14226 assert_eq!(starts[0].level, tracing::Level::WARN);
14227 }
14228
14229 #[tokio::test(flavor = "current_thread")]
14230 async fn route_open_unknown_module_escapes_target_module_id() {
14231 let handler = ControlHandler::new(Arc::new(Registry::default()));
14232 let capture = EventCapture::default();
14233 let _subscriber =
14234 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14235 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
14236 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14237 let response = handler
14238 .handle_control_frame(
14239 &ctx,
14240 route_open_frame(
14241 395,
14242 hostile_module_id,
14243 unique_project_root("hostile-target-module-id"),
14244 ),
14245 )
14246 .await
14247 .unwrap();
14248
14249 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14250 let event = capture
14251 .events()
14252 .into_iter()
14253 .find(|event| {
14254 event.target == "control"
14255 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14256 })
14257 .expect("route.open unknown-module refusal event");
14258 let logged = event.fields.get("module_id").expect("module_id field");
14259 assert!(!logged.bytes().any(|byte| byte < 0x20));
14260 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14261 }
14262
14263 #[tokio::test(flavor = "current_thread")]
14266 async fn route_open_refusal_carries_daemon_origin() {
14267 let handler = ControlHandler::new(Arc::new(Registry::default()));
14268 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14269 let response = handler
14270 .handle_control_frame(
14271 &ctx,
14272 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
14273 )
14274 .await
14275 .unwrap();
14276
14277 assert_eq!(response.len(), 1);
14278 assert_eq!(response[0].header.ty, FrameType::Error);
14279 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14280 assert!(
14281 response[0].header.flags.is_daemon_origin(),
14282 "route.open refusal must carry daemon origin: {:?}",
14283 response[0].header.flags
14284 );
14285 }
14286
14287 #[tokio::test(flavor = "current_thread")]
14291 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
14292 let registry = Arc::new(Registry::default());
14293 let forwarding = Arc::new(ForwardingTable::default());
14294 let handler =
14295 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14296 let module_connection = ConnectionId::new(95);
14297 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14298 hello_via_sink(
14299 &handler,
14300 &module_ctx,
14301 &mut module_rx,
14302 hello_frame("aft", PROTOCOL_VERSION, 395),
14303 )
14304 .await;
14305 forwarding
14306 .begin_module_drain("aft", RouteCloseReason::Restart)
14307 .unwrap()
14308 .expect("the module endpoint drains");
14309
14310 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
14311 let response = handler
14312 .handle_control_frame(
14313 &client_ctx,
14314 route_open_frame(396, "aft", unique_project_root("drain-origin")),
14315 )
14316 .await
14317 .unwrap();
14318
14319 assert_eq!(response.len(), 1);
14320 assert_eq!(response[0].header.ty, FrameType::Error);
14321 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14322 assert!(
14323 response[0].header.flags.is_daemon_origin(),
14324 "drain module_reloading must carry daemon origin: {:?}",
14325 response[0].header.flags
14326 );
14327 }
14328
14329 #[tokio::test(flavor = "current_thread")]
14333 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
14334 let registry = Arc::new(Registry::default());
14335 let forwarding = Arc::new(ForwardingTable::default());
14336 let handler =
14337 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14338 let module_connection = ConnectionId::new(95);
14339 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14340 hello_via_sink(
14341 &handler,
14342 &module_ctx,
14343 &mut module_rx,
14344 hello_frame("aft", PROTOCOL_VERSION, 395),
14345 )
14346 .await;
14347
14348 let client_connection = ConnectionId::new(96);
14349 let (client_ctx, _client_rx) = route_ctx(client_connection);
14350 let (route_task, bind) = relay_route_open(
14351 &handler,
14352 client_connection,
14353 &client_ctx.egress,
14354 &mut module_rx,
14355 396,
14356 "aft",
14357 "spoofed-reloading",
14358 )
14359 .await;
14360 let rejection = Frame::build(
14363 FrameType::Error,
14364 crate::router::daemon_error_flags(),
14365 0,
14366 0,
14367 bind.header.corr,
14368 serde_json::to_vec(&ErrorBody::new(
14369 "module_reloading",
14370 "module refused route.bind",
14371 ))
14372 .unwrap(),
14373 )
14374 .unwrap();
14375 handler
14376 .handle_control_frame(&module_ctx, rejection)
14377 .await
14378 .unwrap();
14379
14380 let response = route_task.await.unwrap();
14381 assert_eq!(response.len(), 1);
14382 assert_eq!(response[0].header.ty, FrameType::Error);
14383 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14384 assert!(
14385 !response[0].header.flags.is_daemon_origin(),
14386 "a module's own rejection must not carry daemon origin: {:?}",
14387 response[0].header.flags
14388 );
14389 }
14390
14391 #[tokio::test(flavor = "current_thread")]
14392 async fn route_open_module_rejection_uses_daemon_counter_key() {
14393 let registry = Arc::new(Registry::default());
14394 let forwarding = Arc::new(ForwardingTable::default());
14395 let handler =
14396 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14397 let module_connection = ConnectionId::new(95);
14398 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14399 hello_via_sink(
14400 &handler,
14401 &module_ctx,
14402 &mut module_rx,
14403 hello_frame("aft", PROTOCOL_VERSION, 395),
14404 )
14405 .await;
14406
14407 let client_connection = ConnectionId::new(96);
14408 let (client_ctx, _client_rx) = route_ctx(client_connection);
14409 let capture = EventCapture::default();
14410 let _subscriber =
14411 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14412 let (route_task, bind) = relay_route_open(
14413 &handler,
14414 client_connection,
14415 &client_ctx.egress,
14416 &mut module_rx,
14417 396,
14418 "aft",
14419 "hostile-module-code",
14420 )
14421 .await;
14422 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
14423 let rejection = Frame::build(
14424 FrameType::Error,
14425 control_flags(),
14426 0,
14427 0,
14428 bind.header.corr,
14429 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
14430 )
14431 .unwrap();
14432 handler
14433 .handle_control_frame(&module_ctx, rejection)
14434 .await
14435 .unwrap();
14436
14437 let response = route_task.await.unwrap();
14438 assert_eq!(parse_error(&response[0])["code"], hostile_code);
14439 let counters = handler.counters().snapshot();
14440 assert_eq!(
14441 counters["route_open_refused_by_code"],
14442 json!({ "module_rejected": 1 })
14443 );
14444 assert!(counters["route_open_refused_by_code"]
14445 .get(hostile_code)
14446 .is_none());
14447
14448 let event = capture
14449 .events()
14450 .into_iter()
14451 .find(|event| {
14452 event.target == "control"
14453 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
14454 })
14455 .expect("route.open module-rejection refusal event");
14456 let logged = event.fields.get("module_code").expect("module_code field");
14457 assert!(!logged.bytes().any(|byte| byte < 0x20));
14458 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14459 }
14460
14461 #[tokio::test]
14462 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
14463 let registry = Arc::new(Registry::default());
14464 let supervisor_handle = SupervisorHandle::new();
14465 let missing_program = std::env::temp_dir().join(format!(
14466 "subc-route-open-missing-program-{}",
14467 std::process::id()
14468 ));
14469 let supervisor =
14470 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14471 .with_handle(supervisor_handle.clone());
14472 let module = supervisor
14473 .supervise_configured(
14474 ModuleSpec {
14475 module_id: "failed".to_string(),
14476 program: missing_program,
14477 args: Vec::new(),
14478 env: Vec::new(),
14479 reserved: false,
14480 reserved_prefixes: Vec::new(),
14481 protocol: ModuleProtocol::Subc,
14482 overlap: Default::default(),
14483 },
14484 true,
14485 )
14486 .unwrap();
14487 assert_eq!(module.state().unwrap(), ModuleState::Failed);
14488
14489 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14490 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
14491 let response = handler
14492 .handle_control_frame(
14493 &ctx,
14494 route_open_frame(305, "failed", unique_project_root("failed")),
14495 )
14496 .await
14497 .unwrap();
14498
14499 assert_eq!(response[0].header.ty, FrameType::Error);
14500 let error = parse_error(&response[0]);
14501 assert_eq!(error["code"], "target_unavailable");
14502 assert!(error["message"]
14503 .as_str()
14504 .unwrap()
14505 .contains("state=failed, enabled=true, live=false"));
14506 }
14507
14508 #[tokio::test]
14509 async fn route_open_role_mismatch_remains_target_unavailable() {
14510 let registry = Arc::new(Registry::default());
14511 let handler = ControlHandler::new(Arc::clone(®istry));
14512 handler
14513 .handle_control(
14514 ConnectionId::new(41),
14515 non_routable_hello_frame_with_control_ops("health-only", 306, None),
14516 )
14517 .unwrap();
14518
14519 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
14520 let response = handler
14521 .handle_control_frame(
14522 &ctx,
14523 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
14524 )
14525 .await
14526 .unwrap();
14527
14528 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14529 assert!(parse_error(&response[0])["message"]
14530 .as_str()
14531 .unwrap()
14532 .contains("does not provide the requested target"));
14533 }
14534
14535 #[tokio::test]
14536 async fn route_open_inactive_registration_remains_target_unavailable() {
14537 let registry = Arc::new(Registry::default());
14538 let handler = ControlHandler::new(Arc::clone(®istry));
14539 handler
14540 .handle_control(
14541 ConnectionId::new(43),
14542 hello_frame("inactive", PROTOCOL_VERSION, 308),
14543 )
14544 .unwrap();
14545 assert!(registry
14546 .set_module_state_for_test("inactive", ChannelState::Closed)
14547 .unwrap());
14548
14549 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
14550 let response = handler
14551 .handle_control_frame(
14552 &ctx,
14553 route_open_frame(309, "inactive", unique_project_root("inactive")),
14554 )
14555 .await
14556 .unwrap();
14557
14558 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14559 assert!(parse_error(&response[0])["message"]
14560 .as_str()
14561 .unwrap()
14562 .contains("is not active"));
14563 }
14564
14565 #[tokio::test]
14566 async fn late_health_reply_is_recorded_through_the_module_response_path() {
14567 let registry = Arc::new(Registry::default());
14568 let forwarding = Arc::new(ForwardingTable::default());
14569 let supervisor_handle = SupervisorHandle::new();
14570 let supervisor =
14571 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
14572 .with_forwarding(Arc::clone(&forwarding))
14573 .with_handle(supervisor_handle.clone());
14574 let module = supervisor
14575 .supervise_configured(
14576 crate::ModuleSpec {
14577 module_id: "late-health-response".to_string(),
14578 program: PathBuf::from("disabled-module"),
14579 args: Vec::new(),
14580 env: Vec::new(),
14581 reserved: false,
14582 reserved_prefixes: Vec::new(),
14583 protocol: ModuleProtocol::Subc,
14584 overlap: Default::default(),
14585 },
14586 false,
14587 )
14588 .unwrap();
14589 let handler =
14590 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14591 .with_supervisor(supervisor_handle);
14592 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
14593 handler
14594 .handle_control_frame(
14595 &module_ctx,
14596 hello_frame_with_control_ops(
14597 "late-health-response",
14598 PROTOCOL_VERSION,
14599 7,
14600 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14601 ),
14602 )
14603 .await
14604 .unwrap();
14605 let probe_started_at = Instant::now() - Duration::from_millis(80);
14606 let pending = forwarding
14607 .begin_health_probe_rpc_for(
14608 "late-health-response",
14609 MODULE_CONTROL_OP_HEALTH_CHECK,
14610 probe_started_at,
14611 Instant::now() - Duration::from_millis(1),
14612 )
14613 .unwrap();
14614 assert!(forwarding
14615 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
14616 .unwrap());
14617
14618 let responses = handler
14619 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
14620 .await
14621 .unwrap();
14622
14623 assert!(responses.is_empty());
14624 let health = module.status().unwrap().health;
14625 assert_eq!(health.late_answer_count, 1);
14626 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
14627 }
14628
14629 #[tokio::test]
14630 async fn health_probe_timeout_and_module_death_are_typed() {
14631 let registry = Arc::new(Registry::default());
14632 let forwarding = Arc::new(ForwardingTable::default());
14633 let handler =
14634 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14635 .with_health_probe_timeout(Duration::from_millis(50));
14636 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14637 hello_via_sink(
14638 &handler,
14639 &module_ctx,
14640 &mut module_rx,
14641 hello_frame_with_control_ops(
14642 "aft",
14643 PROTOCOL_VERSION,
14644 7,
14645 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14646 ),
14647 )
14648 .await;
14649
14650 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14651 let responses = handler
14652 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
14653 .await
14654 .unwrap();
14655 assert_eq!(responses[0].header.ty, FrameType::Error);
14656 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
14657 let _ = module_rx.try_recv();
14658
14659 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
14660 let health_handler = handler.clone();
14661 let death_task = tokio::spawn(async move {
14662 health_handler
14663 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
14664 .await
14665 .unwrap()
14666 });
14667 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14668 .await
14669 .unwrap()
14670 .unwrap();
14671 handler
14672 .cleanup_connection(module_ctx.connection_id)
14673 .unwrap();
14674 let responses = death_task.await.unwrap();
14675 assert_eq!(responses[0].header.ty, FrameType::Error);
14676 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
14677 }
14678
14679 #[test]
14680 fn hello_requires_exact_protocol_version() {
14681 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
14682 let registry = Arc::new(Registry::default());
14683 let handler = ControlHandler::new(Arc::clone(®istry));
14684 let responses = handler
14685 .handle_control(
14686 ConnectionId::new(connection),
14687 hello_frame("aft", offered, 9),
14688 )
14689 .unwrap();
14690
14691 assert_eq!(responses.len(), 1);
14692 assert_eq!(responses[0].header.ty, FrameType::Error);
14693 let error = parse_error(&responses[0]);
14694 assert_eq!(error["code"], "version_unsupported");
14695 assert!(registry.get_module("aft").unwrap().is_none());
14696 assert_eq!(registry.active_registration_count().unwrap(), 0);
14697 }
14698 }
14699
14700 #[test]
14701 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
14702 let registry = Arc::new(Registry::default());
14703 let forwarding = Arc::new(ForwardingTable::default());
14704 let handler =
14705 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14706 let module_connection = ConnectionId::new(301);
14707 let registration = registry
14708 .register_with_control_ops(
14709 manifest("aft-push", PROTOCOL_VERSION),
14710 PROTOCOL_VERSION,
14711 module_connection,
14712 module_baseline_control_ops(),
14713 )
14714 .unwrap();
14715 let (module_tx, _module_rx) = mpsc::channel(8);
14716 let endpoint = forwarding
14717 .register_module_connection(
14718 module_connection,
14719 "aft-push".to_string(),
14720 PROTOCOL_VERSION,
14721 manifest_concurrency(®istration.manifest),
14722 FrameSink::new(module_tx),
14723 )
14724 .unwrap();
14725
14726 let unknown = Frame::build(
14728 FrameType::Push,
14729 control_flags(),
14730 0,
14731 0,
14732 5,
14733 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
14734 )
14735 .unwrap();
14736 let out = handler.handle_status_update(endpoint, unknown).unwrap();
14737 assert!(
14738 out.is_empty(),
14739 "unknown push op must be ignored, got {out:?}"
14740 );
14741
14742 let malformed = Frame::build(
14744 FrameType::Push,
14745 control_flags(),
14746 0,
14747 0,
14748 6,
14749 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
14750 )
14751 .unwrap();
14752 let out = handler.handle_status_update(endpoint, malformed).unwrap();
14753 assert_eq!(out.len(), 1);
14754 assert_eq!(out[0].header.ty, FrameType::Error);
14755 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
14756 }
14757
14758 #[test]
14759 fn hello_rejected_when_connection_already_owns_client_routes() {
14760 let registry = Arc::new(Registry::default());
14761 let forwarding = Arc::new(ForwardingTable::default());
14762 let handler =
14763 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14764 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
14766 let client_connection = ConnectionId::new(202);
14767
14768 let responses = handler
14771 .handle_control(
14772 client_connection,
14773 hello_frame("aft-second", PROTOCOL_VERSION, 9),
14774 )
14775 .unwrap();
14776 assert_eq!(responses[0].header.ty, FrameType::Error);
14777 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
14778 assert!(registry.get_module("aft-second").unwrap().is_none());
14779 }
14780
14781 #[tokio::test]
14782 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
14783 let registry = Arc::new(Registry::default());
14784 let forwarding = Arc::new(ForwardingTable::default());
14785 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
14786 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
14787 hello_via_sink(
14788 &handler,
14789 &module_ctx,
14790 &mut module_rx,
14791 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
14792 )
14793 .await;
14794 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
14795 let pending = forwarding
14796 .begin_route_bind_relay_for_test(
14797 client_ctx.connection_id,
14798 client_ctx.egress.clone(),
14799 2,
14800 "alpha",
14801 )
14802 .unwrap();
14803 forwarding
14804 .complete_pending_relay(
14805 module_ctx.connection_id,
14806 pending.corr,
14807 RouteBindRelayOutcome::Accepted,
14808 )
14809 .unwrap();
14810 client_rx.try_recv().unwrap();
14811 for module_id in ["beta", "alpha"] {
14812 let replies = handler
14813 .handle_control_frame(
14814 &module_ctx,
14815 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
14816 )
14817 .await
14818 .unwrap();
14819 assert_eq!(replies.len(), 1, "second HELLO must be refused");
14820 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
14821 }
14822 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
14823 assert!(registry.get_module("beta").unwrap().is_none());
14824 assert!(matches!(
14825 forwarding
14826 .lookup_data_route(
14827 client_ctx.connection_id,
14828 pending.client_channel,
14829 pending.client_epoch,
14830 )
14831 .unwrap(),
14832 DataRoute::Client(DataRouteState::Bound(_))
14833 ));
14834 assert!(handler
14835 .hello_launch_nonces
14836 .lock()
14837 .unwrap()
14838 .presented(module_ctx.connection_id, Some("alpha-nonce")));
14839 assert!(module_rx.try_recv().is_err());
14840 }
14841
14842 #[test]
14843 fn reserved_module_hello_requires_matching_launch_nonce() {
14844 let registry = Arc::new(Registry::default());
14845 let supervisor = SupervisorHandle::new();
14846 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
14849 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14850
14851 let no_nonce = handler
14853 .handle_control(
14854 ConnectionId::new(1),
14855 hello_frame("vault", PROTOCOL_VERSION, 1),
14856 )
14857 .unwrap();
14858 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
14859 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
14860 assert!(registry.get_module("vault").unwrap().is_none());
14861
14862 let wrong = handler
14864 .handle_control(
14865 ConnectionId::new(2),
14866 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
14867 )
14868 .unwrap();
14869 assert_eq!(wrong[0].header.ty, FrameType::Error);
14870 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14871 assert!(registry.get_module("vault").unwrap().is_none());
14872
14873 let ok = handler
14875 .handle_control(
14876 ConnectionId::new(3),
14877 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14878 )
14879 .unwrap();
14880 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14881 assert!(registry.get_module("vault").unwrap().is_some());
14882 }
14883
14884 #[test]
14885 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14886 let registry = Arc::new(Registry::default());
14887 let supervisor = SupervisorHandle::new();
14888 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14889 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14890 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14891
14892 let squat = handler
14893 .handle_control(
14894 ConnectionId::new(1),
14895 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14896 )
14897 .unwrap();
14898 assert_eq!(squat[0].header.ty, FrameType::Error);
14899 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14900 assert!(parse_error(&squat[0])["message"]
14901 .as_str()
14902 .unwrap()
14903 .contains("fed:"));
14904
14905 let accepted_peer = handler
14906 .handle_control(
14907 ConnectionId::new(2),
14908 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14909 )
14910 .unwrap();
14911 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14912
14913 let accepted_short = handler
14914 .handle_control(
14915 ConnectionId::new(3),
14916 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14917 )
14918 .unwrap();
14919 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14920
14921 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14922 let response = handler
14923 .handle_control(
14924 ConnectionId::new(conn),
14925 hello_frame(module_id, PROTOCOL_VERSION, conn),
14926 )
14927 .unwrap();
14928 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14929 }
14930 }
14931
14932 #[test]
14933 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14934 let registry = Arc::new(Registry::default());
14935 let supervisor = SupervisorHandle::new();
14936 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14937 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14938 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14939 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14940
14941 let owner_nonce = handler
14942 .handle_control(
14943 ConnectionId::new(1),
14944 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14945 )
14946 .unwrap();
14947 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14948 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14949 assert!(registry.get_module("fed:special").unwrap().is_none());
14950
14951 let exact_nonce = handler
14952 .handle_control(
14953 ConnectionId::new(2),
14954 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14955 )
14956 .unwrap();
14957 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14958 assert!(registry.get_module("fed:special").unwrap().is_some());
14959 }
14960
14961 #[test]
14962 fn non_reserved_module_ignores_launch_nonce() {
14963 let registry = Arc::new(Registry::default());
14964 let handler = ControlHandler::new(Arc::clone(®istry));
14967 let no_nonce = handler
14968 .handle_control(
14969 ConnectionId::new(1),
14970 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14971 )
14972 .unwrap();
14973 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14974 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14975
14976 let echoed_nonce = handler
14977 .handle_control(
14978 ConnectionId::new(2),
14979 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14980 )
14981 .unwrap();
14982 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14983 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14984 }
14985
14986 #[test]
14987 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14988 let handler = ControlHandler::default();
14989 let conn = ConnectionId::new(1);
14990 let malformed = Frame::build(
14991 FrameType::Hello,
14992 control_flags(),
14993 0,
14994 0,
14995 3,
14996 b"{not json".to_vec(),
14997 )
14998 .unwrap();
14999
15000 let error = handler.handle_control(conn, malformed).unwrap();
15001 assert_eq!(error[0].header.ty, FrameType::Error);
15002 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
15003
15004 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15005 let pong = handler.handle_control(conn, ping).unwrap();
15006 assert_eq!(pong[0].header.ty, FrameType::Pong);
15007 assert_eq!(pong[0].header.corr, 4);
15008 }
15009
15010 #[test]
15011 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
15012 let registry = Arc::new(Registry::default());
15013 let handler = ControlHandler::new(Arc::clone(®istry));
15014
15015 handler
15016 .handle_control(
15017 ConnectionId::new(1),
15018 hello_frame("aft", PROTOCOL_VERSION, 1),
15019 )
15020 .unwrap();
15021 let duplicate = handler
15022 .handle_control(
15023 ConnectionId::new(2),
15024 hello_frame("aft", PROTOCOL_VERSION, 2),
15025 )
15026 .unwrap();
15027
15028 assert_eq!(duplicate[0].header.ty, FrameType::Error);
15029 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15030 let registration = registry.get_module("aft").unwrap().unwrap();
15031 assert_eq!(registration.connection_id, ConnectionId::new(1));
15032 }
15033
15034 #[test]
15035 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
15036 let registry = Arc::new(Registry::default());
15037 let forwarding = Arc::new(ForwardingTable::default());
15038 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
15039 let handler =
15040 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15041 .with_process_liveness(process_liveness);
15042 let (ctx, route_channel, route_epoch) =
15043 bind_liveness_route(®istry, &forwarding, "aft-dead");
15044 let responses = handler
15045 .handle_route_poll(
15046 &ctx,
15047 route_poll_frame(41, PollKind::Liveness, route_channel),
15048 route_channel,
15049 route_epoch,
15050 PollKind::Liveness,
15051 )
15052 .unwrap();
15053
15054 assert_eq!(responses.len(), 1);
15055 assert_eq!(responses[0].header.ty, FrameType::Response);
15056 assert_route_poll_liveness(&responses[0], false);
15057 }
15058
15059 #[test]
15060 fn liveness_poll_without_process_source_uses_bound_route() {
15061 let registry = Arc::new(Registry::default());
15062 let forwarding = Arc::new(ForwardingTable::default());
15063 let handler =
15064 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15065 let (ctx, route_channel, route_epoch) =
15066 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
15067 let responses = handler
15068 .handle_route_poll(
15069 &ctx,
15070 route_poll_frame(42, PollKind::Liveness, route_channel),
15071 route_channel,
15072 route_epoch,
15073 PollKind::Liveness,
15074 )
15075 .unwrap();
15076
15077 assert_route_poll_liveness(&responses[0], true);
15078 }
15079
15080 #[test]
15081 fn liveness_poll_untracked_process_source_uses_bound_route() {
15082 let registry = Arc::new(Registry::default());
15083 let forwarding = Arc::new(ForwardingTable::default());
15084 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
15085 let handler =
15086 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15087 .with_process_liveness(process_liveness);
15088 let (ctx, route_channel, route_epoch) =
15089 bind_liveness_route(®istry, &forwarding, "aft-untracked");
15090 let responses = handler
15091 .handle_route_poll(
15092 &ctx,
15093 route_poll_frame(43, PollKind::Liveness, route_channel),
15094 route_channel,
15095 route_epoch,
15096 PollKind::Liveness,
15097 )
15098 .unwrap();
15099
15100 assert_route_poll_liveness(&responses[0], true);
15101 }
15102
15103 #[tokio::test]
15104 async fn unknown_op_returns_unknown_control_op() {
15105 let handler = ControlHandler::default();
15106 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
15107 let request = Frame::build(
15108 FrameType::Request,
15109 control_flags(),
15110 0,
15111 0,
15112 55,
15113 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
15114 )
15115 .unwrap();
15116
15117 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15118
15119 assert_eq!(response.len(), 1);
15120 assert_eq!(response[0].header.ty, FrameType::Error);
15121 assert_eq!(response[0].header.corr, 55);
15122 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
15123 }
15124
15125 #[tokio::test]
15126 async fn supervisor_provenance_rejects_unknown_exact_module() {
15127 let handler = ControlHandler::default();
15128 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
15129 let request = Frame::build(
15130 FrameType::Request,
15131 control_flags(),
15132 0,
15133 0,
15134 57,
15135 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
15136 )
15137 .unwrap();
15138
15139 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15140
15141 assert_eq!(response.len(), 1);
15142 assert_eq!(response[0].header.ty, FrameType::Error);
15143 assert_eq!(response[0].header.corr, 57);
15144 let error = parse_error(&response[0]);
15145 assert_eq!(error["code"], "unknown_module");
15146 assert_eq!(error["message"], "module_id 'missing' is not supervised");
15147 }
15148
15149 #[test]
15150 fn provenance_probe_override_keeps_handler_tests_deterministic() {
15151 let expected = subc_control::RunningImageAgreement::Unavailable {
15152 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15153 };
15154 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
15155 assert_eq!(handler.provenance_probe_override, Some(expected));
15156 }
15157
15158 #[test]
15159 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
15160 let verdict = reload_verdict(
15161 std::path::Path::new("/bin/new"),
15162 Some(std::path::Path::new("/bin/old")),
15163 subc_control::RunningImageAgreement::Unavailable {
15164 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15165 },
15166 );
15167 assert!(matches!(
15168 verdict.path,
15169 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
15170 if configured == std::path::Path::new("/bin/new")
15171 && spawned_from == std::path::Path::new("/bin/old")
15172 ));
15173 }
15174
15175 #[test]
15176 fn launch_spec_without_running_digest_is_unavailable_not_match() {
15177 use subc_control::{
15178 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
15179 };
15180 let configured = crate::supervise::ModuleSpec {
15181 module_id: "legacy".into(),
15182 program: "/module".into(),
15183 args: Vec::new(),
15184 env: Vec::new(),
15185 reserved: false,
15186 reserved_prefixes: Vec::new(),
15187 protocol: subc_control::ModuleProtocol::Subc,
15188 overlap: Default::default(),
15189 };
15190 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
15191 assert_eq!(
15192 launch_spec_agreement(&configured, false, Some(&digest)),
15193 Agreement::Unavailable {
15194 reason: Reason::NotRunning
15195 }
15196 );
15197 assert_eq!(
15198 launch_spec_agreement(&configured, true, None),
15199 Agreement::Unavailable {
15200 reason: Reason::SpawnedSpecUnavailable
15201 }
15202 );
15203 assert_eq!(
15204 launch_spec_agreement(&configured, true, Some(&digest)),
15205 Agreement::Match
15206 );
15207 }
15208
15209 #[cfg(windows)]
15210 #[test]
15211 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
15212 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
15213 let configured = dir.join("module.exe");
15214 std::fs::write(&configured, b"executable path fixture").unwrap();
15215 let spawned = std::fs::canonicalize(&configured).unwrap();
15216 assert_ne!(
15217 configured, spawned,
15218 "fixture must exercise different path spellings"
15219 );
15220 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
15221 let lower_case = configured.to_string_lossy().to_lowercase();
15222 for spelling in [
15223 &configured,
15224 Path::new(&forward_slashes),
15225 Path::new(&lower_case),
15226 ] {
15227 let verdict = reload_verdict(
15228 spelling,
15229 Some(&spawned),
15230 subc_control::RunningImageAgreement::Unavailable {
15231 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15232 },
15233 );
15234 assert_eq!(
15235 verdict.path,
15236 ReloadPathAgreement::Match,
15237 "configured {spelling:?}, spawned {spawned:?}"
15238 );
15239 }
15240 }
15241
15242 #[cfg(windows)]
15243 #[test]
15244 fn windows_reload_path_keeps_different_existing_program_pending() {
15245 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
15246 let old = dir.join("old.exe");
15247 let new = dir.join("new.exe");
15248 std::fs::write(&old, b"same executable content").unwrap();
15249 std::fs::copy(&old, &new).unwrap();
15250 let spawned = std::fs::canonicalize(&old).unwrap();
15251 let image = subc_control::RunningImageAgreement::Unavailable {
15252 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15253 };
15254 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
15255 assert!(
15256 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
15257 if configured == new && spawned_from == spawned)
15258 );
15259 let missing = dir.join("missing.exe");
15260 assert!(matches!(
15261 reload_verdict(&missing, Some(&spawned), image).path,
15262 ReloadPathAgreement::Mismatch { .. }
15263 ));
15264 }
15265
15266 #[test]
15267 fn reload_verdict_detects_replaced_image_at_same_path() {
15268 let image = subc_control::RunningImageAgreement::Mismatch {
15269 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
15270 digest: "old".into(),
15271 },
15272 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
15273 digest: "new".into(),
15274 },
15275 };
15276 let verdict = reload_verdict(
15277 std::path::Path::new("/bin/same"),
15278 Some(std::path::Path::new("/bin/same")),
15279 image.clone(),
15280 );
15281 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15282 assert_eq!(verdict.image, image);
15283 }
15284
15285 #[test]
15286 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
15287 let image = subc_control::RunningImageAgreement::Unavailable {
15288 reason: subc_control::RunningImageUnavailableReason::NotRunning,
15289 };
15290 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
15291 assert_eq!(
15292 verdict.path,
15293 subc_control::ReloadPathAgreement::Unavailable {
15294 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
15295 }
15296 );
15297 assert_eq!(verdict.image, image);
15298
15299 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
15300 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15301 };
15302 let verdict = reload_verdict(
15303 std::path::Path::new("/bin/same"),
15304 Some(std::path::Path::new("/bin/same")),
15305 unconfirmed.clone(),
15306 );
15307 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15308 assert_eq!(verdict.image, unconfirmed);
15309 }
15310
15311 #[test]
15312 fn reload_verdict_preserves_each_image_unavailability_reason() {
15313 use subc_control::RunningImageUnavailableReason as Reason;
15314
15315 for reason in [
15316 Reason::NotRunning,
15317 Reason::UnsupportedPlatform,
15318 Reason::RunningExecutableUnreadable,
15319 Reason::SpawnedPathUnreadable,
15320 Reason::HashFailed,
15321 Reason::ProcessIdentityUnconfirmed,
15322 Reason::Unknown("future_probe_reason".to_string()),
15323 ] {
15324 let image = subc_control::RunningImageAgreement::Unavailable {
15325 reason: reason.clone(),
15326 };
15327 let verdict = reload_verdict(
15328 std::path::Path::new("/bin/same"),
15329 Some(std::path::Path::new("/bin/same")),
15330 image.clone(),
15331 );
15332 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15333 assert_eq!(verdict.image, image, "{reason:?}");
15334 }
15335 }
15336
15337 #[tokio::test]
15338 async fn malformed_control_bodies_return_invalid_control_body() {
15339 let handler = ControlHandler::default();
15340 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
15341
15342 for (corr, body) in [
15343 (56, br#"{"route_channel":1}"#.as_slice()),
15344 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
15345 (
15346 58,
15347 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
15348 ),
15349 ] {
15350 let request = Frame::build(
15351 FrameType::Request,
15352 control_flags(),
15353 0,
15354 0,
15355 corr,
15356 body.to_vec(),
15357 )
15358 .unwrap();
15359 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15360
15361 assert_eq!(response.len(), 1);
15362 assert_eq!(response[0].header.ty, FrameType::Error);
15363 assert_eq!(response[0].header.corr, corr);
15364 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
15365 }
15366 }
15367
15368 #[tokio::test]
15369 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
15370 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15371 let registry = Arc::new(Registry::default());
15372 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15373 let router = Router::with_control_handler(Arc::clone(&control));
15374 let connection = router.begin_connection();
15375 let (ctx, mut rx) = route_ctx(connection.id());
15376
15377 router
15378 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
15379 .await
15380 .unwrap();
15381 let response = rx.recv().await.unwrap();
15382 let ack = parse_ack(&response);
15383 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15384 let channel = 1;
15385
15386 let goodbye =
15387 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
15388 router.route_for_connection(&ctx, goodbye).await.unwrap();
15389 assert!(rx.try_recv().is_err());
15390 assert!(registry.get_module("aft").unwrap().is_none());
15391
15392 router
15393 .route_for_connection(&ctx, channel_request(channel, 13))
15394 .await
15395 .unwrap();
15396 let error_frame = rx.recv().await.unwrap();
15397 assert_eq!(error_frame.header.ty, FrameType::Error);
15398 assert_eq!(error_frame.header.channel, channel);
15399 let captured = crate::router::test_log::captured_logs(&logs);
15400 assert_eq!(
15401 captured
15402 .lines()
15403 .filter(|line| {
15404 line.contains("module_id=aft")
15405 && line.contains("reason=explicit_goodbye")
15406 && line.contains("module registration ended")
15407 })
15408 .count(),
15409 1,
15410 "unexpected GOODBYE registry log: {captured}"
15411 );
15412 }
15413
15414 #[tokio::test]
15415 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
15416 let registry = Arc::new(Registry::default());
15417 let handler = ControlHandler::new(registry).with_capability_config(
15418 [("prov".to_string(), true), ("cons".to_string(), true)],
15419 BTreeMap::new(),
15420 );
15421 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
15422 register_capability_manifest(
15423 &handler,
15424 &provider_ctx,
15425 &mut provider_rx,
15426 capability_manifest("prov", &["thing/v1"], &[]),
15427 1,
15428 )
15429 .await;
15430 let mut consumer = capability_manifest("cons", &[], &[]);
15431 consumer.capabilities.as_mut().unwrap().requires.push(
15432 subc_protocol::manifest::CapabilityRequirement {
15433 capability: "thing/v1".to_string(),
15434 need: subc_protocol::manifest::CapabilityNeed::Required,
15435 },
15436 );
15437 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
15438 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
15439 assert_eq!(
15440 handler.capability_evaluator.verdict("cons", "thing/v1"),
15441 Some(CapabilityVerdict::Provided)
15442 );
15443 let (mut client_rx, _) = open_route_for_capability_test(
15444 &handler,
15445 &provider_ctx,
15446 &mut provider_rx,
15447 703,
15448 3,
15449 "prov",
15450 None,
15451 )
15452 .await;
15453 let goodbye =
15454 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15455 handler
15456 .handle_control_frame(&provider_ctx, goodbye)
15457 .await
15458 .unwrap();
15459 assert_eq!(
15460 handler.capability_evaluator.verdict("cons", "thing/v1"),
15461 Some(CapabilityVerdict::NeverProvided)
15462 );
15463 let closed = client_rx
15464 .try_recv()
15465 .expect("GOODBYE pushes route.closed before route GOODBYE");
15466 assert!(
15467 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
15468 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
15469 );
15470 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
15471 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
15472 }
15473
15474 struct RouteClosedFixture {
15475 handler: ControlHandler,
15476 #[cfg(unix)]
15477 supervisor: Supervisor,
15478 module_ctx: RouteCtx,
15479 #[cfg(unix)]
15480 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15481 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15482 route: Arc<crate::forwarding::RouteBinding>,
15483 }
15484
15485 async fn route_closed_fixture() -> RouteClosedFixture {
15486 let registry = Arc::new(Registry::default());
15487 let handler = ControlHandler::new(Arc::clone(®istry));
15488 #[cfg(unix)]
15489 let supervisor = Supervisor::new(registry, RestartPolicy::default())
15490 .with_forwarding(Arc::clone(&handler.forwarding));
15491 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
15492 register_capability_manifest(
15493 &handler,
15494 &module_ctx,
15495 &mut module_rx,
15496 capability_manifest("prov", &[], &[]),
15497 1,
15498 )
15499 .await;
15500 let (client_rx, bind) = open_route_for_capability_test(
15501 &handler,
15502 &module_ctx,
15503 &mut module_rx,
15504 702,
15505 2,
15506 "prov",
15507 None,
15508 )
15509 .await;
15510 let ModuleControlRequest::RouteBind {
15511 route_channel: channel,
15512 epoch: route_epoch,
15513 ..
15514 } = bind
15515 else {
15516 panic!("expected route.bind");
15517 };
15518 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
15519 handler
15520 .forwarding
15521 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
15522 .unwrap()
15523 else {
15524 panic!("the acknowledged route must be bound");
15525 };
15526 RouteClosedFixture {
15527 handler,
15528 #[cfg(unix)]
15529 supervisor,
15530 module_ctx,
15531 #[cfg(unix)]
15532 module_rx,
15533 client_rx,
15534 route,
15535 }
15536 }
15537
15538 impl RouteClosedFixture {
15539 #[cfg(unix)]
15540 async fn cut(&mut self) {
15541 let notices = async {
15542 let mut draining = self.module_rx.recv().await.unwrap();
15543 assert!(matches!(
15544 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
15545 &draining.body
15546 )
15547 .unwrap(),
15548 subc_protocol::session::ModuleControlCommand::Draining {
15549 reason: RouteCloseReason::Restart,
15550 ..
15551 }
15552 ));
15553 draining.flushed.take().unwrap().send(()).unwrap();
15554 let mut closing = self.client_rx.recv().await.unwrap();
15555 assert!(matches!(
15556 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
15557 ClientControlPush::RouteClosing {
15558 reason: RouteCloseReason::Restart,
15559 ..
15560 }
15561 ));
15562 closing.flushed.take().unwrap().send(()).unwrap();
15563 };
15564 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
15565 result.unwrap();
15566 }
15567
15568 fn disconnect(&mut self) -> ClientControlPush {
15569 self.handler
15570 .cleanup_connection(self.module_ctx.connection_id)
15571 .unwrap();
15572 let closed = self.client_rx.try_recv().unwrap();
15573 assert_eq!(closed.header.ty, FrameType::Push);
15574 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
15575 assert!(matches!(
15576 &push,
15577 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
15578 if channels == &[self.route.client_channel]
15579 ));
15580 assert_eq!(
15581 self.client_rx.try_recv().unwrap().header.ty,
15582 FrameType::Goodbye
15583 );
15584 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
15585 push
15586 }
15587 }
15588
15589 #[cfg(unix)]
15590 #[tokio::test(start_paused = true)]
15591 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
15592 let mut fixture = route_closed_fixture().await;
15593 fixture.cut().await;
15594 assert!(matches!(
15595 fixture.disconnect(),
15596 ClientControlPush::RouteClosed {
15597 reason: RouteCloseReason::Restart,
15598 drained: true,
15599 ..
15600 }
15601 ));
15602 }
15603
15604 #[cfg(unix)]
15605 #[tokio::test(start_paused = true)]
15606 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
15607 let mut fixture = route_closed_fixture().await;
15608 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
15609 let started = tokio::time::Instant::now();
15610 fixture.cut().await;
15611 assert!(started.elapsed() >= Duration::from_secs(2));
15612 assert_eq!(fixture.route.flow.in_flight(), 1);
15613 assert!(matches!(
15614 fixture.disconnect(),
15615 ClientControlPush::RouteClosed {
15616 reason: RouteCloseReason::Restart,
15617 drained: false,
15618 ..
15619 }
15620 ));
15621 }
15622
15623 #[tokio::test]
15624 async fn module_connection_crash_reports_crash_and_not_drained() {
15625 let mut fixture = route_closed_fixture().await;
15626 assert!(matches!(
15630 fixture.disconnect(),
15631 ClientControlPush::RouteClosed {
15632 reason: RouteCloseReason::Crash,
15633 drained: false,
15634 ..
15635 }
15636 ));
15637 }
15638
15639 #[tokio::test]
15640 async fn dropping_router_connection_releases_registration() {
15641 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15642 let registry = Arc::new(Registry::default());
15643 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15644 let router = Router::with_control_handler(Arc::clone(&control));
15645 let connection = router.begin_connection();
15646 let connection_id = connection.id();
15647 let (ctx, mut rx) = route_ctx(connection_id);
15648
15649 router
15650 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
15651 .await
15652 .unwrap();
15653 let response = rx.recv().await.unwrap();
15654 let ack = parse_ack(&response);
15655 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15656 assert!(registry.get_module("aft").unwrap().is_some());
15657
15658 drop(connection);
15659
15660 assert!(registry.get_module("aft").unwrap().is_none());
15661 assert_eq!(registry.active_registration_count().unwrap(), 0);
15662
15663 control
15664 .cleanup_connection(ConnectionId::new(u64::MAX))
15665 .unwrap();
15666 let captured = crate::router::test_log::captured_logs(&logs);
15667 println!("captured registry lifecycle logs:\n{captured}");
15668 let events: Vec<_> = captured
15669 .lines()
15670 .filter(|line| {
15671 line.contains("module registered module_id=aft ")
15672 || line.contains("module registration ended")
15673 })
15674 .collect();
15675 assert_eq!(
15676 events.len(),
15677 2,
15678 "unexpected registry lifecycle logs: {captured}"
15679 );
15680 assert!(events[0].contains("module registered module_id=aft "));
15681 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
15682 assert!(events[1].contains(&format!(
15683 "module_id=aft connection_id={}",
15684 connection_id.get()
15685 )));
15686 assert!(events[1].contains("reason=connection_closed"));
15687 assert!(events[1].contains("module registration ended"));
15688 assert_eq!(
15689 captured
15690 .lines()
15691 .filter(|line| line.contains("module registered module_id=aft "))
15692 .count(),
15693 1,
15694 "legacy registration admission line must appear once: {captured}"
15695 );
15696 }
15697
15698 #[tokio::test]
15699 async fn hello_registration_keeps_legacy_module_registered_line_once() {
15700 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15701 let registry = Arc::new(Registry::default());
15702 let control = ControlHandler::new(Arc::clone(®istry));
15703 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
15704 hello_via_sink(
15705 &control,
15706 &ctx,
15707 &mut rx,
15708 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
15709 )
15710 .await;
15711
15712 let captured = crate::router::test_log::captured_logs(&logs);
15713 let admissions: Vec<_> = captured
15714 .lines()
15715 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
15716 .collect();
15717 assert_eq!(
15718 admissions.len(),
15719 1,
15720 "legacy admission line must remain exactly once: {captured}"
15721 );
15722 assert!(admissions[0].contains("routable_provider=true"));
15723 assert!(admissions[0].contains("connection_id=777"));
15724 }
15725
15726 fn capability_manifest(
15727 module_id: &str,
15728 provides: &[&str],
15729 must_never_reach: &[&str],
15730 ) -> ModuleManifest {
15731 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
15732 manifest.capabilities = Some(CapabilityDeclarations {
15733 provides: provides
15734 .iter()
15735 .map(|capability| (*capability).to_string())
15736 .collect(),
15737 requires: Vec::new(),
15738 must_never_reach: must_never_reach
15739 .iter()
15740 .map(|capability| (*capability).to_string())
15741 .collect(),
15742 });
15743 manifest
15744 }
15745
15746 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
15747 Frame::build(
15748 FrameType::Hello,
15749 control_flags(),
15750 0,
15751 0,
15752 corr,
15753 serde_json::to_vec(&ModuleHelloBody {
15754 protocol_ver: manifest.protocol_ver,
15755 manifest,
15756 control_ops: None,
15757 launch_nonce: None,
15758 })
15759 .expect("capability test HELLO serializes"),
15760 )
15761 .expect("capability test HELLO frame builds")
15762 }
15763
15764 fn catalog_update_with_capabilities_frame(
15765 corr: u64,
15766 capabilities: CapabilityDeclarations,
15767 ) -> Frame {
15768 Frame::build(
15769 FrameType::Request,
15770 control_flags(),
15771 0,
15772 0,
15773 corr,
15774 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15775 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
15776 capabilities: Some(capabilities),
15777 ready: None,
15778 })
15779 .expect("capability catalog.update serializes"),
15780 )
15781 .expect("capability catalog.update frame builds")
15782 }
15783
15784 async fn register_capability_manifest(
15785 handler: &ControlHandler,
15786 ctx: &RouteCtx,
15787 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15788 manifest: ModuleManifest,
15789 corr: u64,
15790 ) {
15791 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
15792 }
15793
15794 async fn open_route_for_capability_test(
15795 handler: &ControlHandler,
15796 target_ctx: &RouteCtx,
15797 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15798 client_connection_id: u64,
15799 corr: u64,
15800 target_module_id: &str,
15801 consumer_identity: Option<ConsumerIdentity>,
15802 ) -> (
15803 mpsc::Receiver<crate::router::OutboundFrame>,
15804 ModuleControlRequest,
15805 ) {
15806 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
15807 let route_handler = handler.clone();
15808 let target_module_id = target_module_id.to_string();
15809 let route_task = tokio::spawn(async move {
15810 route_handler
15811 .handle_control_frame(
15812 &client_ctx,
15813 route_open_frame_with_admission_facts(
15814 corr,
15815 &target_module_id,
15816 unique_project_root("admission-facts"),
15817 consumer_identity,
15818 None,
15819 ),
15820 )
15821 .await
15822 .expect("capability test route.open succeeds")
15823 });
15824 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
15825 .await
15826 .expect("capability test route.open must reach route.bind")
15827 .expect("target control receiver stays open");
15828 let bind_request: ModuleControlRequest =
15829 serde_json::from_slice(&bind.body).expect("route.bind decodes");
15830 handler
15831 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
15832 .await
15833 .expect("capability test route.bind ACK succeeds");
15834 assert!(route_task.await.expect("route.open task joins").is_empty());
15835 let opened = client_rx
15836 .recv()
15837 .await
15838 .expect("successful route.open publishes a response");
15839 assert!(matches!(
15840 serde_json::from_slice::<ClientControlResponse>(&opened.body),
15841 Ok(ClientControlResponse::RouteOpen { .. })
15842 ));
15843 (client_rx, bind_request)
15844 }
15845
15846 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
15847 assert_eq!(frame.header.ty, FrameType::Push);
15848 assert_eq!(frame.header.channel, 0);
15849 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
15850 .expect("route.closed control push decodes");
15851 let ClientControlPush::RouteClosed { channels, .. } = &push else {
15852 panic!("expected route.closed");
15853 };
15854 assert_eq!(channels.len(), 1, "exactly one violating route closed");
15855 let closed_channel = channels[0];
15856 let channels = channels.clone();
15857 assert_eq!(
15858 push,
15859 ClientControlPush::RouteClosed {
15860 module_id: target_module_id.to_string(),
15861 channels,
15862 reason: RouteCloseReason::CapabilityDenied,
15863 drained: false,
15864 abandoned: 0,
15865 excluded_subscriptions: 0,
15866 terminal: Some(false),
15867 }
15868 );
15869 closed_channel
15870 }
15871
15872 #[tokio::test]
15873 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15874 let registry = Arc::new(Registry::default());
15875 let forwarding = Arc::new(ForwardingTable::default());
15876 let supervisor = SupervisorHandle::new();
15877 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15878 let handler =
15879 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15880 .with_supervisor(supervisor);
15881 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15882 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15883 register_capability_manifest(
15884 &handler,
15885 &target_ctx,
15886 &mut target_rx,
15887 capability_manifest("target", &["credentials-provider/v1"], &[]),
15888 1,
15889 )
15890 .await;
15891 register_capability_manifest(
15892 &handler,
15893 &opener_ctx,
15894 &mut opener_rx,
15895 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15896 2,
15897 )
15898 .await;
15899
15900 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15901 let replies = handler
15902 .handle_control_frame(
15903 &client_ctx,
15904 route_open_frame_with_admission_facts(
15905 3,
15906 "target",
15907 unique_project_root("admission-facts"),
15908 Some(ConsumerIdentity {
15909 module_id: "opener".to_string(),
15910 launch_nonce: "opener-nonce".to_string(),
15911 }),
15912 None,
15913 ),
15914 )
15915 .await
15916 .expect("denied route.open returns a typed frame");
15917 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15918 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15919 assert!(
15920 target_rx.try_recv().is_err(),
15921 "forbidden route.open must not relay route.bind"
15922 );
15923 }
15924
15925 #[tokio::test]
15926 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15927 let registry = Arc::new(Registry::default());
15928 let forwarding = Arc::new(ForwardingTable::default());
15929 let supervisor = SupervisorHandle::new();
15930 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15931 let handler =
15932 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15933 .with_supervisor(supervisor);
15934 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15935 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15936 register_capability_manifest(
15937 &handler,
15938 &target_ctx,
15939 &mut target_rx,
15940 capability_manifest("target", &["credentials-provider/v1"], &[]),
15941 1,
15942 )
15943 .await;
15944 register_capability_manifest(
15945 &handler,
15946 &old_opener_ctx,
15947 &mut old_opener_rx,
15948 capability_manifest("opener", &[], &[]),
15949 2,
15950 )
15951 .await;
15952 let (mut client_rx, _) = open_route_for_capability_test(
15953 &handler,
15954 &target_ctx,
15955 &mut target_rx,
15956 712,
15957 3,
15958 "target",
15959 Some(ConsumerIdentity {
15960 module_id: "opener".to_string(),
15961 launch_nonce: "opener-nonce".to_string(),
15962 }),
15963 )
15964 .await;
15965 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15966
15967 handler
15968 .cleanup_connection(old_opener_ctx.connection_id)
15969 .expect("old opener registration cleans up");
15970 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15971 register_capability_manifest(
15972 &handler,
15973 &new_opener_ctx,
15974 &mut new_opener_rx,
15975 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15976 4,
15977 )
15978 .await;
15979
15980 let closed_channel = assert_capability_denied_push(
15981 client_rx
15982 .try_recv()
15983 .expect("HELLO deny addition must emit route.closed")
15984 .frame,
15985 "target",
15986 );
15987 let goodbye = client_rx
15988 .try_recv()
15989 .expect("HELLO deny addition must send the client a route GOODBYE")
15990 .frame;
15991 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15992 assert_eq!(goodbye.header.channel, closed_channel);
15993 assert_ne!(goodbye.header.channel, 0);
15994 assert_ne!(goodbye.header.epoch, 0);
15995 assert!(client_rx.try_recv().is_err());
15996 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15997 assert!(matches!(
15998 target_rx.try_recv(),
15999 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16000 ));
16001 }
16002
16003 #[tokio::test]
16004 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
16005 let registry = Arc::new(Registry::default());
16006 let forwarding = Arc::new(ForwardingTable::default());
16007 let supervisor = SupervisorHandle::new();
16008 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16009 let handler =
16010 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16011 .with_supervisor(supervisor);
16012 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
16013 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
16014 register_capability_manifest(
16015 &handler,
16016 &target_ctx,
16017 &mut target_rx,
16018 capability_manifest("target", &[], &[]),
16019 1,
16020 )
16021 .await;
16022 register_capability_manifest(
16023 &handler,
16024 &opener_ctx,
16025 &mut opener_rx,
16026 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16027 2,
16028 )
16029 .await;
16030 let (mut client_rx, _) = open_route_for_capability_test(
16031 &handler,
16032 &target_ctx,
16033 &mut target_rx,
16034 722,
16035 3,
16036 "target",
16037 Some(ConsumerIdentity {
16038 module_id: "opener".to_string(),
16039 launch_nonce: "opener-nonce".to_string(),
16040 }),
16041 )
16042 .await;
16043 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16044
16045 let replies = handler
16046 .handle_control_frame(
16047 &target_ctx,
16048 catalog_update_with_capabilities_frame(
16049 4,
16050 CapabilityDeclarations {
16051 provides: vec!["credentials-provider/v1".to_string()],
16052 requires: Vec::new(),
16053 must_never_reach: Vec::new(),
16054 },
16055 ),
16056 )
16057 .await
16058 .expect("claim catalog.update succeeds");
16059 assert!(matches!(
16060 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
16061 Ok(ModuleControlResponseToModule::CatalogUpdate {})
16062 ));
16063 let closed_channel = assert_capability_denied_push(
16064 client_rx
16065 .try_recv()
16066 .expect("claim addition must emit route.closed")
16067 .frame,
16068 "target",
16069 );
16070 let goodbye = client_rx
16071 .try_recv()
16072 .expect("claim addition must send the client a route GOODBYE")
16073 .frame;
16074 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16075 assert_eq!(goodbye.header.channel, closed_channel);
16076 assert_ne!(goodbye.header.channel, 0);
16077 assert_ne!(goodbye.header.epoch, 0);
16078 assert!(client_rx.try_recv().is_err());
16079 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16080 assert!(matches!(
16081 target_rx.try_recv(),
16082 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16083 ));
16084 }
16085
16086 #[tokio::test]
16087 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
16088 let registry = Arc::new(Registry::default());
16089 let forwarding = Arc::new(ForwardingTable::default());
16090 let supervisor = SupervisorHandle::new();
16091 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16092 let handler =
16093 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16094 .with_supervisor(supervisor);
16095 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
16096 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
16097 register_capability_manifest(
16098 &handler,
16099 &target_ctx,
16100 &mut target_rx,
16101 capability_manifest("target", &["credentials-provider/v1"], &[]),
16102 1,
16103 )
16104 .await;
16105 register_capability_manifest(
16106 &handler,
16107 &opener_ctx,
16108 &mut opener_rx,
16109 capability_manifest("opener", &[], &[]),
16110 2,
16111 )
16112 .await;
16113 let (mut client_rx, _) = open_route_for_capability_test(
16114 &handler,
16115 &target_ctx,
16116 &mut target_rx,
16117 732,
16118 3,
16119 "target",
16120 Some(ConsumerIdentity {
16121 module_id: "opener".to_string(),
16122 launch_nonce: "opener-nonce".to_string(),
16123 }),
16124 )
16125 .await;
16126 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16127
16128 handler
16129 .handle_control_frame(
16130 &target_ctx,
16131 catalog_update_with_capabilities_frame(
16132 4,
16133 CapabilityDeclarations {
16134 provides: Vec::new(),
16135 requires: Vec::new(),
16136 must_never_reach: Vec::new(),
16137 },
16138 ),
16139 )
16140 .await
16141 .expect("claim removal catalog.update succeeds");
16142 assert_eq!(
16143 forwarding.active_binding_count().unwrap(),
16144 1,
16145 "removing an attested target claim must leave the route census unchanged"
16146 );
16147 assert!(
16148 client_rx.try_recv().is_err(),
16149 "claim removal must not emit route.closed or a client route GOODBYE"
16150 );
16151 assert!(
16152 target_rx.try_recv().is_err(),
16153 "claim removal must not send the target a route GOODBYE"
16154 );
16155 }
16156
16157 #[tokio::test]
16160 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
16161 let registry = Arc::new(Registry::default());
16162 let forwarding = Arc::new(ForwardingTable::default());
16163 let supervisor = SupervisorHandle::new();
16164 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16165 let handler =
16166 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16167 .with_supervisor(supervisor);
16168 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
16169 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
16170 register_capability_manifest(
16171 &handler,
16172 &target_ctx,
16173 &mut target_rx,
16174 capability_manifest("target", &["credentials-provider/v1"], &[]),
16175 1,
16176 )
16177 .await;
16178 register_capability_manifest(
16179 &handler,
16180 &opener_ctx,
16181 &mut opener_rx,
16182 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16183 2,
16184 )
16185 .await;
16186
16187 let (_client_rx, bind) = open_route_for_capability_test(
16188 &handler,
16189 &target_ctx,
16190 &mut target_rx,
16191 742,
16192 3,
16193 "target",
16194 None,
16195 )
16196 .await;
16197 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
16198 panic!("direct scope-honesty route must bind");
16199 };
16200 assert_eq!(principal, Some(Principal::Direct));
16201 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16202 }
16203
16204 #[tokio::test]
16207 async fn must_never_reach_self_route_is_capability_forbidden() {
16208 let registry = Arc::new(Registry::default());
16209 let forwarding = Arc::new(ForwardingTable::default());
16210 let supervisor = SupervisorHandle::new();
16211 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
16212 let handler =
16213 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16214 .with_supervisor(supervisor);
16215 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
16216 register_capability_manifest(
16217 &handler,
16218 &self_ctx,
16219 &mut self_rx,
16220 capability_manifest(
16221 "self-provider",
16222 &["credentials-provider/v1"],
16223 &["credentials-provider/v1"],
16224 ),
16225 1,
16226 )
16227 .await;
16228
16229 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
16230 let replies = handler
16231 .handle_control_frame(
16232 &client_ctx,
16233 route_open_frame_with_admission_facts(
16234 2,
16235 "self-provider",
16236 unique_project_root("admission-facts"),
16237 Some(ConsumerIdentity {
16238 module_id: "self-provider".to_string(),
16239 launch_nonce: "self-nonce".to_string(),
16240 }),
16241 None,
16242 ),
16243 )
16244 .await
16245 .expect("self-route refusal returns a typed frame");
16246 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16247 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16248 assert!(
16249 self_rx.try_recv().is_err(),
16250 "self denial must not relay route.bind"
16251 );
16252 }
16253
16254 #[test]
16255 fn unsupported_channel_zero_frame_returns_error() {
16256 let handler = ControlHandler::default();
16257 let request = Frame::build(
16258 FrameType::Request,
16259 control_flags(),
16260 0,
16261 0,
16262 21,
16263 b"opaque".to_vec(),
16264 )
16265 .unwrap();
16266
16267 let response = handler
16268 .handle_control(ConnectionId::new(1), request)
16269 .unwrap();
16270
16271 assert_eq!(response[0].header.ty, FrameType::Error);
16272 assert_eq!(
16273 parse_error(&response[0])["code"],
16274 "unsupported_control_frame"
16275 );
16276 }
16277
16278 mod swap {
16283 use super::*;
16284
16285 const INCUMBENT: ConnectionId = ConnectionId::new(30);
16286 const CANDIDATE: ConnectionId = ConnectionId::new(40);
16287
16288 struct Swap {
16289 registry: Arc<Registry>,
16290 forwarding: Arc<ForwardingTable>,
16291 handler: ControlHandler,
16292 incumbent_ctx: RouteCtx,
16293 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16294 candidate_ctx: RouteCtx,
16295 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16296 }
16297
16298 async fn swap_with_incumbent() -> Swap {
16299 let registry = Arc::new(Registry::default());
16300 let forwarding = Arc::new(ForwardingTable::default());
16301 let handler =
16302 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16303 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
16304 hello_via_sink(
16305 &handler,
16306 &incumbent_ctx,
16307 &mut incumbent_rx,
16308 hello_frame("aft", PROTOCOL_VERSION, 7),
16309 )
16310 .await;
16311 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
16312 Swap {
16313 registry,
16314 forwarding,
16315 handler,
16316 incumbent_ctx,
16317 incumbent_rx,
16318 candidate_ctx,
16319 candidate_rx,
16320 }
16321 }
16322
16323 fn register_candidate(swap: &Swap, ready: Option<bool>) {
16324 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
16325 candidate_manifest.ready = ready;
16326 let registration = swap
16327 .registry
16328 .register_candidate_with_control_ops(
16329 candidate_manifest,
16330 PROTOCOL_VERSION,
16331 CANDIDATE,
16332 module_baseline_control_ops(),
16333 )
16334 .unwrap();
16335 swap.forwarding
16336 .register_candidate_module_connection(
16337 CANDIDATE,
16338 "aft".to_string(),
16339 PROTOCOL_VERSION,
16340 manifest_concurrency(®istration.manifest),
16341 swap.candidate_ctx.egress.clone(),
16342 )
16343 .unwrap();
16344 }
16345
16346 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
16347 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
16348 swap.registry.promote_candidate("aft").unwrap().unwrap();
16349 cutover.incumbent.unwrap()
16350 }
16351
16352 #[tokio::test]
16361 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
16362 let mut swap = swap_with_incumbent().await;
16363 let hold = Arc::new(std::sync::Barrier::new(2));
16364 let handler = swap
16365 .handler
16366 .clone()
16367 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
16368 handler
16369 .supervisor
16370 .open_swap("aft", "candidate-nonce".to_string());
16371 let hello = {
16372 let handler = handler.clone();
16373 let candidate_ctx = swap.candidate_ctx.clone();
16374 std::thread::spawn(move || {
16375 handler
16376 .handle_hello(
16377 candidate_ctx.connection_id,
16378 Some(candidate_ctx.egress.clone()),
16379 hello_frame_with_nonce(
16380 "aft",
16381 PROTOCOL_VERSION,
16382 8,
16383 Some("candidate-nonce"),
16384 ),
16385 )
16386 .unwrap()
16387 })
16388 };
16389
16390 hold.wait();
16391 assert!(swap
16394 .registry
16395 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
16396 .unwrap()
16397 .is_some());
16398 assert!(swap
16399 .registry
16400 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16401 .unwrap()
16402 .is_none());
16403 assert!(swap
16404 .forwarding
16405 .module_endpoint_for_connection(CANDIDATE)
16406 .unwrap()
16407 .is_none());
16408
16409 hold.wait();
16410 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
16411 assert_eq!(
16412 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
16413 FrameType::HelloAck
16414 );
16415 let published = swap
16416 .registry
16417 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16418 .unwrap()
16419 .expect("candidate is published once its HELLO completes");
16420 assert_eq!(published.connection_id, CANDIDATE);
16421 assert!(swap
16422 .forwarding
16423 .module_endpoint_for_connection(CANDIDATE)
16424 .unwrap()
16425 .is_some());
16426 }
16427
16428 fn keyed_total(counters: &Value, key: &str) -> u64 {
16429 counters[key]
16430 .as_object()
16431 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
16432 .unwrap_or(0)
16433 }
16434
16435 #[tokio::test]
16436 async fn replacement_logs_old_end_and_new_admission_once() {
16437 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16438 let mut swap = swap_with_incumbent().await;
16439 swap.handler
16440 .supervisor
16441 .open_swap("aft", "candidate-nonce".to_string());
16442 hello_via_sink(
16443 &swap.handler,
16444 &swap.candidate_ctx,
16445 &mut swap.candidate_rx,
16446 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
16447 )
16448 .await;
16449 cutover(&swap);
16450 swap.handler.cleanup_connection(INCUMBENT).unwrap();
16451
16452 let captured = crate::router::test_log::captured_logs(&logs);
16453 println!("captured replacement registry logs:\n{captured}");
16454 let new_admissions: Vec<_> = captured
16455 .lines()
16456 .filter(|line| {
16457 line.contains("connection_id=40")
16458 && line.contains("swap candidate registered; not routable until cutover")
16459 })
16460 .collect();
16461 assert_eq!(
16462 new_admissions.len(),
16463 1,
16464 "unexpected admission logs: {captured}"
16465 );
16466 assert!(new_admissions[0].contains("module_id=aft"));
16467 assert!(new_admissions[0].contains("connection_id=40"));
16468 assert!(
16469 new_admissions[0].contains("swap candidate registered; not routable until cutover")
16470 );
16471 assert!(new_admissions[0].contains("ready=true"));
16472
16473 let old_admissions: Vec<_> = captured
16474 .lines()
16475 .filter(|line| {
16476 line.contains("module registered module_id=aft ")
16477 && line.contains("connection_id=30")
16478 })
16479 .collect();
16480 assert_eq!(
16481 old_admissions.len(),
16482 1,
16483 "unexpected incumbent admission: {captured}"
16484 );
16485
16486 let old_ends: Vec<_> = captured
16487 .lines()
16488 .filter(|line| {
16489 line.contains("connection_id=30") && line.contains("module registration ended")
16490 })
16491 .collect();
16492 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
16493 assert!(old_ends[0].contains("module_id=aft"));
16494 assert!(old_ends[0].contains("connection_id=30"));
16495 assert!(old_ends[0].contains("reason=replaced"));
16496 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
16497 assert_eq!(
16498 captured
16499 .lines()
16500 .filter(|line| line.contains("module registration promoted"))
16501 .count(),
16502 1,
16503 "unexpected promotion logs: {captured}"
16504 );
16505 }
16506
16507 #[tokio::test]
16513 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
16514 let mut swap = swap_with_incumbent().await;
16515 let handler = swap.handler.clone();
16516
16517 let cotenant = ConnectionId::new(31);
16519 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
16520 let (cotenant_task, cotenant_bind) = relay_route_open(
16521 &handler,
16522 cotenant,
16523 &cotenant_ctx.egress,
16524 &mut swap.incumbent_rx,
16525 100,
16526 "aft",
16527 "swap-cotenant",
16528 )
16529 .await;
16530 handler
16531 .handle_control_frame(
16532 &swap.incumbent_ctx,
16533 route_bind_ack(cotenant_bind.header.corr),
16534 )
16535 .await
16536 .unwrap();
16537 assert!(cotenant_task.await.unwrap().is_empty());
16538 let (cotenant_channel, cotenant_epoch) =
16539 published_route(&cotenant_rx.recv().await.unwrap());
16540
16541 let caller = ConnectionId::new(32);
16543 let (caller_ctx, mut caller_rx) = route_ctx(caller);
16544 let (caller_task, caller_bind) = relay_route_open(
16545 &handler,
16546 caller,
16547 &caller_ctx.egress,
16548 &mut swap.incumbent_rx,
16549 101,
16550 "aft",
16551 "swap-caller",
16552 )
16553 .await;
16554 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
16555
16556 register_candidate(&swap, None);
16557 cutover(&swap);
16558
16559 let ack = handler
16561 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
16562 .await;
16563 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
16564 if module_loop_error.is_some() {
16565 handler.cleanup_connection(INCUMBENT).unwrap();
16568 }
16569
16570 assert!(
16572 cotenant_rx.try_recv().is_err(),
16573 "the co-tenant route on the incumbent was torn down by one late ack: \
16574 {module_loop_error:?}"
16575 );
16576 assert!(matches!(
16577 swap.forwarding
16578 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16579 .unwrap(),
16580 DataRoute::Client(DataRouteState::Bound(_))
16581 ));
16582 assert_eq!(module_loop_error, None);
16583 assert!(swap
16584 .registry
16585 .get_module_by_connection(INCUMBENT)
16586 .unwrap()
16587 .is_some());
16588
16589 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
16591 .await
16592 .expect("the incumbent is told to drop the abandoned binding")
16593 .unwrap()
16594 .frame;
16595 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16596 assert_eq!(goodbye.header.channel, abandoned_channel);
16597 assert_eq!(goodbye.header.epoch, abandoned_epoch);
16598 assert!(swap.incumbent_rx.try_recv().is_err());
16599
16600 let response = caller_task.await.unwrap();
16602 assert_eq!(response.len(), 1);
16603 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
16604 assert!(caller_rx.try_recv().is_err());
16605
16606 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
16610 let counters = handler.counters().snapshot();
16611 assert_eq!(
16612 keyed_total(&counters, "route_open_accepted_by_principal"),
16613 1
16614 );
16615 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
16616 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
16617 }
16618
16619 #[tokio::test]
16623 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
16624 let mut swap = swap_with_incumbent().await;
16625 register_candidate(&swap, None);
16626 let incumbent = cutover(&swap);
16627 swap.forwarding
16628 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
16629 .unwrap()
16630 .expect("the incumbent is still registered");
16631
16632 let client = ConnectionId::new(33);
16633 let (client_ctx, mut client_rx) = route_ctx(client);
16634 let route_handler = swap.handler.clone();
16635 let open_ctx = RouteCtx {
16636 connection_id: client,
16637 egress: client_ctx.egress.clone(),
16638 };
16639 let mut route_task = tokio::spawn(async move {
16640 route_handler
16641 .handle_control_frame(
16642 &open_ctx,
16643 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
16644 )
16645 .await
16646 .unwrap()
16647 });
16648 let bind = tokio::select! {
16649 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
16650 response = &mut route_task => {
16651 let response = response.unwrap();
16652 panic!(
16653 "post-cutover route.open was refused instead of relayed to the candidate: {}",
16654 parse_error(&response[0])["code"]
16655 );
16656 }
16657 };
16658 swap.handler
16659 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
16660 .await
16661 .unwrap();
16662 assert!(route_task.await.unwrap().is_empty());
16663 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
16664 match swap
16665 .forwarding
16666 .lookup_data_route(client, channel, epoch)
16667 .unwrap()
16668 {
16669 DataRoute::Client(DataRouteState::Bound(route)) => {
16670 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
16671 }
16672 other => panic!("expected a bound route on the candidate, got {other:?}"),
16673 }
16674 assert!(swap.incumbent_rx.try_recv().is_err());
16675 }
16676
16677 #[tokio::test]
16682 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
16683 let swap = swap_with_incumbent().await;
16684 register_candidate(&swap, Some(false));
16685 let update = Frame::build(
16686 FrameType::Request,
16687 control_flags(),
16688 0,
16689 0,
16690 55,
16691 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16692 provides: manifest("aft", PROTOCOL_VERSION).provides,
16693 capabilities: None,
16694 ready: Some(true),
16695 })
16696 .unwrap(),
16697 )
16698 .unwrap();
16699
16700 let replies = swap
16701 .handler
16702 .handle_control_frame(&swap.candidate_ctx, update)
16703 .await
16704 .unwrap();
16705
16706 assert_eq!(replies.len(), 1);
16707 assert_eq!(
16708 replies[0].header.ty,
16709 FrameType::Response,
16710 "candidate catalog.update was refused: {:?}",
16711 serde_json::from_slice::<Value>(&replies[0].body).ok()
16712 );
16713 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
16714 assert_eq!(
16715 swap.registry
16716 .get_module("aft")
16717 .unwrap()
16718 .unwrap()
16719 .connection_id,
16720 INCUMBENT
16721 );
16722 }
16723 }
16724
16725 mod swap_admission {
16729 use super::*;
16730
16731 const INCUMBENT_NONCE: &str = "incumbent-nonce";
16732 const CANDIDATE_NONCE: &str = "candidate-nonce";
16733
16734 fn handler_with_incumbent(
16735 module_id: &str,
16736 reserved: bool,
16737 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
16738 let registry = Arc::new(Registry::default());
16739 let supervisor = SupervisorHandle::new();
16740 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
16741 if reserved {
16742 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
16743 }
16744 let handler =
16745 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
16746 let incumbent = handler
16747 .handle_control(
16748 ConnectionId::new(1),
16749 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
16750 )
16751 .unwrap();
16752 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
16753 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
16754 (registry, supervisor, handler)
16755 }
16756
16757 #[test]
16763 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
16764 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
16765
16766 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
16767 let replies = handler
16768 .handle_control(
16769 ConnectionId::new(connection),
16770 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
16771 )
16772 .unwrap();
16773 assert_eq!(replies[0].header.ty, FrameType::Error);
16774 assert_eq!(
16775 parse_error(&replies[0])["code"],
16776 "swap_token_invalid",
16777 "nonce {nonce:?}"
16778 );
16779 }
16780 assert!(registry.get_candidate("aft").unwrap().is_none());
16781 assert_eq!(
16782 registry.get_module("aft").unwrap().unwrap().connection_id,
16783 ConnectionId::new(1)
16784 );
16785
16786 let admitted = handler
16789 .handle_control(
16790 ConnectionId::new(4),
16791 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
16792 )
16793 .unwrap();
16794 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
16795 assert_eq!(
16796 registry
16797 .get_candidate("aft")
16798 .unwrap()
16799 .unwrap()
16800 .connection_id,
16801 ConnectionId::new(4)
16802 );
16803 assert_eq!(
16804 registry.get_module("aft").unwrap().unwrap().connection_id,
16805 ConnectionId::new(1),
16806 "the candidate must not take the active slot"
16807 );
16808 let replayed = handler
16809 .handle_control(
16810 ConnectionId::new(5),
16811 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
16812 )
16813 .unwrap();
16814 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
16815
16816 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
16820 let squatter = handler
16821 .handle_control(
16822 ConnectionId::new(6),
16823 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
16824 )
16825 .unwrap();
16826 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
16827 assert!(
16828 registry.get_module("aft").unwrap().is_none(),
16829 "a squatter took the active slot of an id being swapped"
16830 );
16831 }
16832
16833 #[test]
16838 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
16839 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
16840
16841 let replies = handler
16842 .handle_control(
16843 ConnectionId::new(2),
16844 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16845 )
16846 .unwrap();
16847
16848 assert_eq!(
16849 replies[0].header.ty,
16850 FrameType::HelloAck,
16851 "reserved candidate refused: {:?}",
16852 serde_json::from_slice::<Value>(&replies[0].body).ok()
16853 );
16854 assert_eq!(
16855 registry
16856 .get_candidate("vault")
16857 .unwrap()
16858 .unwrap()
16859 .connection_id,
16860 ConnectionId::new(2)
16861 );
16862 }
16863
16864 #[test]
16867 fn without_an_open_swap_the_ordinary_gates_decide() {
16868 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16869 supervisor.close_swap("vault");
16870
16871 let candidate = handler
16872 .handle_control(
16873 ConnectionId::new(2),
16874 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16875 )
16876 .unwrap();
16877 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16878 let duplicate = handler
16879 .handle_control(
16880 ConnectionId::new(3),
16881 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16882 )
16883 .unwrap();
16884 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16885 assert!(registry.get_candidate("vault").unwrap().is_none());
16886 }
16887 }
16888
16889 mod scopes {
16893 mod idle_tests {
16894 use super::*;
16895 include!("scope_idle_tests.rs");
16896 }
16897 mod route_open_timings {
16898 use super::*;
16899 include!("route_open_timing_tests.rs");
16900 }
16901 mod route_open_refusal_log {
16902 use super::*;
16903 include!("route_open_refusal_log_tests.rs");
16904 }
16905 mod activity {
16906 include!("activity_scope_tests.rs");
16907 }
16908 mod delta {
16909 include!("scope_handler_delta_tests.rs");
16910 }
16911 mod call_modes {
16912 include!("scope_call_mode_tests.rs");
16913 }
16914 #[tokio::test]
16915 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16916 let rig = rig().await;
16917 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16918 "s".into(),
16919 1,
16920 "tool".into(),
16921 ))
16922 .unwrap();
16923 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16924 let (direct, _rx) = wide_ctx(999);
16925 for (ctx, expected) in [
16926 (&rig.owner, FrameType::Response),
16927 (&direct, FrameType::Error),
16928 ] {
16929 let frame =
16930 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16931 .unwrap();
16932 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16933 assert_eq!(replies[0].header.ty, expected);
16934 if expected == FrameType::Response {
16935 let reply: subc_control::ScopeServedByReply =
16936 serde_json::from_slice(&replies[0].body).unwrap();
16937 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16938 } else {
16939 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16940 }
16941 }
16942 let reply = call(
16943 &rig.handler,
16944 &rig.owner,
16945 &ModuleControlRequestFromModule::ScopeDescribe {
16946 owner: Principal::Reserved {
16947 module_id: OWNER.into(),
16948 },
16949 scope_ref: "s".into(),
16950 },
16951 )
16952 .await;
16953 assert!(matches!(
16954 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16955 ModuleControlResponseToModule::ScopeDescribe { .. }
16956 ));
16957 }
16958
16959 use subc_protocol::scope::{
16960 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16961 ScopeStatus,
16962 };
16963
16964 use super::*;
16965
16966 const OWNER: &str = "prefrontal-core";
16967
16968 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16969 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16970 }
16971
16972 async fn call(
16973 handler: &ControlHandler,
16974 ctx: &RouteCtx,
16975 request: &ModuleControlRequestFromModule,
16976 ) -> Frame {
16977 let body = serde_json::to_vec(request).unwrap();
16978 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16979 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16980 assert_eq!(replies.len(), 1, "{replies:?}");
16981 replies.pop().unwrap()
16982 }
16983
16984 async fn sync(
16985 handler: &ControlHandler,
16986 ctx: &RouteCtx,
16987 generation: u64,
16988 scopes: Vec<ScopeRecord>,
16989 ) -> Result<ModuleControlResponseToModule, String> {
16990 let reply = call(
16991 handler,
16992 ctx,
16993 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16994 )
16995 .await;
16996 match reply.header.ty {
16997 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16998 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16999 }
17000 }
17001
17002 async fn describe(
17003 handler: &ControlHandler,
17004 ctx: &RouteCtx,
17005 owner: &str,
17006 scope_ref: &str,
17007 ) -> ModuleControlResponseToModule {
17008 let reply = call(
17009 handler,
17010 ctx,
17011 &ModuleControlRequestFromModule::ScopeDescribe {
17012 owner: Principal::Reserved {
17013 module_id: owner.to_string(),
17014 },
17015 scope_ref: scope_ref.to_string(),
17016 },
17017 )
17018 .await;
17019 assert_eq!(
17020 reply.header.ty,
17021 FrameType::Response,
17022 "{:?}",
17023 parse_error(&reply)
17024 );
17025 serde_json::from_slice(&reply.body).unwrap()
17026 }
17027
17028 #[tokio::test]
17029 async fn scope_describe_never_echoes_bind_proof_or_hash() {
17030 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17031 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17032 let mut rig = rig().await;
17033 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17034 .await;
17035 let _bound = rig
17036 .bound(
17037 Some(AFT),
17038 PLEXUS,
17039 Some(
17040 rig.selector("s", Some(1))
17041 .with_bind_proof(Some(PROOF.into())),
17042 ),
17043 )
17044 .await;
17045 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
17046 let output = serde_json::to_string(&reply).unwrap();
17047 assert!(output.contains("\"status\":\"live\""));
17048 assert!(!output.contains(PROOF));
17049 assert!(!output.contains(HASH));
17050 }
17051
17052 async fn module(
17054 handler: &ControlHandler,
17055 connection: u64,
17056 module_id: &str,
17057 nonce: Option<&str>,
17058 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17059 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
17060 hello_via_sink(
17061 handler,
17062 &ctx,
17063 &mut rx,
17064 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
17065 )
17066 .await;
17067 (ctx, rx)
17068 }
17069
17070 #[tokio::test]
17073 async fn a_client_connection_cannot_sync_or_describe() {
17074 let handler = ControlHandler::new(Arc::new(Registry::default()));
17075 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
17076 for request in [
17077 ModuleControlRequestFromModule::ScopeSync {
17078 generation: 1,
17079 scopes: vec![head("s", 1)],
17080 },
17081 ModuleControlRequestFromModule::ScopeDescribe {
17082 owner: Principal::Direct,
17083 scope_ref: "s".to_string(),
17084 },
17085 ] {
17086 let reply = call(&handler, &ctx, &request).await;
17087 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
17088 }
17089 assert!(
17090 !handler
17091 .scopes
17092 .read()
17093 .unwrap()
17094 .describe(
17095 &Principal::Reserved {
17096 module_id: OWNER.to_string()
17097 },
17098 "s"
17099 )
17100 .owner_synced
17101 );
17102 }
17103
17104 #[tokio::test]
17107 async fn a_module_without_a_supervised_launch_cannot_sync() {
17108 let handler = ControlHandler::new(Arc::new(Registry::default()));
17109 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
17110 assert_eq!(
17111 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
17112 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17113 );
17114 }
17115
17116 #[tokio::test]
17117 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
17118 let supervisor = SupervisorHandle::new();
17119 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17120 let handler = ControlHandler::new(Arc::new(Registry::default()))
17121 .with_supervisor(supervisor.clone());
17122 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17123 sync(&handler, &incumbent, 1, vec![head("s", 1)])
17124 .await
17125 .expect("the current launch syncs");
17126
17127 supervisor.open_swap(OWNER, "n2".to_string());
17130 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
17131 assert_eq!(
17132 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17133 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17134 );
17135 sync(&handler, &incumbent, 2, vec![head("s", 1)])
17136 .await
17137 .expect("the serving owner syncs during the swap");
17138
17139 supervisor.close_swap(OWNER);
17142 assert_eq!(
17143 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17144 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17145 );
17146 sync(&handler, &incumbent, 3, vec![head("s", 1)])
17147 .await
17148 .expect("the serving owner syncs after the rollback");
17149 handler.cleanup_connection(candidate.connection_id).unwrap();
17150
17151 supervisor.open_swap(OWNER, "n3".to_string());
17156 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
17157 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
17158 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
17159 .await
17160 .expect("the promoted launch takes authority");
17161 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17162 panic!("unexpected reply {reply:?}");
17163 };
17164 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
17165 assert_eq!(
17166 sync(&handler, &incumbent, 4, Vec::new()).await,
17167 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17168 );
17169 }
17170
17171 #[tokio::test]
17174 async fn closing_the_authority_connection_frees_sync_authority() {
17175 let supervisor = SupervisorHandle::new();
17176 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17177 let handler = ControlHandler::new(Arc::new(Registry::default()))
17178 .with_supervisor(supervisor.clone());
17179 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17180 sync(&handler, &first, 10, vec![head("s", 1)])
17181 .await
17182 .unwrap();
17183 handler.cleanup_connection(first.connection_id).unwrap();
17184
17185 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
17186 sync(&handler, &second, 1, vec![head("s", 1)])
17187 .await
17188 .expect("the next connection takes the released authority");
17189 }
17190
17191 #[tokio::test]
17192 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
17193 let supervisor = SupervisorHandle::new();
17194 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17195 let handler =
17196 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
17197 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
17198 sync(&handler, &first, 10, vec![head("s", 1)])
17199 .await
17200 .unwrap();
17201 handler
17202 .handle_control_frame(
17203 &first,
17204 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
17205 )
17206 .await
17207 .unwrap();
17208 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
17209 sync(&handler, &second, 1, vec![head("s", 1)])
17210 .await
17211 .expect("GOODBYE releases authority even if the old socket remains open");
17212 }
17213
17214 #[tokio::test]
17215 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
17216 let registry = Arc::new(Registry::default());
17217 let supervisor_handle = SupervisorHandle::new();
17218 let supervisor =
17219 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17220 .with_handle(supervisor_handle.clone())
17221 .with_daemon_incarnation("incarnation-7".to_string());
17222 supervisor
17225 .supervise_configured(
17226 ModuleSpec {
17227 module_id: OWNER.to_string(),
17228 program: PathBuf::from("/nonexistent/prefrontal-core"),
17229 args: Vec::new(),
17230 env: Vec::new(),
17231 reserved: false,
17232 reserved_prefixes: Vec::new(),
17233 protocol: ModuleProtocol::Subc,
17234 overlap: Default::default(),
17235 },
17236 false,
17237 )
17238 .unwrap();
17239 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
17240 let handler =
17241 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17242 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
17243
17244 let ModuleControlResponseToModule::ScopeDescribe {
17246 status,
17247 daemon_incarnation,
17248 owner_synced,
17249 owner_configured,
17250 scope,
17251 ..
17252 } = describe(&handler, &reader, OWNER, "s").await
17253 else {
17254 panic!("not a describe reply");
17255 };
17256 assert_eq!(status, ScopeStatus::NotLive);
17257 assert_eq!(daemon_incarnation, "incarnation-7");
17258 assert!(!owner_synced);
17259 assert!(owner_configured);
17260 assert!(scope.is_none());
17261
17262 let ModuleControlResponseToModule::ScopeDescribe {
17265 status,
17266 owner_configured,
17267 ..
17268 } = describe(&handler, &reader, "ghost", "s").await
17269 else {
17270 panic!("not a describe reply");
17271 };
17272 assert_eq!(status, ScopeStatus::NotLive);
17273 assert!(!owner_configured);
17274
17275 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
17277 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
17278 let ModuleControlResponseToModule::ScopeDescribe {
17279 status,
17280 scope_epoch,
17281 owner_synced,
17282 scope,
17283 ..
17284 } = describe(&handler, &reader, OWNER, "s").await
17285 else {
17286 panic!("not a describe reply");
17287 };
17288 assert_eq!(status, ScopeStatus::Live);
17289 assert_eq!(scope_epoch, Some(4));
17290 assert!(owner_synced);
17291 let stamp = scope.expect("a live scope carries its stamp");
17292 assert!(
17293 stamp.owner_authorized,
17294 "prefrontal-core is the default authority"
17295 );
17296 assert_eq!(stamp.kind, ScopeKind::Head);
17297 }
17298
17299 #[tokio::test]
17300 async fn scope_authority_owners_decides_owner_authorized() {
17301 let supervisor = SupervisorHandle::new();
17302 supervisor.set_spawn_nonce("broca", "b1".to_string());
17303 let handler = ControlHandler::new(Arc::new(Registry::default()))
17304 .with_supervisor(supervisor)
17305 .with_scope_authority_owners(vec!["broca".to_string()]);
17306 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
17307 let mut gated = head("s", 1);
17308 gated.attributes.agent_id = Some("agent".to_string());
17309 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
17310 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
17311 describe(&handler, &broca, "broca", "s").await
17312 else {
17313 panic!("not a describe reply");
17314 };
17315 assert!(scope.unwrap().owner_authorized);
17316 }
17317
17318 #[tokio::test]
17322 async fn scope_ops_and_the_scopes_capability_are_advertised() {
17323 let handler = ControlHandler::new(Arc::new(Registry::default()));
17324 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
17325 let ack = hello_via_sink(
17326 &handler,
17327 &ctx,
17328 &mut rx,
17329 hello_frame("m", PROTOCOL_VERSION, 1),
17330 )
17331 .await;
17332 let ack = parse_ack(&ack);
17333 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
17334 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
17335 }
17336 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
17337
17338 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
17339 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
17340 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
17341 let reply = handler
17342 .handle_control_frame(&client, frame)
17343 .await
17344 .unwrap()
17345 .pop()
17346 .unwrap();
17347 let ClientControlResponse::ServerDescribe { capabilities, .. } =
17348 serde_json::from_slice(&reply.body).unwrap()
17349 else {
17350 panic!("not a server.describe reply");
17351 };
17352 assert!(
17353 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
17354 "{capabilities:?}"
17355 );
17356 }
17357
17358 const PLEXUS: &str = "plexus";
17361 const OTHER: &str = "other";
17362 const AFT: &str = "aft";
17363 const BROCA: &str = "broca";
17364 const MAGIC: &str = "magic-context";
17365
17366 fn nonce(module_id: &str) -> String {
17367 format!("nonce-{module_id}")
17368 }
17369
17370 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17371 let (tx, rx) = mpsc::channel(64);
17372 (
17373 RouteCtx {
17374 connection_id: ConnectionId::new(connection),
17375 egress: FrameSink::new(tx),
17376 },
17377 rx,
17378 )
17379 }
17380
17381 struct Rig {
17385 handler: ControlHandler,
17386 forwarding: Arc<ForwardingTable>,
17387 owner: RouteCtx,
17388 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17389 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
17390 generation: u64,
17391 next_connection: u64,
17392 _supervisor: Supervisor,
17393 }
17394
17395 async fn rig() -> Rig {
17396 rig_with_flow_support(true).await
17397 }
17398
17399 async fn rig_with_flow_support(flow_support: bool) -> Rig {
17400 rig_with_scope_capabilities(flow_support, false).await
17401 }
17402
17403 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
17404 let mut provides = Vec::new();
17405 if flow_support {
17406 provides.push("flow-scopes/v1");
17407 }
17408 if run_support {
17409 provides.push("agent-run-scopes/v1");
17410 }
17411 rig_with_target_capabilities(&provides).await
17412 }
17413
17414 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
17415 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
17416 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17417 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17420 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
17421 Frame::build(
17422 FrameType::Hello,
17423 control_flags(),
17424 0,
17425 0,
17426 connection,
17427 serde_json::to_vec(&body).unwrap(),
17428 )
17429 .unwrap()
17430 }
17431
17432 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
17433 let registry = Arc::new(Registry::default());
17434 let forwarding = Arc::new(ForwardingTable::default());
17435 let supervisor_handle = SupervisorHandle::new();
17436 let supervisor =
17437 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17438 .with_handle(supervisor_handle.clone());
17439 supervisor
17440 .supervise_configured(
17441 ModuleSpec {
17442 module_id: OWNER.to_string(),
17443 program: PathBuf::from("/nonexistent/prefrontal-core"),
17444 args: Vec::new(),
17445 env: Vec::new(),
17446 reserved: false,
17447 reserved_prefixes: Vec::new(),
17448 protocol: ModuleProtocol::Subc,
17449 overlap: Default::default(),
17450 },
17451 false,
17452 )
17453 .unwrap();
17454 for module_id in [OWNER, AFT, BROCA, MAGIC] {
17455 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
17456 }
17457 let handler =
17458 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17459 .with_supervisor(supervisor_handle);
17460 let (owner, mut owner_rx) = wide_ctx(1);
17461 hello_via_sink(
17462 &handler,
17463 &owner,
17464 &mut owner_rx,
17465 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
17466 )
17467 .await;
17468 let mut modules = BTreeMap::new();
17469 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
17470 let (ctx, mut rx) = wide_ctx(connection);
17471 let hello = hello_with_capabilities(module_id, connection, provides);
17472 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
17473 modules.insert(module_id.to_string(), (ctx, rx));
17474 }
17475 Rig {
17476 handler,
17477 forwarding,
17478 owner,
17479 _owner_rx: owner_rx,
17480 modules,
17481 generation: 0,
17482 next_connection: 100,
17483 _supervisor: supervisor,
17484 }
17485 }
17486
17487 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
17488 ScopeCarrier::new(Principal::Reserved {
17489 module_id: module_id.to_string(),
17490 })
17491 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
17492 }
17493
17494 fn session(scope_epoch: u64) -> ScopeRecord {
17497 let mut record = head("s", scope_epoch);
17498 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
17499 record.attributes.agent_id = Some("agent-1".to_string());
17500 record.attributes.delegates = true;
17501 record
17502 }
17503
17504 impl Rig {
17505 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
17506 self.generation += 1;
17507 sync(&self.handler, &self.owner, self.generation, scopes)
17508 .await
17509 .expect("the owner's sync is accepted");
17510 }
17511
17512 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17513 ScopeSelector::new(
17514 Principal::Reserved {
17515 module_id: OWNER.to_string(),
17516 },
17517 scope_ref.to_string(),
17518 scope_epoch,
17519 )
17520 }
17521
17522 fn open_frame(
17523 &mut self,
17524 opener: Option<&str>,
17525 target: &str,
17526 scope: Option<ScopeSelector>,
17527 ) -> (
17528 RouteCtx,
17529 mpsc::Receiver<crate::router::OutboundFrame>,
17530 Frame,
17531 ) {
17532 self.next_connection += 1;
17533 let (ctx, rx) = wide_ctx(self.next_connection);
17534 let root = unique_project_root("scoped-open");
17535 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
17536 target: RouteTarget::ToolProvider {
17537 module_id: target.to_string(),
17538 },
17539 identity: BindIdentity::new(
17540 root.path().to_path_buf(),
17541 "unit".to_string(),
17542 "session".to_string(),
17543 ),
17544 consumer_identity: opener.map(|module_id| ConsumerIdentity {
17545 module_id: module_id.to_string(),
17546 launch_nonce: nonce(module_id),
17547 }),
17548 consumer_capabilities: None,
17549 role_versions: None,
17550 admission_facts: None,
17551 scope,
17552 })
17553 .unwrap();
17554 let frame = Frame::build(
17555 FrameType::Request,
17556 control_flags(),
17557 0,
17558 0,
17559 self.next_connection,
17560 body,
17561 )
17562 .unwrap();
17563 (ctx, rx, frame)
17564 }
17565
17566 async fn refused(
17568 &mut self,
17569 opener: Option<&str>,
17570 target: &str,
17571 scope: Option<ScopeSelector>,
17572 ) -> String {
17573 self.refusal_body(opener, target, scope).await["code"]
17574 .as_str()
17575 .unwrap()
17576 .to_string()
17577 }
17578
17579 async fn refusal_body(
17580 &mut self,
17581 opener: Option<&str>,
17582 target: &str,
17583 scope: Option<ScopeSelector>,
17584 ) -> Value {
17585 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
17586 let replies = tokio::time::timeout(
17587 Duration::from_secs(2),
17588 self.handler.handle_control_frame(&ctx, frame),
17589 )
17590 .await
17591 .expect("the open must be refused before waiting for a bind ack")
17592 .unwrap();
17593 assert_eq!(replies.len(), 1, "{replies:?}");
17594 assert_eq!(replies[0].header.ty, FrameType::Error);
17595 let (_, module_rx) = self.modules.get_mut(target).unwrap();
17596 assert!(
17597 module_rx.try_recv().is_err(),
17598 "a refused open relays nothing"
17599 );
17600 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
17601 parse_error(&replies[0])
17602 }
17603
17604 async fn relayed(
17606 &mut self,
17607 opener: Option<&str>,
17608 target: &str,
17609 scope: Option<ScopeSelector>,
17610 ) -> Relayed {
17611 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
17612 let handler = self.handler.clone();
17613 let task_ctx = ctx.clone();
17614 let task = tokio::spawn(async move {
17615 handler
17616 .handle_control_frame(&task_ctx, frame)
17617 .await
17618 .unwrap()
17619 });
17620 let (_, module_rx) = self.modules.get_mut(target).unwrap();
17621 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
17622 .await
17623 .expect("the target receives the relayed route.bind")
17624 .unwrap()
17625 .frame;
17626 Relayed {
17627 target: target.to_string(),
17628 client: ctx,
17629 client_rx: rx,
17630 task,
17631 bind,
17632 }
17633 }
17634
17635 async fn ack(&self, relayed: &Relayed) {
17636 let (module, _) = &self.modules[&relayed.target];
17637 self.handler
17638 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
17639 .await
17640 .unwrap();
17641 }
17642
17643 async fn bound(
17645 &mut self,
17646 opener: Option<&str>,
17647 target: &str,
17648 scope: Option<ScopeSelector>,
17649 ) -> Bound {
17650 let relayed = self.relayed(opener, target, scope).await;
17651 self.ack(&relayed).await;
17652 let Relayed {
17653 target,
17654 client,
17655 mut client_rx,
17656 task,
17657 bind,
17658 } = relayed;
17659 assert!(
17660 task.await.unwrap().is_empty(),
17661 "the open is answered by commit"
17662 );
17663 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
17664 Bound {
17665 target,
17666 client,
17667 client_rx,
17668 channel,
17669 epoch,
17670 bind,
17671 }
17672 }
17673
17674 fn live(&self, route: &Bound) -> bool {
17675 matches!(
17676 self.forwarding
17677 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17678 .unwrap(),
17679 DataRoute::Client(DataRouteState::Bound(_))
17680 )
17681 }
17682 }
17683
17684 struct Relayed {
17685 target: String,
17686 client: RouteCtx,
17687 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17688 task: tokio::task::JoinHandle<Vec<Frame>>,
17689 bind: Frame,
17690 }
17691
17692 struct Bound {
17693 target: String,
17694 client: RouteCtx,
17695 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17696 channel: u16,
17697 epoch: u32,
17698 bind: Frame,
17699 }
17700
17701 impl Bound {
17702 fn closed_reason(&mut self) -> RouteCloseReason {
17705 let mut reason = None;
17706 let mut goodbye = false;
17707 while let Ok(outbound) = self.client_rx.try_recv() {
17708 let frame = outbound.frame;
17709 match frame.header.ty {
17710 FrameType::Goodbye => {
17711 assert_eq!(
17712 (frame.header.channel, frame.header.epoch),
17713 (self.channel, self.epoch)
17714 );
17715 goodbye = true;
17716 }
17717 FrameType::Push => {
17718 let ClientControlPush::RouteClosed {
17719 reason: r,
17720 module_id,
17721 ..
17722 } = serde_json::from_slice(&frame.body).unwrap()
17723 else {
17724 panic!("unexpected push");
17725 };
17726 assert_eq!(module_id, self.target);
17727 reason = Some(r);
17728 }
17729 other => panic!("unexpected frame {other:?}"),
17730 }
17731 }
17732 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
17733 reason.expect("the client is told why the route closed")
17734 }
17735
17736 fn untouched(&mut self) -> bool {
17737 self.client_rx.try_recv().is_err()
17738 }
17739
17740 fn stamp(&self) -> Option<ScopeStamp> {
17741 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
17742 ModuleControlRequest::RouteBind { scope, .. } => scope,
17743 other => panic!("expected a route.bind, got {other:?}"),
17744 }
17745 }
17746 }
17747
17748 #[tokio::test]
17749 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
17750 ) {
17751 let mut rig = rig().await;
17752 let mut record = session(1);
17753 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
17754 record.child_owners = vec![Principal::Reserved {
17755 module_id: MAGIC.to_string(),
17756 }];
17757 rig.sync(vec![record]).await;
17758 let scope = || Some(rig_selector("s", Some(1)));
17759
17760 rig.bound(Some(OWNER), PLEXUS, scope()).await;
17763 rig.bound(Some(AFT), OTHER, scope()).await;
17764 rig.bound(Some(BROCA), PLEXUS, scope()).await;
17765
17766 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
17770 assert_eq!(
17771 rig.refused(opener, target, scope()).await,
17772 error_codes::SCOPE_NOT_CARRIER,
17773 "{opener:?} -> {target}"
17774 );
17775 }
17776 }
17777
17778 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17779 ScopeSelector::new(
17780 Principal::Reserved {
17781 module_id: OWNER.to_string(),
17782 },
17783 scope_ref.to_string(),
17784 scope_epoch,
17785 )
17786 }
17787
17788 #[tokio::test]
17789 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
17790 let mut rig = rig().await;
17791 rig.sync(vec![session(1)]).await;
17792 for opener in [OWNER, AFT] {
17793 assert_eq!(
17794 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
17795 .await,
17796 error_codes::SCOPE_EPOCH_REQUIRED,
17797 "{opener}"
17798 );
17799 }
17800 }
17801
17802 #[tokio::test]
17803 async fn admission_separates_not_synced_not_live_and_ended() {
17804 let mut rig = rig().await;
17805 let code = rig
17807 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17808 .await;
17809 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
17810 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
17811
17812 let ghost = ScopeSelector::new(
17814 Principal::Reserved {
17815 module_id: "ghost".to_string(),
17816 },
17817 "s".to_string(),
17818 Some(1),
17819 );
17820 assert_eq!(
17821 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
17822 error_codes::SCOPE_NOT_LIVE
17823 );
17824
17825 rig.sync(vec![session(2)]).await;
17826 assert_eq!(
17827 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
17828 .await,
17829 error_codes::SCOPE_NOT_LIVE
17830 );
17831 for epoch in [1, 3] {
17832 assert_eq!(
17833 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
17834 .await,
17835 error_codes::SCOPE_ENDED,
17836 "epoch {epoch}"
17837 );
17838 }
17839 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
17841 .await;
17842 }
17843
17844 #[tokio::test]
17845 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
17846 let mut rig = rig().await;
17847 rig.sync(vec![session(1)]).await;
17848 let route = rig
17849 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17850 .await;
17851 let stamp = route.stamp().expect("a scoped bind carries the stamp");
17852 assert_eq!(stamp.scope_ref, "s");
17853 assert_eq!(stamp.scope_epoch, 1);
17854 assert_eq!(stamp.kind, ScopeKind::Head);
17855 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
17856 assert!(stamp.attributes.delegates);
17857 assert!(stamp.owner_authorized);
17858 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17859 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
17860
17861 let (broca, mut broca_rx) = wide_ctx(50);
17864 hello_via_sink(
17865 &rig.handler,
17866 &broca,
17867 &mut broca_rx,
17868 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17869 )
17870 .await;
17871 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17872 .await
17873 .unwrap();
17874 let own = ScopeSelector::new(
17875 Principal::Reserved {
17876 module_id: BROCA.to_string(),
17877 },
17878 "b".to_string(),
17879 Some(1),
17880 );
17881 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17882 assert!(!route.stamp().unwrap().owner_authorized);
17883 }
17884
17885 #[tokio::test]
17886 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17887 let mut rig = rig().await;
17888 let mut record = head("s", 1);
17889 record.carriers = vec![carrier(AFT, None)];
17890 let flow_id = "Flow:run-7/step_2!~";
17891 record.attributes.flow_id = Some(flow_id.to_string());
17892 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17893 .await
17894 .unwrap();
17895 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17896 panic!("not a sync reply");
17897 };
17898 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17899 let route = rig
17900 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17901 .await;
17902 assert!(rig.live(&route), "the stamped bind committed");
17903 let stamp = route.stamp().expect("a flow scope carries a stamp");
17904 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17905 assert_eq!(stamp.attributes.agent_id, None);
17906 assert!(!stamp.attributes.delegates);
17907 assert!(stamp.owner_authorized);
17908 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17909 assert_eq!(unscoped.stamp(), None);
17910 }
17911
17912 #[tokio::test]
17913 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17914 let mut rig = rig_with_flow_support(false).await;
17915 let mut record = session(1);
17916 record.attributes.flow_id = Some("flow:7".to_string());
17917 rig.sync(vec![record]).await;
17918 for opener in [OWNER, AFT] {
17919 let body = rig
17920 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17921 .await;
17922 assert_eq!(body["code"], "target_flow_unsupported");
17923 let message = body["message"].as_str().unwrap();
17924 for required in [PLEXUS, "flow-scopes/v1"] {
17925 assert!(message.contains(required), "{message}");
17926 }
17927 }
17928 }
17929
17930 #[tokio::test]
17931 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17932 let mut rig = rig_with_flow_support(true).await;
17933 let mut record = session(1);
17934 record.attributes.flow_id = Some("flow:7".to_string());
17935 rig.sync(vec![record]).await;
17936 for opener in [OWNER, AFT] {
17937 let route = rig
17938 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17939 .await;
17940 assert!(rig.live(&route));
17941 assert_eq!(
17942 route.stamp().unwrap().attributes.flow_id.as_deref(),
17943 Some("flow:7")
17944 );
17945 }
17946 }
17947
17948 #[tokio::test]
17949 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17950 use std::future::Future;
17951
17952 let mut rig = rig_with_flow_support(true).await;
17953 let mut record = session(1);
17954 record.attributes.flow_id = Some("flow:7".to_string());
17955 rig.sync(vec![record]).await;
17956 let (client, mut client_rx, frame) =
17957 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17958 for _ in 0..64 {
17961 client.egress.try_send(route_bind_ack(1)).unwrap();
17962 }
17963 let handler = rig.handler.clone();
17964 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17965 std::future::poll_fn(|cx| {
17966 assert!(open.as_mut().poll(cx).is_pending());
17967 std::task::Poll::Ready(())
17968 })
17969 .await;
17970 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17971
17972 let old_connection = rig.modules[PLEXUS].0.connection_id;
17973 rig.handler.cleanup_connection(old_connection).unwrap();
17974 let (replacement, mut replacement_rx) = wide_ctx(200);
17975 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17976 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17977 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17978 let hello = Frame::build(
17979 FrameType::Hello,
17980 control_flags(),
17981 0,
17982 0,
17983 200,
17984 serde_json::to_vec(&body).unwrap(),
17985 )
17986 .unwrap();
17987 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17988
17989 client_rx.try_recv().unwrap();
17990 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17991 .await
17992 .expect("the replacement is refused without waiting for a bind ack")
17993 .unwrap();
17994 assert_eq!(replies.len(), 1);
17995 let body = parse_error(&replies[0]);
17996 assert_eq!(body["code"], "target_flow_unsupported");
17997 for required in [PLEXUS, "flow-scopes/v1"] {
17998 assert!(body["message"].as_str().unwrap().contains(required));
17999 }
18000 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
18001 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18002 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18003 assert!(rig
18004 .handler
18005 .registry
18006 .get_module_by_connection(replacement.connection_id)
18007 .unwrap()
18008 .is_some());
18009 }
18010
18011 #[tokio::test]
18012 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
18013 use std::future::Future;
18014 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18015 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18016 const NEXT_HASH: &str =
18017 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18018 let mut rig = rig().await;
18019 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
18020 record.carriers.push(ScopeCarrier::new(Principal::Direct));
18021 rig.sync(vec![record.clone()]).await;
18022 let (client, mut rx, frame) = rig.open_frame(
18023 None,
18024 PLEXUS,
18025 Some(
18026 rig.selector("s", Some(1))
18027 .with_bind_proof(Some(PROOF.into())),
18028 ),
18029 );
18030 for _ in 0..64 {
18033 client.egress.try_send(route_bind_ack(1)).unwrap();
18034 }
18035 let handler = rig.handler.clone();
18036 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18037 std::future::poll_fn(|cx| {
18038 assert!(open.as_mut().poll(cx).is_pending());
18039 std::task::Poll::Ready(())
18040 })
18041 .await;
18042 record.bind_proof_sha256 = Some(NEXT_HASH.into());
18043 rig.sync(vec![record]).await;
18044 rx.try_recv().unwrap();
18045 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18046 .await
18047 .unwrap()
18048 .unwrap();
18049 assert_eq!(
18050 parse_error(&replies[0])["code"],
18051 error_codes::SCOPE_BIND_PROOF_MISMATCH
18052 );
18053 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
18054 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18055 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18056 }
18057
18058 #[tokio::test]
18059 async fn scope_bind_proof_reserved_openers_ignore_proof() {
18060 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18061 let mut rig = rig().await;
18062 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18063 .await;
18064 for opener in [OWNER, AFT] {
18065 for proof in [None, Some("malformed+proof=")] {
18066 let route = rig
18067 .bound(
18068 Some(opener),
18069 PLEXUS,
18070 Some(
18071 rig.selector("s", Some(1))
18072 .with_bind_proof(proof.map(str::to_string)),
18073 ),
18074 )
18075 .await;
18076 assert!(rig.live(&route));
18077 }
18078 }
18079 }
18080
18081 #[tokio::test]
18082 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
18083 use std::future::Future;
18084 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18085 const NEXT_HASH: &str =
18086 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18087 let mut rig = rig().await;
18088 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18089 .await;
18090 let (client, mut rx, frame) =
18091 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
18092 for _ in 0..64 {
18093 client.egress.try_send(route_bind_ack(1)).unwrap();
18094 }
18095 let handler = rig.handler.clone();
18096 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18097 std::future::poll_fn(|cx| {
18098 assert!(open.as_mut().poll(cx).is_pending());
18099 std::task::Poll::Ready(())
18100 })
18101 .await;
18102 rig.sync(vec![
18103 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
18104 ])
18105 .await;
18106 rx.try_recv().unwrap();
18107 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18108 let bind = std::future::poll_fn(|cx| {
18109 assert!(open.as_mut().poll(cx).is_pending());
18110 module_rx.poll_recv(cx)
18111 })
18112 .await
18113 .unwrap()
18114 .frame;
18115 handler
18116 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
18117 .await
18118 .unwrap();
18119 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18120 .await
18121 .unwrap()
18122 .unwrap();
18123 assert_eq!(replies.len(), 1);
18124 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
18127 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18128 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
18129 let route = rig
18130 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
18131 .await;
18132 assert!(rig.live(&route));
18133 }
18134
18135 #[tokio::test]
18136 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
18137 {
18138 let mut rig = rig_with_flow_support(false).await;
18139 rig.sync(vec![session(1)]).await;
18140 let route = rig
18141 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18142 .await;
18143 assert!(rig.live(&route));
18144 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
18145 let mut record = session(1);
18146 record.attributes.flow_id = Some("flow:7".to_string());
18147 rig.sync(vec![record]).await;
18148 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
18149 assert!(rig.live(&unscoped));
18150 assert_eq!(unscoped.stamp(), None);
18151 }
18152
18153 #[tokio::test]
18154 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
18155 let mut rig = rig().await;
18156 let mut record = session(1);
18157 record.attributes.flow_id = Some("flow:7".to_string());
18158 rig.sync(vec![record.clone()]).await;
18159 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18160 let mut owner_route = rig
18161 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18162 .await;
18163 let mut carrier_route = rig
18164 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
18165 .await;
18166 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18167 rig.sync(vec![record.clone()]).await;
18168 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18169 assert!(rig.live(&owner_route) && owner_route.untouched());
18170 assert!(rig.live(&carrier_route) && carrier_route.untouched());
18171
18172 record.attributes.flow_id = Some("flow:8".to_string());
18173 rig.sync(vec![record]).await;
18174 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
18175 let before = before.unwrap();
18176 assert_eq!(after.scope_epoch, before.scope_epoch);
18177 assert!(after.version > before.version);
18178 for route in [&mut owner_route, &mut carrier_route] {
18179 assert!(!rig.live(route));
18180 assert_eq!(
18181 route.closed_reason(),
18182 RouteCloseReason::ScopeDelegationChanged
18183 );
18184 }
18185 assert!(rig.live(&unscoped) && unscoped.untouched());
18186 for target in [PLEXUS, OTHER] {
18189 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
18190 let goodbye = module_rx
18191 .try_recv()
18192 .expect("the provider sees the drain")
18193 .frame;
18194 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18195 assert!(module_rx.try_recv().is_err());
18196 }
18197 let rebound = rig
18198 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18199 .await;
18200 assert_eq!(
18201 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
18202 Some("flow:8")
18203 );
18204 }
18205
18206 #[tokio::test]
18211 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
18212 let mut rig = rig().await;
18213 rig.sync(vec![session(1)]).await;
18214 let mut cotenant = rig
18215 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18216 .await;
18217
18218 let mut changed = session(1);
18219 changed.child_owners.push(Principal::Reserved {
18220 module_id: MAGIC.to_string(),
18221 });
18222 let mut ended = None;
18223 for (code, next) in [
18224 (error_codes::SCOPE_CHANGED, vec![changed]),
18225 (error_codes::SCOPE_ENDED, Vec::new()),
18226 ] {
18227 let relayed = rig
18228 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18229 .await;
18230 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
18231 ended = Some(next.is_empty());
18232 rig.sync(next).await;
18233 rig.ack(&relayed).await;
18234 let replies = relayed.task.await.unwrap();
18235 assert_eq!(replies.len(), 1, "{replies:?}");
18236 assert_eq!(parse_error(&replies[0])["code"], code);
18237 assert_eq!(
18238 subc_protocol::error_codes::is_retryable_route_open(code),
18239 code == error_codes::SCOPE_CHANGED
18240 );
18241 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18242 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18244 let mut goodbyes = Vec::new();
18247 while let Ok(outbound) = plexus_rx.try_recv() {
18248 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
18249 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
18250 }
18251 assert!(
18252 goodbyes.contains(&(bind_channel, bind_epoch)),
18253 "{goodbyes:?}"
18254 );
18255 assert!(rig
18256 .handler
18257 .registry
18258 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
18259 .unwrap()
18260 .is_some());
18261 }
18262 assert_eq!(ended, Some(true));
18263 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
18266 }
18267
18268 #[tokio::test]
18271 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
18272 struct Case {
18273 name: &'static str,
18274 change: fn(&mut ScopeRecord),
18275 closed: [Option<RouteCloseReason>; 4],
18278 }
18279 use RouteCloseReason::*;
18280 let cases = [
18281 Case {
18282 name: "a carrier entry removed",
18283 change: |r| {
18284 r.carriers.retain(|c| {
18285 c.principal
18286 != Principal::Reserved {
18287 module_id: AFT.to_string(),
18288 }
18289 })
18290 },
18291 closed: [None, Some(ScopeCarrierRemoved), None, None],
18292 },
18293 Case {
18294 name: "a target removed from a carrier",
18295 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
18296 closed: [None, None, None, Some(ScopeCarrierRemoved)],
18297 },
18298 Case {
18299 name: "a bare carrier narrowed to targets",
18300 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
18301 closed: [None, Some(ScopeCarrierRemoved), None, None],
18302 },
18303 Case {
18304 name: "delegates turned off",
18305 change: |r| r.attributes.delegates = false,
18306 closed: [Some(ScopeDelegationChanged); 4],
18307 },
18308 Case {
18309 name: "agent_id changed",
18310 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
18311 closed: [Some(ScopeDelegationChanged); 4],
18312 },
18313 Case {
18314 name: "a carrier added, child owners changed, the record re-sent",
18315 change: |r| {
18316 r.carriers.push(carrier(MAGIC, None));
18317 r.child_owners.push(Principal::Reserved {
18318 module_id: MAGIC.to_string(),
18319 });
18320 },
18321 closed: [None; 4],
18322 },
18323 Case {
18324 name: "a target added",
18325 change: |r| {
18326 r.carriers[1]
18327 .targets
18328 .as_mut()
18329 .unwrap()
18330 .push("third".to_string())
18331 },
18332 closed: [None; 4],
18333 },
18334 Case {
18335 name: "delegates turned on",
18336 change: |r| r.attributes.delegates = true,
18337 closed: [None; 4],
18338 },
18339 ];
18340 for case in cases {
18341 let mut rig = rig().await;
18342 rig.sync(vec![session(1)]).await;
18343 let scope = || Some(rig_selector("s", Some(1)));
18344 let mut routes = [
18345 rig.bound(Some(OWNER), PLEXUS, scope()).await,
18346 rig.bound(Some(AFT), PLEXUS, scope()).await,
18347 rig.bound(Some(BROCA), PLEXUS, scope()).await,
18348 rig.bound(Some(BROCA), OTHER, scope()).await,
18349 ];
18350 let mut record = session(1);
18351 (case.change)(&mut record);
18352 rig.sync(vec![record]).await;
18353 for (index, expected) in case.closed.iter().enumerate() {
18354 let route = &mut routes[index];
18355 match expected {
18356 Some(reason) => {
18357 assert!(!rig.live(route), "{}: route {index} still live", case.name);
18358 assert_eq!(
18359 route.closed_reason(),
18360 *reason,
18361 "{}: route {index}",
18362 case.name
18363 );
18364 }
18365 None => {
18366 assert!(rig.live(route), "{}: route {index} closed", case.name);
18367 assert!(
18368 route.untouched(),
18369 "{}: route {index} was told something",
18370 case.name
18371 );
18372 }
18373 }
18374 }
18375 }
18376 }
18377
18378 #[tokio::test]
18379 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
18380 for next in [Vec::new(), vec![session(2)]] {
18382 let mut rig = rig().await;
18383 rig.sync(vec![session(1)]).await;
18384 let mut route = rig
18385 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18386 .await;
18387 rig.sync(next).await;
18388 assert!(!rig.live(&route));
18389 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
18390 }
18391
18392 let mut rig = rig().await;
18395 let mut child = session(1);
18396 child.scope_ref = "child".to_string();
18397 child.kind = ScopeKind::Worker;
18398 child.parent = Some(ScopeParent::new(
18399 Principal::Reserved {
18400 module_id: OWNER.to_string(),
18401 },
18402 "s".to_string(),
18403 1,
18404 ));
18405 rig.sync(vec![session(1), child.clone()]).await;
18406 let mut child_route = rig
18407 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
18408 .await;
18409 assert_eq!(
18410 child_route.stamp().unwrap().parent_state,
18411 Some(ParentState::Linked)
18412 );
18413 rig.sync(vec![child]).await;
18414 assert!(!rig.live(&child_route));
18415 assert_eq!(
18416 child_route.closed_reason(),
18417 RouteCloseReason::ScopeParentEnded
18418 );
18419 }
18420
18421 #[tokio::test]
18422 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
18423 ) {
18424 let mut rig = rig().await;
18425 rig.sync(vec![session(1)]).await;
18426 let mut route = rig
18427 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18428 .await;
18429 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18430 rig.sync(vec![session(1)]).await;
18431 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18432 assert!(rig.live(&route) && route.untouched());
18433
18434 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
18438 .forwarding
18439 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18440 .unwrap()
18441 else {
18442 panic!("the route is bound");
18443 };
18444 binding.flow.acquire_tagged(9, false).await.unwrap();
18445 let mut widened = session(1);
18446 widened.carriers.push(carrier(MAGIC, None));
18447 rig.sync(vec![widened]).await;
18448 assert!(rig.live(&route) && route.untouched());
18449 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18450 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
18451 assert_eq!(binding.flow.in_flight(), 1);
18454 binding
18455 .flow
18456 .acquire_tagged(10, false)
18457 .await
18458 .expect("the flow is still open");
18459 }
18460
18461 #[tokio::test]
18464 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
18465 let mut rig = rig().await;
18466 rig.sync(vec![session(1)]).await;
18467 let mut on_incumbent = rig
18468 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18469 .await;
18470
18471 let (candidate, _candidate_rx) = wide_ctx(9);
18474 let registration = rig
18475 .handler
18476 .registry
18477 .register_candidate_with_control_ops(
18478 manifest(PLEXUS, PROTOCOL_VERSION),
18479 PROTOCOL_VERSION,
18480 candidate.connection_id,
18481 module_baseline_control_ops(),
18482 )
18483 .unwrap();
18484 rig.forwarding
18485 .register_candidate_module_connection(
18486 candidate.connection_id,
18487 PLEXUS.to_string(),
18488 PROTOCOL_VERSION,
18489 manifest_concurrency(®istration.manifest),
18490 candidate.egress.clone(),
18491 )
18492 .unwrap();
18493 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
18494 rig.handler
18495 .registry
18496 .promote_candidate(PLEXUS)
18497 .unwrap()
18498 .unwrap();
18499 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
18500
18501 rig.sync(Vec::new()).await;
18502 assert!(!rig.live(&on_incumbent));
18503 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
18504 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18505 let goodbye = incumbent_rx
18506 .try_recv()
18507 .expect("the superseded endpoint is told")
18508 .frame;
18509 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18510 }
18511 }
18512}
18513
18514#[cfg(test)]
18515mod concurrency_default_exposure_tests {
18516 use super::*;
18517
18518 fn hello_body(role_json: &str) -> Vec<u8> {
18519 format!(
18520 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
18521 )
18522 .into_bytes()
18523 }
18524
18525 fn manifest_from(body: &[u8]) -> ModuleManifest {
18526 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
18527 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
18528 .expect("manifest parses")
18529 }
18530
18531 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
18532
18533 #[test]
18534 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
18535 let body = hello_body(&format!(
18536 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
18537 ));
18538 let manifest = manifest_from(&body);
18539 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18542 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
18543 }
18544
18545 #[test]
18546 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
18547 let body = hello_body(&format!(
18548 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
18549 ));
18550 let manifest = manifest_from(&body);
18551 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18552 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18553 }
18554
18555 #[test]
18556 fn non_management_roles_are_never_reported() {
18557 let body = hello_body(
18558 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
18559 );
18560 let manifest = manifest_from(&body);
18561 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18562 }
18563}