1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_confirm_operations,
25 validate_hello_event_declarations, validate_hello_flow_operations,
26 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
27 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
28 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
29 },
30 peer::{
31 encode_route_bind, ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY,
32 OPERATOR_APP_CAPABILITY,
33 },
34 scope::{
35 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
36 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
37 SCOPE_SYNC_OP,
38 },
39 session::{
40 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
41 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
42 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
43 ROLE_VERSIONS_FIELD,
44 },
45 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
46 Priority, RouteTarget, PROTOCOL_VERSION,
47};
48use tokio::time::{timeout_at, Instant};
49use tracing::{debug, info, warn};
50
51use crate::{
52 caller_identity::CallerIdentity,
53 capability_requirements::{
54 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
55 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
56 RequirementStatus, RuntimeModule,
57 },
58 child_roster::DaemonShutdownFlag,
59 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
60 forwarding::{
61 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
62 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
63 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
64 },
65 observability::{
66 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
67 },
68 privileged_prompt::{prompt_text, PrivilegedOperation},
69 provenance::{
70 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
71 },
72 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
73 router::{daemon_error_flags, RouteCtx, RouterError},
74 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
75 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
76 stderr_tail::{CaptureState, TailEntry},
77 supervise::{
78 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
79 SupervisorHandle, SwapHelloAdmission,
80 },
81 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
82};
83
84pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
90
91const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
92const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
93const CAP_PING_PONG: &str = "ping_pong_v1";
94const CAP_SESSION_ATTACH: &str = "session_attach_v1";
95const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
96
97const SUBC_CONTROL_OPS: &[&str] = &[
98 ops::ACTIVITY_RECORD,
99 ops::ACTIVITY_QUERY,
100 ops::ACTIVITY_SUBSCRIBE,
101 ops::SERVER_DESCRIBE,
102 ops::CATALOG_LIST,
103 ops::ROUTE_OPEN,
104 ops::ROUTE_POLL,
105 ops::ROUTE_CLOSING,
106 ops::ROUTE_CLOSED,
107 ops::SUPERVISOR_LIST,
108 ops::SUPERVISOR_RESTART,
109 ops::SUPERVISOR_SWAP,
110 ops::SUPERVISOR_RELOAD,
111 ops::SUPERVISOR_RESCAN,
112 ops::SUPERVISOR_RELEASE_RESERVED,
113 ops::SUPERVISOR_SET_ENABLED,
114 ops::SUPERVISOR_HEALTH_PROBE,
115 ops::SUPERVISOR_HEALTH,
116 ops::SUPERVISOR_STDERR_TAIL,
117 ops::SUPERVISOR_TERMINALS,
118 ops::SUPERVISOR_ROUTES,
119 ops::SUPERVISOR_PROVENANCE,
120 ops::SUPERVISOR_SPAWN_SNAPSHOT,
121 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
122];
123
124const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
131
132mod rescan_checked;
133use rescan_checked::{RescanMode, RescanPlan};
134
135const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
136 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
137 "supervisor.live_roots",
138 SCOPE_SYNC_OP,
139 SCOPE_APPLY_OP,
140 SCOPE_DESCRIBE_OP,
141 "operator.confirm",
142];
143
144const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
148
149const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
150
151pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
159
160pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
175
176pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
194
195const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
196const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
197
198fn reload_verdict(
199 configured: &Path,
200 spawned_from: Option<&Path>,
201 image: subc_control::RunningImageAgreement,
202) -> PendingReloadVerdict {
203 let path = match spawned_from {
204 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
205 ReloadPathAgreement::Match
206 }
207 Some(spawned_from) => ReloadPathAgreement::Mismatch {
208 configured: configured.to_path_buf(),
209 spawned_from: spawned_from.to_path_buf(),
210 },
211 None => ReloadPathAgreement::Unavailable {
212 reason: if matches!(
213 image,
214 subc_control::RunningImageAgreement::Unavailable {
215 reason: subc_control::RunningImageUnavailableReason::NotRunning
216 }
217 ) {
218 ReloadPathUnavailableReason::NotRunning
219 } else {
220 ReloadPathUnavailableReason::SpawnedPathUnavailable
221 },
222 },
223 };
224 PendingReloadVerdict {
225 path,
226 image,
227 spec: None,
228 }
229}
230
231fn launch_spec_agreement(
232 configured: &crate::supervise::ModuleSpec,
233 running: bool,
234 spawned: Option<&crate::supervise::LaunchSpecDigest>,
235) -> subc_control::LaunchSpecAgreement {
236 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
237 if !running {
238 return Agreement::Unavailable {
239 reason: Reason::NotRunning,
240 };
241 }
242 match spawned {
243 None => Agreement::Unavailable {
244 reason: Reason::SpawnedSpecUnavailable,
245 },
246 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
247 Agreement::Match
248 }
249 Some(_) => Agreement::Mismatch,
250 }
251}
252
253fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
254 if configured == spawned_from {
255 return true;
256 }
257 #[cfg(windows)]
258 {
259 match (
265 std::fs::canonicalize(configured),
266 std::fs::canonicalize(spawned_from),
267 ) {
268 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
269 _ => false,
270 }
271 }
272 #[cfg(not(windows))]
273 {
274 false
275 }
276}
277
278#[derive(Clone)]
279struct DaemonProvenanceFacts {
280 build: DaemonBuildProvenance,
281 pid: Option<u32>,
282 started_at_ms: Option<u64>,
283 start_clock: Option<crate::clock::StartClock>,
284 executable_path: Option<PathBuf>,
285 executable_identity: Option<SpawnedFileIdentity>,
286 process_start_time: Option<u64>,
287 probe: ExecutableIdentityProbe,
288}
289
290impl Default for DaemonProvenanceFacts {
291 fn default() -> Self {
292 Self {
293 build: DaemonBuildProvenance {
294 build_git_sha: None,
295 build_lock_digest: None,
296 },
297 pid: None,
298 started_at_ms: None,
299 start_clock: None,
300 executable_path: None,
301 executable_identity: None,
302 process_start_time: None,
303 probe: ExecutableIdentityProbe::default(),
304 }
305 }
306}
307
308#[derive(Debug, Clone)]
309struct SupervisorRescanContext {
310 supervisor: Supervisor,
311 config_path: PathBuf,
312 configured_port: Option<u16>,
313 storage_config: Option<crate::daemon_config::StorageConfig>,
314 admission_facts_carrier_module_id: Option<String>,
315 admission_facts_targets: Option<Vec<String>>,
316 scope_authority_owners: Vec<String>,
317 privileged_operator_prompt: PrivilegedOperatorPrompt,
318 generation: Arc<std::sync::atomic::AtomicU64>,
324}
325
326const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
347 "reloading",
348 "supervisor_not_live",
349 "registration_not_active",
350 "no_forwarding_connection",
351 "relay_send_failed",
352];
353
354#[derive(Clone)]
356pub struct ControlHandler {
357 registry: Arc<Registry>,
358 activity: Option<crate::activity::ActivityLedger>,
359 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
360 forwarding: Arc<ForwardingTable>,
361 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
362 supervisor: SupervisorHandle,
363 subc_capabilities: Arc<[String]>,
364 route_bind_relay_timeout: Duration,
368 route_bind_relay_timeouts: BTreeMap<String, Duration>,
372 route_bind_breakers: RouteBindBreakers,
375 route_bind_concurrency: RouteBindConcurrency,
378 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
383 route_bind_breaker_threshold: u32,
385 route_bind_breaker_cooldown: Duration,
387 health_probe_timeout: Duration,
388 storage_config: Option<crate::daemon_config::StorageConfig>,
391 machine_id: Option<crate::machine_id::MachineId>,
395 admission_facts_carrier_module_id: Option<String>,
396 admission_facts_targets: Option<Vec<String>>,
397 scopes: Arc<RwLock<ScopeTable>>,
401 scope_deadline_added: Arc<tokio::sync::Notify>,
405 scope_authority_owners: Vec<String>,
408 privileged_operator_prompt: PrivilegedOperatorPrompt,
410 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
413 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
414 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
415 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
416 pub(crate) operator_apps:
417 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
418 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
423 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
429 #[allow(dead_code)]
433 #[cfg(windows)]
434 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
435 #[allow(dead_code)]
436 #[cfg(windows)]
437 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
438 #[allow(dead_code)]
442 #[cfg(windows)]
443 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
444 #[cfg(all(test, windows))]
447 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
448 #[cfg(test)]
449 caller_lookup: Option<TestCallerLookup>,
450 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
453 rescan: Option<SupervisorRescanContext>,
454 connected_clients: ConnectedClients,
455 counters: DaemonCounters,
456 local_endpoint_unavailable: Option<String>,
459 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
460 daemon_shutdown: DaemonShutdownFlag,
465 daemon_provenance: DaemonProvenanceFacts,
466 #[cfg(test)]
467 control_dispatch_delay: Option<Duration>,
468 #[cfg(test)]
469 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
470 #[cfg(test)]
474 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
475 #[cfg(test)]
481 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
482}
483
484impl fmt::Debug for ControlHandler {
485 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
486 f.debug_struct("ControlHandler")
487 .field("registry", &self.registry)
488 .field("forwarding", &self.forwarding)
489 .field("process_liveness", &self.process_liveness.is_some())
490 .field("supervisor", &self.supervisor)
491 .field("subc_capabilities", &self.subc_capabilities)
492 .finish()
493 }
494}
495
496struct RouteOpenRequest {
497 target: RouteTarget,
498 identity: BindIdentity,
499 consumer_identity: Option<ConsumerIdentity>,
500 consumer_capabilities: Option<Vec<String>>,
501 role_versions: Option<BTreeMap<String, String>>,
502 admission_facts: Option<serde_json::Value>,
503 scope: Option<ScopeSelector>,
504}
505
506#[derive(Clone, Copy)]
507enum RouteOpenPhase {
508 Admission,
509 ProjectRoot,
510 EgressReservation,
511 BindSend,
512 BindAckWait,
513}
514
515struct RouteOpenTimings {
519 module_id: String,
520 principal: String,
521 admission: Duration,
522 project_root: Duration,
523 egress_reservation: Duration,
524 bind_send: Duration,
525 bind_ack_wait: Duration,
526 phase: Option<RouteOpenPhase>,
527 phase_started_at: Instant,
528}
529
530impl RouteOpenTimings {
531 fn new() -> Self {
532 Self {
533 module_id: String::new(),
534 principal: "unresolved".to_string(),
536 admission: Duration::ZERO,
537 project_root: Duration::ZERO,
538 egress_reservation: Duration::ZERO,
539 bind_send: Duration::ZERO,
540 bind_ack_wait: Duration::ZERO,
541 phase: Some(RouteOpenPhase::Admission),
542 phase_started_at: Instant::now(),
543 }
544 }
545
546 fn finish(&mut self) {
547 let elapsed = self.phase_started_at.elapsed();
548 match self.phase.take() {
549 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
550 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
551 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
552 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
553 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
554 None => {}
555 }
556 }
557
558 fn enter(&mut self, phase: RouteOpenPhase) {
559 self.finish();
560 self.phase = Some(phase);
561 self.phase_started_at = Instant::now();
562 }
563}
564
565struct RouteBindReservationGuard {
566 forwarding: Arc<ForwardingTable>,
567 endpoint: ModuleEndpointId,
568 relay_corr: u64,
569 armed: bool,
570}
571
572struct ModuleControlRpcGuard {
573 forwarding: Arc<ForwardingTable>,
574 endpoint: ModuleEndpointId,
575 corr: u64,
576 armed: bool,
577}
578
579impl ModuleControlRpcGuard {
580 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
581 Self {
582 forwarding,
583 endpoint,
584 corr,
585 armed: true,
586 }
587 }
588
589 fn disarm(&mut self) {
590 self.armed = false;
591 }
592}
593
594impl Drop for ModuleControlRpcGuard {
595 fn drop(&mut self) {
596 if self.armed {
597 let _ = self
598 .forwarding
599 .cancel_module_control_rpc(self.endpoint, self.corr);
600 }
601 }
602}
603
604impl RouteBindReservationGuard {
605 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
606 Self {
607 forwarding,
608 endpoint,
609 relay_corr,
610 armed: true,
611 }
612 }
613
614 fn release_and_disarm(&mut self) {
615 if !self.armed {
616 return;
617 }
618 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
619 self.endpoint,
620 self.relay_corr,
621 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
622 ) {
623 send_goodbye_target_best_effort(
624 &self.forwarding.counters(),
625 &target,
626 "canceled route.bind",
627 );
628 }
629 self.armed = false;
630 }
631
632 fn disarm(&mut self) {
633 self.armed = false;
634 }
635}
636
637impl Drop for RouteBindReservationGuard {
638 fn drop(&mut self) {
639 self.release_and_disarm();
640 }
641}
642
643#[derive(Debug, Clone, Default)]
673pub(crate) struct RouteBindBreakers {
674 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
675}
676
677#[derive(Debug, Clone, Default)]
678pub(crate) struct RouteBindConcurrency {
679 modules: Arc<Mutex<HashMap<String, usize>>>,
680}
681
682struct RouteBindConcurrencyGuard {
683 concurrency: RouteBindConcurrency,
684 module_id: String,
685}
686
687impl RouteBindConcurrency {
688 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
691 let mut modules = self
692 .modules
693 .lock()
694 .expect("route.bind concurrency mutex poisoned");
695 let in_flight = modules.entry(module_id.to_string()).or_default();
696 if *in_flight >= limit {
697 return Err(*in_flight);
698 }
699 *in_flight += 1;
700 Ok(RouteBindConcurrencyGuard {
701 concurrency: self.clone(),
702 module_id: module_id.to_string(),
703 })
704 }
705}
706
707impl Drop for RouteBindConcurrencyGuard {
708 fn drop(&mut self) {
709 let mut modules = self
710 .concurrency
711 .modules
712 .lock()
713 .expect("route.bind concurrency mutex poisoned");
714 let remove = {
715 let in_flight = modules
716 .get_mut(&self.module_id)
717 .expect("admitted route.bind has a concurrency entry");
718 *in_flight -= 1;
719 *in_flight == 0
720 };
721 if remove {
722 modules.remove(&self.module_id);
723 }
724 }
725}
726
727#[derive(Debug, Default)]
728struct ModuleBreakerState {
729 consecutive_timeouts: u32,
731 cooldown_until: Option<Instant>,
734 probe_in_flight: Option<Arc<()>>,
739}
740
741enum RouteBindAdmission<'a> {
743 Admitted {
744 guard: RouteBindBreakerGuard<'a>,
745 probe: bool,
748 },
749 Refused {
750 consecutive_timeouts: u32,
751 retry_in: Duration,
755 probe_in_flight: bool,
756 },
757}
758
759struct RouteBindBreakerGuard<'a> {
766 breakers: RouteBindBreakers,
767 module_id: &'a str,
768 probe_token: Option<Arc<()>>,
769 settled: bool,
770}
771
772impl RouteBindBreakerGuard<'_> {
773 fn record_accepted(&mut self) -> bool {
777 self.settled = true;
778 self.breakers.record_accepted(self.module_id)
779 }
780
781 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
784 self.settled = true;
785 self.breakers.record_timeout(
786 self.module_id,
787 self.probe_token.as_ref(),
788 threshold,
789 cooldown,
790 )
791 }
792
793 fn record_inconclusive(&mut self) {
802 self.settled = true;
803 self.breakers
804 .record_inconclusive(self.module_id, self.probe_token.as_ref());
805 }
806}
807
808impl Drop for RouteBindBreakerGuard<'_> {
809 fn drop(&mut self) {
810 if !self.settled {
811 self.breakers
812 .record_inconclusive(self.module_id, self.probe_token.as_ref());
813 }
814 }
815}
816
817struct BreakerOpened {
821 consecutive_timeouts: u32,
822 reopened_after_probe: bool,
825}
826
827impl RouteBindBreakers {
828 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
829 self.modules
830 .lock()
831 .expect("route.bind breaker mutex poisoned")
832 }
833
834 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
837 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
838 probe: probe_token.is_some(),
839 guard: RouteBindBreakerGuard {
840 breakers: self.clone(),
841 module_id,
842 probe_token,
843 settled: false,
844 },
845 };
846
847 let mut modules = self.lock();
848 let Some(state) = modules.get_mut(module_id) else {
849 return admitted(None);
850 };
851 let Some(cooldown_until) = state.cooldown_until else {
852 return admitted(None);
853 };
854 if state.probe_in_flight.is_some() {
855 return RouteBindAdmission::Refused {
856 consecutive_timeouts: state.consecutive_timeouts,
857 retry_in: Duration::ZERO,
858 probe_in_flight: true,
859 };
860 }
861 let now = Instant::now();
862 if now < cooldown_until {
863 return RouteBindAdmission::Refused {
864 consecutive_timeouts: state.consecutive_timeouts,
865 retry_in: cooldown_until - now,
866 probe_in_flight: false,
867 };
868 }
869 let token = Arc::new(());
870 state.probe_in_flight = Some(Arc::clone(&token));
871 admitted(Some(token))
872 }
873
874 fn record_accepted(&self, module_id: &str) -> bool {
875 self.lock()
876 .remove(module_id)
877 .is_some_and(|state| state.cooldown_until.is_some())
878 }
879
880 fn record_timeout(
881 &self,
882 module_id: &str,
883 probe_token: Option<&Arc<()>>,
884 threshold: u32,
885 cooldown: Duration,
886 ) -> Option<BreakerOpened> {
887 let mut modules = self.lock();
888 let state = modules.entry(module_id.to_string()).or_default();
889 let was_open = state.cooldown_until.is_some();
890 let was_probe = Self::owns_probe(state, probe_token);
891 if was_probe {
892 state.probe_in_flight = None;
893 }
894 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
895 if state.consecutive_timeouts < threshold {
896 return None;
897 }
898 state.cooldown_until = Some(Instant::now() + cooldown);
899 Some(BreakerOpened {
900 consecutive_timeouts: state.consecutive_timeouts,
901 reopened_after_probe: was_open && was_probe,
902 })
903 }
904
905 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
906 state
910 .probe_in_flight
911 .as_ref()
912 .zip(token)
913 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
914 }
915
916 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
917 if let Some(state) = self.lock().get_mut(module_id) {
918 if Self::owns_probe(state, probe_token) {
919 state.probe_in_flight = None;
920 }
921 }
922 }
923
924 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
939 self.lock()
940 .remove(module_id)
941 .map(|state| state.consecutive_timeouts)
942 .filter(|discarded| *discarded > 0)
943 }
944
945 fn open_snapshot(&self) -> Option<serde_json::Value> {
952 let now = Instant::now();
953 let modules = self.lock();
954 let open = modules
955 .iter()
956 .filter_map(|(module_id, state)| {
957 let cooldown_until = state.cooldown_until?;
958 Some((
959 module_id.clone(),
960 serde_json::json!({
961 "consecutive_timeouts": state.consecutive_timeouts,
962 "cooldown_remaining_ms":
963 cooldown_until.saturating_duration_since(now).as_millis() as u64,
964 "probe_in_flight": state.probe_in_flight.is_some(),
965 }),
966 ))
967 })
968 .collect::<serde_json::Map<String, serde_json::Value>>();
969 (!open.is_empty()).then_some(serde_json::Value::Object(open))
970 }
971}
972
973impl ControlHandler {
974 pub fn new(registry: Arc<Registry>) -> Self {
975 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
976 }
977
978 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
979 let counters = forwarding.counters();
980 let route_bind_breakers = forwarding.route_bind_breakers();
984 let route_bind_concurrency = forwarding.route_bind_concurrency();
985 let route_outages = forwarding.route_outages();
986 Self {
987 registry,
988 activity: None,
989 activity_streams: Arc::new(Mutex::new(HashMap::new())),
990 forwarding,
991 process_liveness: None,
992 supervisor: SupervisorHandle::new(),
993 subc_capabilities: Arc::from([
994 CAP_MANIFEST_REGISTRATION.to_string(),
995 CAP_CHANNEL_LIFECYCLE.to_string(),
996 CAP_PING_PONG.to_string(),
997 CAP_SESSION_ATTACH.to_string(),
998 CAP_ADMISSION_FACTS_RELAY.to_string(),
999 CAP_SCOPES_V1.to_string(),
1000 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1001 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1002 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1003 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1004 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1005 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1006 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1007 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1008 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1009 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1010 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1011 ]),
1012 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1013 route_bind_relay_timeouts: BTreeMap::new(),
1014 route_bind_breakers,
1015 route_bind_concurrency,
1016 route_outages,
1017 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1018 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1019 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1020 storage_config: None,
1021 machine_id: None,
1022 admission_facts_carrier_module_id: None,
1023 admission_facts_targets: None,
1024 scopes: Arc::new(RwLock::new(ScopeTable::new(
1025 crate::daemon_config::default_scope_authority_owners(),
1026 ))),
1027 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1028 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1029 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1030 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1031 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1032 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1033 operator_app_policy: Default::default(),
1034 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1035 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1036 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1037 #[cfg(windows)]
1038 windows_verify: None,
1039 #[cfg(windows)]
1040 windows_flash: None,
1041 #[cfg(windows)]
1042 windows_track: None,
1043 #[cfg(all(test, windows))]
1044 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1045 #[cfg(test)]
1046 caller_lookup: None,
1047 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1048 rescan: None,
1049 connected_clients: ConnectedClients::new(),
1050 local_endpoint_unavailable: None,
1051 counters,
1052 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1053 daemon_shutdown: DaemonShutdownFlag::default(),
1054 daemon_provenance: DaemonProvenanceFacts::default(),
1055 #[cfg(test)]
1056 control_dispatch_delay: None,
1057 #[cfg(test)]
1058 provenance_probe_override: None,
1059 #[cfg(test)]
1060 route_bind_relay_sent_hold: None,
1061 #[cfg(test)]
1062 hello_registry_to_forwarding_hold: None,
1063 }
1064 }
1065
1066 pub fn with_storage_config(
1069 mut self,
1070 storage_config: Option<crate::daemon_config::StorageConfig>,
1071 ) -> Self {
1072 self.storage_config = storage_config;
1073 self
1074 }
1075
1076 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1079 self.activity = Some(crate::activity::ActivityLedger::open(
1080 run_dir.join("activity.jsonl"),
1081 incarnation,
1082 ));
1083 self.supervisor.set_activity(self.activity.clone());
1084 self
1085 }
1086
1087 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1090 self.machine_id = machine_id;
1091 self
1092 }
1093
1094 pub fn with_admission_facts_config(
1098 mut self,
1099 carrier_module_id: Option<String>,
1100 targets: Option<Vec<String>>,
1101 ) -> Self {
1102 self.admission_facts_carrier_module_id = carrier_module_id;
1103 self.admission_facts_targets = targets;
1104 self
1105 }
1106
1107 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1111 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1112 self.scope_authority_owners = owners;
1113 self
1114 }
1115
1116 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1118 self.privileged_operator_prompt = mode;
1119 self
1120 }
1121
1122 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1125 self.route_bind_relay_timeout = timeout;
1126 self
1127 }
1128
1129 pub fn with_route_bind_relay_timeouts(
1135 mut self,
1136 timeouts: impl IntoIterator<Item = (String, Duration)>,
1137 ) -> Self {
1138 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1139 self
1140 }
1141
1142 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1148 self.route_bind_relay_timeouts
1149 .get(module_id)
1150 .copied()
1151 .unwrap_or(self.route_bind_relay_timeout)
1152 }
1153
1154 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1162 self.route_bind_breaker_threshold = threshold.max(1);
1163 self.route_bind_breaker_cooldown = cooldown;
1164 self
1165 }
1166
1167 #[cfg(test)]
1168 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1169 self.health_probe_timeout = timeout;
1170 self
1171 }
1172
1173 #[cfg(test)]
1174 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1175 self.control_dispatch_delay = Some(delay);
1176 self
1177 }
1178
1179 #[cfg(test)]
1183 pub(crate) fn with_route_bind_relay_sent_hold(
1184 mut self,
1185 hold: Arc<tokio::sync::Barrier>,
1186 ) -> Self {
1187 self.route_bind_relay_sent_hold = Some(hold);
1188 self
1189 }
1190
1191 #[cfg(test)]
1196 pub(crate) fn with_hello_registry_to_forwarding_hold(
1197 mut self,
1198 hold: Arc<std::sync::Barrier>,
1199 ) -> Self {
1200 self.hello_registry_to_forwarding_hold = Some(hold);
1201 self
1202 }
1203
1204 #[cfg(test)]
1205 fn hold_between_registry_and_forwarding(&self) {
1206 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1207 hold.wait();
1208 hold.wait();
1209 }
1210 }
1211
1212 pub fn with_process_liveness(
1213 mut self,
1214 process_liveness: Arc<dyn ModuleProcessLiveness>,
1215 ) -> Self {
1216 self.process_liveness = Some(process_liveness);
1217 self
1218 }
1219
1220 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1221 self.supervisor = supervisor;
1222 self.supervisor.set_activity(self.activity.clone());
1223 self
1224 }
1225
1226 pub fn with_daemon_provenance(
1227 mut self,
1228 pid: u32,
1229 started_at_ms: u64,
1230 executable_path: Option<PathBuf>,
1231 build_git_sha: Option<String>,
1232 build_lock_digest: Option<String>,
1233 ) -> Self {
1234 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1235 let process_start_time = process_start_time(pid);
1236 self.daemon_provenance = DaemonProvenanceFacts {
1237 build: DaemonBuildProvenance {
1238 build_git_sha,
1239 build_lock_digest,
1240 },
1241 pid: Some(pid),
1242 started_at_ms: Some(started_at_ms),
1243 start_clock: None,
1244 executable_path,
1245 executable_identity,
1246 process_start_time,
1247 probe: ExecutableIdentityProbe::default(),
1248 };
1249 self
1250 }
1251
1252 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1253 self.daemon_provenance.start_clock = Some(clock);
1254 self
1255 }
1256
1257 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1259 self.scopes
1260 .write()
1261 .unwrap_or_else(|poisoned| poisoned.into_inner())
1262 .set_wall_clock(Arc::new(clock));
1263 self
1264 }
1265
1266 #[cfg(test)]
1267 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1268 self.provenance_probe_override = Some(result);
1269 self
1270 }
1271
1272 pub fn with_capability_config(
1276 self,
1277 modules: impl IntoIterator<Item = (String, bool)>,
1278 reserved_capabilities: BTreeMap<String, String>,
1279 ) -> Self {
1280 self.capability_evaluator
1281 .configure(modules, reserved_capabilities);
1282 self
1283 }
1284
1285 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1289 self.daemon_shutdown = flag;
1290 self
1291 }
1292
1293 pub fn with_supervisor_rescan(
1294 mut self,
1295 supervisor: Supervisor,
1296 config_path: impl Into<PathBuf>,
1297 configured_port: Option<u16>,
1298 ) -> Self {
1299 self.rescan = Some(SupervisorRescanContext {
1300 supervisor,
1301 config_path: config_path.into(),
1302 configured_port,
1303 storage_config: self.storage_config.clone(),
1304 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1305 admission_facts_targets: self.admission_facts_targets.clone(),
1306 scope_authority_owners: self.scope_authority_owners.clone(),
1307 privileged_operator_prompt: self.privileged_operator_prompt,
1308 generation: Arc::default(),
1309 });
1310 self
1311 }
1312
1313 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1314 self.connected_clients = connected_clients;
1315 self
1316 }
1317
1318 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1321 self.local_endpoint_unavailable = reason;
1322 self
1323 }
1324
1325 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1326 Arc::clone(&self.forwarding)
1327 }
1328
1329 pub(crate) fn counters(&self) -> DaemonCounters {
1330 self.counters.clone()
1331 }
1332
1333 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1336 tokio::spawn(async move {
1337 loop {
1338 self.capability_evaluator
1339 .wait_for_change_or_deadline()
1340 .await;
1341 self.refresh_capability_requirements();
1342 }
1343 });
1344 }
1345
1346 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1352 tokio::spawn(async move {
1353 loop {
1354 let has_deadlines = self
1355 .scopes
1356 .read()
1357 .unwrap_or_else(|poisoned| poisoned.into_inner())
1358 .has_deadlines();
1359 if !has_deadlines {
1360 self.scope_deadline_added.notified().await;
1361 continue;
1362 }
1363 tokio::time::sleep(Duration::from_secs(1)).await;
1364 if let Err(error) = self.sweep_expired_scopes() {
1365 warn!(%error, "scope expiry sweep failed");
1366 }
1367 }
1368 });
1369 }
1370
1371 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1372 let mut table = self
1373 .scopes
1374 .write()
1375 .unwrap_or_else(|poisoned| poisoned.into_inner());
1376 let swept = table.sweep_expired();
1377 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1378 drop(table);
1379 self.close_scope_drained_routes(drained);
1380 Ok(())
1381 }
1382
1383 fn publish_scope_changes(
1384 &self,
1385 changes: &[crate::scopes::ScopeTagChange],
1386 expired: &[crate::scopes::ScopeExpired],
1387 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1388 if changes.is_empty() {
1389 return Ok(Vec::new());
1390 }
1391 let drained = self
1392 .forwarding
1393 .publish_scope_changes(changes)
1394 .map_err(RouterError::Forwarding)?;
1395 let mut counts = HashMap::new();
1396 for route in &drained {
1397 *counts
1398 .entry((
1399 route.scope.owner.as_str(),
1400 route.scope.scope_ref.as_str(),
1401 route.scope.tag.scope_epoch,
1402 ))
1403 .or_insert(0usize) += 1;
1404 }
1405 for scope in expired {
1406 let routes_closed = counts
1407 .get(&(
1408 scope.owner.as_str(),
1409 scope.scope_ref.as_str(),
1410 scope.scope_epoch,
1411 ))
1412 .copied()
1413 .unwrap_or(0);
1414 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1415 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1416 routes_closed, "scope expired");
1417 }
1418 Ok(drained)
1419 }
1420
1421 fn runtime_capability_snapshot(
1422 &self,
1423 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1424 let runtime = self
1425 .supervisor
1426 .list()
1427 .into_iter()
1428 .map(|module| {
1429 let status = module.status().map_err(|err| {
1430 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1431 })?;
1432 Ok(RuntimeModule {
1433 module_id: status.module_id,
1434 state: status.state,
1435 enabled: status.enabled,
1436 })
1437 })
1438 .collect::<Result<Vec<_>, RouterError>>()?;
1439 let registrations = self.registry.list_registered_modules().map_err(|err| {
1445 RouterError::backend(
1446 0,
1447 0,
1448 format!("failed to list capability registrations: {err}"),
1449 )
1450 })?;
1451 let registrations = registrations
1452 .into_iter()
1453 .map(|registration| RegisteredModule {
1454 module_id: registration.manifest.module_id,
1455 module_version: registration.manifest.module_version,
1456 capabilities: registration.manifest.capabilities,
1457 })
1458 .collect();
1459 Ok((runtime, registrations))
1460 }
1461
1462 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1469 let cached_registration = RegisteredModule {
1470 module_id: registration.manifest.module_id.clone(),
1471 module_version: registration.manifest.module_version.clone(),
1472 capabilities: registration.manifest.capabilities.clone(),
1473 };
1474 let change = self.capability_evaluator.record_hello(&cached_registration);
1475 log_manifest_change(&cached_registration.module_id, &change);
1476 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1477 self.enforce_capability_denies();
1478 }
1479 self.refresh_capability_requirements();
1480 }
1481
1482 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1486 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1487 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1488 self.supervisor.set_swap_promotion_observer(observer);
1489 }
1490
1491 pub fn refresh_capability_requirements(&self) {
1492 match self.runtime_capability_snapshot() {
1493 Ok((runtime, registrations)) => {
1494 log_requirement_events(
1495 self.capability_evaluator
1496 .evaluate_now(&runtime, ®istrations),
1497 &self.daemon_shutdown,
1498 );
1499 }
1500 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1501 }
1502 }
1503
1504 fn enforce_capability_denies(&self) {
1508 let registrations = match self.registry.list_registered_modules() {
1509 Ok(snapshot) => snapshot,
1510 Err(err) => {
1511 warn!(error = %err, "failed to read registrations for capability deny census");
1512 return;
1513 }
1514 };
1515 let manifests = registrations
1516 .into_iter()
1517 .map(|registration| {
1518 (
1519 registration.manifest.module_id.clone(),
1520 registration.manifest,
1521 )
1522 })
1523 .collect::<BTreeMap<_, _>>();
1524 let census = match self.forwarding.route_census(None) {
1525 Ok(census) => census,
1526 Err(err) => {
1527 warn!(error = %err, "failed to read route census for capability deny enforcement");
1528 return;
1529 }
1530 };
1531
1532 for (target_module_id, routes) in census {
1533 let Some(target_manifest) = manifests.get(&target_module_id) else {
1534 continue;
1535 };
1536 let mut closed_routes = Vec::new();
1537 let mut goodbyes = Vec::new();
1538 for route in routes {
1539 let Principal::Reserved {
1540 module_id: opening_module_id,
1541 } = &route.principal
1542 else {
1543 continue;
1544 };
1545 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1546 continue;
1547 };
1548 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1549 continue;
1550 };
1551
1552 match self.forwarding.release_client_route(
1553 route.goodbye_target.connection_id,
1554 route.goodbye_target.channel,
1555 route.goodbye_target.epoch,
1556 ) {
1557 Ok(RouteRelease::Removed(module_goodbye)) => {
1558 warn!(
1559 opening_module_id,
1560 target_module_id,
1561 capability,
1562 "force-closing route because an attested capability deny edge now matches"
1563 );
1564 goodbyes.push(module_goodbye);
1565 goodbyes.push(route.goodbye_target.clone());
1566 closed_routes.push(route);
1567 }
1568 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1569 Err(err) => warn!(
1570 opening_module_id,
1571 target_module_id,
1572 capability,
1573 error = %err,
1574 "failed to force-close capability-denied route"
1575 ),
1576 }
1577 }
1578
1579 if closed_routes.is_empty() {
1580 continue;
1581 }
1582 send_route_control_pushes(
1583 &self.forwarding,
1584 closed_routes,
1585 ClientControlPush::RouteClosed {
1586 module_id: target_module_id,
1587 channels: Vec::new(),
1588 reason: RouteCloseReason::CapabilityDenied,
1589 drained: false,
1590 abandoned: 0,
1591 excluded_subscriptions: 0,
1592 terminal: Some(false),
1593 },
1594 );
1595 self.emit_route_goodbyes(goodbyes);
1598 }
1599 }
1600
1601 fn not_ready_reason(
1607 &self,
1608 registration: &crate::registry::ModuleRegistration,
1609 ) -> Option<NotReadyReason> {
1610 if !registration.ready {
1611 return Some(NotReadyReason {
1612 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1613 capability: None,
1614 });
1615 }
1616 self.first_unprovided_required_capability(registration)
1617 .map(|capability| NotReadyReason {
1618 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1619 capability: Some(capability),
1620 })
1621 }
1622
1623 fn first_unprovided_required_capability(
1641 &self,
1642 registration: &crate::registry::ModuleRegistration,
1643 ) -> Option<String> {
1644 let required = registration
1645 .manifest
1646 .capabilities
1647 .iter()
1648 .flat_map(|declarations| declarations.requires.iter())
1649 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1650 .map(|requirement| requirement.capability.as_str())
1651 .collect::<BTreeSet<_>>();
1652 if required.is_empty() {
1653 return None;
1654 }
1655 let module_id = registration.manifest.module_id.as_str();
1656 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1657 if required
1658 .iter()
1659 .any(|capability| verdict(capability).is_none())
1660 {
1661 self.refresh_capability_requirements();
1662 }
1663 required
1664 .into_iter()
1665 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1666 .map(str::to_string)
1667 }
1668
1669 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1670 self.capability_evaluator
1671 .statuses()
1672 .into_iter()
1673 .map(capability_requirement_status)
1674 .collect()
1675 }
1676
1677 fn deregister_connection(
1684 &self,
1685 connection_id: ConnectionId,
1686 reason: RegistrationEndReason,
1687 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1688 self.registry
1689 .deregister_connection_with_reason(connection_id, reason)
1690 }
1691
1692 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1693 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1694 return None;
1695 }
1696 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1697 parse_client_control_request(&frame.body)
1698 else {
1699 return None;
1700 };
1701 Some(target_module_id(&target).to_string())
1702 }
1703
1704 pub(crate) fn route_open_capacity_refusal(
1705 &self,
1706 ctx: &RouteCtx,
1707 frame: &Frame,
1708 target_module_id: &str,
1709 in_flight: usize,
1710 limit: usize,
1711 ) -> Result<Frame, RouterError> {
1712 let caller = self.route_open_caller_from_frame(frame);
1715 self.route_open_admission_refusal_frame(
1716 ctx,
1717 &caller,
1718 frame,
1719 target_module_id,
1720 "open_admission_full",
1721 (in_flight, limit),
1722 format!(
1723 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1724 ),
1725 )
1726 }
1727
1728 fn route_open_target_capacity_refusal(
1729 &self,
1730 ctx: &RouteCtx,
1731 caller: &RouteOpenCaller,
1732 frame: &Frame,
1733 target_module_id: &str,
1734 in_flight: usize,
1735 ) -> Result<Frame, RouterError> {
1736 self.route_open_admission_refusal_frame(
1737 ctx,
1738 caller,
1739 frame,
1740 target_module_id,
1741 "target_binds_full",
1742 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1743 format!(
1744 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1745 ),
1746 )
1747 }
1748
1749 #[allow(clippy::too_many_arguments)]
1757 fn route_open_admission_refusal_frame(
1758 &self,
1759 ctx: &RouteCtx,
1760 caller: &RouteOpenCaller,
1761 frame: &Frame,
1762 target_module_id: &str,
1763 reason: &'static str,
1764 (in_flight, limit): (usize, usize),
1765 message: impl Into<String>,
1766 ) -> Result<Frame, RouterError> {
1767 let code = error_codes::TARGET_UNAVAILABLE;
1768 self.counters.increment_route_open_refused(code);
1769 info!(
1770 target: "control",
1771 code,
1772 reason,
1773 module_id = ?target_module_id,
1774 connection_id = ctx.connection_id.get(),
1775 principal = %caller.principal,
1776 scope = %caller.scope,
1777 harness = %caller.harness,
1778 in_flight,
1779 limit,
1780 "route.open refused"
1781 );
1782 control_error_frame(frame, code, message.into())
1783 }
1784
1785 #[cfg(test)]
1791 pub fn handle_control(
1792 &self,
1793 connection_id: ConnectionId,
1794 frame: Frame,
1795 ) -> Result<Vec<Frame>, RouterError> {
1796 match frame.header.ty {
1797 FrameType::Ping => Ok(vec![pong(&frame)?]),
1798 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1799 FrameType::Goodbye => self.handle_goodbye(connection_id),
1800 ty => Ok(vec![control_error_frame(
1801 &frame,
1802 "unsupported_control_frame",
1803 format!("unsupported channel-0 frame {ty:?}"),
1804 )?]),
1805 }
1806 }
1807
1808 pub async fn handle_control_frame(
1809 &self,
1810 ctx: &RouteCtx,
1811 frame: Frame,
1812 ) -> Result<Vec<Frame>, RouterError> {
1813 self.handle_control_frame_timed(ctx, frame, None).await
1814 }
1815
1816 #[cfg(any(test, feature = "test-support"))]
1817 pub fn with_test_operator_app_policy(
1818 mut self,
1819 policy: crate::operator_apps::TestOperatorAppPolicy,
1820 ) -> Self {
1821 self.operator_app_policy = policy.into();
1822 self
1823 }
1824
1825 #[cfg(all(windows, any(test, feature = "test-support")))]
1829 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1830 self.windows_verify = Some(verify);
1831 self
1832 }
1833
1834 #[cfg(all(windows, any(test, feature = "test-support")))]
1836 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1837 self.windows_flash = Some(flash);
1838 self
1839 }
1840
1841 #[cfg(all(windows, any(test, feature = "test-support")))]
1845 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1846 self.windows_track = Some(track);
1847 self
1848 }
1849
1850 pub(crate) fn record_captured_peer(
1851 &self,
1852 connection: ConnectionId,
1853 peer: subc_os::local_peer::CapturedPeer,
1854 ) {
1855 self.record_observed_peer(connection, peer.observation().clone());
1856 #[cfg(windows)]
1857 if let Some(stamp) = self.stamp_windows_launch(&peer) {
1858 self.operator_apps
1859 .lock()
1860 .unwrap_or_else(|p| p.into_inner())
1861 .insert(connection, stamp);
1862 }
1863 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1864 self.operator_apps
1865 .lock()
1866 .unwrap_or_else(|p| p.into_inner())
1867 .insert(connection, stamp);
1868 }
1869 }
1870
1871 pub(crate) fn record_observed_peer(
1872 &self,
1873 connection: ConnectionId,
1874 peer: subc_os::local_peer::ObservedPeer,
1875 ) {
1876 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1877 let peer = match peer {
1878 OsPeer::Unobserved => ObservedPeer::unobserved(),
1879 OsPeer::Observed(peer) => {
1880 let observed = match peer.user {
1881 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1882 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1883 };
1884 observed
1885 .with_exe(
1886 peer.exe
1887 .and_then(|path| path.into_os_string().into_string().ok()),
1888 )
1889 .with_signing_id(peer.signing_id)
1890 }
1891 };
1892 self.observed_peers
1893 .lock()
1894 .unwrap_or_else(|p| p.into_inner())
1895 .insert(connection, peer);
1896 }
1897
1898 pub(crate) fn record_caller_endpoints(
1899 &self,
1900 connection: ConnectionId,
1901 endpoints: Option<(SocketAddr, SocketAddr)>,
1902 ) {
1903 let mut callers = self
1904 .caller_endpoints
1905 .lock()
1906 .unwrap_or_else(|p| p.into_inner());
1907 callers.remove(&connection);
1908 if let Some(endpoints) = endpoints {
1909 callers.insert(connection, endpoints);
1910 }
1911 }
1912
1913 #[cfg(test)]
1914 pub(crate) fn with_caller_lookup_for_test(
1915 mut self,
1916 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1917 ) -> Self {
1918 self.caller_lookup = Some(Arc::new(lookup));
1919 self
1920 }
1921
1922 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1923 let endpoints = self
1924 .caller_endpoints
1925 .lock()
1926 .unwrap_or_else(|p| p.into_inner())
1927 .get(&connection)
1928 .copied();
1929 #[cfg(test)]
1930 let lookup = self.caller_lookup.clone();
1931 tokio::task::spawn_blocking(move || {
1932 #[cfg(test)]
1933 if let Some(lookup) = lookup {
1934 return lookup(endpoints);
1935 }
1936 endpoints
1937 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1938 .unwrap_or_default()
1939 })
1940 .await
1941 .unwrap_or_default()
1942 }
1943
1944 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1945 frame.header.channel == 0
1946 && frame.header.ty == FrameType::Request
1947 && self
1948 .forwarding
1949 .module_endpoint_for_connection(ctx.connection_id)
1950 .is_ok_and(|endpoint| endpoint.is_none())
1951 && parse_client_control_request(&frame.body)
1952 .ok()
1953 .is_some_and(|request| privileged_operation(&request).is_some())
1954 }
1955
1956 pub(crate) async fn handle_control_frame_timed(
1957 &self,
1958 ctx: &RouteCtx,
1959 frame: Frame,
1960 dispatch_started_at: Option<StdInstant>,
1961 ) -> Result<Vec<Frame>, RouterError> {
1962 match frame.header.ty {
1963 FrameType::Ping => Ok(vec![pong(&frame)?]),
1964 FrameType::Hello => {
1965 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1966 }
1967 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1968 FrameType::Cancel => {
1969 if self
1972 .forwarding
1973 .operator_confirms()
1974 .cancel(ctx.connection_id, frame.header.corr)
1975 || self
1976 .supervisor
1977 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1978 || self.cancel_activity_subscription(ctx, &frame)?
1979 {
1980 Ok(Vec::new())
1981 } else {
1982 Ok(vec![control_error_frame(
1983 &frame,
1984 "unknown_subscription",
1985 "no subscription has this correlation id",
1986 )?])
1987 }
1988 }
1989 FrameType::Request => {
1990 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1993 if op
1994 .as_ref()
1995 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1996 {
1997 return self.handle_activity_record(ctx, frame).await;
1998 }
1999 if op.as_ref().is_some_and(|probe| {
2000 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2001 }) {
2002 return self.handle_activity_read(ctx, frame).await;
2003 }
2004 if op
2005 .as_ref()
2006 .is_some_and(|probe| probe.op == "operator.confirm")
2007 {
2008 return self.handle_operator_confirm(ctx, frame);
2009 }
2010 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2011 return self.handle_app_open(ctx, frame).await;
2012 }
2013 if op
2014 .as_ref()
2015 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2016 {
2017 return self.handle_scope_served_by(ctx.connection_id, frame);
2018 }
2019 if self
2020 .forwarding
2021 .module_endpoint_for_connection(ctx.connection_id)
2022 .map_err(RouterError::Forwarding)?
2023 .is_some()
2024 {
2025 if !is_known_module_request_op(&frame.body) {
2026 return Ok(vec![control_error_frame(
2027 &frame,
2028 "unsupported_control_frame",
2029 "module-originated channel-0 REQUEST is not supported",
2030 )?]);
2031 }
2032 let request = match parse_module_control_request_from_module(&frame.body) {
2033 Ok(request) => request,
2034 Err((err, ControlRequestBodyError::UnknownOp)) => {
2035 return Ok(vec![control_error_frame(
2036 &frame,
2037 "unsupported_control_frame",
2038 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2039 )?])
2040 }
2041 Err((err, ControlRequestBodyError::InvalidBody)) => {
2042 return Ok(vec![control_error_frame(
2043 &frame,
2044 "invalid_control_body",
2045 format!("malformed module control body: {err}"),
2046 )?])
2047 }
2048 };
2049 let op = module_control_request_op(&request);
2050 let corr = frame.header.corr;
2051 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2052 let result =
2053 self.handle_module_control_request(ctx.connection_id, frame, request);
2054 log_slow_control_dispatch(
2055 dispatch_started_at,
2056 op,
2057 ctx.connection_id,
2058 corr,
2059 None,
2060 );
2061 return result;
2062 }
2063
2064 if is_known_module_request_op(&frame.body) {
2065 return Ok(vec![control_error_frame(
2066 &frame,
2067 "not_registered",
2068 "catalog.update requires an active module registration owned by this connection",
2069 )?]);
2070 }
2071
2072 let request = match parse_client_control_request(&frame.body) {
2073 Ok(request) => request,
2074 Err((err, ControlRequestBodyError::UnknownOp)) => {
2075 return Ok(vec![control_error_frame(
2076 &frame,
2077 "unknown_control_op",
2078 format!("unknown client control op: {err}"),
2079 )?])
2080 }
2081 Err((err, ControlRequestBodyError::InvalidBody)) => {
2082 return Ok(vec![control_error_frame(
2083 &frame,
2084 "invalid_control_body",
2085 format!("malformed client control body: {err}"),
2086 )?])
2087 }
2088 };
2089 let op = client_control_request_op(&request);
2090 let corr = frame.header.corr;
2091 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2092 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2093 #[cfg(test)]
2094 if let Some(delay) = self.control_dispatch_delay {
2095 tokio::time::sleep(delay).await;
2096 }
2097 let result = self
2098 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2099 .await;
2100 if let Some(timings) = route_open_timings.as_mut() {
2101 timings.finish();
2102 }
2103 log_slow_control_dispatch(
2104 dispatch_started_at,
2105 op,
2106 ctx.connection_id,
2107 corr,
2108 route_open_timings.as_ref(),
2109 );
2110 result
2111 }
2112 FrameType::Push => {
2113 let Some(endpoint) = self
2114 .forwarding
2115 .module_endpoint_for_connection(ctx.connection_id)
2116 .map_err(RouterError::Forwarding)?
2117 else {
2118 return Ok(vec![control_error_frame(
2119 &frame,
2120 "unsupported_control_frame",
2121 "client-originated channel-0 PUSH is not supported",
2122 )?]);
2123 };
2124 self.handle_status_update(endpoint, frame)
2125 }
2126 FrameType::Response | FrameType::Error
2127 if self
2128 .forwarding
2129 .module_endpoint_for_connection(ctx.connection_id)
2130 .map_err(RouterError::Forwarding)?
2131 .is_some() =>
2132 {
2133 self.handle_module_relay_response(ctx.connection_id, frame)
2134 }
2135 ty => Ok(vec![control_error_frame(
2136 &frame,
2137 "unsupported_control_frame",
2138 format!("unsupported channel-0 frame {ty:?}"),
2139 )?]),
2140 }
2141 }
2142
2143 pub fn cleanup_connection(
2144 &self,
2145 connection_id: ConnectionId,
2146 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2147 self.caller_endpoints
2148 .lock()
2149 .unwrap_or_else(|p| p.into_inner())
2150 .remove(&connection_id);
2151 self.build_class_warned
2152 .lock()
2153 .unwrap_or_else(|p| p.into_inner())
2154 .remove(&connection_id);
2155 self.observed_peers
2156 .lock()
2157 .unwrap_or_else(|p| p.into_inner())
2158 .remove(&connection_id);
2159 self.operator_apps
2160 .lock()
2161 .unwrap_or_else(|p| p.into_inner())
2162 .remove(&connection_id);
2163 self.forwarding
2164 .operator_confirms()
2165 .connection_closed(connection_id);
2166 self.cleanup_connection_with_end_reason(
2167 connection_id,
2168 RegistrationEndReason::ConnectionClosed,
2169 )
2170 }
2171
2172 fn cleanup_connection_with_end_reason(
2173 &self,
2174 connection_id: ConnectionId,
2175 requested_reason: RegistrationEndReason,
2176 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2177 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2178 self.registry
2179 .get_module_by_connection(connection_id)?
2180 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2181 .and_then(|module| module.registration_end_reason().ok().flatten())
2182 .unwrap_or(requested_reason)
2183 } else {
2184 requested_reason
2185 };
2186 let crash_closed = self
2187 .registry
2188 .get_module_by_connection(connection_id)?
2189 .and_then(|registration| {
2190 self.forwarding
2191 .module_endpoint_for_connection(connection_id)
2192 .ok()
2193 .flatten()
2194 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2195 .map(|routes| (registration.manifest.module_id, routes))
2196 });
2197 let crash_closed = crash_closed.map(|(module_id, routes)| {
2198 let terminal = match self.supervisor.get(&module_id) {
2199 None => false,
2200 Some(module) => match module.will_recover_after_connection_loss() {
2201 Ok(will_recover) => !will_recover,
2202 Err(err) => {
2203 warn!(
2204 %module_id,
2205 error = %err,
2206 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2207 );
2208 false
2209 }
2210 },
2211 };
2212 let (reason, drained) = match self
2216 .forwarding
2217 .daemon_drain_result_for_connection(connection_id)
2218 {
2219 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2220 Ok(None) => (RouteCloseReason::Crash, false),
2221 Err(err) => {
2222 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2223 (RouteCloseReason::Crash, false)
2224 }
2225 };
2226 (module_id, routes, reason, drained, terminal)
2227 });
2228 let registrations = self.deregister_connection(connection_id, end_reason);
2229 let cleanup = if crash_closed.is_some() {
2230 self.forwarding.cleanup_connection_counted(connection_id)
2231 } else {
2232 self.forwarding
2238 .cleanup_connection(connection_id)
2239 .map(|released| crate::forwarding::ConnectionCleanup {
2240 released,
2241 abandoned_relays: 0,
2242 })
2243 };
2244 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2249 let abandoned = cleanup
2250 .as_ref()
2251 .map_or(0, |cleanup| cleanup.abandoned_relays);
2252 send_route_control_pushes(
2253 &self.forwarding,
2254 routes,
2255 ClientControlPush::RouteClosed {
2256 module_id,
2257 channels: Vec::new(),
2258 reason,
2259 drained,
2260 abandoned,
2261 excluded_subscriptions: 0,
2262 terminal: Some(terminal),
2263 },
2264 );
2265 }
2266 if let Ok(cleanup) = cleanup {
2267 self.emit_route_goodbyes(cleanup.released);
2268 }
2269 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2273 crate::supervise::notify_registration_release();
2274 self.capability_evaluator.wake_deadline_loop();
2275 self.refresh_capability_requirements();
2276 }
2277 self.supervisor.remove_spawn_subscribers(connection_id);
2278 self.activity_streams
2279 .lock()
2280 .unwrap_or_else(|p| p.into_inner())
2281 .retain(|(id, _), task| {
2282 if *id == connection_id {
2283 task.abort();
2284 false
2285 } else {
2286 true
2287 }
2288 });
2289 self.hello_launch_nonces
2292 .lock()
2293 .unwrap_or_else(|poisoned| poisoned.into_inner())
2294 .forget(connection_id);
2295 self.scopes
2296 .write()
2297 .unwrap_or_else(|poisoned| poisoned.into_inner())
2298 .release_connection(connection_id);
2299 registrations
2300 }
2301
2302 pub(crate) fn handle_route_goodbye(
2303 &self,
2304 connection_id: ConnectionId,
2305 route_channel: u16,
2306 route_epoch: u32,
2307 ) -> Result<bool, RouterError> {
2308 debug!(
2309 connection_id = connection_id.get(),
2310 route_channel, route_epoch, "handling route GOODBYE"
2311 );
2312 let RouteRelease::Removed(released_route) = self
2313 .forwarding
2314 .release_client_route(connection_id, route_channel, route_epoch)
2315 .map_err(RouterError::Forwarding)?
2316 else {
2317 return Ok(false);
2318 };
2319 self.emit_route_goodbyes(vec![released_route]);
2320 Ok(true)
2321 }
2322
2323 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2324 for released in released_routes {
2325 let frame = match Frame::build_with_version(
2326 released.negotiated_ver,
2327 FrameType::Goodbye,
2328 control_flags(),
2329 released.channel,
2330 released.epoch,
2331 0,
2332 Vec::new(),
2333 ) {
2334 Ok(frame) => frame,
2335 Err(err) => {
2336 warn!(
2337 route_channel = released.channel,
2338 error = %err,
2339 "failed to build route GOODBYE frame"
2340 );
2341 continue;
2342 }
2343 };
2344 if !released.close_on_delivery_failure() {
2345 crate::forwarding::send_module_route_goodbye(
2346 &self.counters,
2347 &released.sink,
2348 frame,
2349 released.module_id.as_deref(),
2350 "client route released",
2351 );
2352 continue;
2353 }
2354 if let Err(err) = released.sink.try_send(frame) {
2355 warn!(
2356 target_connection_id = released.connection_id.get(),
2357 route_channel = released.channel,
2358 error = %err,
2359 "route GOODBYE was not delivered to client; closing target connection"
2360 );
2361 if self
2362 .forwarding
2363 .escalate_client_delivery_failure(
2364 released.connection_id,
2365 released.channel,
2366 released.epoch,
2367 CloseReason::new(
2368 "route_goodbye_delivery_failed",
2369 format!(
2370 "failed to enqueue route GOODBYE for channel {}: {err}",
2371 released.channel
2372 ),
2373 ),
2374 crate::forwarding::UndeliveredFrame {
2375 module_id: released.module_id.as_deref(),
2376 sink: &released.sink,
2377 },
2378 )
2379 .unwrap_or(false)
2380 {
2381 self.counters.increment_goodbye_relay_client_failed();
2382 }
2383 }
2384 }
2385 }
2386
2387 fn send_abandoned_route_bind_goodbye(
2399 &self,
2400 module_sink: &crate::FrameSink,
2401 negotiated_ver: u8,
2402 module_channel: u16,
2403 module_epoch: u32,
2404 ) {
2405 let frame = match Frame::build_with_version(
2406 negotiated_ver,
2407 FrameType::Goodbye,
2408 control_flags(),
2409 module_channel,
2410 module_epoch,
2411 0,
2412 Vec::new(),
2413 ) {
2414 Ok(frame) => frame,
2415 Err(err) => {
2416 warn!(
2417 route_channel = module_channel,
2418 error = %err,
2419 "failed to build GOODBYE for abandoned route.bind"
2420 );
2421 return;
2422 }
2423 };
2424 crate::forwarding::send_module_route_goodbye(
2425 &self.counters,
2426 module_sink,
2427 frame,
2428 None,
2429 "abandoned route.bind",
2430 );
2431 }
2432
2433 fn handle_hello(
2434 &self,
2435 connection_id: ConnectionId,
2436 sink: Option<crate::FrameSink>,
2437 frame: Frame,
2438 ) -> Result<Vec<Frame>, RouterError> {
2439 debug!(
2440 connection_id = connection_id.get(),
2441 corr = frame.header.corr,
2442 "handling HELLO"
2443 );
2444 if self
2448 .registry
2449 .get_module_by_connection(connection_id)
2450 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2451 .is_some()
2452 {
2453 return Ok(vec![control_error_frame(
2454 &frame,
2455 "invalid_hello",
2456 "connection is already registered as a module",
2457 )?]);
2458 }
2459 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2460 Ok(value) => value,
2461 Err(err) => {
2462 return Ok(vec![control_error_frame(
2463 &frame,
2464 "invalid_hello",
2465 format!("malformed HELLO body: {err}"),
2466 )?])
2467 }
2468 };
2469 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2470 return Ok(vec![control_error_frame(
2471 &frame,
2472 "invalid_capability_grammar",
2473 err.to_string(),
2474 )?]);
2475 }
2476 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2477 return Ok(vec![control_error_frame(
2478 &frame,
2479 "invalid_manifest",
2480 err.to_string(),
2481 )?]);
2482 }
2483 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2484 return Ok(vec![control_error_frame(
2485 &frame,
2486 "invalid_event_declaration",
2487 err.to_string(),
2488 )?]);
2489 }
2490 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2491 return Ok(vec![control_error_frame(
2492 &frame,
2493 "invalid_manifest",
2494 err.to_string(),
2495 )?]);
2496 }
2497 if let Some(provenance) = hello_value
2498 .get("manifest")
2499 .and_then(|manifest| manifest.get("provenance"))
2500 {
2501 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2502 return Ok(vec![control_error_frame(
2503 &frame,
2504 "invalid_manifest",
2505 format!("malformed manifest provenance: {err}"),
2506 )?]);
2507 }
2508 }
2509 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2510 let module_id = hello_value["manifest"]["module_id"]
2511 .as_str()
2512 .unwrap_or("<unknown>");
2513 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2514 }
2515 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2516 let module_id = hello_value["manifest"]["module_id"]
2517 .as_str()
2518 .unwrap_or("<unknown>");
2519 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2520 }
2521 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2522 let module_id = hello_value["manifest"]["module_id"]
2523 .as_str()
2524 .unwrap_or("<unknown>");
2525 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2526 }
2527 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2528 Ok(hello) => hello,
2529 Err(err) => {
2530 return Ok(vec![control_error_frame(
2531 &frame,
2532 "invalid_hello",
2533 format!("malformed HELLO body: {err}"),
2534 )?])
2535 }
2536 };
2537
2538 if hello.protocol_ver != hello.manifest.protocol_ver {
2539 return Ok(vec![control_error_frame(
2540 &frame,
2541 "invalid_manifest",
2542 format!(
2543 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2544 hello.protocol_ver, hello.manifest.protocol_ver
2545 ),
2546 )?]);
2547 }
2548
2549 if hello.manifest.module_id.trim().is_empty() {
2550 return Ok(vec![control_error_frame(
2551 &frame,
2552 "invalid_manifest",
2553 "manifest module_id must not be empty",
2554 )?]);
2555 }
2556
2557 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2558 Ok(negotiated_ver) => negotiated_ver,
2559 Err(message) => {
2560 return Ok(vec![control_error_frame(
2561 &frame,
2562 "version_unsupported",
2563 message,
2564 )?])
2565 }
2566 };
2567
2568 let swap_admission = self
2576 .supervisor
2577 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2578 if swap_admission == SwapHelloAdmission::Refused {
2579 warn!(
2580 module_id = %hello.manifest.module_id,
2581 connection_id = connection_id.get(),
2582 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2583 );
2584 return Ok(vec![control_error_frame(
2585 &frame,
2586 "swap_token_invalid",
2587 format!(
2588 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2589 hello.manifest.module_id
2590 ),
2591 )?]);
2592 }
2593 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2594
2595 let nonce_verified = if swap_candidate {
2600 true
2601 } else {
2602 match self.supervisor.reserved_hello_verification(
2603 &hello.manifest.module_id,
2604 hello.launch_nonce.as_deref(),
2605 ) {
2606 Ok(verified) => verified,
2607 Err(rejection) => {
2608 let message = match rejection {
2609 ReservedHelloRejection::Exact { module_id } => format!(
2610 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2611 ),
2612 ReservedHelloRejection::Prefix {
2613 prefix,
2614 owner_module_id,
2615 } => format!(
2616 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2617 hello.manifest.module_id
2618 ),
2619 };
2620 return Ok(vec![control_error_frame(
2621 &frame,
2622 "reserved_module",
2623 message,
2624 )?]);
2625 }
2626 }
2627 };
2628
2629 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2630 &hello.manifest.module_id,
2631 hello.manifest.capabilities.as_ref(),
2632 );
2633 if let Some(refusal) = reserved_capability_refusals.first() {
2634 let capability = refusal.capability.clone();
2635 let bound_module = refusal.claimants[0].clone();
2636 log_duplicate_claim_events(reserved_capability_refusals);
2637 return Ok(vec![control_error_frame(
2638 &frame,
2639 "reserved_capability",
2640 format!(
2641 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2642 capability, bound_module, hello.manifest.module_id
2643 ),
2644 )?]);
2645 }
2646
2647 if self
2650 .forwarding
2651 .connection_has_client_routes(connection_id)
2652 .map_err(RouterError::Forwarding)?
2653 {
2654 return Ok(vec![control_error_frame(
2655 &frame,
2656 "invalid_hello",
2657 "connection has open client routes and cannot also register as a module",
2658 )?]);
2659 }
2660
2661 self.hello_launch_nonces
2666 .lock()
2667 .unwrap_or_else(|poisoned| poisoned.into_inner())
2668 .record(connection_id, hello.launch_nonce.as_deref());
2669 let control_ops = effective_module_control_ops(hello.control_ops);
2670 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2673 if swap_candidate {
2674 return self.register_swap_candidate(
2675 connection_id,
2676 sink,
2677 &frame,
2678 hello.manifest,
2679 negotiated_ver,
2680 control_ops,
2681 hello_ack,
2682 );
2683 }
2684 let registration = match self
2691 .registry
2692 .register_unpublished_verified_with_control_ops(
2693 hello.manifest,
2694 negotiated_ver,
2695 connection_id,
2696 control_ops,
2697 nonce_verified,
2698 ) {
2699 Ok(registration) => registration,
2700 Err(RegistryError::DuplicateModuleId { module_id }) => {
2701 return Ok(vec![control_error_frame(
2702 &frame,
2703 "duplicate_module_id",
2704 format!(
2705 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2706 ),
2707 )?])
2708 }
2709 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2710 return Ok(vec![control_error_frame(
2711 &frame,
2712 "invalid_module_id",
2713 err.to_string(),
2714 )?])
2715 }
2716 Err(err) => {
2717 return Ok(vec![control_error_frame(
2718 &frame,
2719 "registry_error",
2720 err.to_string(),
2721 )?])
2722 }
2723 };
2724 #[cfg(test)]
2725 self.hold_between_registry_and_forwarding();
2726
2727 let reply = if let Some(sink) = sink {
2728 let concurrency = manifest_concurrency(®istration.manifest);
2744 if let Err(err) = self.forwarding.register_module_connection_acked(
2745 connection_id,
2746 registration.manifest.module_id.clone(),
2747 negotiated_ver,
2748 concurrency,
2749 sink,
2750 hello_ack,
2751 ) {
2752 if matches!(
2757 self.deregister_connection(
2758 connection_id,
2759 RegistrationEndReason::RegistrationFailed,
2760 ),
2761 Ok(r) if !r.is_empty()
2762 ) {
2763 crate::supervise::notify_registration_release();
2764 }
2765 return Ok(vec![control_error_frame(
2766 &frame,
2767 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2768 "invalid_hello"
2769 } else {
2770 forwarding_error_code(&err)
2771 },
2772 err.to_string(),
2773 )?]);
2774 }
2775 Vec::new()
2776 } else {
2777 vec![hello_ack]
2780 };
2781
2782 if !self
2787 .registry
2788 .publish_registration(connection_id)
2789 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2790 {
2791 debug!(
2792 module_id = %registration.manifest.module_id,
2793 connection_id = connection_id.get(),
2794 "registration was removed before it could be published"
2795 );
2796 }
2797
2798 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2807 info!(
2808 module_id = %registration.manifest.module_id,
2809 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2810 );
2811 }
2812
2813 self.apply_registration_capabilities(®istration);
2814
2815 self.supervisor
2816 .record_activity_registration(®istration.manifest.module_id);
2817
2818 info!(
2819 module_id = %registration.manifest.module_id,
2820 module_version = %registration.manifest.module_version,
2821 negotiated_ver,
2822 routable_provider = manifest_provides_routable_role(®istration.manifest),
2823 connection_id = connection_id.get(),
2824 "module registered"
2825 );
2826
2827 Ok(reply)
2828 }
2829
2830 #[allow(clippy::too_many_arguments)]
2847 fn register_swap_candidate(
2848 &self,
2849 connection_id: ConnectionId,
2850 sink: Option<crate::FrameSink>,
2851 frame: &Frame,
2852 manifest: ModuleManifest,
2853 negotiated_ver: u8,
2854 control_ops: Vec<String>,
2855 hello_ack: Frame,
2856 ) -> Result<Vec<Frame>, RouterError> {
2857 let module_id = manifest.module_id.clone();
2858 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2859 manifest,
2860 negotiated_ver,
2861 connection_id,
2862 control_ops,
2863 ) {
2864 Ok(registration) => registration,
2865 Err(RegistryError::DuplicateModuleId { module_id }) => {
2866 return Ok(vec![control_error_frame(
2867 frame,
2868 "duplicate_module_id",
2869 format!(
2870 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2871 ),
2872 )?])
2873 }
2874 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2875 return Ok(vec![control_error_frame(
2876 frame,
2877 "invalid_module_id",
2878 err.to_string(),
2879 )?])
2880 }
2881 Err(err) => {
2882 return Ok(vec![control_error_frame(
2883 frame,
2884 "registry_error",
2885 err.to_string(),
2886 )?])
2887 }
2888 };
2889 #[cfg(test)]
2890 self.hold_between_registry_and_forwarding();
2891 let reply = if let Some(sink) = sink {
2892 let concurrency = manifest_concurrency(®istration.manifest);
2896 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2897 connection_id,
2898 module_id.clone(),
2899 negotiated_ver,
2900 concurrency,
2901 sink,
2902 hello_ack,
2903 ) {
2904 if matches!(
2905 self.deregister_connection(
2906 connection_id,
2907 RegistrationEndReason::RegistrationFailed,
2908 ),
2909 Ok(r) if !r.is_empty()
2910 ) {
2911 crate::supervise::notify_registration_release();
2912 }
2913 return Ok(vec![control_error_frame(
2914 frame,
2915 forwarding_error_code(&err),
2916 err.to_string(),
2917 )?]);
2918 }
2919 Vec::new()
2920 } else {
2921 vec![hello_ack]
2922 };
2923 if !self
2924 .registry
2925 .publish_registration(connection_id)
2926 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2927 {
2928 debug!(
2929 module_id = %module_id,
2930 connection_id = connection_id.get(),
2931 "swap candidate registration was removed before it could be published"
2932 );
2933 }
2934 self.supervisor.mark_swap_candidate_admitted(&module_id);
2935 info!(
2936 module_id = %module_id,
2937 module_version = %registration.manifest.module_version,
2938 negotiated_ver,
2939 ready = registration.ready,
2940 connection_id = connection_id.get(),
2941 "swap candidate registered; not routable until cutover"
2942 );
2943 Ok(reply)
2944 }
2945
2946 fn build_hello_ack(
2947 &self,
2948 frame: &Frame,
2949 negotiated_ver: u8,
2950 module_id: &str,
2951 ) -> Result<Frame, RouterError> {
2952 let ack = ModuleHelloAckBody {
2953 negotiated_ver,
2954 subc_ops: module_subc_ops(),
2955 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2956 storage: self
2957 .storage_config
2958 .as_ref()
2959 .map(|cfg| cfg.descriptor_for(module_id)),
2960 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2961 };
2962 let body = serde_json::to_vec(&ack).map_err(|err| {
2963 RouterError::backend(
2964 0,
2965 frame.header.corr,
2966 format!("failed to encode HELLO_ACK: {err}"),
2967 )
2968 })?;
2969
2970 Frame::build_with_version(
2971 negotiated_ver,
2972 FrameType::HelloAck,
2973 control_flags(),
2974 0,
2975 0,
2976 frame.header.corr,
2977 body,
2978 )
2979 .map_err(RouterError::FrameBuild)
2980 }
2981
2982 async fn handle_client_control_request(
2983 &self,
2984 ctx: &RouteCtx,
2985 frame: Frame,
2986 request: ClientControlRequest,
2987 route_open_timings: &mut Option<RouteOpenTimings>,
2988 ) -> Result<Vec<Frame>, RouterError> {
2989 if let Some(operation) = privileged_operation(&request) {
2990 let caller = self.resolve_caller(ctx.connection_id).await;
2991 let op = client_control_request_op(&request);
2992 let text = prompt_text(operation, &caller);
2993 match self.privileged_operator_prompt {
2994 PrivilegedOperatorPrompt::Log => {
2995 info!(target: "subc_daemon::privileged_prompt", operation = op,
2996 target = ?operation, caller = %caller.description(), summary = %text,
2997 "privileged request would prompt the operator");
2998 }
2999 PrivilegedOperatorPrompt::Enforce => {
3000 let key =
3001 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3002 let outcome = self
3003 .forwarding
3004 .operator_confirms()
3005 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
3006 .await;
3007 if outcome.code != "confirmed" {
3008 return Ok(vec![outcome.frame(&frame)]);
3009 }
3010 }
3011 }
3012 }
3013 match request {
3014 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3015 ClientControlRequest::CatalogList { module_id } => {
3016 self.handle_catalog_list(frame, module_id)
3017 }
3018 ClientControlRequest::RouteOpen {
3019 target,
3020 identity,
3021 consumer_identity,
3022 consumer_capabilities,
3023 role_versions,
3024 admission_facts,
3025 scope,
3026 } => {
3027 self.handle_route_open(
3028 ctx,
3029 frame,
3030 RouteOpenRequest {
3031 target,
3032 identity,
3033 consumer_identity,
3034 consumer_capabilities,
3035 role_versions,
3036 admission_facts,
3037 scope,
3038 },
3039 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3040 )
3041 .await
3042 }
3043 ClientControlRequest::RoutePoll {
3044 route_channel,
3045 route_epoch,
3046 kind,
3047 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3048 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
3049 ClientControlRequest::SupervisorSpawnSnapshot {} => {
3050 self.handle_supervisor_spawn_snapshot(frame)
3051 }
3052 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3053 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3054 }
3055 ClientControlRequest::SupervisorRestart {
3056 module_id,
3057 drain_timeout_ms,
3058 } => {
3059 self.log_supervisor_request_received(
3060 ctx,
3061 frame.header.corr,
3062 ops::SUPERVISOR_RESTART,
3063 Some(&module_id),
3064 None,
3065 )?;
3066 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3067 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3068 .await
3069 }
3070 ClientControlRequest::SupervisorSwap {
3071 module_id,
3072 ready_timeout_ms,
3073 } => {
3074 self.log_supervisor_request_received(
3075 ctx,
3076 frame.header.corr,
3077 ops::SUPERVISOR_SWAP,
3078 Some(&module_id),
3079 None,
3080 )?;
3081 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3082 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3083 .await
3084 }
3085 ClientControlRequest::SupervisorReload { module_id } => {
3086 self.log_supervisor_request_received(
3087 ctx,
3088 frame.header.corr,
3089 ops::SUPERVISOR_RELOAD,
3090 Some(&module_id),
3091 None,
3092 )?;
3093 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3094 self.handle_supervisor_reload(frame, module_id, requester)
3095 .await
3096 }
3097 ClientControlRequest::SupervisorRescan { preview } => {
3098 if !preview {
3099 self.log_supervisor_request_received(
3100 ctx,
3101 frame.header.corr,
3102 ops::SUPERVISOR_RESCAN,
3103 None,
3104 None,
3105 )?;
3106 }
3107 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3108 self.handle_supervisor_rescan(
3109 frame,
3110 if preview {
3111 RescanMode::Preview
3112 } else {
3113 RescanMode::Execute
3114 },
3115 requester,
3116 )
3117 .await
3118 }
3119 ClientControlRequest::SupervisorRescanChecked {
3120 preview,
3121 candidate,
3122 expect,
3123 } => {
3124 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
3125 .await
3126 }
3127 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3128 self.log_supervisor_request_received(
3129 ctx,
3130 frame.header.corr,
3131 ops::SUPERVISOR_RELEASE_RESERVED,
3132 Some(&module_id),
3133 None,
3134 )?;
3135 self.handle_supervisor_release_reserved(frame, module_id)
3136 .await
3137 }
3138 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3139 self.log_supervisor_request_received(
3140 ctx,
3141 frame.header.corr,
3142 ops::SUPERVISOR_SET_ENABLED,
3143 Some(&module_id),
3144 Some(enabled),
3145 )?;
3146 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3147 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3148 .await
3149 }
3150 ClientControlRequest::SupervisorHealthProbe { module_id } => {
3151 self.handle_supervisor_health_probe(frame, module_id).await
3152 }
3153 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3154 ClientControlRequest::SupervisorRoutes { module_id } => {
3155 self.handle_supervisor_routes(frame, module_id)
3156 }
3157 ClientControlRequest::SupervisorProvenance { module_id } => {
3158 self.handle_supervisor_provenance(frame, module_id).await
3159 }
3160 ClientControlRequest::SupervisorStderrTail {
3161 module_id,
3162 max_lines,
3163 max_bytes,
3164 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3165 ClientControlRequest::SupervisorTerminals { module_id } => {
3166 self.handle_supervisor_terminals(frame, module_id).await
3167 }
3168 }
3169 }
3170
3171 fn log_supervisor_request_received(
3172 &self,
3173 ctx: &RouteCtx,
3174 corr: u64,
3175 op: &'static str,
3176 module_id: Option<&str>,
3177 enabled: Option<bool>,
3178 ) -> Result<(), RouterError> {
3179 let caller = self.supervisor_requester(ctx, corr)?;
3180 let caller = principal_label(&caller);
3181 info!(
3184 target: "control",
3185 op,
3186 module_id,
3187 enabled,
3188 connection_id = ctx.connection_id.get(),
3189 caller = %caller,
3190 "supervisor request received"
3191 );
3192 Ok(())
3193 }
3194
3195 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3196 Ok(self
3197 .registry
3198 .get_module_by_connection(ctx.connection_id)
3199 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3200 .map(|registration| Principal::Reserved {
3201 module_id: registration.manifest.module_id,
3202 })
3203 .unwrap_or(Principal::Direct))
3204 }
3205
3206 fn handle_module_control_request(
3207 &self,
3208 connection_id: ConnectionId,
3209 frame: Frame,
3210 request: ModuleControlRequestFromModule,
3211 ) -> Result<Vec<Frame>, RouterError> {
3212 match request {
3213 ModuleControlRequestFromModule::CatalogUpdate {
3214 provides,
3215 capabilities,
3216 ready,
3217 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3218 ModuleControlRequestFromModule::LiveRoots {} => {
3219 let registered = self
3220 .registry
3221 .get_module_by_connection(connection_id)
3222 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3223 let Some(registration) = registered else {
3224 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3225 };
3226 let response = self
3227 .forwarding
3228 .live_roots(®istration.manifest.module_id)
3229 .map_err(RouterError::Forwarding)?;
3230 Ok(vec![control_response_body_frame(
3231 &frame,
3232 &response,
3233 "ModuleControlResponseToModule::LiveRoots",
3234 )?])
3235 }
3236 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3237 self.handle_scope_sync(connection_id, frame, generation, scopes)
3238 }
3239 ModuleControlRequestFromModule::ScopeApply {
3240 generation,
3241 upsert,
3242 end,
3243 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3244 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3245 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3246 }
3247 }
3248 }
3249
3250 fn handle_scope_served_by(
3251 &self,
3252 connection_id: ConnectionId,
3253 frame: Frame,
3254 ) -> Result<Vec<Frame>, RouterError> {
3255 let registered = self
3256 .registry
3257 .get_module_by_connection(connection_id)
3258 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3259 let Some(registered) = registered else {
3260 return Ok(vec![control_error_frame(
3261 &frame,
3262 "not_registered",
3263 "scope.served_by requires an active module registration owned by this connection",
3264 )?]);
3265 };
3266 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3267 {
3268 Ok(request) => request,
3269 Err(err) => {
3270 return Ok(vec![control_error_frame(
3271 &frame,
3272 "invalid_control_body",
3273 format!("malformed scope.served_by body: {err}"),
3274 )?])
3275 }
3276 };
3277 let answer = self
3278 .scopes
3279 .read()
3280 .unwrap_or_else(|poisoned| poisoned.into_inner())
3281 .served_by(
3282 ®istered.manifest.module_id,
3283 &request.scope_ref,
3284 request.scope_epoch,
3285 &request.module_id,
3286 );
3287 let reply = subc_control::ScopeServedByReply::new(
3288 answer,
3289 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3290 );
3291 Ok(vec![control_response_body_frame(
3292 &frame,
3293 &reply,
3294 "ScopeServedByReply",
3295 )?])
3296 }
3297
3298 fn handle_operator_confirm(
3299 &self,
3300 ctx: &RouteCtx,
3301 frame: Frame,
3302 ) -> Result<Vec<Frame>, RouterError> {
3303 use crate::operator_confirm::{audit, Outcome};
3304 let registration = self
3305 .registry
3306 .get_module_by_connection(ctx.connection_id)
3307 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3308 let Some(registration) = registration else {
3309 let outcome = Outcome::refusal("not_registered");
3310 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3311 return Ok(vec![outcome.frame(&frame)]);
3312 };
3313 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3314 Ok(request) => request,
3315 Err(_) => {
3316 let outcome = Outcome::refusal("invalid_control_body");
3317 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3318 return Ok(vec![outcome.frame(&frame)]);
3319 }
3320 };
3321 let module_id = registration.manifest.module_id;
3322 let nonce = self
3326 .hello_launch_nonces
3327 .lock()
3328 .unwrap_or_else(|p| p.into_inner())
3329 .nonce(ctx.connection_id)
3330 .map(str::to_owned);
3331 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3332 self.supervisor
3333 .spawned_consumer_authorized(&module_id, nonce)
3334 });
3335 let confirms = self.forwarding.operator_confirms();
3336 self.forwarding
3337 .with_operator_route(
3338 ctx.connection_id,
3339 request.route_channel,
3340 request.route_epoch,
3341 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3342 )
3343 .map_err(RouterError::Forwarding)
3344 }
3345
3346 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3349 let mut scopes = ScopeRecords::from(scopes);
3350 for (index, record) in scopes.records.iter_mut().enumerate() {
3351 if let Some(refusal) = Self::canonicalize_workdir(record) {
3352 scopes.refusals.insert(index, refusal);
3353 }
3354 }
3355 scopes
3356 }
3357
3358 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3359 let workdir = record.workdir.as_mut()?;
3360 let path = std::path::Path::new(workdir);
3361 if !path.is_absolute() {
3362 return Some((
3363 error_codes::INVALID_WORKDIR,
3364 "workdir must be absolute".to_string(),
3365 ));
3366 }
3367 match ProjectRootId::from_path_allowing_missing(path) {
3368 Ok(root) => match root.as_path().to_str() {
3369 Some(canonical) => {
3370 *workdir = canonical.to_string();
3371 None
3372 }
3373 None => Some((
3374 error_codes::INVALID_WORKDIR,
3375 "canonical workdir is not UTF-8".to_string(),
3376 )),
3377 },
3378 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3379 }
3380 }
3381
3382 fn handle_scope_sync(
3386 &self,
3387 connection_id: ConnectionId,
3388 frame: Frame,
3389 generation: u64,
3390 scopes: Vec<ScopeRecord>,
3391 ) -> Result<Vec<Frame>, RouterError> {
3392 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3393 }
3394
3395 fn handle_scope_change(
3396 &self,
3397 connection_id: ConnectionId,
3398 frame: Frame,
3399 generation: u64,
3400 scopes: Vec<ScopeRecord>,
3401 end: Option<Vec<ScopeEnd>>,
3402 ) -> Result<Vec<Frame>, RouterError> {
3403 let is_apply = end.is_some();
3404 let Some(registration) = self
3405 .registry
3406 .get_module_by_connection(connection_id)
3407 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3408 else {
3409 return Ok(vec![control_error_frame(
3410 &frame,
3411 "not_registered",
3412 "scope.sync and scope.apply require an active module registration owned by this connection",
3413 )?]);
3414 };
3415 let owner = registration.manifest.module_id;
3416 let scopes = Self::prepare_scope_records(scopes);
3417 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3418 let is_current_launch = |connection: ConnectionId| {
3419 self.hello_launch_nonces
3420 .lock()
3421 .unwrap_or_else(|poisoned| poisoned.into_inner())
3422 .presented(connection, current_nonce.as_deref())
3423 };
3424 let mut table = self
3429 .scopes
3430 .write()
3431 .unwrap_or_else(|poisoned| poisoned.into_inner());
3432 let outcome = match end {
3433 Some(end) => table.apply(
3434 &owner,
3435 connection_id,
3436 is_current_launch,
3437 generation,
3438 scopes,
3439 end,
3440 ),
3441 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3442 };
3443 let drained = match &outcome {
3444 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3445 Err(_) => Vec::new(),
3446 };
3447 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3448 drop(table);
3449 if has_deadlines {
3450 self.scope_deadline_added.notify_one();
3451 }
3452 match outcome {
3453 Ok(applied) => {
3454 let counts = ScopeOutcomeCounts::of(&applied.results);
3455 info!(
3456 owner = %owner,
3457 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3458 generation,
3459 records = applied.results.len(),
3460 created = counts.created,
3461 replaced = counts.replaced,
3462 updated = counts.updated,
3463 unchanged = counts.unchanged,
3464 refused = counts.refused,
3465 ended = applied.ended.len(),
3466 tag_changes = applied.tag_changes.len(),
3467 routes_closed = drained.len(),
3468 "scope change accepted"
3469 );
3470 for refused in applied
3476 .results
3477 .iter()
3478 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3479 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3480 {
3481 warn!(
3482 owner = %owner,
3483 generation,
3484 scope_ref = %refused.scope_ref,
3485 scope_epoch = refused.scope_epoch,
3486 code = refused.code.as_deref().unwrap_or(""),
3487 "scope record refused"
3488 );
3489 }
3490 self.close_scope_drained_routes(drained);
3491 let response = if is_apply {
3492 ModuleControlResponseToModule::ScopeApply {
3493 generation,
3494 results: applied.results,
3495 end_results: applied.end_results,
3496 ended: applied.ended,
3497 }
3498 } else {
3499 ModuleControlResponseToModule::ScopeSync {
3500 generation,
3501 results: applied.results,
3502 ended: applied.ended,
3503 }
3504 };
3505 Ok(vec![control_response_body_frame(
3506 &frame,
3507 &response,
3508 "scope change response",
3509 )?])
3510 }
3511 Err(refusal) => {
3512 info!(
3513 owner = %owner,
3514 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3515 generation,
3516 code = refusal.code,
3517 "scope change refused"
3518 );
3519 Ok(vec![control_error_frame(
3520 &frame,
3521 refusal.code,
3522 refusal.message,
3523 )?])
3524 }
3525 }
3526 }
3527
3528 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3534 if drained.is_empty() {
3535 return;
3536 }
3537 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3538 BTreeMap::new();
3539 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3540 for route in drained {
3541 warn!(
3542 module_id = %route.module_id,
3543 reason = ?route.reason,
3544 client_connection_id = route.client.connection_id.get(),
3545 route_channel = route.client.channel,
3546 "closing route because its scope changed"
3547 );
3548 pushes
3549 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3550 .or_insert_with(|| (route.reason, Vec::new()))
3551 .1
3552 .push(EndpointRoute {
3553 goodbye_target: route.client.clone(),
3554 principal: Principal::Unverified,
3555 bound_at: Instant::now(),
3556 draining: false,
3557 drain_reason: None,
3558 });
3559 goodbyes.push(route.module);
3560 goodbyes.push(route.client);
3561 }
3562 for ((module_id, _), (reason, routes)) in pushes {
3563 send_route_control_pushes(
3564 &self.forwarding,
3565 routes,
3566 ClientControlPush::RouteClosed {
3567 module_id,
3568 channels: Vec::new(),
3569 reason,
3570 drained: false,
3571 abandoned: 0,
3572 excluded_subscriptions: 0,
3573 terminal: Some(false),
3574 },
3575 );
3576 }
3577 self.emit_route_goodbyes(goodbyes);
3578 }
3579
3580 fn handle_scope_describe(
3583 &self,
3584 connection_id: ConnectionId,
3585 frame: Frame,
3586 owner: Principal,
3587 scope_ref: String,
3588 ) -> Result<Vec<Frame>, RouterError> {
3589 let registered = self
3590 .registry
3591 .get_module_by_connection(connection_id)
3592 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3593 let Some(registered) = registered else {
3594 return Ok(vec![control_error_frame(
3595 &frame,
3596 "not_registered",
3597 "scope.describe requires an active module registration owned by this connection",
3598 )?]);
3599 };
3600 let mut description = self
3601 .scopes
3602 .read()
3603 .unwrap_or_else(|poisoned| poisoned.into_inner())
3604 .describe(&owner, &scope_ref);
3605 if !registered
3608 .manifest
3609 .capabilities
3610 .as_ref()
3611 .is_some_and(|caps| {
3612 caps.provides
3613 .iter()
3614 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3615 })
3616 {
3617 if let Some(stamp) = description.stamp.as_mut() {
3618 stamp.attributes.on_behalf_of_agent = None;
3619 }
3620 }
3621 if !registered
3622 .manifest
3623 .capabilities
3624 .as_ref()
3625 .is_some_and(|caps| {
3626 caps.provides
3627 .iter()
3628 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3629 })
3630 {
3631 if let Some(stamp) = description.stamp.as_mut() {
3632 stamp.attributes.flow_version = None;
3633 stamp.attributes.flow_code_hash = None;
3634 stamp.attributes.flow_install_id = None;
3635 }
3636 }
3637 let owner_configured = match &owner {
3638 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3643 _ => false,
3644 };
3645 let response = ModuleControlResponseToModule::ScopeDescribe {
3646 status: description.status,
3647 scope_epoch: description.scope_epoch,
3648 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3649 owner_synced: description.owner_synced,
3650 owner_configured,
3651 scope: description.stamp,
3652 };
3653 Ok(vec![control_response_body_frame(
3654 &frame,
3655 &response,
3656 "ModuleControlResponseToModule::ScopeDescribe",
3657 )?])
3658 }
3659
3660 fn handle_catalog_update(
3661 &self,
3662 connection_id: ConnectionId,
3663 frame: Frame,
3664 provides: Vec<ProviderRole>,
3665 capabilities: Option<CapabilityDeclarations>,
3666 ready: Option<bool>,
3667 ) -> Result<Vec<Frame>, RouterError> {
3668 self.refresh_capability_requirements();
3669 let Some(registration) = self
3670 .registry
3671 .get_module_by_connection(connection_id)
3672 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3673 else {
3674 return Ok(vec![control_error_frame(
3675 &frame,
3676 "not_registered",
3677 "catalog.update requires an active module registration owned by this connection",
3678 )?]);
3679 };
3680
3681 if let Some(message) =
3682 catalog_update_frozen_field_message(®istration.manifest, &provides)
3683 {
3684 return Ok(vec![control_error_frame(
3685 &frame,
3686 "catalog_update_frozen_field",
3687 message,
3688 )?]);
3689 }
3690
3691 let mut candidate = registration.manifest.clone();
3692 candidate.provides = provides.clone();
3693 candidate.capabilities = capabilities
3694 .clone()
3695 .or_else(|| registration.manifest.capabilities.clone());
3696 if let Some(rich_ui) = &candidate.rich_ui {
3697 if let Err(err) = rich_ui.validate() {
3698 return Ok(vec![control_error_frame(
3699 &frame,
3700 "invalid_manifest",
3701 err.to_string(),
3702 )?]);
3703 }
3704 }
3705 if let Err(err) = candidate.validate_capability_grammar() {
3706 return Ok(vec![control_error_frame(
3707 &frame,
3708 "invalid_capability_grammar",
3709 err.to_string(),
3710 )?]);
3711 }
3712
3713 let mut conflicts = self
3716 .capability_evaluator
3717 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3718 if let Some(conflict) = conflicts.first() {
3719 let message = format!(
3720 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3721 conflict.capability, conflict.claimants[0], candidate.module_id
3722 );
3723 for conflict in &mut conflicts {
3724 conflict.source = DuplicateClaimSource::CatalogUpdate;
3725 }
3726 log_duplicate_claim_events(conflicts);
3727 return Ok(vec![control_error_frame(
3728 &frame,
3729 "reserved_capability",
3730 message,
3731 )?]);
3732 }
3733
3734 let updated = self
3735 .registry
3736 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3737 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3738 if updated.is_none() {
3739 return Ok(vec![control_error_frame(
3740 &frame,
3741 "not_registered",
3742 "catalog.update requires an active module registration owned by this connection",
3743 )?]);
3744 }
3745 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3746 log_duplicate_claim_events(
3747 self.capability_evaluator
3748 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3749 );
3750 }
3751 if capability_census_trigger(
3752 registration.manifest.capabilities.as_ref(),
3753 updated
3754 .as_ref()
3755 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3756 ) {
3757 self.enforce_capability_denies();
3758 }
3759 self.refresh_capability_requirements();
3760
3761 let response = ModuleControlResponseToModule::CatalogUpdate {};
3762 control_response_body_frame(
3763 &frame,
3764 &response,
3765 "ModuleControlResponseToModule::CatalogUpdate",
3766 )
3767 .map(|frame| vec![frame])
3768 }
3769
3770 pub(crate) fn with_activity_ledger_config(
3771 mut self,
3772 run_dir: &Path,
3773 incarnation: String,
3774 config: crate::activity::ActivityConfig,
3775 ) -> Self {
3776 self.activity = Some(crate::activity::ActivityLedger::open_config(
3777 run_dir.join("activity.jsonl"),
3778 incarnation,
3779 config,
3780 ));
3781 self.supervisor.set_activity(self.activity.clone());
3782 self
3783 }
3784
3785 pub(crate) async fn configure_activity(
3786 &self,
3787 config: crate::activity::ActivityConfig,
3788 ) -> Result<(), &'static str> {
3789 match &self.activity {
3790 Some(ledger) => ledger.configure(config).await,
3791 None => Ok(()),
3792 }
3793 }
3794
3795 fn cancel_activity_subscription(
3796 &self,
3797 ctx: &RouteCtx,
3798 frame: &Frame,
3799 ) -> Result<bool, RouterError> {
3800 let task = self
3801 .activity_streams
3802 .lock()
3803 .unwrap_or_else(|p| p.into_inner())
3804 .remove(&(ctx.connection_id, frame.header.corr));
3805 let Some(task) = task else {
3806 return Ok(false);
3807 };
3808 task.abort();
3809 let end = Frame::build_with_version(
3810 response_version(frame),
3811 FrameType::StreamEnd,
3812 control_flags(),
3813 0,
3814 0,
3815 frame.header.corr,
3816 Vec::new(),
3817 )
3818 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3819 let sink = ctx.egress.clone();
3820 tokio::spawn(async move {
3821 let _ = sink.send(end).await;
3822 });
3823 Ok(true)
3824 }
3825
3826 async fn handle_activity_read(
3827 &self,
3828 ctx: &RouteCtx,
3829 frame: Frame,
3830 ) -> Result<Vec<Frame>, RouterError> {
3831 use subc_control::{
3832 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3833 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3834 };
3835 #[derive(Deserialize)]
3836 struct Identity {
3837 #[serde(default)]
3838 consumer_identity: Option<ConsumerIdentity>,
3839 }
3840 let principal = match self
3844 .forwarding
3845 .module_id_for_connection(ctx.connection_id)
3846 .map_err(RouterError::Forwarding)?
3847 {
3848 Some(module_id) => Ok(Principal::Reserved { module_id }),
3849 None => {
3850 let identity: Identity = match serde_json::from_slice(&frame.body) {
3851 Ok(identity) => identity,
3852 Err(_) => {
3853 return Ok(vec![control_error_frame(
3854 &frame,
3855 ACTIVITY_MALFORMED,
3856 "malformed activity read",
3857 )?])
3858 }
3859 };
3860 self.route_open_principal(&frame, identity.consumer_identity)?
3861 }
3862 };
3863 let admitted = matches!(principal, Ok(Principal::Direct))
3864 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3865 if !admitted {
3866 return Ok(vec![control_error_frame(
3867 &frame,
3868 ACTIVITY_READ_FORBIDDEN,
3869 "activity reads require an operator or trusted mirror principal",
3870 )?]);
3871 }
3872 let Some(ledger) = &self.activity else {
3873 return Ok(vec![control_error_frame(
3874 &frame,
3875 ACTIVITY_UNAVAILABLE,
3876 "activity ledger unavailable",
3877 )?]);
3878 };
3879 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3880 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3881 {
3882 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3883 Ok(request) => ledger.query(request).await,
3884 Err(_) => Err(ACTIVITY_MALFORMED),
3885 };
3886 return match result {
3887 Ok(reply) => Ok(vec![control_response_body_frame(
3888 &frame,
3889 &reply,
3890 ops::ACTIVITY_QUERY,
3891 )?]),
3892 Err(code) => Ok(vec![control_error_frame(
3893 &frame,
3894 code,
3895 "activity query refused",
3896 )?]),
3897 };
3898 }
3899 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3900 Ok(request) => request,
3901 Err(_) => {
3902 return Ok(vec![control_error_frame(
3903 &frame,
3904 ACTIVITY_MALFORMED,
3905 "malformed activity subscription",
3906 )?])
3907 }
3908 };
3909 let mut subscription = match ledger.subscribe(request.since).await {
3910 Ok(subscription) => subscription,
3911 Err(code) => {
3912 return Ok(vec![control_error_frame(
3913 &frame,
3914 code,
3915 "activity subscription refused",
3916 )?])
3917 }
3918 };
3919 let key = (ctx.connection_id, frame.header.corr);
3920 let sink = ctx.egress.clone();
3921 let mut streams = self
3922 .activity_streams
3923 .lock()
3924 .unwrap_or_else(|p| p.into_inner());
3925 streams.retain(|_, task| !task.is_finished());
3926 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3927 return Ok(vec![control_error_frame(
3928 &frame,
3929 "duplicate_subscription",
3930 "activity subscription correlation id already in use",
3931 )?]);
3932 }
3933 let task = tokio::spawn(async move {
3934 loop {
3935 let entry = if let Some(entry) = subscription.replay.pop_front() {
3936 Some(entry)
3937 } else {
3938 subscription.receiver.recv().await
3939 };
3940 let Some(entry) = entry else {
3941 break;
3942 };
3943 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3944 return;
3945 };
3946 let Ok(next) = Frame::build_with_version(
3947 response_version(&frame),
3948 FrameType::StreamData,
3949 control_flags(),
3950 0,
3951 0,
3952 frame.header.corr,
3953 body,
3954 ) else {
3955 return;
3956 };
3957 if sink.send(next).await.is_err() {
3958 return;
3959 }
3960 }
3961 if let Some(code) = subscription.terminal_code() {
3964 if let Ok(end) =
3965 control_error_frame(&frame, code, "activity subscriber fell behind")
3966 {
3967 let _ = sink.send(end).await;
3968 }
3969 }
3970 });
3971 streams.insert(key, task.abort_handle());
3972 Ok(Vec::new())
3973 }
3974
3975 async fn handle_activity_record(
3976 &self,
3977 ctx: &RouteCtx,
3978 frame: Frame,
3979 ) -> Result<Vec<Frame>, RouterError> {
3980 use subc_control::{
3981 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3982 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3983 };
3984 let result = async {
3985 let module = self
3986 .forwarding
3987 .module_id_for_connection(ctx.connection_id)
3988 .ok()
3989 .flatten()
3990 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3991 let request: ActivityRecordRequest =
3992 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3993 crate::activity::validate(&module, &request)?;
3994 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3995 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3996 let at_ms = std::time::SystemTime::now()
3997 .duration_since(std::time::UNIX_EPOCH)
3998 .unwrap_or_default()
3999 .as_millis() as u64;
4000 ledger.record(module, request, actor, at_ms).await
4001 }
4002 .await;
4003 match result {
4004 Ok(entry) => Ok(vec![control_response_body_frame(
4005 &frame,
4006 &ActivityRecordResponse { seq: entry.seq },
4007 "activity.record",
4008 )?]),
4009 Err(code) => Ok(vec![control_error_frame(
4010 &frame,
4011 code,
4012 "activity recording refused",
4013 )?]),
4014 }
4015 }
4016
4017 fn activity_actor(
4018 &self,
4019 connection: ConnectionId,
4020 module: &str,
4021 on: Option<&subc_control::ActivityOn>,
4022 ) -> Result<subc_control::ActivityActor, &'static str> {
4023 use crate::forwarding::{DataRoute, DataRouteState};
4024 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4025 let principal = |id: &str| Principal::Reserved {
4026 module_id: id.to_owned(),
4027 };
4028 let mut actor = ActivityActor {
4029 principal: principal(module),
4030 scope_owner: None,
4031 agent_id: None,
4032 run_id: None,
4033 flow_id: None,
4034 project_root: None,
4035 workdir: None,
4036 project_id: None,
4037 };
4038 let scope = match on {
4039 None => return Ok(actor),
4040 Some(ActivityOn::Scope(scope)) => {
4041 let selector = subc_protocol::scope::ScopeSelector::new(
4042 principal(module),
4043 scope.scope_ref.clone(),
4044 Some(scope.epoch),
4045 );
4046 self.scopes
4047 .read()
4048 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4049 .attribute(&principal(module), module, &selector, true)
4050 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4051 }
4052 Some(ActivityOn::Route(route)) => {
4053 let DataRoute::Module(DataRouteState::Bound(binding)) = self
4054 .forwarding
4055 .lookup_data_route(connection, route.channel, route.epoch)
4056 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4057 else {
4058 return Err(ACTIVITY_ACTOR_UNKNOWN);
4059 };
4060 if binding.module_id != module {
4061 return Err(ACTIVITY_ACTOR_UNKNOWN);
4062 }
4063 actor.principal = binding.principal.clone();
4064 actor.project_root = binding
4065 .project_root
4066 .as_ref()
4067 .map(|root| root.as_path().to_path_buf());
4068 let Some(bound) = &binding.scope else {
4069 return Ok(actor);
4070 };
4071 let owner = principal(&bound.owner);
4072 let selector = subc_protocol::scope::ScopeSelector::new(
4073 owner.clone(),
4074 bound.scope_ref.clone(),
4075 Some(bound.tag.scope_epoch),
4076 );
4077 let admitted = self
4078 .scopes
4079 .read()
4080 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4081 .attribute(&owner, module, &selector, true)
4082 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4083 if admitted.tag != bound.tag {
4084 return Err(ACTIVITY_ACTOR_UNKNOWN);
4085 }
4086 admitted
4087 }
4088 };
4089 actor.scope_owner = Some(scope.stamp.owner);
4090 actor.agent_id = scope.stamp.attributes.agent_id;
4091 actor.run_id = scope.stamp.attributes.run_id;
4092 actor.flow_id = scope.stamp.attributes.flow_id;
4093 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4094 actor.project_id = scope.stamp.project_id;
4095 Ok(actor)
4096 }
4097
4098 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4099 self.refresh_capability_requirements();
4100 let mut counters = self.counters.snapshot();
4106 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4107 obj.insert(
4108 "activity_unparseable_lines".into(),
4109 ledger.read_errors().into(),
4110 );
4111 obj.insert(
4112 "activity_lifecycle_append_failures".into(),
4113 ledger.lifecycle_failures().into(),
4114 );
4115 obj.insert(
4116 "activity_started_digest_skips".into(),
4117 ledger.lifecycle_digest_skips().into(),
4118 );
4119 }
4120 if let (Ok((connections_with_routes, max)), Some(obj)) = (
4121 self.forwarding.client_route_concentration(),
4122 counters.as_object_mut(),
4123 ) {
4124 obj.insert(
4125 "client_connections_with_routes".into(),
4126 connections_with_routes.into(),
4127 );
4128 obj.insert("max_routes_on_one_connection".into(), max.into());
4129 }
4130 if let (Some(open_breakers), Some(obj)) = (
4136 self.route_bind_breakers.open_snapshot(),
4137 counters.as_object_mut(),
4138 ) {
4139 obj.insert("route_bind_breakers_open".into(), open_breakers);
4140 }
4141 if let (Some(reason), Some(obj)) = (
4146 self.local_endpoint_unavailable.clone(),
4147 counters.as_object_mut(),
4148 ) {
4149 obj.insert("local_endpoint_unavailable".into(), reason.into());
4150 }
4151 let response = ClientControlResponse::ServerDescribe {
4152 protocol_ver: PROTOCOL_VERSION,
4153 subc_ops: subc_ops(),
4154 capabilities: self.describe_capabilities(),
4155 connected_clients: self.connected_clients.count(),
4156 counters: Some(counters),
4157 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4158 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4159 capability_requirements: self.capability_requirement_statuses(),
4160 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4161 config_path: self
4162 .rescan
4163 .as_ref()
4164 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4165 };
4166 Ok(vec![control_response_body_frame(
4167 &frame,
4168 &response,
4169 "ClientControlResponse::ServerDescribe",
4170 )?])
4171 }
4172
4173 fn handle_catalog_list(
4174 &self,
4175 frame: Frame,
4176 module_id: Option<String>,
4177 ) -> Result<Vec<Frame>, RouterError> {
4178 let (generation, modules) = self
4179 .registry
4180 .list_modules_with_nonce_verification()
4181 .map_err(|err| {
4182 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4183 })?;
4184 let entries = modules
4185 .into_iter()
4186 .filter(|(registration, _)| {
4187 module_id
4188 .as_deref()
4189 .map(|wanted| registration.manifest.module_id == wanted)
4190 .unwrap_or(true)
4191 })
4192 .map(|(registration, nonce_verified)| {
4193 let not_ready = self.not_ready_reason(®istration);
4194 let roles = registration.manifest.provides;
4195 CatalogEntry::new(
4196 registration.manifest.module_id,
4197 roles,
4198 registration.control_ops,
4199 )
4200 .with_ready(not_ready.is_none())
4201 .with_not_ready(not_ready)
4202 .with_module_version(Some(registration.manifest.module_version))
4203 .with_capabilities(registration.manifest.capabilities)
4204 .with_self_signals(registration.manifest.self_signals)
4205 .with_events(registration.manifest.events)
4206 .with_operation_delivery(registration.manifest.operation_delivery)
4207 .with_operation_payload(registration.manifest.operation_payload)
4208 .with_flow_operations(registration.manifest.flow_operations)
4209 .with_confirm_operations(registration.manifest.confirm_operations)
4210 .with_rich_ui(registration.manifest.rich_ui)
4211 .with_role_declarations(registration.manifest.role_declarations)
4212 .with_nonce_verified(nonce_verified)
4213 })
4214 .collect();
4215 let response = ClientControlResponse::CatalogList {
4216 generation,
4217 modules: entries,
4218 subc_ops: subc_ops(),
4219 };
4220 Ok(vec![control_response_body_frame(
4221 &frame,
4222 &response,
4223 "ClientControlResponse::CatalogList",
4224 )?])
4225 }
4226
4227 fn attest_route_open_consumer(
4232 &self,
4233 consumer_identity: Option<&ConsumerIdentity>,
4234 ) -> Option<Principal> {
4235 let Some(consumer_identity) = consumer_identity else {
4236 return Some(Principal::Direct);
4237 };
4238 self.supervisor
4239 .spawned_consumer_authorized(
4240 &consumer_identity.module_id,
4241 &consumer_identity.launch_nonce,
4242 )
4243 .then(|| Principal::Reserved {
4244 module_id: consumer_identity.module_id.clone(),
4245 })
4246 }
4247
4248 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4252 let Ok(ClientControlRequest::RouteOpen {
4253 identity,
4254 consumer_identity,
4255 scope,
4256 ..
4257 }) = parse_client_control_request(&frame.body)
4258 else {
4259 return RouteOpenCaller::unknown();
4260 };
4261 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4262 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4263 }
4264
4265 fn route_open_principal(
4266 &self,
4267 frame: &Frame,
4268 consumer_identity: Option<ConsumerIdentity>,
4269 ) -> Result<Result<Principal, Frame>, RouterError> {
4270 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4271 self.route_open_principal_attested(frame, consumer_identity, attested)
4272 }
4273
4274 fn route_open_principal_attested(
4278 &self,
4279 frame: &Frame,
4280 consumer_identity: Option<ConsumerIdentity>,
4281 attested: Option<Principal>,
4282 ) -> Result<Result<Principal, Frame>, RouterError> {
4283 if let Some(principal) = attested {
4284 return Ok(Ok(principal));
4285 }
4286 let Some(consumer_identity) = consumer_identity else {
4287 return Ok(Ok(Principal::Direct));
4288 };
4289
4290 Ok(Err(control_error_frame(
4291 frame,
4292 "bad_consumer_identity",
4293 format!(
4294 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4295 consumer_identity.module_id
4296 ),
4297 )?))
4298 }
4299
4300 #[allow(clippy::too_many_arguments)]
4307 fn route_open_refusal_frame(
4308 &self,
4309 ctx: &RouteCtx,
4310 caller: &RouteOpenCaller,
4311 frame: &Frame,
4312 module_id: &str,
4313 reason: &'static str,
4314 code: &'static str,
4315 message: impl Into<String>,
4316 ) -> Result<Frame, RouterError> {
4317 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4318 control_error_frame(frame, code, message.into())
4319 }
4320
4321 #[allow(clippy::too_many_arguments)]
4330 fn route_bind_rejection(
4331 &self,
4332 ctx: &RouteCtx,
4333 caller: &RouteOpenCaller,
4334 frame: &Frame,
4335 target_module_id: &str,
4336 body: ErrorBody,
4337 daemon_authored: bool,
4338 ) -> Result<Vec<Frame>, RouterError> {
4339 if daemon_authored {
4345 let scope_code = match body.code.as_str() {
4346 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4347 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4348 _ => None,
4349 };
4350 if let Some(code) = scope_code {
4351 self.observe_route_open_refusal(
4352 ctx,
4353 caller,
4354 target_module_id,
4355 "scope_changed_before_commit",
4356 code,
4357 );
4358 return Ok(vec![control_error_body_frame(frame, body)?]);
4359 }
4360 }
4361 self.counters
4362 .increment_route_open_refused("module_rejected");
4363 info!(
4364 target: "control",
4365 code = "module_rejected",
4366 module_code = ?body.code,
4367 module_id = ?target_module_id,
4368 connection_id = ctx.connection_id.get(),
4369 principal = %caller.principal,
4370 scope = %caller.scope,
4371 harness = %caller.harness,
4372 "route.open refused"
4373 );
4374 let refusal = if daemon_authored {
4375 control_error_body_frame(frame, body)?
4376 } else {
4377 control_relayed_error_body_frame(frame, body)?
4378 };
4379 Ok(vec![refusal])
4380 }
4381
4382 #[allow(clippy::too_many_arguments)]
4400 fn route_open_breaker_refusal_frame(
4401 &self,
4402 ctx: &RouteCtx,
4403 caller: &RouteOpenCaller,
4404 frame: &Frame,
4405 module_id: &str,
4406 consecutive_timeouts: u32,
4407 retry_in: Duration,
4408 probe_in_flight: bool,
4409 ) -> Result<Frame, RouterError> {
4410 self.counters
4411 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4412 debug!(
4413 target: "control",
4414 code = "module_timeout",
4415 module_id = ?module_id,
4416 connection_id = ctx.connection_id.get(),
4417 principal = %caller.principal,
4418 scope = %caller.scope,
4419 harness = %caller.harness,
4420 consecutive_timeouts,
4421 retry_in_ms = retry_in.as_millis() as u64,
4422 probe_in_flight,
4423 "route.open refused by open bind-relay breaker"
4424 );
4425 let detail = if probe_in_flight {
4432 "one test route is already being tried; retry once it settles".to_string()
4433 } else {
4434 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4435 };
4436 control_error_frame(
4437 frame,
4438 "module_timeout",
4439 format!(
4440 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4441 timed out in a row); {detail}; its established routes are unaffected"
4442 ),
4443 )
4444 }
4445
4446 fn observe_route_open_refusal(
4459 &self,
4460 ctx: &RouteCtx,
4461 caller: &RouteOpenCaller,
4462 module_id: &str,
4463 reason: &'static str,
4464 code: &'static str,
4465 ) {
4466 self.counters.increment_route_open_refused(code);
4467 info!(
4468 target: "control",
4469 code,
4470 reason,
4471 module_id = ?module_id,
4472 connection_id = ctx.connection_id.get(),
4473 principal = %caller.principal,
4474 scope = %caller.scope,
4475 harness = %caller.harness,
4476 "route.open refused"
4477 );
4478 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4479 self.route_outages.record_not_serving(module_id, reason);
4480 }
4481 }
4482
4483 fn observe_route_open_accept(
4528 &self,
4529 ctx: &RouteCtx,
4530 module_id: &str,
4531 principal: &str,
4532 scope: &str,
4533 ) {
4534 self.route_outages.record_accepted(module_id);
4535 self.counters.increment_route_open_accepted(principal);
4536 info!(
4537 target: "control",
4538 principal,
4539 module_id,
4540 connection_id = ctx.connection_id.get(),
4541 scope = %scope,
4542 "route.open accepted"
4543 );
4544 }
4545
4546 fn supervised_absent_route_open_refusal_frame(
4547 &self,
4548 ctx: &RouteCtx,
4549 caller: &RouteOpenCaller,
4550 frame: &Frame,
4551 module_id: &str,
4552 code: &'static str,
4553 status: &crate::supervise::ModuleStatus,
4554 ) -> Result<Frame, RouterError> {
4555 self.counters.increment_route_open_refused(code);
4556 info!(
4557 target: "control",
4558 code,
4559 reason = "supervised_not_registered",
4560 module_id = ?module_id,
4561 connection_id = ctx.connection_id.get(),
4562 principal = %caller.principal,
4563 scope = %caller.scope,
4564 harness = %caller.harness,
4565 state = %status.state,
4566 enabled = status.enabled,
4567 live = status.live,
4568 "route.open refused"
4569 );
4570 self.route_outages
4574 .record_not_serving(module_id, "supervised_not_registered");
4575 let message = if let Some(pid) = status.waiting_for_previous_pid {
4576 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4577 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4578 })
4579 } else {
4580 format!(
4581 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4582 status.state, status.enabled, status.live
4583 )
4584 };
4585 control_error_frame(frame, code, message)
4586 }
4587
4588 async fn handle_route_open(
4589 &self,
4590 ctx: &RouteCtx,
4591 frame: Frame,
4592 request: RouteOpenRequest,
4593 timings: &mut RouteOpenTimings,
4594 ) -> Result<Vec<Frame>, RouterError> {
4595 let RouteOpenRequest {
4596 target,
4597 mut identity,
4598 consumer_identity,
4599 consumer_capabilities,
4600 role_versions,
4601 admission_facts,
4602 scope,
4603 } = request;
4604 let target_module_id = target_module_id(&target).to_string();
4605 timings.module_id.clone_from(&target_module_id);
4606 if self
4607 .registry
4608 .get_module_by_connection(ctx.connection_id)
4609 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4610 .is_some()
4611 {
4612 return Ok(vec![control_error_frame(
4613 &frame,
4614 "invalid_request",
4615 "module connections cannot open client routes",
4616 )?]);
4617 }
4618 debug!(
4619 connection_id = ctx.connection_id.get(),
4620 corr = frame.header.corr,
4621 module_id = %target_module_id,
4622 "handling route.open"
4623 );
4624
4625 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4630 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4631
4632 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4638 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4639 self.observe_route_open_refusal(
4640 ctx,
4641 &caller,
4642 &target_module_id,
4643 "invalid_role_versions",
4644 error_codes::INVALID_REQUEST,
4645 );
4646 return Ok(vec![control_error_body_frame(
4647 &frame,
4648 ErrorBody {
4649 code: error_codes::INVALID_REQUEST.to_string(),
4650 message: error.to_string(),
4651 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4652 },
4653 )?]);
4654 }
4655
4656 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4657 if status.waiting_for_previous_pid.is_some() {
4658 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4659 ctx,
4660 &caller,
4661 &frame,
4662 &target_module_id,
4663 "module_warming",
4664 &status,
4665 )?]);
4666 }
4667 }
4668
4669 let Some(registration) = self
4688 .registry
4689 .get_module(&target_module_id)
4690 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4691 else {
4692 if let Some((status, warming)) =
4693 self.supervisor_status(&target_module_id, frame.header.corr)?
4694 {
4695 if status.protocol == ModuleProtocol::None {
4702 return Ok(vec![self.route_open_refusal_frame(
4703 ctx,
4704 &caller,
4705 &frame,
4706 &target_module_id,
4707 "protocol_none",
4708 error_codes::MODULE_NO_PROTOCOL,
4709 format!(
4710 "module_id '{target_module_id}' is declared protocol: none; \
4711 it speaks no subc wire and serves no routes"
4712 ),
4713 )?]);
4714 }
4715 let code = if warming {
4716 "module_warming"
4717 } else {
4718 "target_unavailable"
4719 };
4720 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4721 ctx,
4722 &caller,
4723 &frame,
4724 &target_module_id,
4725 code,
4726 &status,
4727 )?]);
4728 }
4729 if let Some(removed_ago_ms) =
4730 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4731 {
4732 return Ok(vec![self.route_open_refusal_frame(
4733 ctx,
4734 &caller,
4735 &frame,
4736 &target_module_id,
4737 "removed",
4738 error_codes::MODULE_REMOVED,
4739 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4740 )?]);
4741 }
4742 return Ok(vec![self.route_open_refusal_frame(
4743 ctx,
4744 &caller,
4745 &frame,
4746 &target_module_id,
4747 "not_registered",
4748 error_codes::UNKNOWN_MODULE,
4749 format!("module_id '{target_module_id}' is not registered"),
4750 )?]);
4751 };
4752
4753 if !registration.ready {
4757 self.counters
4758 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4759 info!(
4760 target: "control",
4761 code = error_codes::MODULE_WARMING,
4762 module_id = ?target_module_id,
4763 connection_id = ctx.connection_id.get(),
4764 principal = %caller.principal,
4765 scope = %caller.scope,
4766 harness = %caller.harness,
4767 reason = "declared_not_ready",
4768 "route.open refused"
4769 );
4770 self.route_outages
4773 .record_not_serving(&target_module_id, "declared_not_ready");
4774 return Ok(vec![control_error_body_frame(
4775 &frame,
4776 ErrorBody {
4777 code: error_codes::MODULE_WARMING.to_string(),
4778 message: format!(
4779 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4780 ),
4781 detail: Some(serde_json::json!({
4782 "reason": "declared_not_ready"
4783 })),
4784 },
4785 )?]);
4786 }
4787
4788 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4807 self.counters
4808 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4809 info!(
4810 target: "control",
4811 code = error_codes::MODULE_WARMING,
4812 module_id = ?target_module_id,
4813 connection_id = ctx.connection_id.get(),
4814 principal = %caller.principal,
4815 scope = %caller.scope,
4816 harness = %caller.harness,
4817 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4818 capability = %capability,
4819 "route.open refused"
4820 );
4821 return Ok(vec![control_error_body_frame(
4822 &frame,
4823 ErrorBody {
4824 code: error_codes::MODULE_WARMING.to_string(),
4825 message: format!(
4826 "module_id '{target_module_id}' requires capability '{capability}', \
4827 which no registered module provides; retry"
4828 ),
4829 detail: Some(serde_json::json!({
4830 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4831 "capability": capability,
4832 })),
4833 },
4834 )?]);
4835 }
4836
4837 if !target_has_required_role(&target, ®istration.manifest.provides) {
4838 return Ok(vec![self.route_open_refusal_frame(
4839 ctx,
4840 &caller,
4841 &frame,
4842 &target_module_id,
4843 "role_not_provided",
4844 "target_unavailable",
4845 format!("module_id '{target_module_id}' does not provide the requested target"),
4846 )?]);
4847 }
4848
4849 if registration.state != ChannelState::Active {
4850 return Ok(vec![self.route_open_refusal_frame(
4851 ctx,
4852 &caller,
4853 &frame,
4854 &target_module_id,
4855 "registration_not_active",
4856 "target_unavailable",
4857 format!("module_id '{target_module_id}' is not active"),
4858 )?]);
4859 }
4860
4861 if self
4862 .forwarding
4863 .module_is_draining(&target_module_id)
4864 .map_err(RouterError::Forwarding)?
4865 {
4866 return Ok(vec![self.route_open_refusal_frame(
4867 ctx,
4868 &caller,
4869 &frame,
4870 &target_module_id,
4871 "reloading",
4872 "module_reloading",
4873 format!("module_id '{target_module_id}' is reloading"),
4874 )?]);
4875 }
4876
4877 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4878 process_liveness.process_live(&target_module_id) == Some(false)
4879 }) {
4880 if process_liveness.process_replacing(&target_module_id) {
4887 return Ok(vec![self.route_open_refusal_frame(
4888 ctx,
4889 &caller,
4890 &frame,
4891 &target_module_id,
4892 "reloading",
4893 "module_reloading",
4894 format!("module_id '{target_module_id}' is reloading"),
4895 )?]);
4896 }
4897 return Ok(vec![self.route_open_refusal_frame(
4898 ctx,
4899 &caller,
4900 &frame,
4901 &target_module_id,
4902 "supervisor_not_live",
4903 "target_unavailable",
4904 format!("module_id '{target_module_id}' is not live"),
4905 )?]);
4906 }
4907
4908 if !self
4909 .forwarding
4910 .has_live_module_connection(&target_module_id)
4911 .map_err(RouterError::Forwarding)?
4912 {
4913 return Ok(vec![self.route_open_refusal_frame(
4914 ctx,
4915 &caller,
4916 &frame,
4917 &target_module_id,
4918 "no_forwarding_connection",
4919 "target_unavailable",
4920 format!("module_id '{target_module_id}' has no live forwarding connection"),
4921 )?]);
4922 }
4923
4924 if let Some(error) =
4925 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4926 {
4927 self.observe_route_open_refusal(
4928 ctx,
4929 &caller,
4930 &target_module_id,
4931 "op_not_allowed",
4932 "op_not_allowed",
4933 );
4934 return Ok(vec![error]);
4935 }
4936
4937 let principal =
4938 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4939 Ok(principal) => principal,
4940 Err(error) => {
4941 self.observe_route_open_refusal(
4942 ctx,
4943 &caller,
4944 &target_module_id,
4945 "bad_consumer_identity",
4946 "bad_consumer_identity",
4947 );
4948 return Ok(vec![error]);
4949 }
4950 };
4951 timings.principal = principal_label(&principal);
4952
4953 let build_class = identity.build_class.take().filter(|build_class| {
4956 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
4957 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
4958 } else {
4959 if self
4960 .build_class_warned
4961 .lock()
4962 .unwrap_or_else(|p| p.into_inner())
4963 .insert(ctx.connection_id)
4964 {
4965 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
4966 "ignoring build_class from a non-Callosum opener");
4967 }
4968 false
4969 }
4970 });
4971
4972 if let Principal::Reserved {
4976 module_id: opening_module_id,
4977 } = &principal
4978 {
4979 if let Some(opening_registration) = self
4980 .registry
4981 .get_module(opening_module_id)
4982 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4983 {
4984 if let Some(capability) =
4985 denied_capability(&opening_registration.manifest, ®istration.manifest)
4986 {
4987 warn!(
4988 opening_module_id,
4989 target_module_id,
4990 capability,
4991 "refusing route.open because an attested capability deny edge matches"
4992 );
4993 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4994 &frame,
4995 &target_module_id,
4996 "capability_deny_edge",
4997 "capability_forbidden",
4998 format!(
4999 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5000 ),
5001 )?]);
5002 }
5003 }
5004 }
5005
5006 if admission_facts.is_some() {
5007 let carrier_matches = matches!(
5008 &principal,
5009 Principal::Reserved { module_id }
5010 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5011 );
5012 if !carrier_matches {
5013 return Ok(vec![self.route_open_refusal_frame(
5014 ctx,
5015 &caller,
5016 &frame,
5017 &target_module_id,
5018 "admission_facts_carrier_not_permitted",
5019 "admission_facts_not_permitted",
5020 "admission facts may only be carried by the configured reserved module",
5021 )?]);
5022 }
5023
5024 let target_allowed = self
5025 .admission_facts_targets
5026 .as_ref()
5027 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5028 if !target_allowed {
5029 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5030 &frame,
5031 &target_module_id,
5032 "admission_facts_target_not_listed",
5033 "admission_facts_target_not_allowed",
5034 format!(
5035 "admission facts are not permitted for target module_id '{target_module_id}'"
5036 ),
5037 )?]);
5038 }
5039
5040 }
5044
5045 let (bound_scope, mut scope_stamp) = match &scope {
5051 None => (None, None),
5052 Some(selector) => {
5053 let owner_configured = match &selector.owner {
5054 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5060 _ => false,
5061 };
5062 let mut scopes = self
5063 .scopes
5064 .write()
5065 .unwrap_or_else(|poisoned| poisoned.into_inner());
5066 let admitted = scopes
5067 .admit(&principal, &target_module_id, selector, owner_configured)
5068 .and_then(|admission| {
5069 crate::scopes::check_target_flow_support(
5070 &admission.stamp,
5071 &target_module_id,
5072 registration.manifest.capabilities.as_ref(),
5073 )?;
5074 crate::scopes::check_target_agent_run_support(
5075 &admission.stamp,
5076 &target_module_id,
5077 registration.manifest.capabilities.as_ref(),
5078 )?;
5079 let bound = scopes.reserve_bind(
5080 &admission,
5081 &principal,
5082 ctx.connection_id,
5083 selector.hold_presence,
5084 );
5085 Ok((admission, bound))
5086 });
5087 match admitted {
5088 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5089 Err(refusal) => {
5090 return Ok(vec![self.route_open_refusal_frame(
5091 ctx,
5092 &caller,
5093 &frame,
5094 &target_module_id,
5095 refusal.code,
5096 refusal.code,
5097 refusal.message,
5098 )?]);
5099 }
5100 }
5101 }
5102 };
5103 if let Some(stamp) = &scope_stamp {
5106 caller.scope = route_open_scope_label(stamp);
5107 }
5108
5109 timings.enter(RouteOpenPhase::ProjectRoot);
5131 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
5132 Ok(project_root) => project_root,
5133 Err(err) => {
5134 return Ok(vec![control_error_frame(
5135 &frame,
5136 "invalid_project_root",
5137 err.to_string(),
5138 )?])
5139 }
5140 };
5141 timings.enter(RouteOpenPhase::Admission);
5142 identity.project_root = project_root.as_path().to_path_buf();
5143
5144 let _concurrency_guard = match self
5155 .route_bind_concurrency
5156 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5157 {
5158 Ok(guard) => guard,
5159 Err(in_flight) => {
5160 return Ok(vec![self.route_open_target_capacity_refusal(
5161 ctx,
5162 &caller,
5163 &frame,
5164 &target_module_id,
5165 in_flight,
5166 )?]);
5167 }
5168 };
5169
5170 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5173 RouteBindAdmission::Admitted { guard, probe } => {
5174 if probe {
5175 info!(
5176 module_id = %target_module_id,
5177 connection_id = ctx.connection_id.get(),
5178 "route.bind breaker half-open: admitting one probe"
5179 );
5180 }
5181 guard
5182 }
5183 RouteBindAdmission::Refused {
5184 consecutive_timeouts,
5185 retry_in,
5186 probe_in_flight,
5187 } => {
5188 return Ok(vec![self.route_open_breaker_refusal_frame(
5189 ctx,
5190 &caller,
5191 &frame,
5192 &target_module_id,
5193 consecutive_timeouts,
5194 retry_in,
5195 probe_in_flight,
5196 )?]);
5197 }
5198 };
5199
5200 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5205 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5206 let operator_app = self
5207 .operator_apps
5208 .lock()
5209 .unwrap_or_else(|p| p.into_inner())
5210 .get(&ctx.connection_id)
5211 .cloned();
5212 timings.enter(RouteOpenPhase::EgressReservation);
5213 let pending = match self
5214 .forwarding
5215 .begin_route_bind_relay_for(
5216 ctx.connection_id,
5217 ctx.egress.clone(),
5218 response_version(&frame),
5219 frame.header.corr,
5220 &target_module_id,
5221 principal.clone(),
5222 bound_scope,
5223 Some(project_root),
5224 identity.harness.clone(),
5225 build_class,
5226 operator_app.clone(),
5227 relay_deadline,
5228 )
5229 .await
5230 {
5231 Ok(pending) => pending,
5232 Err(err) => {
5233 return Ok(vec![self.route_open_refusal_frame(
5234 ctx,
5235 &caller,
5236 &frame,
5237 &target_module_id,
5238 "relay_reservation_failed",
5239 forwarding_error_code(&err),
5240 err.to_string(),
5241 )?])
5242 }
5243 };
5244 timings.enter(RouteOpenPhase::Admission);
5245 let crate::forwarding::PendingRouteBindRelay {
5246 endpoint,
5247 module_sink,
5248 negotiated_ver,
5249 client_channel,
5250 client_epoch,
5251 module_channel,
5252 module_epoch,
5253 corr: relay_corr,
5254 receiver,
5255 } = pending;
5256 let mut reservation =
5257 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5258
5259 let relay_registration = self
5264 .registry
5265 .get_module_by_connection(endpoint.connection_id)
5266 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5267 let relay_capabilities = relay_registration
5268 .as_ref()
5269 .and_then(|registration| registration.manifest.capabilities.as_ref());
5270 if let Some(selector) = &scope {
5271 if let Err(refusal) = self
5274 .scopes
5275 .read()
5276 .unwrap_or_else(|poisoned| poisoned.into_inner())
5277 .check_bind_proof(&principal, selector)
5278 {
5279 reservation.release_and_disarm();
5280 return Ok(vec![self.route_open_refusal_frame(
5281 ctx,
5282 &caller,
5283 &frame,
5284 &target_module_id,
5285 refusal.code,
5286 refusal.code,
5287 refusal.message,
5288 )?]);
5289 }
5290 }
5291 if let Some(stamp) = scope_stamp
5292 .as_ref()
5293 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5294 {
5295 if let Err(refusal) = crate::scopes::check_target_flow_support(
5296 stamp,
5297 &target_module_id,
5298 relay_capabilities,
5299 )
5300 .and_then(|()| {
5301 crate::scopes::check_target_agent_run_support(
5302 stamp,
5303 &target_module_id,
5304 relay_capabilities,
5305 )
5306 }) {
5307 reservation.release_and_disarm();
5308 return Ok(vec![self.route_open_refusal_frame(
5309 ctx,
5310 &caller,
5311 &frame,
5312 &target_module_id,
5313 refusal.code,
5314 refusal.code,
5315 refusal.message,
5316 )?]);
5317 }
5318 }
5319
5320 if !relay_capabilities.is_some_and(|caps| {
5323 caps.provides
5324 .iter()
5325 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5326 }) {
5327 if let Some(stamp) = scope_stamp.as_mut() {
5328 stamp.workdir = None;
5329 stamp.project_id = None;
5330 }
5331 }
5332 let worker_run = scope_stamp
5337 .as_ref()
5338 .is_some_and(crate::scopes::is_worker_run)
5339 && relay_capabilities.is_some_and(|caps| {
5340 caps.provides
5341 .iter()
5342 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5343 });
5344 if !worker_run
5345 && !relay_capabilities.is_some_and(|caps| {
5346 caps.provides
5347 .iter()
5348 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5349 })
5350 {
5351 if let Some(stamp) = scope_stamp.as_mut() {
5352 stamp.attributes.on_behalf_of_agent = None;
5353 }
5354 }
5355 if !relay_capabilities.is_some_and(|caps| {
5356 caps.provides
5357 .iter()
5358 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5359 }) {
5360 if let Some(stamp) = scope_stamp.as_mut() {
5361 stamp.attributes.flow_version = None;
5362 stamp.attributes.flow_code_hash = None;
5363 stamp.attributes.flow_install_id = None;
5364 }
5365 }
5366 let call_mode = if worker_run
5367 || relay_capabilities
5368 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5369 {
5370 Some(derive_call_mode(scope_stamp.as_ref()))
5371 } else {
5372 if let Some(stamp) = scope_stamp.as_mut() {
5373 stamp.expires_at_ms = None;
5374 }
5375 None
5376 };
5377
5378 debug!(
5379 connection_id = ctx.connection_id.get(),
5380 client_channel,
5381 client_epoch,
5382 module_channel,
5383 module_epoch,
5384 "reserved route handle pair"
5385 );
5386 let route_principal_label = principal_label(&principal);
5389 let relay = ModuleControlRequest::RouteBind {
5390 route_channel: module_channel,
5391 epoch: module_epoch,
5392 target,
5393 identity,
5394 principal: Some(principal),
5395 consumer_capabilities,
5396 role_versions,
5397 admission_facts,
5398 scope: scope_stamp,
5399 call_mode,
5400 };
5401 let peer = RouteBindPeer::new()
5402 .with_observed_peer(
5403 relay_capabilities
5404 .is_some_and(|caps| {
5405 caps.provides
5406 .iter()
5407 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5408 })
5409 .then(|| {
5410 self.observed_peers
5411 .lock()
5412 .unwrap_or_else(|p| p.into_inner())
5413 .get(&ctx.connection_id)
5414 .cloned()
5415 .unwrap_or_else(ObservedPeer::unobserved)
5416 }),
5417 )
5418 .with_operator_app(
5419 operator_app
5420 .filter(|stamp| {
5421 stamp.delivery_enabled(&self.operator_app_policy)
5422 && relay_capabilities.is_some_and(|caps| {
5423 caps.provides
5424 .iter()
5425 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5426 })
5427 })
5428 .map(|stamp| stamp.app),
5429 );
5430 let relay_body = encode_route_bind(&relay, &peer).map_err(|err| {
5431 RouterError::backend(
5432 0,
5433 frame.header.corr,
5434 format!("failed to encode route.bind request: {err}"),
5435 )
5436 })?;
5437 let relay_frame = Frame::build_with_version(
5438 negotiated_ver,
5439 FrameType::Request,
5440 control_flags(),
5441 0,
5442 0,
5443 relay_corr,
5444 relay_body,
5445 )
5446 .map_err(RouterError::FrameBuild)?;
5447
5448 timings.enter(RouteOpenPhase::BindSend);
5449 let sent = module_sink.send(relay_frame).await;
5450 timings.enter(RouteOpenPhase::Admission);
5451 if let Err(err) = sent {
5452 reservation.release_and_disarm();
5453 return Ok(vec![self.route_open_refusal_frame(
5454 ctx,
5455 &caller,
5456 &frame,
5457 &target_module_id,
5458 "relay_send_failed",
5459 "target_unavailable",
5460 err.to_string(),
5461 )?]);
5462 }
5463
5464 #[cfg(test)]
5465 if let Some(hold) = &self.route_bind_relay_sent_hold {
5466 hold.wait().await;
5470 hold.wait().await;
5471 }
5472
5473 if self
5478 .forwarding
5479 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5480 .map_err(RouterError::Forwarding)?
5481 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5482 {
5483 self.send_abandoned_route_bind_goodbye(
5484 &module_sink,
5485 negotiated_ver,
5486 module_channel,
5487 module_epoch,
5488 );
5489 }
5490
5491 timings.enter(RouteOpenPhase::BindAckWait);
5492 let outcome = timeout_at(relay_deadline, receiver).await;
5493 timings.enter(RouteOpenPhase::Admission);
5494 match outcome {
5495 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5496 reservation.disarm();
5497 if breaker.record_accepted() {
5498 info!(
5499 module_id = %target_module_id,
5500 "route.bind breaker closed: the probe was accepted"
5501 );
5502 }
5503 self.observe_route_open_accept(
5504 ctx,
5505 &target_module_id,
5506 &route_principal_label,
5507 &caller.scope,
5508 );
5509 Ok(Vec::new())
5510 }
5511 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5512 reservation.release_and_disarm();
5513 breaker.record_inconclusive();
5517 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5518 }
5519 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5520 reservation.release_and_disarm();
5521 breaker.record_inconclusive();
5522 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5523 }
5524 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5525 reservation.release_and_disarm();
5526 breaker.record_inconclusive();
5527 tracing::warn!(
5534 module_id = %target_module_id,
5535 "route.bind relay abandoned: {message}"
5536 );
5537 Ok(vec![self.route_open_refusal_frame(
5538 ctx,
5539 &caller,
5540 &frame,
5541 &target_module_id,
5542 "relay_abandoned",
5543 "target_unavailable",
5544 message,
5545 )?])
5546 }
5547 Ok(Err(_)) => {
5548 reservation.release_and_disarm();
5549 breaker.record_inconclusive();
5550 Ok(vec![self.route_open_refusal_frame(
5551 ctx,
5552 &caller,
5553 &frame,
5554 &target_module_id,
5555 "relay_waiter_canceled",
5556 "target_unavailable",
5557 "route.bind relay waiter was canceled before the module responded",
5558 )?])
5559 }
5560 Err(_) => {
5561 reservation.release_and_disarm();
5562 if let Some(opened) = breaker.record_timeout(
5566 self.route_bind_breaker_threshold,
5567 self.route_bind_breaker_cooldown,
5568 ) {
5569 warn!(
5570 module_id = %target_module_id,
5571 consecutive_timeouts = opened.consecutive_timeouts,
5572 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5573 reopened_after_probe = opened.reopened_after_probe,
5574 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5575 );
5576 }
5577 tracing::warn!(
5585 module_id = %target_module_id,
5586 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5587 "route.bind relay timed out: module did not ack within budget"
5588 );
5589 Ok(vec![self.route_open_refusal_frame(
5590 ctx,
5591 &caller,
5592 &frame,
5593 &target_module_id,
5594 "relay_timed_out",
5595 "module_timeout",
5596 format!(
5597 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5598 route_bind_relay_timeout
5599 ),
5600 )?])
5601 }
5602 }
5603 }
5604
5605 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5606 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5607 snapshot: self.supervisor.spawn_snapshot(),
5608 };
5609 Ok(vec![control_response_body_frame(
5610 &frame,
5611 &response,
5612 "ClientControlResponse::SupervisorSpawnSnapshot",
5613 )?])
5614 }
5615
5616 fn handle_supervisor_spawn_subscribe(
5617 &self,
5618 ctx: &RouteCtx,
5619 frame: Frame,
5620 since: Option<SpawnCursor>,
5621 ) -> Result<Vec<Frame>, RouterError> {
5622 match self.supervisor.subscribe_spawns(
5623 ctx.connection_id,
5624 frame.header.corr,
5625 response_version(&frame),
5626 since,
5627 ctx.egress.clone(),
5628 ) {
5629 Ok(()) => Ok(Vec::new()),
5630 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5631 Ok(vec![control_error_body_frame(
5632 &frame,
5633 ErrorBody {
5634 code: "spawn_cursor_incarnation_mismatch".to_string(),
5635 message: "spawn cursor belongs to a different daemon incarnation"
5636 .to_string(),
5637 detail: Some(serde_json::json!({
5638 "current_daemon_incarnation": current
5639 })),
5640 },
5641 )?])
5642 }
5643 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5644 &frame,
5645 ErrorBody {
5646 code: "spawn_cursor_too_old".to_string(),
5647 message: "spawn cursor predates the retained event ring".to_string(),
5648 detail: Some(serde_json::json!({
5649 "oldest_retained_cursor": oldest
5650 })),
5651 },
5652 )?]),
5653 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5654 0,
5655 frame.header.corr,
5656 format!("failed to open supervisor spawn subscription: {error}"),
5657 )),
5658 }
5659 }
5660
5661 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5662 let generation = self
5663 .registry
5664 .generation()
5665 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5666 let mut modules = Vec::new();
5667 for module in self.supervisor.list() {
5668 let status = module.status_for_control("list").map_err(|err| {
5669 RouterError::backend(
5670 0,
5671 frame.header.corr,
5672 format!("failed to read supervisor status: {err}"),
5673 )
5674 })?;
5675 let (configured, _) = module.configuration().map_err(|err| {
5676 RouterError::backend(
5677 0,
5678 frame.header.corr,
5679 format!("failed to read module configuration: {err}"),
5680 )
5681 })?;
5682 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5683 RouterError::backend(
5684 0,
5685 frame.header.corr,
5686 format!("failed to read retry time: {err}"),
5687 )
5688 })?;
5689 let image = module.running_image_agreement().await;
5691 let resources = Some(module.child_resource_usage());
5694 let mut verdict =
5695 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5696 verdict.spec = Some(launch_spec_agreement(
5697 &configured,
5698 status.process_alive,
5699 status.launch_spec_digest.as_ref(),
5700 ));
5701 let pending_reload = Some(verdict);
5702 let configured_enabled = self
5703 .capability_evaluator
5704 .configured_enabled(&status.module_id);
5705 modules.push(
5708 SupervisorEntry::new(
5709 status.module_id,
5710 status.state.to_string(),
5711 status.enabled,
5712 status.live,
5713 status.health.status,
5714 )
5715 .with_configured_enabled(configured_enabled)
5716 .with_next_retry_at_ms(next_retry_at_ms)
5717 .with_launch_nonce_env(Some(!cfg!(unix)))
5718 .with_launch_nonce_source(status.launch_nonce_source)
5719 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5720 .with_protocol(status.protocol)
5721 .with_pending_reload(pending_reload)
5722 .with_last_probe_ms(status.health.last_probe_ms)
5723 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5724 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5725 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5726 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5727 .with_restart_count(Some(status.restart_count))
5728 .with_max_restarts(Some(status.max_restarts))
5729 .with_lifetime_restarts(Some(status.lifetime_restarts))
5730 .with_spawn_generation(Some(status.spawn_generation))
5731 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5732 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5733 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5734 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5735 .with_resources(resources),
5736 );
5737 }
5738 let response = ClientControlResponse::SupervisorList {
5739 generation,
5740 modules,
5741 };
5742 Ok(vec![control_response_body_frame(
5743 &frame,
5744 &response,
5745 "ClientControlResponse::SupervisorList",
5746 )?])
5747 }
5748
5749 fn handle_supervisor_stderr_tail(
5750 &self,
5751 frame: Frame,
5752 module_id: String,
5753 max_lines: Option<u32>,
5754 max_bytes: Option<u32>,
5755 ) -> Result<Vec<Frame>, RouterError> {
5756 let Some(module) = self.supervisor.get(&module_id) else {
5757 return Ok(vec![control_error_frame(
5758 &frame,
5759 "unknown_module",
5760 format!("module_id '{module_id}' is not supervised"),
5761 )?]);
5762 };
5763
5764 let snapshot = module.stderr_tail(
5765 max_lines.map(|value| value as usize),
5766 max_bytes.map(|value| value as usize),
5767 );
5768
5769 let response = ClientControlResponse::SupervisorStderrTail {
5770 module_id,
5771 tail: StderrTail {
5772 capture: match snapshot.capture {
5773 CaptureState::Captured => StderrCaptureState::Captured,
5774 CaptureState::Incomplete { reason } => {
5775 StderrCaptureState::Incomplete { reason }
5776 }
5777 CaptureState::NotCaptured { reason } => {
5778 StderrCaptureState::NotCaptured { reason }
5779 }
5780 },
5781 entries: snapshot
5782 .entries
5783 .into_iter()
5784 .map(|entry| match entry {
5785 TailEntry::Line {
5786 text,
5787 truncated,
5788 at_ms,
5789 } => StderrTailEntry::Line {
5790 text,
5791 truncated,
5792 at_ms,
5793 },
5794 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5795 })
5796 .collect(),
5797 dropped_lines: snapshot.dropped_lines,
5798 },
5799 };
5800 Ok(vec![control_response_body_frame(
5801 &frame,
5802 &response,
5803 "ClientControlResponse::SupervisorStderrTail",
5804 )?])
5805 }
5806
5807 async fn handle_supervisor_terminals(
5808 &self,
5809 frame: Frame,
5810 module_id: String,
5811 ) -> Result<Vec<Frame>, RouterError> {
5812 let Some(module) = self.supervisor.get(&module_id) else {
5813 return Ok(vec![control_error_frame(
5814 &frame,
5815 "unknown_module",
5816 format!("module_id '{module_id}' is not supervised"),
5817 )?]);
5818 };
5819
5820 let terminals = module
5823 .read_durable_terminal_history()
5824 .await
5825 .map_err(|error| {
5826 RouterError::backend(
5827 0,
5828 frame.header.corr,
5829 format!("failed to read terminal history: {error}"),
5830 )
5831 })?;
5832 let response = ClientControlResponse::SupervisorTerminals {
5833 module_id,
5834 terminals,
5835 };
5836 Ok(vec![control_response_body_frame(
5837 &frame,
5838 &response,
5839 "ClientControlResponse::SupervisorTerminals",
5840 )?])
5841 }
5842
5843 fn handle_supervisor_routes(
5844 &self,
5845 frame: Frame,
5846 module_id: Option<String>,
5847 ) -> Result<Vec<Frame>, RouterError> {
5848 let modules = self
5849 .forwarding
5850 .route_census(module_id.as_deref())
5851 .map_err(RouterError::Forwarding)?
5852 .into_iter()
5853 .map(|(module_id, routes)| SupervisorRouteModule {
5854 module_id,
5855 routes: routes
5856 .into_iter()
5857 .map(|route| SupervisorRoute {
5858 consumer: match route.principal {
5859 Principal::Reserved { module_id } => {
5860 SupervisorRouteConsumer::Reserved { module_id }
5861 }
5862 Principal::Direct | Principal::Unverified => {
5863 SupervisorRouteConsumer::Direct {
5864 connection_id: route.goodbye_target.connection_id.get(),
5865 }
5866 }
5867 },
5868 age_ms: Instant::now()
5869 .saturating_duration_since(route.bound_at)
5870 .as_millis()
5871 .try_into()
5872 .unwrap_or(u64::MAX),
5873 draining: route.draining,
5874 drain_reason: route.drain_reason,
5875 })
5876 .collect(),
5877 })
5878 .collect();
5879 let response = ClientControlResponse::SupervisorRoutes { modules };
5880 Ok(vec![control_response_body_frame(
5881 &frame,
5882 &response,
5883 "ClientControlResponse::SupervisorRoutes",
5884 )?])
5885 }
5886
5887 async fn handle_supervisor_provenance(
5888 &self,
5889 frame: Frame,
5890 module_id: Option<String>,
5891 ) -> Result<Vec<Frame>, RouterError> {
5892 let mut selected = if let Some(module_id) = module_id {
5893 let Some(module) = self.supervisor.get(&module_id) else {
5894 return Ok(vec![control_error_frame(
5895 &frame,
5896 "unknown_module",
5897 format!("module_id '{module_id}' is not supervised"),
5898 )?]);
5899 };
5900 vec![module]
5901 } else {
5902 self.supervisor.list()
5903 };
5904
5905 let mut modules = Vec::with_capacity(selected.len());
5906 for module in selected.drain(..) {
5907 let (status, observed_image) = module
5908 .status_and_running_image_agreement()
5909 .await
5910 .map_err(|err| {
5911 RouterError::backend(
5912 0,
5913 frame.header.corr,
5914 format!("failed to read supervisor status: {err}"),
5915 )
5916 })?;
5917 let module_declared = self
5918 .registry
5919 .get_module(&status.module_id)
5920 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5921 .and_then(|registration| registration.manifest.provenance)
5922 .map(|build| ModuleDeclaredProvenance::Reported { build })
5923 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5924 #[cfg(test)]
5925 let running_image = match &self.provenance_probe_override {
5926 Some(result) => result.clone(),
5927 None => observed_image,
5928 };
5929 #[cfg(not(test))]
5930 let running_image = observed_image;
5931 modules.push(SupervisorModuleProvenance {
5932 module_id: status.module_id,
5933 module_declared,
5934 daemon_observed: SupervisorObservedProcess {
5935 launch_nonce_source: status.launch_nonce_source,
5936 pid: status.pid,
5937 spawned_at_ms: status.spawned_at_ms,
5938 spawned_from: status.spawned_from,
5939 running_image,
5940 },
5941 });
5942 }
5943 let daemon = SupervisorDaemonProvenance {
5944 daemon_build: self.daemon_provenance.build.clone(),
5945 daemon_observed: DaemonObservedProcess {
5946 pid: self.daemon_provenance.pid,
5947 started_at_ms: self
5948 .daemon_provenance
5949 .start_clock
5950 .map(|clock| clock.started_at_ms())
5951 .or(self.daemon_provenance.started_at_ms),
5952 running_image: self
5953 .daemon_provenance
5954 .probe
5955 .observe(
5956 self.daemon_provenance.pid,
5957 self.daemon_provenance.executable_path.as_deref(),
5958 self.daemon_provenance.executable_identity.clone(),
5959 self.daemon_provenance.process_start_time,
5960 )
5961 .await,
5962 },
5963 };
5964 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5965 Ok(vec![control_response_body_frame(
5966 &frame,
5967 &response,
5968 "ClientControlResponse::SupervisorProvenance",
5969 )?])
5970 }
5971
5972 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5973 self.refresh_capability_requirements();
5974 let generation = self
5975 .registry
5976 .generation()
5977 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5978 let modules = self
5979 .supervisor
5980 .list()
5981 .into_iter()
5982 .map(|module| {
5983 let status = module.status_for_control("health").map_err(|err| {
5984 RouterError::backend(
5985 0,
5986 frame.header.corr,
5987 format!("failed to read supervisor health: {err}"),
5988 )
5989 })?;
5990 let module_id = status.module_id;
5991 let capability_detail = self
5992 .capability_evaluator
5993 .required_problem_detail(&module_id);
5994 Ok(SupervisorHealthEntry {
5995 module_id,
5996 status: status.health.status,
5997 detail: append_capability_problem_detail(
5998 status.health.detail,
5999 capability_detail,
6000 ),
6001 metrics: status.health.metrics,
6002 consecutive_failures: status.health.consecutive_failures,
6003 late_answer_count: status.health.late_answer_count,
6004 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6005 last_action: status.health.last_action,
6006 last_action_ms: status.health.last_action_ms,
6007 last_probe_ms: status.health.last_probe_ms,
6008 })
6009 })
6010 .collect::<Result<Vec<_>, RouterError>>()?;
6011 let response = ClientControlResponse::SupervisorHealth {
6012 generation,
6013 modules,
6014 };
6015 Ok(vec![control_response_body_frame(
6016 &frame,
6017 &response,
6018 "ClientControlResponse::SupervisorHealth",
6019 )?])
6020 }
6021
6022 async fn handle_supervisor_restart(
6023 &self,
6024 frame: Frame,
6025 module_id: String,
6026 drain_timeout_ms: Option<u64>,
6027 requester: Principal,
6028 ) -> Result<Vec<Frame>, RouterError> {
6029 let operation_lock = self.supervisor.operation_lock();
6030 let _operation_guard = operation_lock.lock().await;
6031 let Some(module) = self.supervisor.get(&module_id) else {
6032 return Ok(vec![control_error_frame(
6033 &frame,
6034 "unknown_module",
6035 format!("module_id '{module_id}' is not supervised"),
6036 )?]);
6037 };
6038
6039 self.route_outages.mark_operator_action(&module_id);
6040 if let Err(err) = module
6041 .restart_requested(drain_timeout_ms, Some(requester))
6042 .await
6043 {
6044 self.route_outages
6045 .operator_action_ended_unrefused(&module_id);
6046 let (code, message) = match err {
6047 crate::supervise::SuperviseError::Disabled { .. } => {
6048 ("module_disabled", err.to_string())
6049 }
6050 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6051 ("swap_in_progress", err.to_string())
6052 }
6053 _ => (
6054 "target_unavailable",
6055 format!("failed to restart module_id '{module_id}': {err}"),
6056 ),
6057 };
6058 return Ok(vec![control_error_frame(&frame, code, message)?]);
6059 }
6060
6061 let response = ClientControlResponse::SupervisorAck {
6062 module_id,
6063 applied: true,
6064 };
6065 Ok(vec![control_response_body_frame(
6066 &frame,
6067 &response,
6068 "ClientControlResponse::SupervisorAck",
6069 )?])
6070 }
6071
6072 async fn handle_supervisor_swap(
6076 &self,
6077 frame: Frame,
6078 module_id: String,
6079 ready_timeout_ms: Option<u64>,
6080 requester: Principal,
6081 ) -> Result<Vec<Frame>, RouterError> {
6082 let module = {
6089 let operation_lock = self.supervisor.operation_lock();
6090 let _operation_guard = operation_lock.lock().await;
6091 self.supervisor.get(&module_id)
6092 };
6093 let Some(module) = module else {
6094 return Ok(vec![control_error_frame(
6095 &frame,
6096 "unknown_module",
6097 format!("module_id '{module_id}' is not supervised"),
6098 )?]);
6099 };
6100
6101 self.route_outages.mark_operator_action(&module_id);
6102 if let Err(err) = module
6103 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6104 .await
6105 {
6106 self.route_outages
6107 .operator_action_ended_unrefused(&module_id);
6108 use crate::supervise::SuperviseError;
6109 let message = err.to_string();
6110 let error = match err {
6111 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6112 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6113 code: "swap_refused".to_string(),
6114 message,
6115 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6116 },
6117 SuperviseError::SwapFailed {
6118 arm,
6119 candidate_exit,
6120 ..
6121 } => ErrorBody {
6122 code: "swap_failed".to_string(),
6123 message,
6124 detail: Some(serde_json::json!({
6125 "arm": arm.as_str(),
6126 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6127 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6128 })),
6129 },
6130 _ => ErrorBody::new(
6131 "target_unavailable",
6132 format!("failed to swap module_id '{module_id}': {message}"),
6133 ),
6134 };
6135 return Ok(vec![control_error_body_frame(&frame, error)?]);
6136 }
6137 self.route_outages
6141 .operator_action_ended_unrefused(&module_id);
6142
6143 let response = ClientControlResponse::SupervisorAck {
6144 module_id,
6145 applied: true,
6146 };
6147 Ok(vec![control_response_body_frame(
6148 &frame,
6149 &response,
6150 "ClientControlResponse::SupervisorAck",
6151 )?])
6152 }
6153
6154 async fn handle_supervisor_reload(
6155 &self,
6156 frame: Frame,
6157 module_id: String,
6158 requester: Principal,
6159 ) -> Result<Vec<Frame>, RouterError> {
6160 let operation_lock = self.supervisor.operation_lock();
6161 let _operation_guard = operation_lock.lock().await;
6162 let Some(module) = self.supervisor.get(&module_id) else {
6163 return Ok(vec![control_error_frame(
6164 &frame,
6165 "unknown_module",
6166 format!("module_id '{module_id}' is not supervised"),
6167 )?]);
6168 };
6169
6170 self.route_outages.mark_operator_action(&module_id);
6171 if let Err(err) = module.reload_requested(Some(requester)).await {
6172 self.route_outages
6173 .operator_action_ended_unrefused(&module_id);
6174 let (code, message) = match err {
6175 crate::supervise::SuperviseError::Disabled { .. } => {
6176 ("module_disabled", err.to_string())
6177 }
6178 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6179 ("swap_in_progress", err.to_string())
6180 }
6181 _ => (
6182 "reload_failed",
6183 format!("failed to reload module_id '{module_id}': {err}"),
6184 ),
6185 };
6186 return Ok(vec![control_error_frame(&frame, code, message)?]);
6187 }
6188
6189 let response = ClientControlResponse::SupervisorAck {
6190 module_id,
6191 applied: true,
6192 };
6193 Ok(vec![control_response_body_frame(
6194 &frame,
6195 &response,
6196 "ClientControlResponse::SupervisorAck",
6197 )?])
6198 }
6199
6200 async fn handle_supervisor_rescan(
6204 &self,
6205 frame: Frame,
6206 mode: RescanMode,
6207 requester: Principal,
6208 ) -> Result<Vec<Frame>, RouterError> {
6209 let preview = mode.previews();
6210 let Some(context) = self.rescan.clone() else {
6211 return Ok(vec![control_error_frame(
6212 &frame,
6213 "rescan_unavailable",
6214 "the daemon was not started with a reloadable config path".to_string(),
6215 )?]);
6216 };
6217
6218 let operation_lock = self.supervisor.operation_lock();
6219 let _operation_guard = operation_lock.lock().await;
6220 let loaded = match self.load_rescan_config(&mode, &context) {
6224 Ok(config) => config,
6225 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6226 };
6227 let Some((config, activity_config)) = loaded else {
6235 return Ok(vec![control_error_frame(
6236 &frame,
6237 "invalid_daemon_config",
6238 format!(
6239 "daemon config not found at {}; refusing to rescan (an absent config would \
6240 retire every supervised module)",
6241 context.config_path.display()
6242 ),
6243 )?]);
6244 };
6245 let (
6246 configured_port,
6247 storage_config,
6248 admission_facts_carrier_module_id,
6249 admission_facts_targets,
6250 scope_authority_owners,
6251 privileged_operator_prompt,
6252 modules,
6253 reserved_capabilities,
6254 ) = (
6255 config.port,
6256 config.storage,
6257 config.admission_facts_carrier_module_id,
6258 config.admission_facts_targets,
6259 config.scope_authority_owners,
6260 config.privileged_operator_prompt,
6261 config.modules,
6262 config.reserved_capabilities,
6263 );
6264
6265 let mut restart_required = Vec::new();
6274 for section in RestartRequiredSection::ALL {
6275 let changed = match section {
6276 RestartRequiredSection::Port => configured_port != context.configured_port,
6277 RestartRequiredSection::Storage => storage_config != context.storage_config,
6278 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6279 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6280 }
6281 RestartRequiredSection::AdmissionFactsTargets => {
6282 admission_facts_targets != context.admission_facts_targets
6283 }
6284 RestartRequiredSection::ScopeAuthorityOwners => {
6285 scope_authority_owners != context.scope_authority_owners
6286 }
6287 RestartRequiredSection::PrivilegedOperatorPrompt => {
6288 privileged_operator_prompt != context.privileged_operator_prompt
6289 }
6290 };
6291 if changed {
6292 restart_required.push(section.label().to_string());
6293 }
6294 }
6295 if !restart_required.is_empty() {
6296 warn!(
6297 config_path = %context.config_path.display(),
6298 sections = %restart_required.join(", "),
6299 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6300 );
6301 }
6302
6303 for configured in &modules {
6304 if let Err(err) = validate_spec(&configured.module_spec()) {
6305 return Ok(vec![control_error_frame(
6306 &frame,
6307 "invalid_daemon_config",
6308 format!("supervisor rescan rejected daemon config: {err}"),
6309 )?]);
6310 }
6311 }
6312
6313 let configured_capabilities = modules
6314 .iter()
6315 .map(|module| (module.module_id.clone(), module.enabled))
6316 .collect::<Vec<_>>();
6317 let preview_capability_warnings = if preview {
6318 let (_, registrations) = self.runtime_capability_snapshot()?;
6319 let current_modules = self
6320 .supervisor
6321 .list()
6322 .into_iter()
6323 .map(|module| module.module_id().to_string())
6324 .collect::<BTreeSet<_>>();
6325 let resulting_modules = configured_capabilities.clone();
6326 let removed = current_modules
6327 .into_iter()
6328 .filter(|module_id| {
6329 !resulting_modules
6330 .iter()
6331 .any(|(configured_id, _)| configured_id == module_id)
6332 })
6333 .collect::<Vec<_>>();
6334 let disabled = match &mode {
6337 RescanMode::Candidate(_) => resulting_modules
6338 .iter()
6339 .filter(|(_, enabled)| !enabled)
6340 .map(|(module_id, _)| module_id.clone())
6341 .collect::<Vec<_>>(),
6342 _ => Vec::new(),
6343 };
6344 self.capability_evaluator.preview_warnings(
6345 resulting_modules,
6346 &removed,
6347 &disabled,
6348 ®istrations,
6349 )
6350 } else {
6351 Vec::new()
6352 };
6353 let reconciled = match &mode {
6354 RescanMode::Candidate(candidate) => self
6355 .preview_candidate(&context, modules, candidate)
6356 .map(|result| (result, Vec::new())),
6357 _ => {
6358 self.reconcile_supervised_modules(
6359 &context.supervisor,
6360 modules,
6361 preview,
6362 requester.clone(),
6363 )
6364 .await
6365 }
6366 };
6367 let (result, changed) = match reconciled {
6368 Ok(result) => result,
6369 Err(message) => {
6370 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6371 }
6372 };
6373 if !preview {
6374 if let Some(ledger) = &self.activity {
6375 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6376 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6377 }
6378 if let Err(code) = self.configure_activity(activity_config).await {
6379 warn!(
6380 code,
6381 "activity retention could not be applied during rescan"
6382 );
6383 }
6384 self.capability_evaluator
6385 .configure(configured_capabilities, reserved_capabilities);
6386 self.capability_evaluator.wake_deadline_loop();
6387 self.refresh_capability_requirements();
6388 }
6389 let mut result = result;
6390 result.restart_required = restart_required;
6391 result.capability_warnings = preview_capability_warnings;
6392 let response = ClientControlResponse::SupervisorRescan { result };
6393 Ok(vec![control_response_body_frame(
6394 &frame,
6395 &response,
6396 "ClientControlResponse::SupervisorRescan",
6397 )?])
6398 }
6399
6400 async fn handle_supervisor_release_reserved(
6401 &self,
6402 frame: Frame,
6403 module_id: String,
6404 ) -> Result<Vec<Frame>, RouterError> {
6405 let Some(context) = self.rescan.clone() else {
6406 return Ok(vec![control_error_frame(
6407 &frame,
6408 "release_unavailable",
6409 "reserved-id release requires a daemon started with a reloadable config path",
6410 )?]);
6411 };
6412 let operation_lock = self.supervisor.operation_lock();
6413 let _operation_guard = operation_lock.lock().await;
6414 let loaded = match crate::daemon_config::load(&context.config_path) {
6415 Ok(Some(config)) => config,
6416 Ok(None) => {
6417 return Ok(vec![control_error_frame(
6418 &frame,
6419 "invalid_daemon_config",
6420 format!(
6421 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6422 context.config_path.display()
6423 ),
6424 )?])
6425 }
6426 Err(err) => {
6427 return Ok(vec![control_error_frame(
6428 &frame,
6429 "invalid_daemon_config",
6430 format!("unable to verify reserved-id release against daemon config: {err}"),
6431 )?])
6432 }
6433 };
6434 if loaded
6435 .modules
6436 .iter()
6437 .any(|configured| configured.module_id == module_id)
6438 {
6439 return Ok(vec![control_error_frame(
6440 &frame,
6441 "reserved_module_configured",
6442 format!(
6443 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6444 ),
6445 )?]);
6446 }
6447 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6448 return Ok(vec![control_error_frame(
6449 &frame,
6450 "reserved_gate_not_retained",
6451 format!(
6452 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6453 ),
6454 )?]);
6455 }
6456
6457 let response = ClientControlResponse::SupervisorAck {
6458 module_id,
6459 applied: true,
6460 };
6461 Ok(vec![control_response_body_frame(
6462 &frame,
6463 &response,
6464 "ClientControlResponse::SupervisorAck",
6465 )?])
6466 }
6467
6468 fn plan_rescan(
6472 &self,
6473 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6474 ) -> Result<RescanPlan, String> {
6475 let mut current = BTreeMap::new();
6476 for module in self.supervisor.list() {
6477 let (spec, health) = module.configuration().map_err(|err| {
6478 format!(
6479 "failed to read configuration for module_id '{}': {err}",
6480 module.module_id()
6481 )
6482 })?;
6483 let enabled = module
6484 .status()
6485 .map_err(|err| {
6486 format!(
6487 "failed to read status for module_id '{}': {err}",
6488 module.module_id()
6489 )
6490 })?
6491 .enabled;
6492 current.insert(
6493 module.module_id().to_string(),
6494 (module, spec, health, enabled),
6495 );
6496 }
6497 let configured = configured_modules
6498 .into_iter()
6499 .map(|module| (module.module_id.clone(), module))
6500 .collect::<BTreeMap<_, _>>();
6501
6502 let added = configured
6503 .keys()
6504 .filter(|module_id| !current.contains_key(*module_id))
6505 .cloned()
6506 .collect::<Vec<_>>();
6507 let removed = current
6508 .keys()
6509 .filter(|module_id| !configured.contains_key(*module_id))
6510 .cloned()
6511 .collect::<Vec<_>>();
6512 let mut changed_pending_reload = Vec::new();
6513 let mut configuration_changes = BTreeSet::new();
6514 let mut enabled_changes = BTreeSet::new();
6515 let mut unchanged = 0_u32;
6516
6517 for (module_id, configured_module) in &configured {
6518 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6519 else {
6520 continue;
6521 };
6522 let launch_changed = *current_spec != configured_module.module_spec();
6526 let configuration_changed =
6527 launch_changed || *current_health != configured_module.health;
6528 let enabled_changed = *current_enabled != configured_module.enabled;
6529 if configuration_changed {
6530 configuration_changes.insert(module_id.clone());
6531 }
6532 if launch_changed {
6533 changed_pending_reload.push(module_id.clone());
6534 }
6535 if enabled_changed {
6536 enabled_changes.insert(module_id.clone());
6537 }
6538 if !configuration_changed && !enabled_changed {
6539 unchanged = unchanged.saturating_add(1);
6540 }
6541 }
6542 Ok(RescanPlan {
6543 current,
6544 configured,
6545 added,
6546 removed,
6547 changed_pending_reload,
6548 configuration_changes,
6549 enabled_changes,
6550 unchanged,
6551 })
6552 }
6553
6554 async fn reconcile_supervised_modules(
6563 &self,
6564 supervisor: &Supervisor,
6565 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6566 preview: bool,
6567 requester: Principal,
6568 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6569 let plan = self.plan_rescan(configured_modules)?;
6570
6571 if preview {
6577 return Ok((plan.preview_result(), Vec::new()));
6578 }
6579 let changed = plan
6580 .configuration_changes
6581 .union(&plan.enabled_changes)
6582 .cloned()
6583 .collect::<Vec<_>>();
6584 let RescanPlan {
6585 current,
6586 configured,
6587 added,
6588 removed,
6589 changed_pending_reload,
6590 configuration_changes,
6591 enabled_changes,
6592 unchanged,
6593 } = plan;
6594 if !(added.is_empty()
6599 && removed.is_empty()
6600 && configuration_changes.is_empty()
6601 && enabled_changes.is_empty())
6602 {
6603 if let Some(context) = &self.rescan {
6604 context
6605 .generation
6606 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6607 }
6608 }
6609
6610 for module_id in &removed {
6611 let module = ¤t
6612 .get(module_id)
6613 .expect("removed module came from current supervisor state")
6614 .0;
6615 module
6616 .retire_requested(Some(requester.clone()))
6617 .await
6618 .map_err(|err| {
6619 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6620 })?;
6621 self.supervisor.record_rescan_removal(module_id);
6655 self.supervisor.retire(module_id);
6656 self.route_outages.forget(module_id);
6657 }
6658
6659 for module_id in configured.keys() {
6660 let Some((module, _, _, _)) = current.get(module_id) else {
6661 continue;
6662 };
6663 let configured_module = configured
6664 .get(module_id)
6665 .expect("configured module id came from configured map");
6666 if configuration_changes.contains(module_id) {
6667 module
6668 .update_configuration(
6669 configured_module.module_spec(),
6670 configured_module.health.clone(),
6671 configured_module.drain_timeout_ms,
6672 )
6673 .await
6674 .map_err(|err| {
6675 format!(
6676 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6677 )
6678 })?;
6679 }
6680 if enabled_changes.contains(module_id) {
6681 self.route_outages.mark_operator_action(module_id);
6684 module
6685 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6686 .await
6687 .map_err(|err| {
6688 self.route_outages.operator_action_ended_unrefused(module_id);
6689 format!(
6690 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6691 configured_module.enabled
6692 )
6693 })?;
6694 }
6695 }
6696
6697 for module_id in &added {
6698 let configured_module = configured
6699 .get(module_id)
6700 .expect("added module id came from configured map");
6701 supervisor
6702 .supervise_configured_requested(
6703 configured_module.module_spec(),
6704 configured_module.enabled,
6705 configured_module.health.clone(),
6706 configured_module.drain_timeout_ms,
6707 configured_module.restart,
6708 configured_module.register_timeout,
6709 Some(requester.clone()),
6710 )
6711 .map_err(|err| {
6712 format!("failed to add module_id '{module_id}' during rescan: {err}")
6713 })?;
6714 }
6715
6716 Ok((
6717 SupervisorRescanResult {
6718 added,
6719 removed,
6720 changed_pending_reload,
6721 enabled_changes: enabled_changes.iter().cloned().collect(),
6722 unchanged,
6723 preview: false,
6724 restart_required: Vec::new(),
6727 capability_warnings: Vec::new(),
6728 module_actions: Vec::new(),
6729 earlier_pending: Vec::new(),
6730 expect: None,
6731 },
6732 changed,
6733 ))
6734 }
6735
6736 async fn handle_supervisor_set_enabled(
6737 &self,
6738 frame: Frame,
6739 module_id: String,
6740 enabled: bool,
6741 requester: Principal,
6742 ) -> Result<Vec<Frame>, RouterError> {
6743 let operation_lock = self.supervisor.operation_lock();
6744 let _operation_guard = operation_lock.lock().await;
6745 let Some(module) = self.supervisor.get(&module_id) else {
6746 return Ok(vec![control_error_frame(
6747 &frame,
6748 "unknown_module",
6749 format!("module_id '{module_id}' is not supervised"),
6750 )?]);
6751 };
6752
6753 self.route_outages.mark_operator_action(&module_id);
6756 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6757 Ok(applied) => applied,
6758 Err(err) => {
6759 self.route_outages
6760 .operator_action_ended_unrefused(&module_id);
6761 return Ok(vec![control_error_frame(
6762 &frame,
6763 "target_unavailable",
6764 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6765 )?]);
6766 }
6767 };
6768 if !applied {
6769 self.route_outages
6772 .operator_action_ended_unrefused(&module_id);
6773 }
6774
6775 self.capability_evaluator.wake_deadline_loop();
6776 self.refresh_capability_requirements();
6777 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6778 Ok(vec![control_response_body_frame(
6779 &frame,
6780 &response,
6781 "ClientControlResponse::SupervisorAck",
6782 )?])
6783 }
6784
6785 async fn handle_supervisor_health_probe(
6786 &self,
6787 frame: Frame,
6788 module_id: String,
6789 ) -> Result<Vec<Frame>, RouterError> {
6790 self.refresh_capability_requirements();
6791 let Some(registration) = self
6792 .registry
6793 .get_module(&module_id)
6794 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6795 else {
6796 return Ok(vec![control_error_frame(
6797 &frame,
6798 "unknown_module",
6799 format!("module_id '{module_id}' is not registered"),
6800 )?]);
6801 };
6802
6803 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6816 {
6817 return Ok(vec![control_error_frame(
6818 &frame,
6819 "health_not_advertised",
6820 format!("module_id '{module_id}' did not advertise health.check"),
6821 )?]);
6822 }
6823
6824 let deadline = Instant::now() + self.health_probe_timeout;
6825 let pending = match self.forwarding.begin_module_control_rpc_for(
6826 &module_id,
6827 MODULE_CONTROL_OP_HEALTH_CHECK,
6828 deadline,
6829 ) {
6830 Ok(pending) => pending,
6831 Err(err) => {
6832 return Ok(vec![control_error_frame(
6833 &frame,
6834 forwarding_error_code(&err),
6835 err.to_string(),
6836 )?])
6837 }
6838 };
6839
6840 let PendingModuleControlRpc {
6841 endpoint,
6842 module_sink,
6843 negotiated_ver,
6844 corr: probe_corr,
6845 receiver,
6846 } = pending;
6847 let mut guard =
6848 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6849 let probe_body =
6850 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6851 RouterError::backend(
6852 0,
6853 frame.header.corr,
6854 format!("failed to encode health.check request: {err}"),
6855 )
6856 })?;
6857 let probe_frame = Frame::build_with_version(
6858 negotiated_ver,
6859 FrameType::Request,
6860 control_flags(),
6861 0,
6862 0,
6863 probe_corr,
6864 probe_body,
6865 )
6866 .map_err(RouterError::FrameBuild)?;
6867
6868 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6871 Ok(Ok(())) => {}
6872 Ok(Err(err)) => {
6873 return Ok(vec![control_error_frame(
6874 &frame,
6875 "target_unavailable",
6876 err.to_string(),
6877 )?]);
6878 }
6879 Err(_) => {
6880 return Ok(vec![control_error_frame(
6881 &frame,
6882 "module_timeout",
6883 format!(
6884 "module_id '{module_id}' did not answer health.check within {:?}",
6885 self.health_probe_timeout
6886 ),
6887 )?]);
6888 }
6889 }
6890
6891 match timeout_at(deadline, receiver).await {
6892 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6893 guard.disarm();
6894 let Some(report) = response.health_report() else {
6895 return Ok(vec![control_error_frame(
6896 &frame,
6897 "invalid_control_body",
6898 "health.check RPC returned a non-health response",
6899 )?]);
6900 };
6901 let HealthReport {
6906 status,
6907 detail,
6908 metrics,
6909 } = report;
6910 let capability_detail = self
6911 .capability_evaluator
6912 .required_problem_detail(&module_id);
6913 let response = ClientControlResponse::SupervisorHealthProbe {
6914 module_id,
6915 status,
6916 detail: append_capability_problem_detail(detail, capability_detail),
6917 metrics,
6918 };
6919 Ok(vec![control_response_body_frame(
6920 &frame,
6921 &response,
6922 "ClientControlResponse::SupervisorHealthProbe",
6923 )?])
6924 }
6925 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6926 guard.disarm();
6927 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
6930 }
6931 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6932 guard.disarm();
6933 Ok(vec![control_error_frame(
6934 &frame,
6935 "target_unavailable",
6936 message,
6937 )?])
6938 }
6939 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6940 guard.disarm();
6941 Ok(vec![control_error_frame(
6942 &frame,
6943 "invalid_control_body",
6944 message,
6945 )?])
6946 }
6947 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6948 guard.disarm();
6949 Ok(vec![control_error_frame(
6950 &frame,
6951 "invalid_control_body",
6952 format!("expected module-control op '{expected}', got '{actual}'"),
6953 )?])
6954 }
6955 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6956 guard.disarm();
6957 Ok(vec![control_error_frame(
6958 &frame,
6959 "module_timeout",
6960 format!(
6961 "module_id '{module_id}' answered health.check after {:?}",
6962 self.health_probe_timeout
6963 ),
6964 )?])
6965 }
6966 Ok(Err(_)) => Ok(vec![control_error_frame(
6967 &frame,
6968 "target_unavailable",
6969 "health.check waiter was canceled before the module responded",
6970 )?]),
6971 Err(_) => Ok(vec![control_error_frame(
6972 &frame,
6973 "module_timeout",
6974 format!(
6975 "module_id '{module_id}' did not answer health.check within {:?}",
6976 self.health_probe_timeout
6977 ),
6978 )?]),
6979 }
6980 }
6981
6982 fn supervisor_status(
6983 &self,
6984 module_id: &str,
6985 corr: u64,
6986 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6987 self.supervisor
6988 .get(module_id)
6989 .map(|module| {
6990 let warming = module.is_warming_for_control("status").map_err(|err| {
6991 RouterError::backend(
6992 0,
6993 corr,
6994 format!(
6995 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6996 ),
6997 )
6998 })?;
6999 module.status_for_control("status").map_err(|err| {
7000 RouterError::backend(
7001 0,
7002 corr,
7003 format!(
7004 "failed to read supervisor status for module_id '{module_id}': {err}"
7005 ),
7006 )
7007 }).map(|status| (status, warming))
7008 })
7009 .transpose()
7010 }
7011
7012 fn guard_module_control_op(
7013 &self,
7014 frame: &Frame,
7015 module_id: &str,
7016 op: &str,
7017 ) -> Result<Option<Frame>, RouterError> {
7018 if self.module_grants_op(module_id, op, frame.header.corr)? {
7019 return Ok(None);
7020 }
7021
7022 Ok(Some(control_error_frame(
7023 frame,
7024 "op_not_allowed",
7025 format!("module_id '{module_id}' did not grant control op '{op}'"),
7026 )?))
7027 }
7028
7029 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7030 let Some(registration) = self
7031 .registry
7032 .get_module(module_id)
7033 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7034 else {
7035 return Ok(false);
7036 };
7037 Ok(module_registration_grants_op(®istration.control_ops, op))
7038 }
7039
7040 fn handle_status_update(
7041 &self,
7042 endpoint: ModuleEndpointId,
7043 frame: Frame,
7044 ) -> Result<Vec<Frame>, RouterError> {
7045 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7046 Ok(update) => update,
7047 Err(err) => {
7048 if is_known_module_push_op(&frame.body) {
7053 return Ok(vec![control_error_frame(
7054 &frame,
7055 "invalid_control_body",
7056 format!("malformed module control push body: {err}"),
7057 )?]);
7058 }
7059 return Ok(Vec::new());
7060 }
7061 };
7062
7063 match update {
7064 ModuleControlPush::RouteStatus {
7065 route_channel,
7066 route_epoch,
7067 status,
7068 } => {
7069 self.forwarding
7070 .cache_status(endpoint, route_channel, route_epoch, status)
7071 .map_err(RouterError::Forwarding)?;
7072 }
7073 }
7074 Ok(Vec::new())
7075 }
7076
7077 fn handle_route_poll(
7078 &self,
7079 ctx: &RouteCtx,
7080 frame: Frame,
7081 route_channel: u16,
7082 route_epoch: u32,
7083 kind: PollKind,
7084 ) -> Result<Vec<Frame>, RouterError> {
7085 let snapshot = self
7086 .forwarding
7087 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7088 .map_err(RouterError::Forwarding)?;
7089 let response = match (kind, snapshot) {
7090 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7091 ClientControlResponse::RoutePoll {
7092 route_channel,
7093 route_epoch,
7094 status,
7095 live: None,
7096 }
7097 }
7098 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7099 route_channel,
7100 route_epoch,
7101 status: None,
7102 live: None,
7103 },
7104 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7105 let live = self
7131 .process_liveness
7132 .as_ref()
7133 .and_then(|source| source.process_live(&module_id))
7134 .unwrap_or(true);
7135 ClientControlResponse::RoutePoll {
7136 route_channel,
7137 route_epoch,
7138 status: None,
7139 live: Some(live),
7140 }
7141 }
7142 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7143 route_channel,
7144 route_epoch,
7145 status: None,
7146 live: Some(false),
7147 },
7148 };
7149
7150 Ok(vec![control_response_body_frame(
7151 &frame,
7152 &response,
7153 "ClientControlResponse::RoutePoll",
7154 )?])
7155 }
7156
7157 pub(crate) fn observe_module_control_completion(
7158 &self,
7159 completion: ModuleControlRpcCompletion,
7160 ) -> bool {
7161 match completion {
7162 ModuleControlRpcCompletion::Unknown => false,
7163 ModuleControlRpcCompletion::Settled => true,
7164 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7165 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7166 info!(
7167 module_id = %module_id,
7168 latency_ms,
7169 "late health.check answer proves the module is alive"
7170 );
7171 match self
7172 .supervisor
7173 .record_late_health_answer(&module_id, latency_ms)
7174 {
7175 Ok(true) => {}
7176 Ok(false) => debug!(
7177 module_id = %module_id,
7178 latency_ms,
7179 "late health.check answer has no active supervisor snapshot"
7180 ),
7181 Err(err) => warn!(
7182 module_id = %module_id,
7183 latency_ms,
7184 error = %err,
7185 "failed to record late health.check answer"
7186 ),
7187 }
7188 true
7189 }
7190 }
7191 }
7192
7193 fn refuse_to_end_module_connection_for_a_client(
7214 &self,
7215 module_connection_id: ConnectionId,
7216 corr: u64,
7217 err: ForwardingError,
7218 ) -> Result<(), RouterError> {
7219 if let ForwardingError::ConnectionClosing { connection_id } = err {
7220 if connection_id != module_connection_id {
7221 warn!(
7222 module_connection_id = module_connection_id.get(),
7223 client_connection_id = connection_id.get(),
7224 corr,
7225 "dropping a route.bind response for a closing client; the module connection keeps serving"
7226 );
7227 return Ok(());
7228 }
7229 }
7230 Err(RouterError::Forwarding(err))
7231 }
7232
7233 fn handle_module_relay_response(
7234 &self,
7235 connection_id: ConnectionId,
7236 frame: Frame,
7237 ) -> Result<Vec<Frame>, RouterError> {
7238 let mut secondary_error = None;
7239 let outcome = match frame.header.ty {
7240 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7241 Ok(probe) if probe.op == "route.bind" => {
7242 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7243 Ok(ModuleControlResponse::RouteBindAck {}) => {
7244 RouteBindRelayOutcome::Accepted
7245 }
7246 Ok(other) => {
7247 let message =
7248 format!("route.bind response carried unexpected body: {other:?}");
7249 secondary_error = Some(control_error_frame(
7250 &frame,
7251 "invalid_control_body",
7252 message.clone(),
7253 )?);
7254 RouteBindRelayOutcome::ModuleGone(message)
7255 }
7256 Err(err) => {
7257 let message = format!("malformed route.bind response body: {err}");
7258 secondary_error = Some(control_error_frame(
7259 &frame,
7260 "invalid_control_body",
7261 message.clone(),
7262 )?);
7263 RouteBindRelayOutcome::ModuleGone(message)
7264 }
7265 }
7266 }
7267 Ok(probe) => {
7268 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7269 {
7270 Ok(response) => ModuleControlRpcOutcome::Response(response),
7271 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7272 "malformed {} response body: {err}",
7273 probe.op
7274 )),
7275 };
7276 let completion = self
7277 .forwarding
7278 .complete_module_control_rpc(
7279 connection_id,
7280 frame.header.corr,
7281 Some(&probe.op),
7282 outcome,
7283 )
7284 .map_err(RouterError::Forwarding)?;
7285 if !self.observe_module_control_completion(completion) {
7286 debug!(
7287 connection_id = connection_id.get(),
7288 corr = frame.header.corr,
7289 op = %probe.op,
7290 "dropping late or unknown module-control RPC response"
7291 );
7292 }
7293 return Ok(Vec::new());
7294 }
7295 Err(err) => {
7296 if let Some(expected_op) = self
7297 .forwarding
7298 .pending_module_control_op(connection_id, frame.header.corr)
7299 .map_err(RouterError::Forwarding)?
7300 {
7301 let completion = self
7302 .forwarding
7303 .complete_module_control_rpc(
7304 connection_id,
7305 frame.header.corr,
7306 None,
7307 ModuleControlRpcOutcome::MalformedResponse(format!(
7308 "malformed {expected_op} response body: {err}"
7309 )),
7310 )
7311 .map_err(RouterError::Forwarding)?;
7312 if !self.observe_module_control_completion(completion) {
7313 debug!(
7314 connection_id = connection_id.get(),
7315 corr = frame.header.corr,
7316 "dropping late malformed module-control RPC response"
7317 );
7318 }
7319 return Ok(Vec::new());
7320 }
7321 let message = format!("malformed route.bind response body: {err}");
7322 secondary_error = Some(control_error_frame(
7323 &frame,
7324 "invalid_control_body",
7325 message.clone(),
7326 )?);
7327 RouteBindRelayOutcome::ModuleGone(message)
7328 }
7329 },
7330 FrameType::Error => {
7331 if self
7332 .forwarding
7333 .pending_module_control_op(connection_id, frame.header.corr)
7334 .map_err(RouterError::Forwarding)?
7335 .is_some()
7336 {
7337 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7338 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7339 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7340 "malformed module-control ERROR body: {err}"
7341 )),
7342 };
7343 let completion = self
7344 .forwarding
7345 .complete_module_control_rpc(
7346 connection_id,
7347 frame.header.corr,
7348 None,
7349 outcome,
7350 )
7351 .map_err(RouterError::Forwarding)?;
7352 if !self.observe_module_control_completion(completion) {
7353 debug!(
7354 connection_id = connection_id.get(),
7355 corr = frame.header.corr,
7356 "dropping late or unknown module-control RPC error"
7357 );
7358 }
7359 return Ok(Vec::new());
7360 }
7361 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7362 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7363 Err(err) => {
7364 let message = format!("malformed route.bind ERROR body: {err}");
7365 secondary_error = Some(control_error_frame(
7366 &frame,
7367 "invalid_control_body",
7368 message.clone(),
7369 )?);
7370 RouteBindRelayOutcome::ModuleGone(message)
7371 }
7372 }
7373 }
7374 ty => {
7375 return Ok(vec![control_error_frame(
7376 &frame,
7377 "unsupported_control_frame",
7378 format!("unsupported module channel-0 frame {ty:?}"),
7379 )?])
7380 }
7381 };
7382
7383 let settled =
7384 self.forwarding
7385 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7386 let completion = match settled {
7387 Ok(completion) => completion,
7388 Err(err) => {
7389 self.refuse_to_end_module_connection_for_a_client(
7390 connection_id,
7391 frame.header.corr,
7392 err,
7393 )?;
7394 return Ok(secondary_error.into_iter().collect());
7395 }
7396 };
7397 if let Some(target) = completion.abandoned.as_ref() {
7398 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7399 }
7400 if !completion.settled {
7401 debug!(
7402 connection_id = connection_id.get(),
7403 corr = frame.header.corr,
7404 frame_type = ?frame.header.ty,
7405 "dropping late or unknown route.bind relay response"
7406 );
7407 }
7408 Ok(secondary_error.into_iter().collect())
7409 }
7410
7411 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7412 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7413 self.cleanup_connection_with_end_reason(
7417 connection_id,
7418 RegistrationEndReason::ExplicitGoodbye,
7419 )
7420 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7421 Ok(Vec::new())
7422 }
7423}
7424
7425impl Default for ControlHandler {
7426 fn default() -> Self {
7427 Self::new(Arc::new(Registry::default()))
7428 }
7429}
7430
7431impl crate::supervise::SwapPromotionObserver for ControlHandler {
7432 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7433 self.apply_registration_capabilities(registration);
7434 }
7435}
7436
7437fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7438 CapabilityRequirementStatus {
7439 consumer: status.consumer,
7440 capability: status.capability,
7441 need: match status.need {
7442 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7443 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7444 },
7445 verdict: status.verdict.as_str().to_string(),
7446 episode_seq: status.episode_seq,
7447 config_satisfiable: status.config_satisfiable,
7448 runtime_available: status.runtime_available,
7449 detail: status.detail,
7450 }
7451}
7452
7453fn append_capability_problem_detail(
7454 detail: Option<String>,
7455 capability_detail: Option<String>,
7456) -> Option<String> {
7457 match (detail, capability_detail) {
7458 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7459 (Some(detail), None) => Some(detail),
7460 (None, Some(capability_detail)) => Some(capability_detail),
7461 (None, None) => None,
7462 }
7463}
7464
7465fn subc_ops() -> Vec<String> {
7466 SUBC_CONTROL_OPS
7467 .iter()
7468 .map(|op| (*op).to_string())
7469 .collect()
7470}
7471
7472fn module_subc_ops() -> Vec<String> {
7473 SUBC_CONTROL_OPS
7474 .iter()
7475 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7476 .map(|op| (*op).to_string())
7477 .collect()
7478}
7479
7480#[cfg(test)]
7481fn module_baseline_control_ops() -> Vec<String> {
7482 MODULE_BASELINE_CONTROL_OPS
7483 .iter()
7484 .map(|op| (*op).to_string())
7485 .collect()
7486}
7487
7488fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7489 let mut seen = HashSet::new();
7490 let mut effective = Vec::new();
7491 for op in MODULE_BASELINE_CONTROL_OPS {
7492 if seen.insert((*op).to_string()) {
7493 effective.push((*op).to_string());
7494 }
7495 }
7496 for op in declared.unwrap_or_default() {
7497 if seen.insert(op.clone()) {
7498 effective.push(op);
7499 }
7500 }
7501 effective
7502}
7503
7504fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7505 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7506}
7507
7508fn target_module_id(target: &RouteTarget) -> &str {
7509 match target {
7510 RouteTarget::ToolProvider { module_id }
7511 | RouteTarget::ManagementSurface { module_id }
7512 | RouteTarget::InternalService { module_id, .. } => module_id,
7513 }
7514}
7515
7516fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7517 roles.iter().any(|role| match (target, role) {
7518 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7519 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7520 (
7521 RouteTarget::InternalService { service_id, .. },
7522 ProviderRole::InternalService {
7523 service_id: provided,
7524 ..
7525 },
7526 ) => service_id == provided,
7527 _ => false,
7528 })
7529}
7530
7531fn is_routable_role(role: &ProviderRole) -> bool {
7532 matches!(
7533 role,
7534 ProviderRole::ToolProvider { .. }
7535 | ProviderRole::ManagementSurface { .. }
7536 | ProviderRole::InternalService { .. }
7537 )
7538}
7539
7540#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7541enum ControlRequestBodyError {
7542 UnknownOp,
7543 InvalidBody,
7544}
7545
7546#[derive(Debug, Deserialize)]
7547struct ControlOpProbe {
7548 op: String,
7549}
7550
7551const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7554
7555fn is_known_module_push_op(body: &[u8]) -> bool {
7556 serde_json::from_slice::<ControlOpProbe>(body)
7557 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7558 .unwrap_or(false)
7559}
7560
7561fn is_known_module_request_op(body: &[u8]) -> bool {
7562 serde_json::from_slice::<ControlOpProbe>(body)
7563 .map(|probe| is_module_to_subc_op(&probe.op))
7564 .unwrap_or(false)
7565}
7566
7567fn is_module_to_subc_op(op: &str) -> bool {
7568 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7569}
7570
7571fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7572 debug!(
7573 op = %op,
7574 connection_id = connection_id.get(),
7575 corr,
7576 "control dispatch"
7577 );
7578}
7579
7580fn log_slow_control_dispatch(
7581 dispatch_started_at: Option<StdInstant>,
7582 op: &'static str,
7583 connection_id: ConnectionId,
7584 corr: u64,
7585 route_open_timings: Option<&RouteOpenTimings>,
7586) {
7587 let Some(dispatch_started_at) = dispatch_started_at else {
7588 return;
7589 };
7590 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7592 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7593 if let Some(timings) = route_open_timings {
7594 warn!(
7595 op = %op,
7596 connection_id = connection_id.get(),
7597 corr,
7598 elapsed_ms = elapsed.as_millis() as u64,
7599 module_id = ?timings.module_id,
7600 principal = %timings.principal,
7601 admission_ms = timings.admission.as_millis() as u64,
7602 project_root_ms = timings.project_root.as_millis() as u64,
7603 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7604 bind_send_ms = timings.bind_send.as_millis() as u64,
7605 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7606 "slow control dispatch"
7607 );
7608 return;
7609 }
7610 warn!(
7611 op = %op,
7612 connection_id = connection_id.get(),
7613 corr,
7614 elapsed_ms = elapsed.as_millis() as u64,
7615 "slow control dispatch"
7616 );
7617 }
7618}
7619
7620#[cfg(test)]
7622type TestCallerLookup =
7623 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7624
7625fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7627 match request {
7628 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7629 Some(PrivilegedOperation::Restart { module_id })
7630 }
7631 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7632 Some(PrivilegedOperation::Swap { module_id })
7633 }
7634 ClientControlRequest::SupervisorReload { module_id } => {
7635 Some(PrivilegedOperation::Reload { module_id })
7636 }
7637 ClientControlRequest::SupervisorRescan { preview } => {
7638 if *preview {
7639 None
7640 } else {
7641 Some(PrivilegedOperation::Rescan)
7642 }
7643 }
7644 ClientControlRequest::SupervisorRescanChecked {
7649 preview,
7650 candidate,
7651 expect,
7652 } => (!*preview && candidate.is_none() && expect.is_some())
7653 .then_some(PrivilegedOperation::Rescan),
7654 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7655 Some(PrivilegedOperation::SetEnabled {
7656 module_id,
7657 enabled: *enabled,
7658 })
7659 }
7660 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7661 Some(PrivilegedOperation::ReleaseReserved { module_id })
7662 }
7663 ClientControlRequest::ServerDescribe {}
7664 | ClientControlRequest::CatalogList { .. }
7665 | ClientControlRequest::RouteOpen { .. }
7666 | ClientControlRequest::RoutePoll { .. }
7667 | ClientControlRequest::SupervisorList {}
7668 | ClientControlRequest::SupervisorHealth {}
7669 | ClientControlRequest::SupervisorHealthProbe { .. }
7670 | ClientControlRequest::SupervisorRoutes { .. }
7671 | ClientControlRequest::SupervisorProvenance { .. }
7672 | ClientControlRequest::SupervisorStderrTail { .. }
7673 | ClientControlRequest::SupervisorTerminals { .. }
7674 | ClientControlRequest::SupervisorSpawnSnapshot {}
7675 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7676 }
7677}
7678
7679fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7680 match request {
7681 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7682 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7683 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7684 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7685 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7686 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7687 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7688 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7689 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7690 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7691 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7692 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7693 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7694 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7695 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7696 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7697 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7698 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7699 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7700 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7701 }
7702}
7703
7704struct RouteOpenCaller {
7712 principal: String,
7716 scope: String,
7719 harness: String,
7722}
7723
7724impl RouteOpenCaller {
7725 fn new(
7726 principal: Option<&Principal>,
7727 scope: Option<&ScopeSelector>,
7728 identity: &BindIdentity,
7729 ) -> Self {
7730 Self {
7731 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7732 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7733 harness: bounded_log_text(&identity.harness),
7734 }
7735 }
7736
7737 fn unknown() -> Self {
7740 Self {
7741 principal: "unknown".to_string(),
7742 scope: "unknown".to_string(),
7743 harness: "unknown".to_string(),
7744 }
7745 }
7746}
7747
7748const MAX_LOGGED_TEXT_CHARS: usize = 64;
7751
7752fn bounded_log_text(value: &str) -> String {
7756 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7757 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7758 bounded.push('…');
7759 }
7760 format!("{bounded:?}")
7761}
7762
7763fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7769 use subc_protocol::scope::ScopeKind;
7770 let kind = match stamp.kind {
7771 ScopeKind::Head => "head",
7772 ScopeKind::Worker => "worker",
7773 ScopeKind::Ephemeral => "ephemeral",
7774 };
7775 format!(
7776 "{kind}/{}/{}@{}",
7777 principal_label(&stamp.owner),
7778 bounded_log_text(&stamp.scope_ref),
7779 stamp.scope_epoch
7780 )
7781}
7782
7783fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7790 let epoch = selector
7791 .scope_epoch
7792 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7793 format!(
7794 "requested/{}/{}@{epoch}",
7795 bounded_log_text(&principal_label(&selector.owner)),
7796 bounded_log_text(&selector.scope_ref),
7797 )
7798}
7799
7800fn principal_label(principal: &Principal) -> String {
7801 match principal {
7802 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7803 Principal::Direct => "direct".to_string(),
7804 other => format!("{other:?}"),
7805 }
7806}
7807
7808fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7809 match request {
7810 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7811 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7812 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7813 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7814 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7815 }
7816}
7817
7818fn parse_client_control_request(
7819 body: &[u8],
7820) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7821 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7822 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7823 Ok(probe)
7824 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7825 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7826 {
7827 ControlRequestBodyError::InvalidBody
7828 }
7829 Ok(_) => ControlRequestBodyError::UnknownOp,
7830 Err(_) => ControlRequestBodyError::InvalidBody,
7831 };
7832 (err, classification)
7833 })
7834}
7835
7836fn parse_module_control_request_from_module(
7837 body: &[u8],
7838) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7839 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7840 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7841 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7842 Ok(_) => ControlRequestBodyError::UnknownOp,
7843 Err(_) => ControlRequestBodyError::InvalidBody,
7844 };
7845 (err, classification)
7846 })
7847}
7848
7849#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7850enum ProviderRoleKind {
7851 ToolProvider,
7852 PipelineStage,
7853 ManagementSurface,
7854 InternalService,
7855}
7856
7857fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7858 match role {
7859 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7860 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7861 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7862 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7863 }
7864}
7865
7866fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7867 roles.iter().map(provider_role_kind).collect()
7868}
7869
7870const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7873
7874#[derive(Debug, Default, PartialEq, Eq)]
7876struct ScopeOutcomeCounts {
7877 created: usize,
7878 replaced: usize,
7879 updated: usize,
7880 unchanged: usize,
7881 refused: usize,
7882}
7883
7884impl ScopeOutcomeCounts {
7885 fn of(results: &[ScopeRecordResult]) -> Self {
7886 let mut counts = Self::default();
7887 for result in results {
7888 let slot = match result.outcome {
7889 ScopeRecordOutcome::Created => &mut counts.created,
7890 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7891 ScopeRecordOutcome::Updated => &mut counts.updated,
7892 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7893 ScopeRecordOutcome::Refused => &mut counts.refused,
7894 };
7895 *slot += 1;
7896 }
7897 counts
7898 }
7899}
7900
7901#[cfg(test)]
7902mod scope_outcome_count_tests {
7903 use super::*;
7904
7905 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7906 ScopeRecordResult {
7907 scope_ref: "r".to_string(),
7908 scope_epoch: 1,
7909 outcome,
7910 code: None,
7911 message: None,
7912 version: None,
7913 parent_state: None,
7914 }
7915 }
7916
7917 #[test]
7920 fn every_outcome_is_counted_in_its_own_field() {
7921 let results = [
7922 result(ScopeRecordOutcome::Created),
7923 result(ScopeRecordOutcome::Created),
7924 result(ScopeRecordOutcome::Replaced),
7925 result(ScopeRecordOutcome::Updated),
7926 result(ScopeRecordOutcome::Unchanged),
7927 result(ScopeRecordOutcome::Refused),
7928 result(ScopeRecordOutcome::Refused),
7929 result(ScopeRecordOutcome::Refused),
7930 ];
7931 assert_eq!(
7932 ScopeOutcomeCounts::of(&results),
7933 ScopeOutcomeCounts {
7934 created: 2,
7935 replaced: 1,
7936 updated: 1,
7937 unchanged: 1,
7938 refused: 3,
7939 }
7940 );
7941 }
7942}
7943
7944fn capability_census_trigger(
7948 old: Option<&CapabilityDeclarations>,
7949 new: Option<&CapabilityDeclarations>,
7950) -> bool {
7951 let old_provides = old
7952 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7953 .unwrap_or_default();
7954 let old_denies = old
7955 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7956 .unwrap_or_default();
7957 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7958 provides: Vec::new(),
7959 requires: Vec::new(),
7960 must_never_reach: Vec::new(),
7961 });
7962
7963 new.provides
7964 .iter()
7965 .any(|capability| !old_provides.contains(capability))
7966 || new
7967 .must_never_reach
7968 .iter()
7969 .any(|capability| !old_denies.contains(capability))
7970}
7971
7972fn denied_capability<'a>(
7975 opening_manifest: &'a ModuleManifest,
7976 target_manifest: &ModuleManifest,
7977) -> Option<&'a str> {
7978 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7979 let target_capabilities = target_manifest.capabilities.as_ref()?;
7980 opening_capabilities
7981 .must_never_reach
7982 .iter()
7983 .find(|denied| {
7984 target_capabilities
7985 .provides
7986 .iter()
7987 .any(|provided| provided == *denied)
7988 })
7989 .map(String::as_str)
7990}
7991
7992fn catalog_update_frozen_field_message(
7993 registered: &ModuleManifest,
7994 provides: &[ProviderRole],
7995) -> Option<String> {
7996 let old_has_provides = !registered.provides.is_empty();
7997 let new_has_provides = !provides.is_empty();
7998 if old_has_provides != new_has_provides {
7999 return Some(format!(
8000 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8001 registered.module_id
8002 ));
8003 }
8004
8005 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
8006 return Some(format!(
8007 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8008 registered.module_id
8009 ));
8010 }
8011
8012 let registered_concurrency = manifest_concurrency(registered);
8013 let mut candidate = registered.clone();
8014 candidate.provides = provides.to_vec();
8015 let candidate_concurrency = manifest_concurrency(&candidate);
8016 if candidate_concurrency != registered_concurrency {
8017 return Some(format!(
8018 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8019 registered.module_id, registered_concurrency, candidate_concurrency
8020 ));
8021 }
8022
8023 None
8026}
8027
8028fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8029 manifest.provides.iter().any(is_routable_role)
8030}
8031
8032fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8038 manifest
8039 .provides
8040 .iter()
8041 .find_map(|provider| match provider {
8042 ProviderRole::ToolProvider { concurrency, .. }
8043 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8044 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8045 })
8046 .unwrap_or(Concurrency::ModuleManaged)
8047}
8048
8049fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8056 let has_management_surface = manifest
8057 .provides
8058 .iter()
8059 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8060 if !has_management_surface {
8061 return false;
8062 }
8063 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8064 return false;
8065 };
8066 let Some(provides) = raw
8067 .get("manifest")
8068 .and_then(|manifest| manifest.get("provides"))
8069 .and_then(serde_json::Value::as_array)
8070 else {
8071 return false;
8072 };
8073 provides.iter().any(|role| {
8078 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8079 && role.get("concurrency").is_none()
8080 })
8081}
8082
8083fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8084 if peer_version != PROTOCOL_VERSION {
8085 return Err(format!(
8086 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8087 ));
8088 }
8089 Ok(PROTOCOL_VERSION)
8090}
8091
8092fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8093 Frame::build_with_version(
8094 response_version(frame),
8095 FrameType::Pong,
8096 frame.header.flags,
8097 0,
8098 0,
8099 frame.header.corr,
8100 Vec::new(),
8101 )
8102 .map_err(RouterError::FrameBuild)
8103}
8104
8105pub(crate) fn control_error_frame(
8106 frame: &Frame,
8107 code: &'static str,
8108 message: impl Into<String>,
8109) -> Result<Frame, RouterError> {
8110 control_error_body_frame(
8111 frame,
8112 ErrorBody {
8113 code: code.to_string(),
8114 message: message.into(),
8115 detail: None,
8116 },
8117 )
8118}
8119
8120fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8128 control_error_frame_with_flags(frame, error, daemon_error_flags())
8129}
8130
8131fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8137 control_error_frame_with_flags(frame, error, control_flags())
8138}
8139
8140fn control_error_frame_with_flags(
8141 frame: &Frame,
8142 error: ErrorBody,
8143 flags: Flags,
8144) -> Result<Frame, RouterError> {
8145 let body = serde_json::to_vec(&error).map_err(|err| {
8146 RouterError::backend(
8147 0,
8148 frame.header.corr,
8149 format!("failed to encode control ERROR: {err}"),
8150 )
8151 })?;
8152
8153 Frame::build_with_version(
8154 response_version(frame),
8155 FrameType::Error,
8156 flags,
8157 0,
8158 0,
8159 frame.header.corr,
8160 body,
8161 )
8162 .map_err(RouterError::FrameBuild)
8163}
8164
8165pub(crate) fn control_response_body_frame<T: Serialize>(
8166 frame: &Frame,
8167 reply: &T,
8168 label: &'static str,
8169) -> Result<Frame, RouterError> {
8170 let body = serde_json::to_vec(reply).map_err(|err| {
8171 RouterError::backend(
8172 0,
8173 frame.header.corr,
8174 format!("failed to encode {label}: {err}"),
8175 )
8176 })?;
8177
8178 Frame::build_with_version(
8179 response_version(frame),
8180 FrameType::Response,
8181 control_flags(),
8182 0,
8183 0,
8184 frame.header.corr,
8185 body,
8186 )
8187 .map_err(RouterError::FrameBuild)
8188}
8189
8190fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8220 match err {
8221 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8222 ForwardingError::NoModuleConnection => "target_unavailable",
8223 ForwardingError::ModuleReloading { .. } => "module_reloading",
8224 ForwardingError::ClientRouteChannelExhausted { .. }
8225 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8226 ForwardingError::StaleModuleEndpoint
8227 | ForwardingError::UnknownReservation { .. }
8228 | ForwardingError::ConnectionClosing { .. }
8229 | ForwardingError::ClientEgressClosed { .. }
8230 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8231 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8234 ForwardingError::RelayCorrelationExhausted
8235 | ForwardingError::RouteOpenBuild(_)
8236 | ForwardingError::Poisoned => "forwarding_error",
8237 }
8238}
8239
8240fn response_version(frame: &Frame) -> u8 {
8241 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8242 frame.header.ver
8243 } else {
8244 PROTOCOL_VERSION
8245 }
8246}
8247
8248fn control_flags() -> Flags {
8249 Flags::new(false, Priority::Passive, false)
8250}
8251
8252fn send_goodbye_target_best_effort(
8257 counters: &DaemonCounters,
8258 target: &GoodbyeTarget,
8259 context: &'static str,
8260) {
8261 let Ok(frame) = Frame::build_with_version(
8262 target.negotiated_ver,
8263 FrameType::Goodbye,
8264 control_flags(),
8265 target.channel,
8266 target.epoch,
8267 0,
8268 Vec::new(),
8269 ) else {
8270 return;
8271 };
8272 crate::forwarding::send_module_route_goodbye(
8273 counters,
8274 &target.sink,
8275 frame,
8276 target.module_id.as_deref(),
8277 context,
8278 );
8279}
8280
8281pub(crate) fn send_route_control_pushes(
8282 forwarding: &ForwardingTable,
8283 routes: Vec<EndpointRoute>,
8284 push: ClientControlPush,
8285) {
8286 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8287 for route in routes {
8288 let target = route.goodbye_target;
8289 if let Some((existing, channels)) = targets
8290 .iter_mut()
8291 .find(|(existing, _)| existing.connection_id == target.connection_id)
8292 {
8293 debug_assert_eq!(
8294 existing.negotiated_ver, target.negotiated_ver,
8295 "one connection cannot negotiate multiple frame versions"
8296 );
8297 if !channels.contains(&target.channel) {
8298 channels.push(target.channel);
8299 }
8300 continue;
8301 }
8302 let channel = target.channel;
8303 targets.push((target, vec![channel]));
8304 }
8305 for (target, mut channels) in targets {
8306 channels.sort_unstable();
8307 let mut push = push.clone();
8308 match &mut push {
8309 ClientControlPush::RouteClosing {
8310 channels: covered, ..
8311 }
8312 | ClientControlPush::RouteClosed {
8313 channels: covered, ..
8314 } => *covered = channels,
8315 }
8316 let body = match serde_json::to_vec(&push) {
8317 Ok(body) => body,
8318 Err(err) => {
8319 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8320 continue;
8321 }
8322 };
8323 let frame = match Frame::build_with_version(
8324 target.negotiated_ver,
8325 FrameType::Push,
8326 control_flags(),
8327 0,
8328 0,
8329 0,
8330 body.clone(),
8331 ) {
8332 Ok(frame) => frame,
8333 Err(err) => {
8334 warn!(
8335 route_channel = target.channel,
8336 error = %err,
8337 "failed to build route lifecycle control PUSH frame"
8338 );
8339 continue;
8340 }
8341 };
8342 if let Err(err) = target.sink.try_send(frame) {
8343 if target.close_on_delivery_failure() {
8344 warn!(
8345 target_connection_id = target.connection_id.get(),
8346 route_channel = target.channel,
8347 error = %err,
8348 "route lifecycle control PUSH was not delivered to client; closing target connection"
8349 );
8350 let _ = forwarding.escalate_client_delivery_failure(
8351 target.connection_id,
8352 target.channel,
8353 target.epoch,
8354 CloseReason::new(
8355 "route_lifecycle_push_delivery_failed",
8356 format!(
8357 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8358 target.channel
8359 ),
8360 ),
8361 crate::forwarding::UndeliveredFrame {
8362 module_id: target.module_id.as_deref(),
8363 sink: &target.sink,
8364 },
8365 );
8366 }
8367 }
8368 }
8369}
8370
8371#[cfg(test)]
8372mod tests {
8373 mod peer_manifest {
8374 include!("peer_manifest_tests.rs");
8375 }
8376 mod activity_read {
8377 include!("activity_read_tests.rs");
8378 }
8379
8380 mod lifecycle_activity {
8381 include!("control_lifecycle_tests.rs");
8382 }
8383 #[cfg(unix)]
8384 #[tokio::test]
8385 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8386 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8387 let path = dir.join("subc.jsonc");
8388 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8389 "program":"/bin/sleep","args":["60"],"protocol":"none",
8390 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8391 }}}).to_string()).unwrap();
8392 let mut configured = crate::daemon_config::load(&path)
8393 .unwrap()
8394 .unwrap()
8395 .modules
8396 .pop()
8397 .unwrap();
8398 let registry = std::sync::Arc::new(crate::Registry::default());
8399 let handle = crate::SupervisorHandle::new();
8400 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8401 .with_handle(handle.clone());
8402 let module = supervisor
8403 .supervise_configured_with_health(
8404 configured.module_spec(),
8405 true,
8406 configured.health.clone(),
8407 None,
8408 configured.restart,
8409 )
8410 .unwrap();
8411 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8412 let before = module.status().unwrap().pid;
8413 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8414 configured.health.cadence = std::time::Duration::from_secs(3600);
8415 let health_only = handler
8416 .reconcile_supervised_modules(
8417 &supervisor,
8418 vec![configured.clone()],
8419 false,
8420 Principal::Direct,
8421 )
8422 .await
8423 .unwrap();
8424 assert!(
8425 health_only.0.changed_pending_reload.is_empty(),
8426 "health policy is already applied live"
8427 );
8428 assert_eq!(module.status().unwrap().pid, before);
8429 assert_eq!(
8430 module.configuration().unwrap().1.http,
8431 configured.health.http
8432 );
8433 configured.args = vec!["61".into()];
8434 let launch = handler
8435 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8436 .await
8437 .unwrap();
8438 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8439 assert_eq!(
8440 module.status().unwrap().pid,
8441 before,
8442 "a launch edit is stored until reload"
8443 );
8444 module.drain().await.unwrap();
8445 }
8446 use cortexkit_test_support::ScratchDir;
8447 use std::{
8448 collections::BTreeMap,
8449 fmt,
8450 path::PathBuf,
8451 sync::{Arc, Mutex},
8452 time::Duration,
8453 };
8454
8455 use serde_json::{json, Value};
8456 use subc_protocol::{
8457 manifest::{
8458 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8459 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8460 },
8461 session::HealthStatus,
8462 FrameType,
8463 };
8464
8465 use super::*;
8466 use crate::{
8467 forwarding::{DataRoute, DataRouteState},
8468 registry::ChannelState,
8469 router::FrameSink,
8470 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8471 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8472 RouteCtx, Router,
8473 };
8474 use tokio::{
8475 sync::mpsc,
8476 time::{sleep, Instant},
8477 };
8478 use tracing::{
8479 field::{Field, Visit},
8480 Event, Subscriber,
8481 };
8482 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8483
8484 fn fake_aft_stub_path() -> PathBuf {
8503 let mut path = std::env::current_exe().expect("current_exe available in tests");
8504 path.pop(); path.pop(); path.push(if cfg!(windows) {
8507 "fake-aft-stub.exe"
8508 } else {
8509 "fake-aft-stub"
8510 });
8511 assert!(
8512 path.exists(),
8513 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8514 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8515 path.display()
8516 );
8517 path
8518 }
8519
8520 fn client_retries(code: &str) -> bool {
8525 subc_protocol::error_codes::is_retryable_route_open(code)
8526 }
8527
8528 #[test]
8539 fn retryability_of_forwarding_codes_matches_the_failure() {
8540 let transient = [
8543 ForwardingError::NoModuleConnection,
8544 ForwardingError::ModuleReloading {
8545 module_id: "m".into(),
8546 },
8547 ForwardingError::StaleModuleEndpoint,
8548 ForwardingError::UnknownReservation {
8549 client_channel: 1,
8550 module_channel: 1,
8551 },
8552 ForwardingError::ConnectionClosing {
8553 connection_id: ConnectionId::new(1),
8554 },
8555 ForwardingError::ClientEgressClosed {
8556 connection_id: ConnectionId::new(1),
8557 },
8558 ForwardingError::ModuleEgressUnavailable {
8559 connection_id: ConnectionId::new(1),
8560 },
8561 ];
8562 for err in transient {
8563 let code = forwarding_error_code(&err);
8564 assert!(
8565 client_retries(code),
8566 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8567 );
8568 }
8569
8570 let permanent = [
8575 ForwardingError::ConnectionRoleConflict {
8576 connection_id: ConnectionId::new(1),
8577 },
8578 ForwardingError::ClientRouteChannelExhausted {
8579 connection_id: ConnectionId::new(1),
8580 },
8581 ForwardingError::ModuleRouteChannelExhausted {
8582 endpoint: ModuleEndpointId {
8583 connection_id: ConnectionId::new(1),
8584 generation: 1,
8585 },
8586 },
8587 ForwardingError::RelayCorrelationExhausted,
8588 ForwardingError::RouteOpenBuild("x".into()),
8589 ForwardingError::Poisoned,
8590 ];
8591 for err in permanent {
8592 let code = forwarding_error_code(&err);
8593 assert!(
8594 !client_retries(code),
8595 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8596 );
8597 }
8598 }
8599
8600 #[tokio::test]
8628 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8629 let handler = ControlHandler::default();
8630 let frame =
8631 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8632
8633 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8637 assert_eq!(
8638 stamped,
8639 Principal::Direct,
8640 "a caller that proved nothing must not be stamped as a supervised module"
8641 );
8642
8643 let forged = handler
8648 .route_open_principal(
8649 &frame,
8650 Some(ConsumerIdentity {
8651 module_id: "aft".to_string(),
8652 launch_nonce: "not-a-real-nonce".to_string(),
8653 }),
8654 )
8655 .unwrap();
8656 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8657 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8658 }
8659
8660 #[test]
8677 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8678 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8679 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8680 let ClientControlRequest::RouteOpen {
8681 consumer_identity, ..
8682 } = parsed
8683 else {
8684 panic!("route.open body must parse as RouteOpen");
8685 };
8686 assert_eq!(
8687 consumer_identity,
8688 Some(ConsumerIdentity {
8689 module_id: "aft".to_string(),
8690 launch_nonce: "n".to_string(),
8691 }),
8692 "the wire field name must reach the value route_open_principal reads"
8693 );
8694 }
8695
8696 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8697 ModuleManifest::builder(module_id, "0.1.0")
8698 .protocol_ver(protocol_ver)
8699 .provides(vec![ProviderRole::ToolProvider {
8700 tools: vec![Tool {
8701 name: "read".to_string(),
8702 description: None,
8703 execution_mode: ExecutionMode::Pure,
8704 schema: json!({"type": "object"}),
8705 }],
8706 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8707 concurrency: Concurrency::ModuleManaged,
8708 emits_push: true,
8709 sub_supervises: true,
8710 }])
8711 .build()
8712 }
8713
8714 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8715 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8716 }
8717
8718 fn hello_frame_with_control_ops(
8719 module_id: &str,
8720 protocol_ver: u8,
8721 corr: u64,
8722 control_ops: Option<Vec<String>>,
8723 ) -> Frame {
8724 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8725 }
8726
8727 fn hello_frame_with_nonce(
8728 module_id: &str,
8729 protocol_ver: u8,
8730 corr: u64,
8731 launch_nonce: Option<&str>,
8732 ) -> Frame {
8733 hello_frame_full(
8734 module_id,
8735 protocol_ver,
8736 corr,
8737 None,
8738 launch_nonce.map(ToOwned::to_owned),
8739 )
8740 }
8741
8742 fn hello_frame_full(
8743 module_id: &str,
8744 protocol_ver: u8,
8745 corr: u64,
8746 control_ops: Option<Vec<String>>,
8747 launch_nonce: Option<String>,
8748 ) -> Frame {
8749 let body = serde_json::to_vec(&ModuleHelloBody {
8750 manifest: manifest(module_id, protocol_ver),
8751 protocol_ver,
8752 control_ops,
8753 launch_nonce,
8754 })
8755 .unwrap();
8756 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8757 }
8758
8759 fn hello_frame_with_rich_ui(
8760 module_id: &str,
8761 protocol_ver: u8,
8762 corr: u64,
8763 launch_nonce: Option<&str>,
8764 rich_ui: Value,
8765 ) -> Frame {
8766 let mut body = serde_json::to_value(ModuleHelloBody {
8767 manifest: manifest(module_id, protocol_ver),
8768 protocol_ver,
8769 control_ops: None,
8770 launch_nonce: launch_nonce.map(str::to_string),
8771 })
8772 .unwrap();
8773 body["manifest"]["rich_ui"] = rich_ui;
8774 Frame::build(
8775 FrameType::Hello,
8776 control_flags(),
8777 0,
8778 0,
8779 corr,
8780 serde_json::to_vec(&body).unwrap(),
8781 )
8782 .unwrap()
8783 }
8784
8785 fn non_routable_hello_frame_with_control_ops(
8786 module_id: &str,
8787 corr: u64,
8788 control_ops: Option<Vec<String>>,
8789 ) -> Frame {
8790 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8791 manifest.provides.clear();
8792 let body = serde_json::to_vec(&ModuleHelloBody {
8793 manifest,
8794 protocol_ver: PROTOCOL_VERSION,
8795 control_ops,
8796 launch_nonce: None,
8797 })
8798 .unwrap();
8799 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8800 }
8801
8802 fn capability_grammar_hello_frame(
8803 capabilities: Value,
8804 runtime_computed: Option<Value>,
8805 corr: u64,
8806 ) -> Frame {
8807 let mut body = serde_json::to_value(ModuleHelloBody {
8808 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8809 protocol_ver: PROTOCOL_VERSION,
8810 control_ops: None,
8811 launch_nonce: None,
8812 })
8813 .expect("HELLO body serializes");
8814 body["manifest"]["capabilities"] = capabilities;
8815 if let Some(runtime_computed) = runtime_computed {
8816 body["runtime_computed"] = runtime_computed;
8817 }
8818 Frame::build(
8819 FrameType::Hello,
8820 control_flags(),
8821 0,
8822 0,
8823 corr,
8824 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8825 )
8826 .expect("HELLO frame builds")
8827 }
8828
8829 fn channel_request(channel: u16, corr: u64) -> Frame {
8830 Frame::build(
8831 FrameType::Request,
8832 Flags::new(true, Priority::Interactive, false),
8833 channel,
8834 0,
8835 corr,
8836 b"opaque".to_vec(),
8837 )
8838 .unwrap()
8839 }
8840
8841 fn route_ctx(
8842 connection_id: ConnectionId,
8843 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8844 let (tx, rx) = mpsc::channel(8);
8845 (
8846 RouteCtx {
8847 connection_id,
8848 egress: FrameSink::new(tx),
8849 },
8850 rx,
8851 )
8852 }
8853
8854 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8855 serde_json::from_slice(&frame.body).unwrap()
8856 }
8857
8858 async fn hello_via_sink(
8863 handler: &ControlHandler,
8864 ctx: &RouteCtx,
8865 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8866 hello: Frame,
8867 ) -> Frame {
8868 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8869 assert!(
8870 replies.is_empty(),
8871 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8872 );
8873 let ack = rx
8874 .try_recv()
8875 .expect("HELLO_ACK is queued on the module sink")
8876 .frame;
8877 assert_eq!(ack.header.ty, FrameType::HelloAck);
8878 ack
8879 }
8880
8881 fn parse_error(frame: &Frame) -> Value {
8882 serde_json::from_slice(&frame.body).unwrap()
8883 }
8884
8885 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8886 serde_json::from_slice(&frame.body).unwrap()
8887 }
8888
8889 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8890 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8891 route_channel,
8892 route_epoch: 0,
8893 kind,
8894 })
8895 .unwrap();
8896 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8897 }
8898
8899 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8900 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8901 module_id: module_id.to_string(),
8902 })
8903 .unwrap();
8904 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8905 }
8906
8907 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8908 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8909 }
8910
8911 fn route_open_frame_with_consumer_capabilities(
8912 corr: u64,
8913 module_id: &str,
8914 project_root: ScratchDir,
8915 consumer_capabilities: Option<Vec<String>>,
8916 ) -> Frame {
8917 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8918 target: RouteTarget::ToolProvider {
8919 module_id: module_id.to_string(),
8920 },
8921 identity: BindIdentity::new(
8922 project_root.path().to_path_buf(),
8923 "unit".to_string(),
8924 "session".to_string(),
8925 ),
8926 consumer_identity: None,
8927 consumer_capabilities,
8928 role_versions: None,
8929 admission_facts: None,
8930 scope: None,
8931 })
8932 .unwrap();
8933 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8934 }
8935
8936 fn route_open_frame_with_role_versions(
8937 corr: u64,
8938 module_id: &str,
8939 project_root: ScratchDir,
8940 role_versions: Option<BTreeMap<String, String>>,
8941 ) -> Frame {
8942 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8943 target: RouteTarget::ToolProvider {
8944 module_id: module_id.to_string(),
8945 },
8946 identity: BindIdentity::new(
8947 project_root.path().to_path_buf(),
8948 "unit".to_string(),
8949 format!("session-{corr}"),
8950 ),
8951 consumer_identity: None,
8952 consumer_capabilities: None,
8953 role_versions,
8954 admission_facts: None,
8955 scope: None,
8956 })
8957 .unwrap();
8958 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8959 }
8960
8961 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8962 entries
8963 .iter()
8964 .map(|(role, version)| (role.to_string(), version.to_string()))
8965 .collect()
8966 }
8967
8968 fn route_open_frame_with_admission_facts(
8969 corr: u64,
8970 module_id: &str,
8971 project_root: ScratchDir,
8972 consumer_identity: Option<subc_control::ConsumerIdentity>,
8973 facts: Option<Value>,
8974 ) -> Frame {
8975 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8976 target: RouteTarget::ToolProvider {
8977 module_id: module_id.to_string(),
8978 },
8979 identity: BindIdentity::new(
8980 project_root.path().to_path_buf(),
8981 "unit".to_string(),
8982 format!("session-{corr}"),
8983 ),
8984 consumer_identity,
8985 consumer_capabilities: None,
8986 role_versions: None,
8987 admission_facts: facts,
8988 scope: None,
8989 })
8990 .unwrap();
8991 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8992 }
8993
8994 #[derive(Clone, Default)]
8995 struct EventCapture {
8996 events: Arc<Mutex<Vec<CapturedEvent>>>,
8997 }
8998
8999 #[derive(Clone, Debug)]
9000 struct CapturedEvent {
9001 target: String,
9002 level: tracing::Level,
9003 fields: BTreeMap<String, String>,
9004 }
9005
9006 impl EventCapture {
9007 fn events(&self) -> Vec<CapturedEvent> {
9008 self.events.lock().unwrap().clone()
9009 }
9010 }
9011
9012 impl<S> Layer<S> for EventCapture
9013 where
9014 S: Subscriber,
9015 {
9016 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9017 let mut visitor = EventFieldVisitor::default();
9018 event.record(&mut visitor);
9019 self.events.lock().unwrap().push(CapturedEvent {
9020 target: event.metadata().target().to_string(),
9021 level: *event.metadata().level(),
9022 fields: visitor.fields,
9023 });
9024 }
9025 }
9026
9027 #[derive(Default)]
9028 struct EventFieldVisitor {
9029 fields: BTreeMap<String, String>,
9030 }
9031
9032 impl Visit for EventFieldVisitor {
9033 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9034 self.fields
9035 .insert(field.name().to_string(), format!("{value:?}"));
9036 }
9037 }
9038
9039 fn health_response(corr: u64, status: HealthStatus) -> Frame {
9040 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9041 status,
9042 detail: Some("warming".to_string()),
9043 metrics: Some(json!({"queue_depth": 3})),
9044 })
9045 .unwrap();
9046 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9047 }
9048
9049 fn route_bind_ack(corr: u64) -> Frame {
9050 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9051 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9052 }
9053
9054 fn unique_project_root(label: &str) -> ScratchDir {
9055 ScratchDir::new(label)
9056 }
9057
9058 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9059 match parse_route_poll(frame) {
9060 ClientControlResponse::RoutePoll {
9061 status: None,
9062 live: Some(live),
9063 ..
9064 } => assert_eq!(live, expected_live),
9065 other => panic!("unexpected route.poll response: {other:?}"),
9066 }
9067 }
9068
9069 fn bind_liveness_route(
9070 registry: &Registry,
9071 forwarding: &ForwardingTable,
9072 module_id: &str,
9073 ) -> (RouteCtx, u16, u32) {
9074 let module_connection = ConnectionId::new(101);
9075 let client_connection = ConnectionId::new(202);
9076 let registration = registry
9077 .register_with_control_ops(
9078 manifest(module_id, PROTOCOL_VERSION),
9079 PROTOCOL_VERSION,
9080 module_connection,
9081 module_baseline_control_ops(),
9082 )
9083 .unwrap();
9084 let (module_tx, _module_rx) = mpsc::channel(8);
9085 let endpoint = forwarding
9086 .register_module_connection(
9087 module_connection,
9088 module_id.to_string(),
9089 PROTOCOL_VERSION,
9090 manifest_concurrency(®istration.manifest),
9091 FrameSink::new(module_tx),
9092 )
9093 .unwrap();
9094 let (client_ctx, _client_rx) = route_ctx(client_connection);
9095 let pending = forwarding
9096 .begin_route_bind_relay_for_test(
9097 client_connection,
9098 client_ctx.egress.clone(),
9099 1,
9100 module_id,
9101 )
9102 .unwrap();
9103 assert_eq!(pending.endpoint, endpoint);
9104 let route_channel = pending.client_channel;
9105 let route_epoch = pending.client_epoch;
9106 forwarding
9107 .complete_pending_relay(
9108 module_connection,
9109 pending.corr,
9110 RouteBindRelayOutcome::Accepted,
9111 )
9112 .unwrap();
9113 (client_ctx, route_channel, route_epoch)
9114 }
9115
9116 struct FakeProcessLiveness {
9117 live: Option<bool>,
9118 }
9119
9120 impl ModuleProcessLiveness for FakeProcessLiveness {
9121 fn process_live(&self, _module_id: &str) -> Option<bool> {
9122 self.live
9123 }
9124 }
9125
9126 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9127 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9128 {
9129 let registry = Arc::new(Registry::default());
9130 let supervisor_handle = SupervisorHandle::new();
9131 let supervisor = Supervisor::new_for_test(
9132 Arc::clone(®istry),
9133 RestartPolicy::new(1, Duration::from_millis(10)),
9134 )
9135 .with_handle(supervisor_handle.clone());
9136 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9137 let module = supervisor
9138 .spawn(ModuleSpec {
9139 module_id: "stderr-tail-wire".to_string(),
9140 program: fake_aft_stub_path(),
9141 args: Vec::new(),
9142 env: vec![
9143 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9144 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9145 ],
9146 reserved: false,
9147 reserved_prefixes: Vec::new(),
9148 protocol: ModuleProtocol::Subc,
9149 overlap: Default::default(),
9150 })
9151 .unwrap();
9152
9153 let deadline = Instant::now() + Duration::from_secs(5);
9154 loop {
9155 let tail = module.stderr_tail(None, None);
9156 if tail
9157 .entries
9158 .iter()
9159 .any(|entry| matches!(entry, TailEntry::ProcessStart))
9160 && tail.entries.iter().any(|entry| {
9161 matches!(
9162 entry,
9163 TailEntry::Line {
9164 truncated: true,
9165 ..
9166 }
9167 )
9168 })
9169 {
9170 break;
9171 }
9172 assert!(
9173 Instant::now() < deadline,
9174 "module did not produce a truncated line and restart boundary: {tail:?}"
9175 );
9176 sleep(Duration::from_millis(10)).await;
9177 }
9178
9179 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9180 let request = ClientControlRequest::SupervisorStderrTail {
9181 module_id: "stderr-tail-wire".to_string(),
9182 max_lines: None,
9183 max_bytes: None,
9184 };
9185 let frame = Frame::build(
9186 FrameType::Request,
9187 control_flags(),
9188 0,
9189 0,
9190 1,
9191 serde_json::to_vec(&request).unwrap(),
9192 )
9193 .unwrap();
9194 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9195 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9196 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9197 serde_json::from_slice(&responses[0].body).unwrap()
9198 else {
9199 panic!("expected supervisor.stderr_tail response");
9200 };
9201
9202 assert!(
9203 tail.entries
9204 .iter()
9205 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9206 "the control response lost the restart boundary"
9207 );
9208 let Some(StderrTailEntry::Line {
9209 text,
9210 truncated,
9211 at_ms,
9212 }) = tail.entries.iter().find(|entry| {
9213 matches!(
9214 entry,
9215 StderrTailEntry::Line {
9216 truncated: true,
9217 ..
9218 }
9219 )
9220 })
9221 else {
9222 panic!("the control response lost the truncated line");
9223 };
9224 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9225 assert!(*truncated);
9226 assert!(
9227 at_ms.is_some(),
9228 "the control response lost the line's capture time"
9229 );
9230 }
9231
9232 #[tokio::test(flavor = "current_thread")]
9237 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9238 let dir = ScratchDir::new("terminals-off-worker");
9239 let journal_path = dir.join("terminals.jsonl");
9240 let registry = Arc::new(Registry::default());
9241 let supervisor_handle = SupervisorHandle::new();
9242 let supervisor =
9243 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9244 .with_handle(supervisor_handle.clone())
9245 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9246 let module = supervisor
9247 .spawn(ModuleSpec {
9248 module_id: "terminal-off-worker".to_string(),
9249 program: fake_aft_stub_path(),
9250 args: Vec::new(),
9251 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9252 reserved: false,
9253 reserved_prefixes: Vec::new(),
9254 protocol: ModuleProtocol::Subc,
9255 overlap: Default::default(),
9256 })
9257 .unwrap();
9258 let deadline = Instant::now() + Duration::from_secs(5);
9259 while module.terminal_history().entries.len() != 2 {
9260 assert!(Instant::now() < deadline, "module did not record two exits");
9261 sleep(Duration::from_millis(10)).await;
9262 }
9263
9264 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9265 let handler =
9266 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
9267 let frame = Frame::build(
9268 FrameType::Request,
9269 control_flags(),
9270 0,
9271 0,
9272 1,
9273 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9274 module_id: "terminal-off-worker".to_string(),
9275 })
9276 .unwrap(),
9277 )
9278 .unwrap();
9279 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9280 let spawned_at = std::time::Instant::now();
9281 let read = tokio::spawn({
9282 let handler = Arc::clone(&handler);
9283 async move { handler.handle_control_frame(&ctx, frame).await }
9284 });
9285 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9288 .await
9289 .unwrap()
9290 .expect("the history read reached its pause");
9291 let elapsed = spawned_at.elapsed();
9292 assert!(
9293 elapsed < Duration::from_secs(2) && !read.is_finished(),
9294 "this task could not run while the history read was paused \
9295 (resumed after {elapsed:?}, read finished: {})",
9296 read.is_finished()
9297 );
9298
9299 drop(release);
9300 let responses = read.await.unwrap().unwrap();
9301 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9302 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9303 panic!("expected supervisor.terminals response");
9304 };
9305 assert_eq!(terminals.entries.len(), 2);
9306 assert_eq!(terminals.journal_skipped_lines, 0);
9307 assert_eq!(terminals.journal_read_errors, 0);
9308 }
9309
9310 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9311 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9312 let registry = Arc::new(Registry::default());
9313 let supervisor_handle = SupervisorHandle::new();
9314 let supervisor =
9315 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9316 .with_handle(supervisor_handle.clone());
9317 let module = supervisor
9318 .spawn(ModuleSpec {
9319 module_id: "terminal-golden".to_string(),
9320 program: fake_aft_stub_path(),
9321 args: Vec::new(),
9322 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9323 reserved: false,
9324 reserved_prefixes: Vec::new(),
9325 protocol: ModuleProtocol::Subc,
9326 overlap: Default::default(),
9327 })
9328 .unwrap();
9329
9330 let deadline = Instant::now() + Duration::from_secs(5);
9331 while module.terminal_history().entries.len() != 2 {
9332 assert!(
9333 Instant::now() < deadline,
9334 "module did not retain two terminal exits: {:?}",
9335 module.terminal_history()
9336 );
9337 sleep(Duration::from_millis(10)).await;
9338 }
9339
9340 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9341 let request = ClientControlRequest::SupervisorTerminals {
9342 module_id: "terminal-golden".to_string(),
9343 };
9344 let frame = Frame::build(
9345 FrameType::Request,
9346 control_flags(),
9347 0,
9348 0,
9349 1,
9350 serde_json::to_vec(&request).unwrap(),
9351 )
9352 .unwrap();
9353 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9354 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9355 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9356 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9357 panic!("expected supervisor.terminals response");
9358 };
9359 assert_eq!(terminals.entries.len(), 2);
9360 assert_eq!(terminals.dropped, 0);
9361
9362 let mut rendered = serde_json::to_value(response).unwrap();
9363 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9366 for (index, entry) in rendered["entries"]
9367 .as_array_mut()
9368 .expect("terminal response entries array")
9369 .iter_mut()
9370 .enumerate()
9371 {
9372 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9373 }
9374
9375 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9376 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9377 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9378 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9379 std::fs::write(&golden_path, &serialized).unwrap();
9380 }
9381 let expected: Value =
9382 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9383 assert_eq!(rendered, expected);
9384 }
9385
9386 #[test]
9387 fn hello_registers_manifest_and_returns_ack() {
9388 let registry = Arc::new(Registry::default());
9389 let handler = ControlHandler::new(Arc::clone(®istry));
9390 let conn = ConnectionId::new(1);
9391
9392 let responses = handler
9393 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9394 .unwrap();
9395
9396 assert_eq!(responses.len(), 1);
9397 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9398 assert_eq!(responses[0].header.channel, 0);
9399 assert_eq!(responses[0].header.corr, 7);
9400 let ack = parse_ack(&responses[0]);
9401 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9402 assert!(ack
9403 .subc_capabilities
9404 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9405 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9406 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9407 assert!(ack
9408 .subc_ops
9409 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9410 assert!(ack
9411 .subc_ops
9412 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9413
9414 let registration = registry.get_module("aft").unwrap().unwrap();
9415 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9416 assert_eq!(registration.state, ChannelState::Active);
9417 assert_eq!(registration.connection_id, conn);
9418 assert_eq!(registration.control_ops, module_baseline_control_ops());
9419 }
9420
9421 #[test]
9422 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9423 let invalid_identifiers = [
9424 ("case_change", "credentials-Provider/v1"),
9425 ("leading_zero", "credentials-provider/v01"),
9426 ("trailing_hyphen", "credentials-provider-/v1"),
9427 ("consecutive_hyphens", "credentials--provider/v1"),
9428 ("uppercase", "Credentials-provider/v1"),
9429 ("missing_v", "credentials-provider/1"),
9430 ("whitespace", "credentials provider/v1"),
9431 ("zero_version", "credentials-provider/v0"),
9432 ("out_of_range_version", "credentials-provider/v4294967296"),
9433 (
9434 "overlength_name",
9435 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9436 ),
9437 ];
9438 let mut cases = invalid_identifiers
9439 .into_iter()
9440 .map(|(name, identifier)| {
9441 (
9442 format!("identifier_{name}"),
9443 "capabilities.provides[0]".to_string(),
9444 identifier.to_string(),
9445 json!({ "provides": [identifier] }),
9446 None,
9447 )
9448 })
9449 .collect::<Vec<_>>();
9450 cases.extend([
9451 (
9452 "unknown_need".to_string(),
9453 "capabilities.requires[0].need".to_string(),
9454 "deferred".to_string(),
9455 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9456 None,
9457 ),
9458 (
9459 "duplicate_provides".to_string(),
9460 "capabilities.provides[1]".to_string(),
9461 "credentials-provider/v1".to_string(),
9462 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9463 None,
9464 ),
9465 (
9466 "duplicate_must_never_reach".to_string(),
9467 "capabilities.must_never_reach[1]".to_string(),
9468 "credentials-provider/v1".to_string(),
9469 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9470 None,
9471 ),
9472 (
9473 "duplicate_requires_same_need".to_string(),
9474 "capabilities.requires[1]".to_string(),
9475 "credentials-provider/v1".to_string(),
9476 json!({ "requires": [
9477 { "capability": "credentials-provider/v1", "need": "required" },
9478 { "capability": "credentials-provider/v1", "need": "required" }
9479 ] }),
9480 None,
9481 ),
9482 (
9483 "duplicate_requires_conflicting_need".to_string(),
9484 "capabilities.requires[1]".to_string(),
9485 "credentials-provider/v1".to_string(),
9486 json!({ "requires": [
9487 { "capability": "credentials-provider/v1", "need": "required" },
9488 { "capability": "credentials-provider/v1", "need": "optional" }
9489 ] }),
9490 None,
9491 ),
9492 (
9493 "capabilities_root_pointer".to_string(),
9494 "runtime_computed[0]".to_string(),
9495 "/capabilities".to_string(),
9496 json!({}),
9497 Some(json!(["/capabilities"])),
9498 ),
9499 (
9500 "capabilities_descendant_pointer".to_string(),
9501 "runtime_computed[0]".to_string(),
9502 "/capabilities/provides".to_string(),
9503 json!({}),
9504 Some(json!(["/capabilities/provides"])),
9505 ),
9506 (
9507 "malformed_pointer_without_leading_slash".to_string(),
9508 "runtime_computed[0]".to_string(),
9509 "capabilities".to_string(),
9510 json!({}),
9511 Some(json!(["capabilities"])),
9512 ),
9513 (
9514 "malformed_pointer_escape".to_string(),
9515 "runtime_computed[0]".to_string(),
9516 "/roles/~2/tools".to_string(),
9517 json!({}),
9518 Some(json!(["/roles/~2/tools"])),
9519 ),
9520 (
9521 "unknown_capabilities_field".to_string(),
9522 "capabilities.future".to_string(),
9523 "<array>".to_string(),
9524 json!({ "future": [] }),
9525 None,
9526 ),
9527 ]);
9528
9529 for (index, (name, field, value, capabilities, runtime_computed)) in
9530 cases.into_iter().enumerate()
9531 {
9532 let registry = Arc::new(Registry::default());
9533 let handler = ControlHandler::new(Arc::clone(®istry));
9534 let response = handler
9535 .handle_control(
9536 ConnectionId::new((index + 1) as u64),
9537 capability_grammar_hello_frame(
9538 capabilities,
9539 runtime_computed,
9540 index as u64 + 1,
9541 ),
9542 )
9543 .expect("invalid HELLO returns a refusal");
9544
9545 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9546 let error = parse_error(&response[0]);
9547 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9548 let message = error["message"]
9549 .as_str()
9550 .expect("error message is a string");
9551 assert!(
9552 message.contains(&field),
9553 "{name}: field missing from {message}"
9554 );
9555 assert!(
9556 message.contains(&value),
9557 "{name}: value missing from {message}"
9558 );
9559 assert_eq!(
9560 registry
9561 .active_registration_count()
9562 .expect("registry reads"),
9563 0,
9564 "{name}: refused HELLO must not create a catalog entry"
9565 );
9566 }
9567 }
9568
9569 #[test]
9570 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9571 let registry = Arc::new(Registry::default());
9572 let handler = ControlHandler::new(Arc::clone(®istry));
9573 let capabilities = json!({
9574 "provides": ["credentials-provider/v1"],
9575 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9576 "must_never_reach": ["federation-transport/v1"]
9577 });
9578 let response = handler
9579 .handle_control(
9580 ConnectionId::new(99),
9581 capability_grammar_hello_frame(
9582 capabilities.clone(),
9583 Some(json!(["/roles/0/tools"])),
9584 99,
9585 ),
9586 )
9587 .expect("valid HELLO registers");
9588 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9589
9590 let request = Frame::build(
9591 FrameType::Request,
9592 control_flags(),
9593 0,
9594 0,
9595 100,
9596 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9597 .expect("catalog request serializes"),
9598 )
9599 .expect("catalog request frame builds");
9600 let response = handler
9601 .handle_catalog_list(request, None)
9602 .expect("catalog list succeeds");
9603 let ClientControlResponse::CatalogList { modules, .. } =
9604 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9605 else {
9606 panic!("catalog request must return catalog.list");
9607 };
9608 assert_eq!(modules.len(), 1);
9609 assert_eq!(
9610 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9611 capabilities
9612 );
9613 }
9614
9615 #[test]
9616 fn catalog_list_mirrors_management_operation_description() {
9617 let registry = Arc::new(Registry::default());
9618 let handler = ControlHandler::new(Arc::clone(®istry));
9619 let description = "List managed records and return their identifiers and metadata.";
9620 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9621 manifest.provides = vec![ProviderRole::ManagementSurface {
9622 operations: vec![ManagementOperation {
9623 name: "records.list".to_string(),
9624 kind: ManagementOperationKind::Query,
9625 description: Some(description.to_string()),
9626 }],
9627 config_schema: json!({"type": "object"}),
9628 observability: vec![ObservabilitySurface {
9629 name: "records.stats".to_string(),
9630 kind: ObservabilityKind::Snapshot,
9631 }],
9632 identity_scope: vec![IdentityScope::Project],
9633 concurrency: Concurrency::ModuleManaged,
9634 }];
9635 registry
9636 .register_with_control_ops(
9637 manifest,
9638 PROTOCOL_VERSION,
9639 ConnectionId::new(99),
9640 Vec::new(),
9641 )
9642 .expect("described management manifest registers");
9643
9644 let request = Frame::build(
9645 FrameType::Request,
9646 control_flags(),
9647 0,
9648 0,
9649 100,
9650 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9651 .expect("catalog request serializes"),
9652 )
9653 .expect("catalog request frame builds");
9654 let response = handler
9655 .handle_catalog_list(request, None)
9656 .expect("catalog list succeeds");
9657 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9658 assert_eq!(
9659 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9660 "catalog.list must preserve the declared operation description verbatim"
9661 );
9662 }
9663
9664 #[test]
9665 fn catalog_list_mirrors_registered_operation_delivery() {
9666 let registry = Arc::new(Registry::default());
9667 let handler = ControlHandler::new(Arc::clone(®istry));
9668 let delivery =
9669 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9670 let mut body = serde_json::to_value(ModuleHelloBody {
9671 manifest: manifest("watch-management", PROTOCOL_VERSION),
9672 protocol_ver: PROTOCOL_VERSION,
9673 control_ops: None,
9674 launch_nonce: None,
9675 })
9676 .unwrap();
9677 body["manifest"]["provides"] = json!([{
9678 "role": "management_surface",
9679 "operations": [
9680 {"name": "status", "kind": "query"},
9681 {"name": "future", "kind": "query"},
9682 {"name": "reset", "kind": "mutate"}
9683 ],
9684 "config_schema": {}, "observability": [], "identity_scope": []
9685 }]);
9686 body["manifest"]["operation_delivery"] = delivery.clone();
9687 let hello = Frame::build(
9688 FrameType::Hello,
9689 control_flags(),
9690 0,
9691 0,
9692 99,
9693 serde_json::to_vec(&body).unwrap(),
9694 )
9695 .unwrap();
9696 let response = handler
9697 .handle_control(ConnectionId::new(99), hello)
9698 .unwrap();
9699 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9700
9701 let (_, registrations) = registry.list_modules().unwrap();
9702 let registered = ®istrations[0].manifest;
9703 assert!(registered.is_watch_operation("status"));
9704 assert!(!registered.is_watch_operation("future"));
9705 assert!(!registered.is_watch_operation("reset"));
9706 assert!(!registered.is_watch_operation("missing"));
9707
9708 let request = Frame::build(
9709 FrameType::Request,
9710 control_flags(),
9711 0,
9712 0,
9713 100,
9714 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9715 )
9716 .unwrap();
9717 let response = handler.handle_catalog_list(request, None).unwrap();
9718 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9719 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9720 let ClientControlResponse::CatalogList { modules, .. } =
9721 serde_json::from_slice(&response[0].body).unwrap()
9722 else {
9723 panic!("expected catalog.list");
9724 };
9725 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9726 }
9727
9728 #[test]
9729 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9730 let registry = Arc::new(Registry::default());
9731 let supervisor = SupervisorHandle::new();
9732 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9733 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9734 let descriptor = json!({
9735 "schema": "ckios.richui.manifest/1",
9736 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9737 "bytes": 4096,
9738 "chunks": 4
9739 });
9740 let connection = ConnectionId::new(201);
9741 let hello = handler
9742 .handle_control(
9743 connection,
9744 hello_frame_with_rich_ui(
9745 "vault",
9746 PROTOCOL_VERSION,
9747 201,
9748 Some("the-real-nonce"),
9749 descriptor.clone(),
9750 ),
9751 )
9752 .unwrap();
9753 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9754
9755 let request =
9756 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9757 let provides = registry
9758 .get_module_by_connection(connection)
9759 .unwrap()
9760 .unwrap()
9761 .manifest
9762 .provides;
9763 handler
9764 .handle_catalog_update(connection, request, provides, None, Some(false))
9765 .unwrap();
9766
9767 let request = Frame::build(
9768 FrameType::Request,
9769 control_flags(),
9770 0,
9771 0,
9772 203,
9773 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9774 )
9775 .unwrap();
9776 let response = handler.handle_catalog_list(request, None).unwrap();
9777 let ClientControlResponse::CatalogList { modules, .. } =
9778 serde_json::from_slice(&response[0].body).unwrap()
9779 else {
9780 panic!("expected catalog.list");
9781 };
9782 assert_eq!(
9783 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9784 descriptor
9785 );
9786 assert!(modules[0].nonce_verified);
9787 }
9788
9789 fn hello_with_role_declarations(declarations: Value) -> Frame {
9790 let mut body = serde_json::to_value(ModuleHelloBody {
9791 manifest: manifest("role-test", PROTOCOL_VERSION),
9792 protocol_ver: PROTOCOL_VERSION,
9793 control_ops: None,
9794 launch_nonce: None,
9795 })
9796 .unwrap();
9797 body["manifest"]["role_declarations"] = declarations;
9798 Frame::build(
9799 FrameType::Hello,
9800 control_flags(),
9801 0,
9802 0,
9803 301,
9804 serde_json::to_vec(&body).unwrap(),
9805 )
9806 .unwrap()
9807 }
9808
9809 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9810 let request = Frame::build(
9811 FrameType::Request,
9812 control_flags(),
9813 0,
9814 0,
9815 302,
9816 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9817 )
9818 .unwrap();
9819 let response = handler.handle_catalog_list(request, None).unwrap();
9820 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9821 let ClientControlResponse::CatalogList { mut modules, .. } =
9822 serde_json::from_slice(&response[0].body).unwrap()
9823 else {
9824 panic!("expected catalog.list");
9825 };
9826 (raw["modules"][0].clone(), modules.remove(0))
9827 }
9828
9829 #[test]
9830 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9831 let registry = Arc::new(Registry::default());
9832 let handler = ControlHandler::new(Arc::clone(®istry));
9833 let declarations = json!({
9834 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9835 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9836 });
9837 let connection = ConnectionId::new(301);
9838 let response = handler
9839 .handle_control(
9840 connection,
9841 hello_with_role_declarations(declarations.clone()),
9842 )
9843 .unwrap();
9844 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9845 let registered = registry
9846 .get_module_by_connection(connection)
9847 .unwrap()
9848 .unwrap()
9849 .manifest;
9850 assert_eq!(
9854 serde_json::to_value(®istered.role_declarations).unwrap(),
9855 declarations
9856 );
9857 let (raw, entry) = role_declarations_catalog(&handler);
9858 assert_eq!(raw["role_declarations"], declarations);
9859 assert_eq!(entry.role_declarations, registered.role_declarations);
9860
9861 let request =
9862 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9863 handler
9864 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9865 .unwrap();
9866 let (raw, entry) = role_declarations_catalog(&handler);
9867 assert_eq!(raw["role_declarations"], declarations);
9868 assert_eq!(entry.role_declarations, registered.role_declarations);
9869 }
9870
9871 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9872 let registry = Arc::new(Registry::default());
9873 let handler = ControlHandler::new(Arc::clone(®istry));
9874 let capture = EventCapture::default();
9875 let _subscriber =
9876 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9877 let response = handler
9878 .handle_control(
9879 ConnectionId::new(301),
9880 hello_with_role_declarations(declarations),
9881 )
9882 .unwrap();
9883 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9884 assert!(registry
9885 .get_module_by_connection(ConnectionId::new(301))
9886 .unwrap()
9887 .unwrap()
9888 .manifest
9889 .role_declarations
9890 .is_empty());
9891 let (raw, entry) = role_declarations_catalog(&handler);
9892 assert!(raw.get("role_declarations").is_none());
9893 assert!(entry.role_declarations.is_empty());
9894 let warnings: Vec<_> = capture
9895 .events()
9896 .into_iter()
9897 .filter(|event| {
9898 event.fields.get("message").map(String::as_str)
9899 == Some("dropping whole role_declarations map from HELLO")
9900 })
9901 .collect();
9902 assert_eq!(warnings.len(), 1);
9903 assert_eq!(warnings[0].level, tracing::Level::WARN);
9904 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9905 assert_eq!(warnings[0].fields["reason"], reason);
9906 }
9907
9908 #[test]
9909 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9910 for value in [
9911 json!(null),
9912 json!(false),
9913 json!(0),
9914 json!("opaque"),
9915 json!([]),
9916 ] {
9917 assert_hello_drops_role_declarations_with_warning(
9918 json!({"routing/v1": {}, "broken/v1": value}),
9919 "role_declarations values must be objects",
9920 );
9921 }
9922 assert_hello_drops_role_declarations_with_warning(
9923 json!([]),
9924 "role_declarations must be an object",
9925 );
9926 }
9927
9928 #[test]
9929 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9930 assert_hello_drops_role_declarations_with_warning(
9931 json!({"routing/v1": {}, "bad key": {}}),
9932 "role_declarations key must use capability-name grammar",
9933 );
9934 }
9935
9936 #[test]
9937 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9938 assert_hello_drops_role_declarations_with_warning(
9939 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9940 "role_declarations exceeds 16384 compact JSON bytes",
9941 );
9942 }
9943
9944 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
9945 let mut body = serde_json::to_value(ModuleHelloBody {
9946 manifest: manifest("flow-ops", PROTOCOL_VERSION),
9947 protocol_ver: PROTOCOL_VERSION,
9948 control_ops: None,
9949 launch_nonce: None,
9950 })
9951 .unwrap();
9952 body["manifest"]["provides"] = json!([{
9953 "role": "management_surface",
9954 "operations": [
9955 {"name": "diagnostics.findings", "kind": "query"},
9956 {"name": "reset", "kind": "mutate"}
9957 ],
9958 "config_schema": {}, "observability": [], "identity_scope": []
9959 }]);
9960 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
9961 if let Some(flow_operations) = flow_operations {
9962 body["manifest"]["flow_operations"] = flow_operations;
9963 }
9964 Frame::build(
9965 FrameType::Hello,
9966 control_flags(),
9967 0,
9968 0,
9969 401,
9970 serde_json::to_vec(&body).unwrap(),
9971 )
9972 .unwrap()
9973 }
9974
9975 #[test]
9976 fn catalog_list_mirrors_flow_operations_unchanged() {
9977 for (capability, flow_operations) in [
9980 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
9981 ("call-modes/v1", json!(["diagnostics.findings"])),
9982 ("flow-scopes/v1", json!([])),
9983 ] {
9984 let registry = Arc::new(Registry::default());
9985 let handler = ControlHandler::new(Arc::clone(®istry));
9986 let connection = ConnectionId::new(401);
9987 let response = handler
9988 .handle_control(
9989 connection,
9990 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
9991 )
9992 .unwrap();
9993 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9994 let registered = registry
9995 .get_module_by_connection(connection)
9996 .unwrap()
9997 .unwrap()
9998 .manifest;
9999 assert_eq!(
10000 serde_json::to_value(®istered.flow_operations).unwrap(),
10001 flow_operations
10002 );
10003 let (raw, entry) = role_declarations_catalog(&handler);
10004 assert_eq!(raw["flow_operations"], flow_operations);
10005 assert_eq!(entry.flow_operations, registered.flow_operations);
10006 }
10007
10008 let registry = Arc::new(Registry::default());
10009 let handler = ControlHandler::new(Arc::clone(®istry));
10010 let response = handler
10011 .handle_control(
10012 ConnectionId::new(401),
10013 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10014 )
10015 .unwrap();
10016 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10017 let (raw, entry) = role_declarations_catalog(&handler);
10018 assert!(raw.get("flow_operations").is_none());
10019 assert_eq!(entry.flow_operations, None);
10020 }
10021
10022 fn assert_hello_drops_flow_operations_with_warning(
10023 capabilities: Value,
10024 flow_operations: Value,
10025 reason: &str,
10026 ) {
10027 let registry = Arc::new(Registry::default());
10028 let handler = ControlHandler::new(Arc::clone(®istry));
10029 let capture = EventCapture::default();
10030 let _subscriber =
10031 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10032 let response = handler
10033 .handle_control(
10034 ConnectionId::new(401),
10035 hello_with_flow_operations(capabilities, Some(flow_operations)),
10036 )
10037 .unwrap();
10038 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10039 let registered = registry
10040 .get_module_by_connection(ConnectionId::new(401))
10041 .unwrap()
10042 .expect("the module still registers")
10043 .manifest;
10044 assert_eq!(registered.module_id, "flow-ops");
10045 assert_eq!(registered.flow_operations, None);
10046 let (raw, entry) = role_declarations_catalog(&handler);
10047 assert_eq!(raw["module_id"], "flow-ops");
10048 assert!(raw.get("flow_operations").is_none());
10049 assert_eq!(entry.flow_operations, None);
10050 let warnings: Vec<_> = capture
10051 .events()
10052 .into_iter()
10053 .filter(|event| {
10054 event.fields.get("message").map(String::as_str)
10055 == Some("dropping flow_operations from HELLO")
10056 })
10057 .collect();
10058 assert_eq!(warnings.len(), 1);
10059 assert_eq!(warnings[0].level, tracing::Level::WARN);
10060 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10061 assert_eq!(warnings[0].fields["reason"], reason);
10062 }
10063
10064 #[test]
10065 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10066 assert_hello_drops_flow_operations_with_warning(
10067 json!(["flow-scopes/v1"]),
10068 json!(["diagnostics.findings", "missing"]),
10069 "flow_operations names 'missing', which is not a declared management operation",
10070 );
10071 }
10072
10073 #[test]
10074 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10075 assert_hello_drops_flow_operations_with_warning(
10076 json!([]),
10077 json!(["diagnostics.findings"]),
10078 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10079 );
10080 assert_hello_drops_flow_operations_with_warning(
10082 json!(["agent-run-scopes/v1"]),
10083 json!([]),
10084 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10085 );
10086 }
10087
10088 #[test]
10089 fn hello_malformed_flow_operations_is_dropped_with_warning() {
10090 assert_hello_drops_flow_operations_with_warning(
10091 json!(["flow-scopes/v1"]),
10092 json!(["reset", "reset"]),
10093 "flow_operations names 'reset' more than once",
10094 );
10095 assert_hello_drops_flow_operations_with_warning(
10096 json!(["flow-scopes/v1"]),
10097 json!(vec!["reset"; 257]),
10098 "flow_operations lists 257 names, more than 256",
10099 );
10100 assert_hello_drops_flow_operations_with_warning(
10101 json!(["flow-scopes/v1"]),
10102 json!("reset"),
10103 "flow_operations must be an array of operation names",
10104 );
10105 }
10106
10107 #[test]
10108 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10109 let registry = Arc::new(Registry::default());
10110 let handler = ControlHandler::new(registry.clone());
10111 let descriptor = json!({
10112 "schema": "ckios.richui.manifest/1",
10113 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10114 "bytes": 128,
10115 "chunks": 2
10116 });
10117 let malformed = [
10118 ("schema", json!("")),
10119 ("schema", json!("a".repeat(65))),
10120 ("schema", json!("Bad schema")),
10121 (
10122 "manifest_digest",
10123 json!(format!("sha256:{}", "A".repeat(64))),
10124 ),
10125 ("manifest_digest", json!("sha256:abc")),
10126 ("bytes", json!(0)),
10127 ("bytes", json!(16_777_217)),
10128 ("bytes", json!("128")),
10129 ("chunks", json!(0)),
10130 ("chunks", json!(1025)),
10131 ("chunks", json!("2")),
10132 ];
10133 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10134 let mut invalid = descriptor.clone();
10135 invalid[field] = invalid_value;
10136 let response = handler
10137 .handle_control(
10138 ConnectionId::new(210 + index as u64),
10139 hello_frame_with_rich_ui(
10140 &format!("bad-rich-ui-{index}"),
10141 PROTOCOL_VERSION,
10142 210 + index as u64,
10143 None,
10144 invalid,
10145 ),
10146 )
10147 .unwrap();
10148 assert_eq!(response[0].header.ty, FrameType::Error);
10149 let error = parse_error(&response[0]);
10150 assert_eq!(error["code"], "invalid_manifest");
10151 assert!(error["message"]
10152 .as_str()
10153 .unwrap()
10154 .contains(&format!("rich_ui.{field}")));
10155 assert!(registry
10156 .get_module(&format!("bad-rich-ui-{index}"))
10157 .unwrap()
10158 .is_none());
10159 }
10160 }
10161
10162 #[test]
10163 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10164 let registry = Arc::new(Registry::default());
10165 let handler = ControlHandler::new(Arc::clone(®istry));
10166 let connection = ConnectionId::new(219);
10167 registry
10168 .register_with_control_ops(
10169 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10170 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10171 "bad schema",
10172 format!("sha256:{}", "a".repeat(64)),
10173 128,
10174 2,
10175 )))
10176 .build(),
10177 PROTOCOL_VERSION,
10178 connection,
10179 Vec::new(),
10180 )
10181 .unwrap();
10182 let request =
10183 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10184 let response = handler
10185 .handle_catalog_update(connection, request, Vec::new(), None, None)
10186 .unwrap();
10187 assert_eq!(response[0].header.ty, FrameType::Error);
10188 let error = parse_error(&response[0]);
10189 assert_eq!(error["code"], "invalid_manifest");
10190 assert!(error["message"]
10191 .as_str()
10192 .unwrap()
10193 .contains("rich_ui.schema"));
10194 }
10195
10196 #[test]
10197 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10198 let registry = Arc::new(Registry::default());
10199 let supervisor = SupervisorHandle::new();
10200 supervisor.open_swap("provider", "candidate-nonce".to_string());
10201 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10202 let connection = ConnectionId::new(218);
10203 let hello = handler
10204 .handle_control(
10205 connection,
10206 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10207 )
10208 .unwrap();
10209 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10210 assert!(registry.list_modules().unwrap().1.is_empty());
10211
10212 registry.promote_candidate("provider").unwrap().unwrap();
10213 let request = Frame::build(
10214 FrameType::Request,
10215 control_flags(),
10216 0,
10217 0,
10218 219,
10219 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10220 )
10221 .unwrap();
10222 let response = handler.handle_catalog_list(request, None).unwrap();
10223 let ClientControlResponse::CatalogList { modules, .. } =
10224 serde_json::from_slice(&response[0].body).unwrap()
10225 else {
10226 panic!("expected catalog.list");
10227 };
10228 assert_eq!(modules.len(), 1);
10229 assert!(modules[0].nonce_verified);
10230 }
10231
10232 #[test]
10233 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10234 let registry = Arc::new(Registry::default());
10235 let handler = ControlHandler::new(Arc::clone(®istry));
10236 let hello = Frame::build(
10237 FrameType::Hello,
10238 control_flags(),
10239 0,
10240 0,
10241 220,
10242 serde_json::to_vec(&ModuleHelloBody {
10243 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10244 protocol_ver: PROTOCOL_VERSION,
10245 control_ops: None,
10246 launch_nonce: Some("unverified".to_string()),
10247 })
10248 .unwrap(),
10249 )
10250 .unwrap();
10251 assert_eq!(
10252 handler
10253 .handle_control(ConnectionId::new(220), hello)
10254 .unwrap()[0]
10255 .header
10256 .ty,
10257 FrameType::HelloAck
10258 );
10259 let request = Frame::build(
10260 FrameType::Request,
10261 control_flags(),
10262 0,
10263 0,
10264 221,
10265 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10266 )
10267 .unwrap();
10268 let response = handler.handle_catalog_list(request, None).unwrap();
10269 let ClientControlResponse::CatalogList { modules, .. } =
10270 serde_json::from_slice(&response[0].body).unwrap()
10271 else {
10272 panic!("expected catalog.list");
10273 };
10274 assert_eq!(
10275 serde_json::to_vec(&modules[0]).unwrap(),
10276 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10277 );
10278 assert!(!modules[0].nonce_verified);
10279 }
10280
10281 #[test]
10282 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10283 let registry = Arc::new(Registry::default());
10284 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10285 [("vault".to_string(), true), ("squatter".to_string(), true)],
10286 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10287 );
10288 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10289 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10290 provides: vec!["credentials-provider/v1".to_string()],
10291 requires: Vec::new(),
10292 must_never_reach: Vec::new(),
10293 });
10294 let frame = Frame::build(
10295 FrameType::Hello,
10296 control_flags(),
10297 0,
10298 0,
10299 77,
10300 serde_json::to_vec(&ModuleHelloBody {
10301 manifest: squatter,
10302 protocol_ver: PROTOCOL_VERSION,
10303 control_ops: None,
10304 launch_nonce: None,
10305 })
10306 .expect("HELLO serializes"),
10307 )
10308 .expect("HELLO frame builds");
10309 let response = handler
10310 .handle_control(ConnectionId::new(77), frame)
10311 .expect("reserved claim receives a typed refusal");
10312 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10313 assert_eq!(
10314 registry
10315 .active_registration_count()
10316 .expect("registry reads"),
10317 0,
10318 "a reserved capability refusal must not leave a catalog entry"
10319 );
10320 }
10321
10322 #[test]
10323 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10324 for settlement in ["timeout", "inconclusive", "drop"] {
10325 let breakers = RouteBindBreakers::default();
10326 let RouteBindAdmission::Admitted {
10327 guard: mut old,
10328 probe: false,
10329 } = breakers.admit("prov")
10330 else {
10331 panic!("ordinary relay admitted")
10332 };
10333 let RouteBindAdmission::Admitted {
10334 guard: mut opener, ..
10335 } = breakers.admit("prov")
10336 else {
10337 panic!("second relay admitted")
10338 };
10339 assert!(
10340 !opener
10341 .record_timeout(1, Duration::ZERO)
10342 .unwrap()
10343 .reopened_after_probe
10344 );
10345 let RouteBindAdmission::Admitted {
10346 guard: mut probe,
10347 probe: true,
10348 } = breakers.admit("prov")
10349 else {
10350 panic!("one cooldown probe admitted")
10351 };
10352 match settlement {
10353 "timeout" => assert!(
10354 !old.record_timeout(1, Duration::ZERO)
10355 .unwrap()
10356 .reopened_after_probe
10357 ),
10358 "inconclusive" => old.record_inconclusive(),
10359 "drop" => drop(old),
10360 _ => unreachable!(),
10361 }
10362 assert!(
10363 matches!(
10364 breakers.admit("prov"),
10365 RouteBindAdmission::Refused {
10366 probe_in_flight: true,
10367 ..
10368 }
10369 ),
10370 "{settlement} of a pre-open relay cannot release the real probe"
10371 );
10372 assert!(
10373 probe
10374 .record_timeout(1, Duration::ZERO)
10375 .unwrap()
10376 .reopened_after_probe
10377 );
10378 assert!(matches!(
10379 breakers.admit("prov"),
10380 RouteBindAdmission::Admitted { probe: true, .. }
10381 ));
10382 }
10383 let breakers = RouteBindBreakers::default();
10384 let admit = || match breakers.admit("prov") {
10385 RouteBindAdmission::Admitted { guard, .. } => guard,
10386 _ => panic!("relay admitted"),
10387 };
10388 admit().record_timeout(1, Duration::ZERO);
10389 let mut old_probe = admit();
10390 breakers.reset_for_new_module_connection("prov");
10391 admit().record_timeout(1, Duration::ZERO);
10392 let _new_probe = admit();
10393 old_probe.record_inconclusive();
10394 assert!(matches!(
10395 breakers.admit("prov"),
10396 RouteBindAdmission::Refused {
10397 probe_in_flight: true,
10398 ..
10399 }
10400 ));
10401 }
10402
10403 #[tokio::test]
10404 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10405 for candidate in [false, true] {
10406 let registry = Arc::new(Registry::default());
10407 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10408 [("vault".to_string(), true), ("squatter".to_string(), true)],
10409 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10410 );
10411 let conn = ConnectionId::new(77);
10412 let (ctx, mut rx) = route_ctx(conn);
10413 let initial = capability_manifest("squatter", &[], &[]);
10414 if candidate {
10415 registry
10416 .register_candidate_with_control_ops(
10417 initial.clone(),
10418 PROTOCOL_VERSION,
10419 conn,
10420 module_baseline_control_ops(),
10421 )
10422 .unwrap();
10423 handler
10424 .forwarding
10425 .register_candidate_module_connection(
10426 conn,
10427 "squatter".to_string(),
10428 PROTOCOL_VERSION,
10429 manifest_concurrency(&initial),
10430 ctx.egress.clone(),
10431 )
10432 .unwrap();
10433 } else {
10434 hello_via_sink(
10435 &handler,
10436 &ctx,
10437 &mut rx,
10438 hello_frame_with_manifest(initial.clone(), 1),
10439 )
10440 .await;
10441 }
10442 let response = handler
10443 .handle_control_frame(
10444 &ctx,
10445 catalog_update_with_capabilities_frame(
10446 2,
10447 capability_manifest("squatter", &["credentials-provider/v1"], &[])
10448 .capabilities
10449 .unwrap(),
10450 ),
10451 )
10452 .await
10453 .unwrap();
10454 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10455 assert_eq!(
10456 registry
10457 .get_module_by_connection(conn)
10458 .unwrap()
10459 .unwrap()
10460 .manifest,
10461 initial
10462 );
10463 }
10464 }
10465
10466 #[test]
10467 fn server_describe_surfaces_required_capability_verdict_fields() {
10468 let registry = Arc::new(Registry::default());
10469 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10470 [
10471 ("consumer".to_string(), true),
10472 ("provider".to_string(), false),
10473 ],
10474 BTreeMap::new(),
10475 );
10476 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10477 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10478 provides: Vec::new(),
10479 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10480 capability: "credentials-provider/v1".to_string(),
10481 need: subc_protocol::manifest::CapabilityNeed::Required,
10482 }],
10483 must_never_reach: Vec::new(),
10484 });
10485 let hello = Frame::build(
10486 FrameType::Hello,
10487 control_flags(),
10488 0,
10489 0,
10490 78,
10491 serde_json::to_vec(&ModuleHelloBody {
10492 manifest: consumer,
10493 protocol_ver: PROTOCOL_VERSION,
10494 control_ops: None,
10495 launch_nonce: None,
10496 })
10497 .expect("HELLO serializes"),
10498 )
10499 .expect("HELLO frame builds");
10500 handler
10501 .handle_control(ConnectionId::new(78), hello)
10502 .expect("consumer registers");
10503 let describe = Frame::build(
10504 FrameType::Request,
10505 control_flags(),
10506 0,
10507 0,
10508 79,
10509 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10510 .expect("request serializes"),
10511 )
10512 .expect("describe frame builds");
10513 let response = handler
10514 .handle_server_describe(describe)
10515 .expect("server.describe succeeds");
10516 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10517 let requirement = &rendered["capability_requirements"][0];
10518 assert_eq!(requirement["consumer"], "consumer");
10519 assert_eq!(requirement["verdict"], "never_provided");
10520 assert_eq!(requirement["episode_seq"], 1);
10521 assert_eq!(requirement["config_satisfiable"], false);
10522 assert_eq!(requirement["runtime_available"], false);
10523 assert!(requirement["detail"]
10524 .as_str()
10525 .expect("detail string")
10526 .contains("credentials-provider/v1"));
10527 }
10528
10529 fn deregister_required_provider(
10535 daemon_shutdown_begun: bool,
10536 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10537 const CAPABILITY: &str = "project-identity/v1";
10538 let registry = Arc::new(Registry::default());
10539 let daemon_shutdown = DaemonShutdownFlag::default();
10540 let handler = ControlHandler::new(Arc::clone(®istry))
10541 .with_capability_config(
10542 [
10543 ("consumer".to_string(), true),
10544 ("provider".to_string(), true),
10545 ],
10546 BTreeMap::new(),
10547 )
10548 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10549 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10550 consumer.capabilities = Some(CapabilityDeclarations {
10551 provides: Vec::new(),
10552 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10553 capability: CAPABILITY.to_string(),
10554 need: subc_protocol::manifest::CapabilityNeed::Required,
10555 }],
10556 must_never_reach: Vec::new(),
10557 });
10558 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10559 let provider_conn = ConnectionId::new(182);
10560 for (conn, manifest, corr) in [
10561 (ConnectionId::new(181), consumer, 181),
10562 (provider_conn, provider, 182),
10563 ] {
10564 let hello = handler
10565 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10566 .expect("module registers");
10567 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10568 }
10569 assert_eq!(
10570 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10571 Some(CapabilityVerdict::Provided),
10572 "the registered provider must satisfy the requirement first"
10573 );
10574 if daemon_shutdown_begun {
10575 daemon_shutdown.set_for_test();
10576 }
10577
10578 let capture = EventCapture::default();
10579 let _subscriber =
10580 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10581 handler
10582 .cleanup_connection(provider_conn)
10583 .expect("provider deregisters");
10584 let events = capture
10585 .events()
10586 .into_iter()
10587 .filter(|event| {
10588 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10589 })
10590 .collect();
10591 (
10592 events,
10593 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10594 )
10595 }
10596
10597 #[test]
10598 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10599 let (events, verdict) = deregister_required_provider(true);
10600 assert_eq!(
10601 verdict,
10602 Some(CapabilityVerdict::NeverProvided),
10603 "shutdown must not change the verdict route.open refusals read"
10604 );
10605 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10606 let event = &events[0];
10607 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10608 assert_eq!(
10609 event.fields.get("cause").map(String::as_str),
10610 Some("\"daemon_shutdown\""),
10611 "{event:?}"
10612 );
10613 assert_eq!(
10614 event.fields.get("verdict").map(String::as_str),
10615 Some("\"never_provided\""),
10616 "{event:?}"
10617 );
10618 }
10619
10620 #[test]
10621 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10622 let (events, verdict) = deregister_required_provider(false);
10623 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10624 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10625 let event = &events[0];
10626 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10627 assert_eq!(event.fields.get("cause"), None, "{event:?}");
10628 assert_eq!(
10629 event.fields.get("verdict").map(String::as_str),
10630 Some("\"never_provided\""),
10631 "{event:?}"
10632 );
10633 }
10634
10635 #[test]
10636 fn catalog_list_omits_capabilities_for_legacy_manifest() {
10637 let registry = Arc::new(Registry::default());
10638 let handler = ControlHandler::new(Arc::clone(®istry));
10639 let hello = handler
10640 .handle_control(
10641 ConnectionId::new(101),
10642 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10643 )
10644 .expect("legacy HELLO registers");
10645 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10646
10647 let request = Frame::build(
10648 FrameType::Request,
10649 control_flags(),
10650 0,
10651 0,
10652 102,
10653 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10654 .expect("catalog request serializes"),
10655 )
10656 .expect("catalog request frame builds");
10657 let response = handler
10658 .handle_catalog_list(request, None)
10659 .expect("catalog list succeeds");
10660 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10661 assert!(
10662 body["modules"][0].get("capabilities").is_none(),
10663 "legacy manifest must retain an absent capabilities field on catalog.list"
10664 );
10665 }
10666
10667 #[test]
10668 fn hello_ack_omits_storage_when_no_storage_config() {
10669 let registry = Arc::new(Registry::default());
10670 let handler = ControlHandler::new(Arc::clone(®istry));
10671 let responses = handler
10672 .handle_control(
10673 ConnectionId::new(1),
10674 hello_frame("aft", PROTOCOL_VERSION, 7),
10675 )
10676 .unwrap();
10677 let ack = parse_ack(&responses[0]);
10678 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10679 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10680 }
10681
10682 #[tokio::test]
10683 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10684 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10685 let registry = Arc::new(Registry::default());
10686 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
10687 let responses = handler
10688 .handle_control(
10689 ConnectionId::new(1),
10690 hello_frame("aft", PROTOCOL_VERSION, 7),
10691 )
10692 .unwrap();
10693 let ack = parse_ack(&responses[0]);
10694 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10695
10696 let described = handler
10697 .handle_control_frame(
10698 &route_ctx(ConnectionId::new(2)).0,
10699 Frame::build(
10700 FrameType::Request,
10701 control_flags(),
10702 0,
10703 0,
10704 9,
10705 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10706 )
10707 .unwrap(),
10708 )
10709 .await
10710 .unwrap();
10711 let ClientControlResponse::ServerDescribe { machine_id, .. } =
10712 serde_json::from_slice(&described[0].body).unwrap()
10713 else {
10714 panic!("server.describe answered with another shape");
10715 };
10716 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10717 }
10718
10719 #[test]
10720 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10721 let registry = Arc::new(Registry::default());
10724 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
10725 crate::daemon_config::StorageConfig::Sqlite {
10726 data_home: std::path::PathBuf::from("/data"),
10727 },
10728 ));
10729
10730 let responses = handler
10731 .handle_control(
10732 ConnectionId::new(1),
10733 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10734 )
10735 .unwrap();
10736 let ack = parse_ack(&responses[0]);
10737 assert_eq!(
10738 ack.storage,
10739 Some(serde_json::json!({
10740 "module_id": "alfonso-routing",
10741 "storage_namespace": "default",
10742 "isolation": { "kind": "module" },
10743 "backend": {
10744 "backend": "sqlite",
10745 "path": "/data/cortexkit/alfonso-routing/store.db"
10746 }
10747 })),
10748 "the delivered descriptor is the module's own sqlite store path"
10749 );
10750 }
10751
10752 #[test]
10753 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10754 let registry = Arc::new(Registry::default());
10755 let handler = ControlHandler::new(Arc::clone(®istry));
10756 let conn = ConnectionId::new(1);
10757 let responses = handler
10758 .handle_control(
10759 conn,
10760 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10761 )
10762 .unwrap();
10763 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10764 let registration = registry.get_module("aft").unwrap().unwrap();
10765 assert_eq!(registration.control_ops, module_baseline_control_ops());
10766
10767 let frame =
10768 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10769 assert!(handler
10770 .guard_module_control_op(&frame, "aft", "route.bind")
10771 .unwrap()
10772 .is_none());
10773 let error = handler
10774 .guard_module_control_op(&frame, "aft", "test.synthetic")
10775 .unwrap()
10776 .expect("synthetic ungranted op should be rejected");
10777 assert_eq!(error.header.ty, FrameType::Error);
10778 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10779 }
10780
10781 #[test]
10782 fn hello_control_ops_some_adds_optional_grants() {
10783 let registry = Arc::new(Registry::default());
10784 let handler = ControlHandler::new(Arc::clone(®istry));
10785 handler
10786 .handle_control(
10787 ConnectionId::new(1),
10788 hello_frame_with_control_ops(
10789 "aft",
10790 PROTOCOL_VERSION,
10791 7,
10792 Some(vec![
10793 "future.synthetic".to_string(),
10794 "route.bind".to_string(),
10795 ]),
10796 ),
10797 )
10798 .unwrap();
10799 let registration = registry.get_module("aft").unwrap().unwrap();
10800 assert_eq!(
10801 registration.control_ops,
10802 vec![
10803 "route.bind".to_string(),
10804 "route.status".to_string(),
10805 "future.synthetic".to_string(),
10806 ]
10807 );
10808 let frame =
10809 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10810 assert!(handler
10811 .guard_module_control_op(&frame, "aft", "future.synthetic")
10812 .unwrap()
10813 .is_none());
10814 }
10815
10816 #[tokio::test]
10817 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10818 let registry = Arc::new(Registry::default());
10819 let forwarding = Arc::new(ForwardingTable::default());
10820 let handler =
10821 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10822 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10823 hello_via_sink(
10824 &handler,
10825 &module_ctx,
10826 &mut module_rx,
10827 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10828 )
10829 .await;
10830
10831 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10832 let responses = handler
10833 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10834 .await
10835 .unwrap();
10836 assert_eq!(responses.len(), 1);
10837 assert_eq!(responses[0].header.ty, FrameType::Error);
10838 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10839 assert!(module_rx.try_recv().is_err());
10840 }
10841
10842 #[tokio::test]
10843 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10844 let registry = Arc::new(Registry::default());
10845 let forwarding = Arc::new(ForwardingTable::default());
10846 let handler =
10847 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10848 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10849 hello_via_sink(
10850 &handler,
10851 &module_ctx,
10852 &mut module_rx,
10853 hello_frame_with_control_ops(
10854 "aft",
10855 PROTOCOL_VERSION,
10856 7,
10857 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10858 ),
10859 )
10860 .await;
10861
10862 let project_root = unique_project_root("demux");
10863 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10864 let route_handler = handler.clone();
10865 let route_task = tokio::spawn(async move {
10866 route_handler
10867 .handle_control_frame(
10868 &route_client_ctx,
10869 route_open_frame(100, "aft", project_root),
10870 )
10871 .await
10872 .unwrap()
10873 });
10874 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10875 .await
10876 .unwrap()
10877 .unwrap();
10878 assert!(matches!(
10879 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10880 ModuleControlRequest::RouteBind { .. }
10881 ));
10882
10883 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10884 let health_handler = handler.clone();
10885 let health_task = tokio::spawn(async move {
10886 health_handler
10887 .handle_control_frame(
10888 &health_client_ctx,
10889 supervisor_health_probe_frame(101, "aft"),
10890 )
10891 .await
10892 .unwrap()
10893 });
10894 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10895 .await
10896 .unwrap()
10897 .unwrap();
10898 assert_eq!(
10899 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10900 ModuleControlRequest::HealthCheck {}
10901 );
10902
10903 handler
10904 .handle_control_frame(
10905 &module_ctx,
10906 health_response(health_frame.header.corr, HealthStatus::Degraded),
10907 )
10908 .await
10909 .unwrap();
10910 let health_response = health_task.await.unwrap();
10911 assert_eq!(health_response.len(), 1);
10912 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10913 ClientControlResponse::SupervisorHealthProbe {
10914 module_id,
10915 status,
10916 detail,
10917 metrics,
10918 } => {
10919 assert_eq!(module_id, "aft");
10920 assert_eq!(status, HealthStatus::Degraded);
10921 assert_eq!(detail.as_deref(), Some("warming"));
10922 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10923 }
10924 other => panic!("unexpected health response: {other:?}"),
10925 }
10926
10927 handler
10928 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10929 .await
10930 .unwrap();
10931 let route_response = route_task.await.unwrap();
10932 assert!(route_response.is_empty());
10933 let published = route_client_rx.recv().await.unwrap();
10934 assert!(matches!(
10935 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10936 ClientControlResponse::RouteOpen { .. }
10937 ));
10938 }
10939
10940 async fn relay_route_open(
10945 handler: &ControlHandler,
10946 client_connection: ConnectionId,
10947 client_egress: &FrameSink,
10948 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10949 corr: u64,
10950 module_id: &str,
10951 project_root_label: &str,
10952 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10953 let ctx = RouteCtx {
10954 connection_id: client_connection,
10955 egress: client_egress.clone(),
10956 };
10957 let handler = handler.clone();
10958 let project_root = unique_project_root(project_root_label);
10959 let module_id = module_id.to_string();
10960 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10961 let task = tokio::spawn(async move {
10962 let _guard = tracing::dispatcher::set_default(&dispatch);
10963 handler
10964 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10965 .await
10966 .unwrap()
10967 });
10968 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10969 .await
10970 .expect("module receives the relayed route.bind")
10971 .expect("module egress is open");
10972 (task, bind.frame)
10973 }
10974
10975 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10976 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10977 ModuleControlRequest::RouteBind {
10978 route_channel,
10979 epoch,
10980 ..
10981 } => (route_channel, epoch),
10982 other => panic!("expected a route.bind request, got {other:?}"),
10983 }
10984 }
10985
10986 fn published_route(frame: &Frame) -> (u16, u32) {
10987 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10988 ClientControlResponse::RouteOpen {
10989 route_channel,
10990 route_epoch,
10991 } => (route_channel, route_epoch),
10992 other => panic!("expected a route.open response, got {other:?}"),
10993 }
10994 }
10995
10996 #[tokio::test]
11016 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11017 let registry = Arc::new(Registry::default());
11018 let forwarding = Arc::new(ForwardingTable::default());
11019 let handler =
11020 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11021
11022 let module_connection = ConnectionId::new(30);
11023 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11024 hello_via_sink(
11025 &handler,
11026 &module_ctx,
11027 &mut module_rx,
11028 hello_frame("aft", PROTOCOL_VERSION, 7),
11029 )
11030 .await;
11031
11032 let dying_client = ConnectionId::new(31);
11033 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11034
11035 let (first_task, first_bind) = relay_route_open(
11038 &handler,
11039 dying_client,
11040 &dying_ctx.egress,
11041 &mut module_rx,
11042 100,
11043 "aft",
11044 "closing-first",
11045 )
11046 .await;
11047 handler
11048 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11049 .await
11050 .unwrap();
11051 assert!(first_task.await.unwrap().is_empty());
11052 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11053
11054 let (second_task, second_bind) = relay_route_open(
11056 &handler,
11057 dying_client,
11058 &dying_ctx.egress,
11059 &mut module_rx,
11060 101,
11061 "aft",
11062 "closing-second",
11063 )
11064 .await;
11065 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11066
11067 assert!(forwarding
11070 .escalate_client_delivery_failure(
11071 dying_client,
11072 first_channel,
11073 first_epoch,
11074 CloseReason::new(
11075 "module_to_client_delivery_failed",
11076 "client egress refused a module frame",
11077 ),
11078 crate::forwarding::UndeliveredFrame {
11079 module_id: None,
11080 sink: &dying_ctx.egress,
11081 },
11082 )
11083 .unwrap());
11084 assert!(!dying_ctx.egress.is_closed());
11085
11086 let ack = handler
11088 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11089 .await;
11090 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11091 if module_loop_error.is_some() {
11092 handler.cleanup_connection(module_connection).unwrap();
11096 }
11097 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11101 .await
11102 .ok()
11103 .flatten();
11104
11105 assert!(
11107 registry
11108 .get_module_by_connection(module_connection)
11109 .unwrap()
11110 .is_some(),
11111 "one client's closing connection ended the shared module connection: \
11112 {module_loop_error:?}"
11113 );
11114 let cotenant = ConnectionId::new(32);
11116 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11117 let (cotenant_task, cotenant_bind) = relay_route_open(
11118 &handler,
11119 cotenant,
11120 &cotenant_ctx.egress,
11121 &mut module_rx,
11122 102,
11123 "aft",
11124 "closing-cotenant",
11125 )
11126 .await;
11127 handler
11128 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11129 .await
11130 .unwrap();
11131 assert!(cotenant_task.await.unwrap().is_empty());
11132 let (cotenant_channel, cotenant_epoch) =
11133 published_route(&cotenant_rx.recv().await.unwrap());
11134 assert!(matches!(
11135 forwarding
11136 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11137 .unwrap(),
11138 DataRoute::Client(DataRouteState::Bound(_))
11139 ));
11140
11141 let goodbye = post_ack_module_frame
11144 .expect("module receives a GOODBYE for the abandoned route channel");
11145 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11146 assert_eq!(goodbye.header.channel, abandoned_channel);
11147 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11148
11149 assert!(dying_rx.try_recv().is_err());
11153 let second_response = second_task.await.unwrap();
11154 assert_eq!(second_response.len(), 1);
11155 assert_eq!(
11156 parse_error(&second_response[0])["code"],
11157 "target_unavailable"
11158 );
11159 }
11160
11161 async fn catalog_snapshot(
11164 handler: &ControlHandler,
11165 client_ctx: &RouteCtx,
11166 corr: u64,
11167 ) -> (u64, Vec<(String, bool)>) {
11168 let body =
11169 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11170 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11171 let reply = handler
11172 .handle_control_frame(client_ctx, request)
11173 .await
11174 .unwrap()
11175 .pop()
11176 .unwrap();
11177 let ClientControlResponse::CatalogList {
11178 generation,
11179 modules,
11180 ..
11181 } = serde_json::from_slice(&reply.body).unwrap()
11182 else {
11183 panic!("not a catalog.list reply");
11184 };
11185 (
11186 generation,
11187 modules
11188 .into_iter()
11189 .map(|entry| (entry.module_id, entry.ready))
11190 .collect(),
11191 )
11192 }
11193
11194 fn hello_held_before_forwarding() -> (
11200 ControlHandler,
11201 Arc<std::sync::Barrier>,
11202 std::thread::JoinHandle<Vec<Frame>>,
11203 RouteCtx,
11204 mpsc::Receiver<crate::router::OutboundFrame>,
11205 ) {
11206 let hold = Arc::new(std::sync::Barrier::new(2));
11207 let handler = ControlHandler::with_forwarding(
11208 Arc::new(Registry::default()),
11209 Arc::new(ForwardingTable::default()),
11210 )
11211 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11212 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11213 let hello = {
11214 let handler = handler.clone();
11215 let module_ctx = module_ctx.clone();
11216 std::thread::spawn(move || {
11217 handler
11218 .handle_hello(
11219 module_ctx.connection_id,
11220 Some(module_ctx.egress.clone()),
11221 hello_frame("aft", PROTOCOL_VERSION, 7),
11222 )
11223 .unwrap()
11224 })
11225 };
11226 (handler, hold, hello, module_ctx, module_rx)
11227 }
11228
11229 #[tokio::test]
11238 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11239 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11240 let client = ConnectionId::new(41);
11241 let (client_ctx, mut client_rx) = route_ctx(client);
11242
11243 hold.wait();
11244 assert!(handler.registry.get_module("aft").unwrap().is_some());
11246 assert!(!handler
11247 .forwarding
11248 .has_live_module_connection("aft")
11249 .unwrap());
11250 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11251 assert!(
11252 listed.is_empty(),
11253 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11254 );
11255 let refused = handler
11256 .handle_control_frame(
11257 &client_ctx,
11258 route_open_frame(301, "aft", unique_project_root("listed-window")),
11259 )
11260 .await
11261 .unwrap();
11262 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11263 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11264 let duplicate = handler
11265 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11266 .await
11267 .unwrap();
11268 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11269
11270 hold.wait();
11271 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11272 let ack = module_rx.try_recv().unwrap().frame;
11273 assert_eq!(ack.header.ty, FrameType::HelloAck);
11274 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11275 assert_eq!(listed, vec![("aft".to_string(), true)]);
11276
11277 let (open_task, bind) = relay_route_open(
11278 &handler,
11279 client,
11280 &client_ctx.egress,
11281 &mut module_rx,
11282 303,
11283 "aft",
11284 "listed-after",
11285 )
11286 .await;
11287 assert!(handler
11288 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11289 .await
11290 .unwrap()
11291 .is_empty());
11292 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11293 published_route(&client_rx.try_recv().unwrap().frame);
11294 }
11295
11296 #[tokio::test]
11303 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11304 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11305 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11306 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11307
11308 hold.wait();
11309 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11310 assert!(listed.is_empty());
11311 assert_eq!(
11312 during, before,
11313 "the generation moved while the module was not listed"
11314 );
11315
11316 hold.wait();
11317 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11318 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11319 assert_eq!(listed, vec![("aft".to_string(), true)]);
11320 assert_eq!(
11321 after,
11322 before + 1,
11323 "the module appeared without exactly one generation bump"
11324 );
11325 }
11326
11327 #[tokio::test]
11332 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11333 let registry = Arc::new(Registry::default());
11334 let handler = ControlHandler::with_forwarding(
11335 Arc::clone(®istry),
11336 Arc::new(ForwardingTable::default()),
11337 );
11338 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11339 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11340 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11341 drop(module_rx);
11342
11343 let replies = handler
11344 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11345 .await
11346 .unwrap();
11347 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11348 assert!(registry.get_module("aft").unwrap().is_none());
11349 assert!(registry
11350 .get_module_by_connection(module_ctx.connection_id)
11351 .unwrap()
11352 .is_none());
11353 assert_eq!(
11354 catalog_snapshot(&handler, &client_ctx, 301).await,
11355 (before, Vec::new())
11356 );
11357 }
11358
11359 async fn route_open_held_after_bind_send() -> (
11363 ControlHandler,
11364 Arc<tokio::sync::Barrier>,
11365 RouteCtx,
11366 mpsc::Receiver<crate::router::OutboundFrame>,
11367 ) {
11368 let hold = Arc::new(tokio::sync::Barrier::new(2));
11369 let handler = ControlHandler::with_forwarding(
11370 Arc::new(Registry::default()),
11371 Arc::new(ForwardingTable::default()),
11372 )
11373 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11374 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11375 hello_via_sink(
11376 &handler,
11377 &module_ctx,
11378 &mut module_rx,
11379 hello_frame("aft", PROTOCOL_VERSION, 7),
11380 )
11381 .await;
11382 (handler, hold, module_ctx, module_rx)
11383 }
11384
11385 #[tokio::test]
11392 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11393 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11394 let client = ConnectionId::new(41);
11395 let (client_ctx, mut client_rx) = route_ctx(client);
11396
11397 let (open_task, bind) = relay_route_open(
11398 &handler,
11399 client,
11400 &client_ctx.egress,
11401 &mut module_rx,
11402 200,
11403 "aft",
11404 "acked-before-mark",
11405 )
11406 .await;
11407 let (module_channel, module_epoch) = route_bind_channel(&bind);
11408 hold.wait().await;
11410
11411 assert!(handler
11413 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11414 .await
11415 .unwrap()
11416 .is_empty());
11417 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11418
11419 hold.wait().await;
11420 assert!(
11421 open_task.await.unwrap().is_empty(),
11422 "route.open answers accepted"
11423 );
11424
11425 match module_rx.try_recv() {
11428 Err(mpsc::error::TryRecvError::Empty) => {}
11429 Ok(outbound) => panic!(
11430 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11431 outbound.frame.header.ty,
11432 outbound.frame.header.channel,
11433 outbound.frame.header.epoch
11434 ),
11435 Err(err) => panic!("module egress closed: {err}"),
11436 }
11437
11438 let router = Router::with_control_handler(Arc::new(handler.clone()));
11441 let request = Frame::build(
11442 FrameType::Request,
11443 Flags::new(true, Priority::Interactive, false),
11444 client_channel,
11445 client_epoch,
11446 201,
11447 b"first request".to_vec(),
11448 )
11449 .unwrap();
11450 router
11451 .route_for_connection(&client_ctx, request)
11452 .await
11453 .unwrap();
11454 let delivered = module_rx.try_recv().unwrap().frame;
11455 assert_eq!(delivered.header.ty, FrameType::Request);
11456 assert_eq!(
11457 (delivered.header.channel, delivered.header.epoch),
11458 (module_channel, module_epoch)
11459 );
11460 assert_eq!(delivered.body, b"first request");
11461 }
11462
11463 #[tokio::test]
11469 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11470 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11471 let client = ConnectionId::new(42);
11472 let (client_ctx, _client_rx) = route_ctx(client);
11473
11474 let (open_task, bind) = relay_route_open(
11475 &handler,
11476 client,
11477 &client_ctx.egress,
11478 &mut module_rx,
11479 300,
11480 "aft",
11481 "released-before-mark",
11482 )
11483 .await;
11484 let (module_channel, module_epoch) = route_bind_channel(&bind);
11485 hold.wait().await;
11486
11487 handler.cleanup_connection(client).unwrap();
11488 assert!(
11489 module_rx.try_recv().is_err(),
11490 "client teardown must not GOODBYE a bind route.open is still queueing"
11491 );
11492
11493 hold.wait().await;
11494 let response = open_task.await.unwrap();
11495 assert_eq!(response.len(), 1);
11496 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11497
11498 let goodbye = module_rx
11499 .try_recv()
11500 .expect("module is told to drop the binding the daemon abandoned")
11501 .frame;
11502 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11503 assert_eq!(
11504 (goodbye.header.channel, goodbye.header.epoch),
11505 (module_channel, module_epoch)
11506 );
11507 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11508 }
11509
11510 #[tokio::test]
11515 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11516 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11517 let client = ConnectionId::new(43);
11518 let (client_ctx, client_rx) = route_ctx(client);
11519
11520 let (open_task, bind) = relay_route_open(
11521 &handler,
11522 client,
11523 &client_ctx.egress,
11524 &mut module_rx,
11525 400,
11526 "aft",
11527 "gone-client-before-mark",
11528 )
11529 .await;
11530 let (module_channel, module_epoch) = route_bind_channel(&bind);
11531 hold.wait().await;
11532
11533 drop(client_rx);
11534 assert!(client_ctx.egress.is_closed());
11535 handler
11536 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11537 .await
11538 .unwrap();
11539
11540 hold.wait().await;
11541 let response = open_task.await.unwrap();
11542 assert_eq!(response.len(), 1);
11543 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11544
11545 let goodbye = module_rx
11546 .try_recv()
11547 .expect("module is told to drop the binding it acked")
11548 .frame;
11549 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11550 assert_eq!(
11551 (goodbye.header.channel, goodbye.header.epoch),
11552 (module_channel, module_epoch)
11553 );
11554 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11555 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11556 }
11557
11558 #[test]
11565 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11566 let handler = ControlHandler::default();
11567 let module_connection = ConnectionId::new(30);
11568 let client_connection = ConnectionId::new(31);
11569
11570 handler
11571 .refuse_to_end_module_connection_for_a_client(
11572 module_connection,
11573 77,
11574 ForwardingError::ConnectionClosing {
11575 connection_id: client_connection,
11576 },
11577 )
11578 .expect("a closing client must never end the module connection");
11579
11580 assert!(matches!(
11581 handler.refuse_to_end_module_connection_for_a_client(
11582 module_connection,
11583 78,
11584 ForwardingError::ConnectionClosing {
11585 connection_id: module_connection,
11586 },
11587 ),
11588 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11589 connection_id
11590 })) if connection_id == module_connection
11591 ));
11592 assert!(matches!(
11593 handler.refuse_to_end_module_connection_for_a_client(
11594 module_connection,
11595 79,
11596 ForwardingError::Poisoned,
11597 ),
11598 Err(RouterError::Forwarding(ForwardingError::Poisoned))
11599 ));
11600 assert!(matches!(
11601 handler.refuse_to_end_module_connection_for_a_client(
11602 module_connection,
11603 80,
11604 ForwardingError::StaleModuleEndpoint,
11605 ),
11606 Err(RouterError::Forwarding(
11607 ForwardingError::StaleModuleEndpoint
11608 ))
11609 ));
11610 }
11611
11612 #[tokio::test]
11625 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11626 let registry = Arc::new(Registry::default());
11627 let forwarding = Arc::new(ForwardingTable::default());
11628 let supervisor = SupervisorHandle::new();
11629 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11630 let handler =
11631 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11632 .with_supervisor(supervisor);
11633
11634 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11635 hello_via_sink(
11636 &handler,
11637 &target_ctx,
11638 &mut target_rx,
11639 hello_frame("target", PROTOCOL_VERSION, 1),
11640 )
11641 .await;
11642
11643 let wrong_nonce = handler
11647 .handle_control_frame(
11648 &route_ctx(ConnectionId::new(91)).0,
11649 route_open_frame_with_admission_facts(
11650 20,
11651 "target",
11652 unique_project_root("admission-facts"),
11653 Some(subc_control::ConsumerIdentity {
11654 module_id: "fed".to_string(),
11655 launch_nonce: "not-the-real-nonce".to_string(),
11656 }),
11657 None,
11658 ),
11659 )
11660 .await
11661 .unwrap();
11662 assert_eq!(
11663 parse_error(&wrong_nonce[0])["code"],
11664 "bad_consumer_identity",
11665 "a mismatched launch nonce must be refused, not stamped Reserved"
11666 );
11667
11668 let never_spawned = handler
11672 .handle_control_frame(
11673 &route_ctx(ConnectionId::new(92)).0,
11674 route_open_frame_with_admission_facts(
11675 21,
11676 "target",
11677 unique_project_root("admission-facts"),
11678 Some(subc_control::ConsumerIdentity {
11679 module_id: "never-spawned".to_string(),
11680 launch_nonce: "any-nonce".to_string(),
11681 }),
11682 None,
11683 ),
11684 )
11685 .await
11686 .unwrap();
11687 assert_eq!(
11688 parse_error(&never_spawned[0])["code"],
11689 "bad_consumer_identity",
11690 "an unspawned module_id must be refused rather than accepted for lack of a record"
11691 );
11692 }
11693
11694 #[tokio::test]
11717 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11718 let registry = Arc::new(Registry::default());
11719 let forwarding = Arc::new(ForwardingTable::default());
11720 let supervisor = SupervisorHandle::new();
11721 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11722 let handler =
11723 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11724 .with_supervisor(supervisor);
11725
11726 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11727 hello_via_sink(
11728 &handler,
11729 &target_ctx,
11730 &mut target_rx,
11731 hello_frame("target", PROTOCOL_VERSION, 1),
11732 )
11733 .await;
11734
11735 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11736 let route_handler = handler.clone();
11737 let route_task = tokio::spawn(async move {
11738 route_handler
11739 .handle_control_frame(
11740 &client_ctx,
11741 route_open_frame_with_admission_facts(
11742 30,
11743 "target",
11744 unique_project_root("admission-facts"),
11745 Some(subc_control::ConsumerIdentity {
11746 module_id: "fed".to_string(),
11747 launch_nonce: "fed-nonce".to_string(),
11748 }),
11749 None,
11750 ),
11751 )
11752 .await
11753 .unwrap()
11754 });
11755
11756 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11763 .await
11764 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11765 .expect("module control channel closed before route.bind");
11766 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11767 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11768 panic!("expected route.bind")
11769 };
11770 assert_eq!(
11771 principal,
11772 Some(Principal::Reserved {
11773 module_id: "fed".to_string()
11774 }),
11775 "a correctly attested consumer must be stamped Reserved for its own id"
11776 );
11777
11778 handler
11779 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11780 .await
11781 .unwrap();
11782 assert!(route_task.await.unwrap().is_empty());
11783 assert!(
11784 matches!(
11785 serde_json::from_slice::<ClientControlResponse>(
11786 &client_rx.recv().await.unwrap().body
11787 )
11788 .unwrap(),
11789 ClientControlResponse::RouteOpen { .. }
11790 ),
11791 "the route must actually open, not merely avoid an error"
11792 );
11793 }
11794
11795 #[tokio::test(start_paused = true)]
11796 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11797 let registry = Arc::new(Registry::default());
11798 let forwarding = Arc::new(ForwardingTable::default());
11799 let supervisor = SupervisorHandle::new();
11800 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11801 let handler =
11802 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11803 .with_supervisor(supervisor);
11804
11805 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11806 hello_via_sink(
11807 &handler,
11808 &target_ctx,
11809 &mut target_rx,
11810 hello_frame("target", PROTOCOL_VERSION, 1),
11811 )
11812 .await;
11813
11814 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11815 let direct_handler = handler.clone();
11816 let direct_open = tokio::spawn(async move {
11817 direct_handler
11818 .handle_control_frame(
11819 &direct_ctx,
11820 route_open_frame(2, "target", unique_project_root("route-census-direct")),
11821 )
11822 .await
11823 .unwrap()
11824 });
11825 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11826 .await
11827 .expect("no direct route.bind within 5s")
11828 .expect("target control channel closed before direct route.bind");
11829 handler
11830 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11831 .await
11832 .unwrap();
11833 assert!(direct_open.await.unwrap().is_empty());
11834 let _ = direct_rx.recv().await.unwrap();
11835
11836 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11837 let reserved_handler = handler.clone();
11838 let reserved_open = tokio::spawn(async move {
11839 reserved_handler
11840 .handle_control_frame(
11841 &reserved_ctx,
11842 route_open_frame_with_admission_facts(
11843 3,
11844 "target",
11845 unique_project_root("admission-facts"),
11846 Some(ConsumerIdentity {
11847 module_id: "fed".to_string(),
11848 launch_nonce: "fed-nonce".to_string(),
11849 }),
11850 None,
11851 ),
11852 )
11853 .await
11854 .unwrap()
11855 });
11856 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11857 .await
11858 .expect("no reserved route.bind within 5s")
11859 .expect("target control channel closed before reserved route.bind");
11860 handler
11861 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11862 .await
11863 .unwrap();
11864 assert!(reserved_open.await.unwrap().is_empty());
11865 let _ = reserved_rx.recv().await.unwrap();
11866
11867 forwarding
11868 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11869 .unwrap();
11870 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11871 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11872 module_id: Some("target".to_string()),
11873 })
11874 .unwrap();
11875 let census_frame =
11876 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11877 let response = handler
11878 .handle_control_frame(&census_ctx, census_frame)
11879 .await
11880 .unwrap()
11881 .pop()
11882 .unwrap();
11883 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11884 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11885 assert!(matches!(
11886 decoded,
11887 ClientControlResponse::SupervisorRoutes { .. }
11888 ));
11889 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11890 assert_eq!(routes.len(), 2);
11891 assert!(routes.iter().all(|route| route["draining"] == true));
11892 assert!(
11895 routes.iter().all(|route| route["drain_reason"] == "reload"),
11896 "draining routes must name the drain's reason: {routes:?}"
11897 );
11898 assert!(routes.iter().any(|route| {
11899 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11900 }));
11901 assert!(routes.iter().any(|route| {
11902 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11903 }));
11904
11905 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11906 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11907 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11908 std::fs::write(
11909 &golden_path,
11910 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11911 )
11912 .unwrap();
11913 }
11914 let expected: Value =
11915 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11916 assert_eq!(actual, expected);
11917 }
11918
11919 async fn query_live_roots(
11920 handler: &ControlHandler,
11921 module_ctx: &RouteCtx,
11922 ) -> ModuleControlResponseToModule {
11923 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11924 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11925 let response = handler
11926 .handle_control_frame(module_ctx, frame)
11927 .await
11928 .unwrap()
11929 .pop()
11930 .unwrap();
11931 serde_json::from_slice(&response.body).unwrap()
11932 }
11933
11934 #[tokio::test(start_paused = true)]
11935 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11936 let registry = Arc::new(Registry::default());
11937 let forwarding = Arc::new(ForwardingTable::default());
11938 let handler = ControlHandler::with_forwarding(registry, forwarding);
11939 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11940 hello_via_sink(
11941 &handler,
11942 &target_ctx,
11943 &mut target_rx,
11944 hello_frame("target", PROTOCOL_VERSION, 1),
11945 )
11946 .await;
11947 let root = unique_project_root("live-roots-known");
11948 let path = ProjectRootId::from_path_allowing_missing(root.path())
11949 .unwrap()
11950 .as_path()
11951 .to_path_buf();
11952 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11953 let open_handler = handler.clone();
11954 let opened = tokio::spawn(async move {
11955 open_handler
11956 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11957 .await
11958 .unwrap()
11959 });
11960 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11961 .await
11962 .unwrap()
11963 .unwrap();
11964 handler
11965 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11966 .await
11967 .unwrap();
11968 assert!(opened.await.unwrap().is_empty());
11969 let _ = client_rx.recv().await.unwrap();
11970
11971 let root = unique_project_root("live-roots-pending");
11972 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11973 .unwrap()
11974 .as_path()
11975 .to_path_buf();
11976 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11977 let open_handler = handler.clone();
11978 let pending = tokio::spawn(async move {
11979 open_handler
11980 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11981 .await
11982 .unwrap()
11983 });
11984 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11985 .await
11986 .unwrap()
11987 .unwrap();
11988 let actual = query_live_roots(&handler, &target_ctx).await;
11989 let ModuleControlResponseToModule::LiveRoots {
11990 roots,
11991 unknown_root_bindings,
11992 total_bindings,
11993 } = actual
11994 else {
11995 panic!("expected live roots")
11996 };
11997 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11998 assert_eq!(unknown_root_bindings, 0);
11999 assert_eq!(
12000 roots.len(),
12001 2,
12002 "root-known arm must retain each canonical root"
12003 );
12004 assert_eq!(
12005 total_bindings,
12006 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12007 );
12008 let counts = roots
12009 .iter()
12010 .map(|root| (root.project_root.clone(), root.bound, root.pending))
12011 .collect::<Vec<_>>();
12012 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12013 expected.sort_by(|a, b| a.0.cmp(&b.0));
12014 assert_eq!(
12015 counts, expected,
12016 "roots must sort by path and count pending separately"
12017 );
12018 handler
12019 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12020 .await
12021 .unwrap();
12022 assert!(pending.await.unwrap().is_empty());
12023 }
12024
12025 #[tokio::test(start_paused = true)]
12026 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12027 let forwarding = Arc::new(ForwardingTable::default());
12028 let handler =
12029 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12030 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12031 hello_via_sink(
12032 &handler,
12033 &target_ctx,
12034 &mut target_rx,
12035 hello_frame("target", PROTOCOL_VERSION, 1),
12036 )
12037 .await;
12038 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12039 let pending = forwarding
12040 .begin_route_bind_relay_for_test(
12041 client_ctx.connection_id,
12042 client_ctx.egress.clone(),
12043 2,
12044 "target",
12045 )
12046 .unwrap();
12047 forwarding
12048 .complete_pending_relay(
12049 target_ctx.connection_id,
12050 pending.corr,
12051 RouteBindRelayOutcome::Accepted,
12052 )
12053 .unwrap();
12054 let actual = query_live_roots(&handler, &target_ctx).await;
12055 let ModuleControlResponseToModule::LiveRoots {
12056 roots,
12057 unknown_root_bindings,
12058 total_bindings,
12059 } = actual
12060 else {
12061 panic!("expected live roots")
12062 };
12063 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12064 assert_eq!(
12065 unknown_root_bindings, 1,
12066 "unknown-root arm must not read as no bindings"
12067 );
12068 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12069 assert_eq!(
12070 total_bindings,
12071 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12072 );
12073 }
12074
12075 #[tokio::test(start_paused = true)]
12083 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12084 let forwarding = Arc::new(ForwardingTable::default());
12085 let handler =
12086 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12087 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12088 let replies = handler
12089 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12090 .await
12091 .unwrap();
12092 let queued_by_hello = module_rx.len();
12093
12094 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12095 let open_handler = handler.clone();
12096 let open = tokio::spawn(async move {
12097 open_handler
12098 .handle_control_frame(
12099 &client_ctx,
12100 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12101 )
12102 .await
12103 .unwrap()
12104 });
12105 let mut spins = 0;
12107 while module_rx.len() == queued_by_hello {
12108 spins += 1;
12109 assert!(spins < 10_000, "route.open never queued a route.bind");
12110 tokio::task::yield_now().await;
12111 }
12112
12113 for reply in replies {
12115 module_ctx.egress.send(reply).await.unwrap();
12116 }
12117
12118 let first = module_rx.recv().await.unwrap().frame;
12119 assert_eq!(
12120 first.header.ty,
12121 FrameType::HelloAck,
12122 "the first frame a registering module reads must be its HELLO_ACK"
12123 );
12124 assert_eq!(first.header.corr, 7);
12125 let second = module_rx.recv().await.unwrap().frame;
12126 assert_eq!(second.header.ty, FrameType::Request);
12127 assert!(
12128 matches!(
12129 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12130 ModuleControlRequest::RouteBind { .. }
12131 ),
12132 "the route.bind follows the ack"
12133 );
12134 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12135
12136 handler
12137 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12138 .await
12139 .unwrap();
12140 assert!(open.await.unwrap().is_empty());
12141 let _ = client_rx.recv().await.unwrap();
12142 }
12143
12144 #[tokio::test(start_paused = true)]
12145 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12146 let handler = ControlHandler::with_forwarding(
12147 Arc::new(Registry::default()),
12148 Arc::new(ForwardingTable::default()),
12149 );
12150 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12151 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12152 hello_via_sink(
12153 &handler,
12154 &first_ctx,
12155 &mut first_rx,
12156 hello_frame("first", PROTOCOL_VERSION, 1),
12157 )
12158 .await;
12159 hello_via_sink(
12160 &handler,
12161 &second_ctx,
12162 &mut second_rx,
12163 hello_frame("second", PROTOCOL_VERSION, 2),
12164 )
12165 .await;
12166 let root = unique_project_root("second-only");
12167 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12168 let cloned = handler.clone();
12169 let open = tokio::spawn(async move {
12170 cloned
12171 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12172 .await
12173 .unwrap()
12174 });
12175 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12176 .await
12177 .unwrap()
12178 .unwrap();
12179 let first = query_live_roots(&handler, &first_ctx).await;
12180 let second = query_live_roots(&handler, &second_ctx).await;
12181 assert!(
12182 matches!(
12183 first,
12184 ModuleControlResponseToModule::LiveRoots {
12185 total_bindings: 0,
12186 ..
12187 }
12188 ),
12189 "cross-module scope must not expose another module's roots"
12190 );
12191 assert!(
12192 matches!(
12193 second,
12194 ModuleControlResponseToModule::LiveRoots {
12195 total_bindings: 1,
12196 ..
12197 }
12198 ),
12199 "second module must see its pending route"
12200 );
12201 handler
12202 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12203 .await
12204 .unwrap();
12205 assert!(open.await.unwrap().is_empty());
12206 }
12207
12208 #[tokio::test(start_paused = true)]
12209 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12210 let handler = ControlHandler::with_forwarding(
12211 Arc::new(Registry::default()),
12212 Arc::new(ForwardingTable::default()),
12213 );
12214 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12215 hello_via_sink(
12216 &handler,
12217 &target_ctx,
12218 &mut target_rx,
12219 hello_frame("target", PROTOCOL_VERSION, 1),
12220 )
12221 .await;
12222 let actual = query_live_roots(&handler, &target_ctx).await;
12223 let ModuleControlResponseToModule::LiveRoots {
12224 roots,
12225 unknown_root_bindings,
12226 total_bindings,
12227 } = actual
12228 else {
12229 panic!("expected live roots")
12230 };
12231 assert!(roots.is_empty());
12232 assert_eq!(unknown_root_bindings, 0);
12233 assert_eq!(total_bindings, 0);
12234 assert_eq!(
12235 total_bindings,
12236 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12237 );
12238 }
12239
12240 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12247 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12248 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12249 let text = std::fs::read_to_string(&path)
12250 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12251 let vectors: Vec<(String, Value)> = text
12252 .lines()
12253 .filter(|line| !line.trim().is_empty())
12254 .map(|line| {
12255 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12256 let id = entry["corpus_id"]
12257 .as_str()
12258 .expect("every vector carries a corpus_id")
12259 .to_string();
12260 (id, entry["package"].clone())
12261 })
12262 .collect();
12263 assert_eq!(
12267 vectors.len(),
12268 3,
12269 "vendored fed corpus changed size; re-sync from subc-federation"
12270 );
12271
12272 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12287 let richest = vectors
12288 .iter()
12289 .filter_map(|(_, package)| package.as_object())
12290 .map(|object| {
12291 object
12292 .keys()
12293 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12294 .count()
12295 })
12296 .max()
12297 .unwrap_or(0);
12298 assert!(
12299 richest >= 2,
12300 "vendored corpus no longer carries a package with unmodelled fields, \
12301 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12302 );
12303
12304 vectors
12305 }
12306
12307 #[tokio::test]
12315 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12316 for (corpus_id, package) in fed_admission_facts_vectors() {
12317 let registry = Arc::new(Registry::default());
12318 let forwarding = Arc::new(ForwardingTable::default());
12319 let supervisor = SupervisorHandle::new();
12320 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12321 let handler =
12322 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12323 .with_supervisor(supervisor)
12324 .with_admission_facts_config(
12325 Some("fed".to_string()),
12326 Some(vec!["target".to_string()]),
12327 );
12328
12329 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12330 hello_via_sink(
12331 &handler,
12332 &target_ctx,
12333 &mut target_rx,
12334 hello_frame("target", PROTOCOL_VERSION, 1),
12335 )
12336 .await;
12337
12338 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12339 let route_handler = handler.clone();
12340 let expected = package.clone();
12341 let route_task = tokio::spawn(async move {
12342 route_handler
12343 .handle_control_frame(
12344 &client_ctx,
12345 route_open_frame_with_admission_facts(
12346 20,
12347 "target",
12348 unique_project_root("admission-facts"),
12349 Some(subc_control::ConsumerIdentity {
12350 module_id: "fed".to_string(),
12351 launch_nonce: "fed-nonce".to_string(),
12352 }),
12353 Some(package),
12354 ),
12355 )
12356 .await
12357 .unwrap()
12358 });
12359
12360 let bind_frame = target_rx.recv().await.unwrap();
12361 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12362 let ModuleControlRequest::RouteBind {
12363 admission_facts, ..
12364 } = bind
12365 else {
12366 panic!("{corpus_id}: expected route.bind")
12367 };
12368 assert_eq!(
12369 admission_facts,
12370 Some(expected),
12371 "{corpus_id}: relay must not add, drop or reshape any field"
12372 );
12373
12374 handler
12375 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12376 .await
12377 .unwrap();
12378 route_task.await.unwrap();
12379 }
12380 }
12381
12382 #[tokio::test]
12383 async fn route_bind_never_relays_the_openers_build_class() {
12384 let registry = Arc::new(Registry::default());
12385 let forwarding = Arc::new(ForwardingTable::default());
12386 let handler =
12387 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12388 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12389 hello_via_sink(
12390 &handler,
12391 &target_ctx,
12392 &mut target_rx,
12393 hello_frame("target", PROTOCOL_VERSION, 1),
12394 )
12395 .await;
12396
12397 let root = unique_project_root("build-class");
12398 let mut identity = BindIdentity::new(
12399 root.path().to_path_buf(),
12400 "unit".to_string(),
12401 "s".to_string(),
12402 );
12403 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12404 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12405 target: RouteTarget::ToolProvider {
12406 module_id: "target".to_string(),
12407 },
12408 identity,
12409 consumer_identity: None,
12410 consumer_capabilities: None,
12411 role_versions: None,
12412 admission_facts: None,
12413 scope: None,
12414 })
12415 .unwrap();
12416 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12417
12418 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12419 let route_handler = handler.clone();
12420 let route_task = tokio::spawn(async move {
12421 route_handler
12422 .handle_control_frame(&client_ctx, open)
12423 .await
12424 .unwrap()
12425 });
12426 let bind_frame = target_rx.recv().await.unwrap();
12427 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12428 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12429 panic!("expected route.bind")
12430 };
12431 assert_eq!(
12432 identity.build_class, None,
12433 "route.bind carried the build class"
12434 );
12435 assert!(
12436 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12437 "the route.bind bytes still name build_class"
12438 );
12439 handler
12440 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12441 .await
12442 .unwrap();
12443 route_task.await.unwrap();
12444 }
12445
12446 #[tokio::test]
12447 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12448 let registry = Arc::new(Registry::default());
12449 let forwarding = Arc::new(ForwardingTable::default());
12450 let supervisor = SupervisorHandle::new();
12451 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12452 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12453 let handler =
12454 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12455 .with_supervisor(supervisor)
12456 .with_admission_facts_config(
12457 Some("fed".to_string()),
12458 Some(vec!["target".to_string()]),
12459 );
12460
12461 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12462 hello_via_sink(
12463 &handler,
12464 &target_ctx,
12465 &mut target_rx,
12466 hello_frame("target", PROTOCOL_VERSION, 1),
12467 )
12468 .await;
12469 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12470 hello_via_sink(
12471 &handler,
12472 &other_ctx,
12473 &mut other_rx,
12474 hello_frame("other", PROTOCOL_VERSION, 2),
12475 )
12476 .await;
12477
12478 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12479 let expected_facts = facts.clone();
12480 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12481 let route_handler = handler.clone();
12482 let route_task = tokio::spawn(async move {
12483 route_handler
12484 .handle_control_frame(
12485 &client_ctx,
12486 route_open_frame_with_admission_facts(
12487 10,
12488 "target",
12489 unique_project_root("admission-facts"),
12490 Some(subc_control::ConsumerIdentity {
12491 module_id: "fed".to_string(),
12492 launch_nonce: "fed-nonce".to_string(),
12493 }),
12494 Some(facts.clone()),
12495 ),
12496 )
12497 .await
12498 .unwrap()
12499 });
12500 let bind_frame = target_rx.recv().await.unwrap();
12501 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12502 let ModuleControlRequest::RouteBind {
12503 admission_facts, ..
12504 } = bind
12505 else {
12506 panic!("expected route.bind")
12507 };
12508 assert_eq!(admission_facts, Some(expected_facts));
12509 handler
12510 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12511 .await
12512 .unwrap();
12513 assert!(route_task.await.unwrap().is_empty());
12514 assert!(matches!(
12515 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12516 .unwrap(),
12517 ClientControlResponse::RouteOpen { .. }
12518 ));
12519
12520 let direct = handler
12521 .handle_control_frame(
12522 &route_ctx(ConnectionId::new(73)).0,
12523 route_open_frame_with_admission_facts(
12524 11,
12525 "target",
12526 unique_project_root("admission-facts"),
12527 None,
12528 Some(json!({"x": 1})),
12529 ),
12530 )
12531 .await
12532 .unwrap();
12533 assert_eq!(
12534 parse_error(&direct[0])["code"],
12535 "admission_facts_not_permitted"
12536 );
12537
12538 let different_reserved = handler
12539 .handle_control_frame(
12540 &route_ctx(ConnectionId::new(77)).0,
12541 route_open_frame_with_admission_facts(
12542 15,
12543 "target",
12544 unique_project_root("admission-facts"),
12545 Some(subc_control::ConsumerIdentity {
12546 module_id: "other".to_string(),
12547 launch_nonce: "other-nonce".to_string(),
12548 }),
12549 Some(json!({"x": 1})),
12550 ),
12551 )
12552 .await
12553 .unwrap();
12554 assert_eq!(
12555 parse_error(&different_reserved[0])["code"],
12556 "admission_facts_not_permitted"
12557 );
12558
12559 let other_target = handler
12560 .handle_control_frame(
12561 &route_ctx(ConnectionId::new(74)).0,
12562 route_open_frame_with_admission_facts(
12563 12,
12564 "other",
12565 unique_project_root("admission-facts"),
12566 Some(subc_control::ConsumerIdentity {
12567 module_id: "fed".to_string(),
12568 launch_nonce: "fed-nonce".to_string(),
12569 }),
12570 Some(json!({"x": 1})),
12571 ),
12572 )
12573 .await
12574 .unwrap();
12575 assert_eq!(
12576 parse_error(&other_target[0])["code"],
12577 "admission_facts_target_not_allowed"
12578 );
12579
12580 let nonexistent = handler
12581 .handle_control_frame(
12582 &route_ctx(ConnectionId::new(75)).0,
12583 route_open_frame_with_admission_facts(
12584 13,
12585 "missing",
12586 unique_project_root("admission-facts"),
12587 None,
12588 Some(json!({"x": 1})),
12589 ),
12590 )
12591 .await
12592 .unwrap();
12593 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12594
12595 let described = handler
12596 .handle_control_frame(
12597 &route_ctx(ConnectionId::new(76)).0,
12598 Frame::build(
12599 FrameType::Request,
12600 control_flags(),
12601 0,
12602 0,
12603 14,
12604 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12605 )
12606 .unwrap(),
12607 )
12608 .await
12609 .unwrap();
12610 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12611 serde_json::from_slice(&described[0].body).unwrap()
12612 else {
12613 panic!("expected server.describe response")
12614 };
12615 assert!(capabilities
12616 .iter()
12617 .any(|cap| cap == "admission_facts_relay_v1"));
12618 }
12619
12620 #[tokio::test]
12621 async fn admission_facts_without_configured_carrier_are_rejected() {
12622 let registry = Arc::new(Registry::default());
12623 let forwarding = Arc::new(ForwardingTable::default());
12624 let handler = ControlHandler::with_forwarding(registry, forwarding);
12625 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12626 hello_via_sink(
12627 &handler,
12628 &target_ctx,
12629 &mut target_rx,
12630 hello_frame("target", PROTOCOL_VERSION, 1),
12631 )
12632 .await;
12633
12634 let responses = handler
12635 .handle_control_frame(
12636 &route_ctx(ConnectionId::new(79)).0,
12637 route_open_frame_with_admission_facts(
12638 16,
12639 "target",
12640 unique_project_root("admission-facts"),
12641 None,
12642 Some(json!({"x": 1})),
12643 ),
12644 )
12645 .await
12646 .unwrap();
12647 assert_eq!(
12648 parse_error(&responses[0])["code"],
12649 "admission_facts_not_permitted"
12650 );
12651 }
12652
12653 #[tokio::test]
12654 async fn route_open_relays_consumer_capabilities_verbatim() {
12655 let registry = Arc::new(Registry::default());
12656 let forwarding = Arc::new(ForwardingTable::default());
12657 let handler =
12658 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12659 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12660 hello_via_sink(
12661 &handler,
12662 &module_ctx,
12663 &mut module_rx,
12664 hello_frame("aft", PROTOCOL_VERSION, 7),
12665 )
12666 .await;
12667
12668 let expected = vec!["elicitation".to_string(), "roots".to_string()];
12669 let expected_for_request = expected.clone();
12670 let project_root = unique_project_root("consumer-capabilities-present");
12671 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12672 let route_handler = handler.clone();
12673 let route_task = tokio::spawn(async move {
12674 route_handler
12675 .handle_control_frame(
12676 &client_ctx,
12677 route_open_frame_with_consumer_capabilities(
12678 401,
12679 "aft",
12680 project_root,
12681 Some(expected_for_request),
12682 ),
12683 )
12684 .await
12685 .unwrap()
12686 });
12687 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12688 .await
12689 .unwrap()
12690 .unwrap();
12691 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12692 let ModuleControlRequest::RouteBind {
12693 consumer_capabilities,
12694 ..
12695 } = bind
12696 else {
12697 panic!("expected route.bind request, got {bind:?}");
12698 };
12699 assert_eq!(consumer_capabilities, Some(expected.clone()));
12700
12701 handler
12702 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12703 .await
12704 .unwrap();
12705 let route_response = route_task.await.unwrap();
12706 assert!(route_response.is_empty());
12707 let published = client_rx.recv().await.unwrap();
12708 assert!(matches!(
12709 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12710 ClientControlResponse::RouteOpen { .. }
12711 ));
12712 }
12713
12714 #[tokio::test]
12715 async fn route_open_without_consumer_capabilities_relays_none() {
12716 let registry = Arc::new(Registry::default());
12717 let forwarding = Arc::new(ForwardingTable::default());
12718 let handler =
12719 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12720 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12721 hello_via_sink(
12722 &handler,
12723 &module_ctx,
12724 &mut module_rx,
12725 hello_frame("aft", PROTOCOL_VERSION, 7),
12726 )
12727 .await;
12728
12729 let project_root = unique_project_root("consumer-capabilities-absent");
12730 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12731 let route_handler = handler.clone();
12732 let route_task = tokio::spawn(async move {
12733 route_handler
12734 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12735 .await
12736 .unwrap()
12737 });
12738 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12739 .await
12740 .unwrap()
12741 .unwrap();
12742 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12743 let ModuleControlRequest::RouteBind {
12744 consumer_capabilities,
12745 ..
12746 } = bind
12747 else {
12748 panic!("expected route.bind request, got {bind:?}");
12749 };
12750 assert_eq!(consumer_capabilities, None);
12751
12752 handler
12753 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12754 .await
12755 .unwrap();
12756 let route_response = route_task.await.unwrap();
12757 assert!(route_response.is_empty());
12758 let published = client_rx.recv().await.unwrap();
12759 assert!(matches!(
12760 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12761 ClientControlResponse::RouteOpen { .. }
12762 ));
12763 }
12764
12765 async fn bind_role_versions_for(
12769 sent: Option<BTreeMap<String, String>>,
12770 connection: u64,
12771 ) -> Option<BTreeMap<String, String>> {
12772 let registry = Arc::new(Registry::default());
12773 let forwarding = Arc::new(ForwardingTable::default());
12774 let handler =
12775 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12776 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12777 hello_via_sink(
12778 &handler,
12779 &module_ctx,
12780 &mut module_rx,
12781 hello_frame("aft", PROTOCOL_VERSION, 7),
12782 )
12783 .await;
12784 let project_root = unique_project_root("role-versions");
12785 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12786 let route_handler = handler.clone();
12787 let route_task = tokio::spawn(async move {
12788 route_handler
12789 .handle_control_frame(
12790 &client_ctx,
12791 route_open_frame_with_role_versions(403, "aft", project_root, sent),
12792 )
12793 .await
12794 .unwrap()
12795 });
12796 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12797 .await
12798 .expect("a well-formed route.open reaches the module as a bind")
12799 .unwrap();
12800 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12801 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12802 panic!("expected route.bind request, got {bind:?}");
12803 };
12804 handler
12805 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12806 .await
12807 .unwrap();
12808 assert!(route_task.await.unwrap().is_empty());
12809 let published = client_rx.recv().await.unwrap();
12810 assert!(matches!(
12811 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12812 ClientControlResponse::RouteOpen { .. }
12813 ));
12814 role_versions
12815 }
12816
12817 #[tokio::test]
12818 async fn route_open_relays_role_versions_verbatim() {
12819 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12820 assert_eq!(
12821 bind_role_versions_for(Some(sent.clone()), 141).await,
12822 Some(sent)
12823 );
12824 }
12825
12826 #[tokio::test]
12829 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12830 assert_eq!(bind_role_versions_for(None, 143).await, None);
12831 assert_eq!(
12832 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12833 None
12834 );
12835 }
12836
12837 #[tokio::test]
12838 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12839 let registry = Arc::new(Registry::default());
12840 let forwarding = Arc::new(ForwardingTable::default());
12841 let handler =
12842 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12843 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12844 hello_via_sink(
12845 &handler,
12846 &module_ctx,
12847 &mut module_rx,
12848 hello_frame("aft", PROTOCOL_VERSION, 7),
12849 )
12850 .await;
12851
12852 let nine: BTreeMap<String, String> = (0..9)
12853 .map(|index| (format!("role-{index}"), "v1".to_string()))
12854 .collect();
12855 for (label, malformed) in [
12856 (
12857 "invalid role name",
12858 role_versions(&[("Tool_Provider", "v1")]),
12859 ),
12860 ("invalid version", role_versions(&[("tool-provider", "v0")])),
12861 ("nine entries", nine),
12862 ] {
12863 let responses = tokio::time::timeout(
12866 Duration::from_secs(1),
12867 handler.handle_control_frame(
12868 &route_ctx(ConnectionId::new(148)).0,
12869 route_open_frame_with_role_versions(
12870 404,
12871 "aft",
12872 unique_project_root("role-versions-malformed"),
12873 Some(malformed),
12874 ),
12875 ),
12876 )
12877 .await
12878 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12879 .unwrap();
12880 assert_eq!(responses.len(), 1, "{label}");
12881 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12882 let error = parse_error(&responses[0]);
12883 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12884 assert_eq!(
12885 error["detail"]["field"], "role_versions",
12886 "{label}: {error}"
12887 );
12888 assert!(
12889 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12890 "{label}: a malformed declaration is terminal"
12891 );
12892 assert!(
12893 module_rx.try_recv().is_err(),
12894 "{label}: the module must never see a bind"
12895 );
12896 }
12897 }
12898
12899 #[tokio::test]
12903 async fn route_role_versions_capability_is_advertised() {
12904 let handler = ControlHandler::new(Arc::new(Registry::default()));
12905 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12906 let ack = hello_via_sink(
12907 &handler,
12908 &ctx,
12909 &mut rx,
12910 hello_frame("m", PROTOCOL_VERSION, 1),
12911 )
12912 .await;
12913 let ack = parse_ack(&ack);
12914 assert!(
12915 ack.subc_capabilities
12916 .iter()
12917 .any(|c| c == "route-role-versions/v1"),
12918 "{:?}",
12919 ack.subc_capabilities
12920 );
12921
12922 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12923 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12924 let reply = handler
12925 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12926 .await
12927 .unwrap()
12928 .pop()
12929 .unwrap();
12930 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12931 serde_json::from_slice(&reply.body).unwrap()
12932 else {
12933 panic!("not a server.describe reply");
12934 };
12935 assert!(
12936 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12937 "{capabilities:?}"
12938 );
12939 }
12940
12941 #[tokio::test]
12945 async fn catalog_events_capability_is_advertised() {
12946 let handler = ControlHandler::new(Arc::new(Registry::default()));
12947 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12948 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12949 let reply = handler
12950 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12951 .await
12952 .unwrap()
12953 .pop()
12954 .unwrap();
12955 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12956 serde_json::from_slice(&reply.body).unwrap()
12957 else {
12958 panic!("not a server.describe reply");
12959 };
12960 assert!(
12961 capabilities
12962 .iter()
12963 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12964 "{capabilities:?}"
12965 );
12966 }
12967
12968 #[tokio::test]
12969 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12970 let handler = ControlHandler::new(Arc::new(Registry::default()));
12971 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12972 let ack = hello_via_sink(
12973 &handler,
12974 &module,
12975 &mut module_rx,
12976 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12977 )
12978 .await;
12979 let ack = parse_ack(&ack);
12980 assert!(
12981 ack.subc_capabilities
12982 .iter()
12983 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12984 "{:?}",
12985 ack.subc_capabilities
12986 );
12987 assert!(ack
12988 .subc_capabilities
12989 .iter()
12990 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12991
12992 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12993 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12994 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12995 let reply = handler
12996 .handle_control_frame(&client, frame)
12997 .await
12998 .unwrap()
12999 .pop()
13000 .unwrap();
13001 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13002 serde_json::from_slice(&reply.body).unwrap()
13003 else {
13004 panic!("not a server.describe reply");
13005 };
13006 assert!(
13007 capabilities
13008 .iter()
13009 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13010 "{capabilities:?}"
13011 );
13012 assert!(capabilities
13013 .iter()
13014 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13015 }
13016
13017 #[tokio::test]
13018 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13019 let handler = ControlHandler::new(Arc::new(Registry::default()));
13020 let (module, mut rx) = route_ctx(ConnectionId::new(156));
13021 let ack = hello_via_sink(
13022 &handler,
13023 &module,
13024 &mut rx,
13025 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13026 )
13027 .await;
13028 assert!(parse_ack(&ack)
13029 .subc_capabilities
13030 .iter()
13031 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13032 assert!(parse_ack(&ack)
13033 .subc_capabilities
13034 .iter()
13035 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13036 let (client, _rx) = route_ctx(ConnectionId::new(157));
13037 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13038 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13039 let reply = handler
13040 .handle_control_frame(&client, frame)
13041 .await
13042 .unwrap()
13043 .pop()
13044 .unwrap();
13045 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13046 serde_json::from_slice(&reply.body).unwrap()
13047 else {
13048 panic!("server.describe reply")
13049 };
13050 assert!(capabilities
13051 .iter()
13052 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13053 assert!(capabilities
13054 .iter()
13055 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13056 }
13057
13058 #[tokio::test]
13059 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13060 let handler = ControlHandler::new(Arc::new(Registry::default()));
13061 let (module, mut rx) = route_ctx(ConnectionId::new(158));
13062 let ack = hello_via_sink(
13063 &handler,
13064 &module,
13065 &mut rx,
13066 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13067 )
13068 .await;
13069 assert!(parse_ack(&ack)
13070 .subc_capabilities
13071 .iter()
13072 .any(|cap| cap == "scope-workdir/v1"));
13073 let (client, _rx) = route_ctx(ConnectionId::new(159));
13074 let frame = Frame::build(
13075 FrameType::Request,
13076 control_flags(),
13077 0,
13078 0,
13079 5,
13080 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13081 )
13082 .unwrap();
13083 let reply = handler
13084 .handle_control_frame(&client, frame)
13085 .await
13086 .unwrap()
13087 .remove(0);
13088 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13089 serde_json::from_slice(&reply.body).unwrap()
13090 else {
13091 panic!("server.describe reply")
13092 };
13093 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13094 }
13095
13096 #[tokio::test]
13097 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13098 let handler = ControlHandler::new(Arc::new(Registry::default()));
13099 let (module, mut rx) = route_ctx(ConnectionId::new(160));
13100 let ack = hello_via_sink(
13101 &handler,
13102 &module,
13103 &mut rx,
13104 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13105 )
13106 .await;
13107 assert!(parse_ack(&ack)
13108 .subc_capabilities
13109 .iter()
13110 .any(|cap| cap == "worker-runs/v1"));
13111 let (client, _rx) = route_ctx(ConnectionId::new(161));
13112 let frame = Frame::build(
13113 FrameType::Request,
13114 control_flags(),
13115 0,
13116 0,
13117 5,
13118 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13119 )
13120 .unwrap();
13121 let reply = handler
13122 .handle_control_frame(&client, frame)
13123 .await
13124 .unwrap()
13125 .remove(0);
13126 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13127 serde_json::from_slice(&reply.body).unwrap()
13128 else {
13129 panic!("server.describe reply")
13130 };
13131 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13132 }
13133
13134 #[tokio::test]
13135 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13136 let handler = ControlHandler::new(Arc::new(Registry::default()));
13137 let (module, mut rx) = route_ctx(ConnectionId::new(154));
13138 let ack = hello_via_sink(
13139 &handler,
13140 &module,
13141 &mut rx,
13142 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13143 )
13144 .await;
13145 assert!(parse_ack(&ack)
13146 .subc_capabilities
13147 .iter()
13148 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13149 let (client, _rx) = route_ctx(ConnectionId::new(155));
13150 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13151 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13152 let reply = handler
13153 .handle_control_frame(&client, frame)
13154 .await
13155 .unwrap()
13156 .pop()
13157 .unwrap();
13158 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13159 serde_json::from_slice(&reply.body).unwrap()
13160 else {
13161 panic!("not a server.describe reply")
13162 };
13163 assert!(capabilities
13164 .iter()
13165 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13166 }
13167
13168 #[tokio::test]
13169 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13170 let registry = Arc::new(Registry::default());
13171 let forwarding = Arc::new(ForwardingTable::default());
13172 let handler =
13173 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13174 .with_health_probe_timeout(Duration::from_secs(5));
13175 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13176 hello_via_sink(
13177 &handler,
13178 &module_ctx,
13179 &mut module_rx,
13180 non_routable_hello_frame_with_control_ops(
13181 "mcp",
13182 300,
13183 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13184 ),
13185 )
13186 .await;
13187 assert!(registry
13188 .get_module("mcp")
13189 .unwrap()
13190 .unwrap()
13191 .manifest
13192 .provides
13193 .is_empty());
13194
13195 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13196 let route_response = handler
13197 .handle_control_frame(
13198 &route_client_ctx,
13199 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13200 )
13201 .await
13202 .unwrap();
13203 assert_eq!(route_response[0].header.ty, FrameType::Error);
13204 assert_eq!(
13205 parse_error(&route_response[0])["code"],
13206 "target_unavailable"
13207 );
13208 assert!(parse_error(&route_response[0])["message"]
13209 .as_str()
13210 .unwrap()
13211 .contains("does not provide the requested target"));
13212 assert!(module_rx.try_recv().is_err());
13213
13214 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13215 let health_handler = handler.clone();
13216 let health_task = tokio::spawn(async move {
13217 health_handler
13218 .handle_control_frame(
13219 &health_client_ctx,
13220 supervisor_health_probe_frame(302, "mcp"),
13221 )
13222 .await
13223 .unwrap()
13224 });
13225 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13226 .await
13227 .unwrap()
13228 .unwrap();
13229 assert_eq!(
13230 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13231 ModuleControlRequest::HealthCheck {}
13232 );
13233 handler
13234 .handle_control_frame(
13235 &module_ctx,
13236 health_response(health_frame.header.corr, HealthStatus::Ok),
13237 )
13238 .await
13239 .unwrap();
13240 let health_response = health_task.await.unwrap();
13241 assert_eq!(health_response[0].header.ty, FrameType::Response);
13242 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13243 ClientControlResponse::SupervisorHealthProbe {
13244 module_id, status, ..
13245 } => {
13246 assert_eq!(module_id, "mcp");
13247 assert_eq!(status, HealthStatus::Ok);
13248 }
13249 other => panic!("unexpected health response: {other:?}"),
13250 }
13251
13252 forwarding
13257 .cleanup_connection(module_ctx.connection_id)
13258 .unwrap();
13259 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13260 let cleanup_response = tokio::time::timeout(
13261 Duration::from_millis(200),
13262 handler.handle_control_frame(
13263 &cleanup_probe_ctx,
13264 supervisor_health_probe_frame(303, "mcp"),
13265 ),
13266 )
13267 .await
13268 .expect("probe should fail immediately when the control lane is gone")
13269 .unwrap();
13270 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13271 assert_eq!(
13272 parse_error(&cleanup_response[0])["code"],
13273 "target_unavailable"
13274 );
13275 assert!(parse_error(&cleanup_response[0])["message"]
13276 .as_str()
13277 .unwrap()
13278 .contains("no module connection"));
13279
13280 handler
13281 .cleanup_connection(module_ctx.connection_id)
13282 .unwrap();
13283 }
13284
13285 #[tokio::test]
13286 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13287 let registry = Arc::new(Registry::default());
13288 let supervisor_handle = SupervisorHandle::new();
13289 let supervisor =
13290 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13291 .with_handle(supervisor_handle.clone())
13292 .with_connection_file_path(
13293 std::env::temp_dir()
13294 .join(format!("subc-route-open-warming-{}", std::process::id())),
13295 );
13296 let module = supervisor
13297 .supervise_configured(
13298 ModuleSpec {
13299 module_id: "warming".to_string(),
13300 program: fake_aft_stub_path(),
13301 args: Vec::new(),
13302 env: Vec::new(),
13303 reserved: false,
13304 reserved_prefixes: Vec::new(),
13305 protocol: ModuleProtocol::Subc,
13306 overlap: Default::default(),
13307 },
13308 true,
13309 )
13310 .unwrap();
13311 assert_eq!(module.state().unwrap(), ModuleState::Running);
13312
13313 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13314 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13315 let response = handler
13316 .handle_control_frame(
13317 &ctx,
13318 route_open_frame(304, "warming", unique_project_root("warming")),
13319 )
13320 .await
13321 .unwrap();
13322 module.stop().await.unwrap();
13323
13324 assert_eq!(response[0].header.ty, FrameType::Error);
13325 let error = parse_error(&response[0]);
13326 assert_eq!(error["code"], "module_warming");
13327 assert!(error["message"]
13328 .as_str()
13329 .unwrap()
13330 .contains("state=running, enabled=true, live=false"));
13331 }
13332
13333 #[test]
13334 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13335 let handler = ControlHandler::new(Arc::new(Registry::default()));
13336 let capture = EventCapture::default();
13337 let _subscriber =
13338 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13339 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13340 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13341 let pending = (0..limit).collect::<Vec<_>>();
13342 let response = handler
13343 .route_open_capacity_refusal(
13344 &ctx,
13345 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13346 "busy",
13347 pending.len(),
13348 limit,
13349 )
13350 .unwrap();
13351 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13352 let event = capture
13353 .events()
13354 .into_iter()
13355 .find(|event| {
13356 event.target == "control"
13357 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13358 })
13359 .expect("connection admission refusal event");
13360 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13361 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13362 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13365 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13366 }
13367
13368 #[test]
13369 fn route_open_target_cap_logs_admission_reason_and_capacity() {
13370 let handler = ControlHandler::new(Arc::new(Registry::default()));
13371 let capture = EventCapture::default();
13372 let _subscriber =
13373 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13374 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13375 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13376 let guards = (0..limit)
13377 .map(|_| {
13378 handler
13379 .route_bind_concurrency
13380 .try_admit("busy", limit)
13381 .unwrap()
13382 })
13383 .collect::<Vec<_>>();
13384 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13385 Err(in_flight) => in_flight,
13386 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13387 };
13388 let response = handler
13389 .route_open_target_capacity_refusal(
13390 &ctx,
13391 &RouteOpenCaller::unknown(),
13392 &route_open_frame(397, "busy", unique_project_root("target-cap")),
13393 "busy",
13394 in_flight,
13395 )
13396 .unwrap();
13397 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13398 let event = capture
13399 .events()
13400 .into_iter()
13401 .find(|event| {
13402 event.target == "control"
13403 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13404 })
13405 .expect("target admission refusal event");
13406 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13407 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13408 drop(guards);
13409 }
13410
13411 #[cfg(unix)]
13412 #[tokio::test(start_paused = true)]
13413 async fn supervisor_list_shows_a_failed_modules_next_retry() {
13414 let registry = Arc::new(Registry::default());
13415 let handle = SupervisorHandle::new();
13416 let supervisor =
13417 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13418 .with_handle(handle.clone());
13419 let module = supervisor
13420 .spawn(crate::ModuleSpec {
13421 module_id: "retry-list".into(),
13422 program: PathBuf::from("/bin/sh"),
13423 args: vec!["-c".into(), "exit 17".into()],
13424 env: vec![],
13425 reserved: false,
13426 reserved_prefixes: vec![],
13427 protocol: ModuleProtocol::None,
13428 overlap: Default::default(),
13429 })
13430 .unwrap();
13431 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13432 while module.state().unwrap() != crate::ModuleState::Failed {
13433 assert!(
13434 std::time::Instant::now() < deadline,
13435 "waiting for budget failure"
13436 );
13437 tokio::task::yield_now().await;
13438 }
13439 let handler = ControlHandler::new(registry).with_supervisor(handle);
13440 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13441 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13442 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13443 assert_eq!(body["modules"][0]["state"], "failed");
13444 let retry = body["modules"][0]["next_retry_at_ms"]
13445 .as_u64()
13446 .expect("scheduled retry on the wire");
13447 let now = std::time::SystemTime::now()
13448 .duration_since(std::time::UNIX_EPOCH)
13449 .unwrap()
13450 .as_millis() as u64;
13451 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13452 module.set_enabled(false).await.unwrap();
13453 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13454 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13455 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13456 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13457 }
13458
13459 #[tokio::test]
13460 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
13461 let registry = Arc::new(Registry::default());
13462 let handle = SupervisorHandle::new();
13463 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13464 .with_handle(handle.clone());
13465 supervisor
13466 .supervise_configured(
13467 crate::ModuleSpec {
13468 module_id: "drift".into(),
13469 program: PathBuf::from("disabled-module"),
13470 args: Vec::new(),
13471 env: Vec::new(),
13472 reserved: false,
13473 reserved_prefixes: Vec::new(),
13474 protocol: ModuleProtocol::Subc,
13475 overlap: Default::default(),
13476 },
13477 false,
13478 )
13479 .unwrap();
13480 let handler = ControlHandler::new(registry).with_supervisor(handle);
13481 for configured in [None, Some(false), Some(true)] {
13482 handler.capability_evaluator.configure(
13483 configured.map(|enabled| ("drift".to_string(), enabled)),
13484 BTreeMap::new(),
13485 );
13486 let frame =
13487 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13488 let replies = handler
13489 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
13490 .await
13491 .unwrap();
13492 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13493 assert!(body.get("configured_enabled").is_none());
13494 assert_eq!(body["op"], "supervisor.ack");
13495 let frame =
13496 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13497 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13498 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13499 assert_eq!(
13500 body["modules"][0].get("configured_enabled"),
13501 configured.map(Value::Bool).as_ref()
13502 );
13503 }
13504 }
13505
13506 #[tokio::test(flavor = "current_thread")]
13507 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
13508 let handler = ControlHandler::new(Arc::new(Registry::default()));
13509 let capture = EventCapture::default();
13510 let _subscriber =
13511 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13512 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
13513 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13514
13515 let _ = handler
13516 .handle_client_control_request(
13517 &ctx,
13518 frame,
13519 ClientControlRequest::SupervisorSetEnabled {
13520 module_id: "broca".to_string(),
13521 enabled: false,
13522 },
13523 &mut None,
13524 )
13525 .await
13526 .unwrap();
13527
13528 let events = capture
13529 .events()
13530 .into_iter()
13531 .filter(|event| {
13532 event.target == "control"
13533 && event.fields.get("message").map(String::as_str)
13534 == Some("supervisor request received")
13535 })
13536 .collect::<Vec<_>>();
13537 assert_eq!(events.len(), 1, "one supervisor request log line");
13538 let event = &events[0];
13539 assert_eq!(event.level, tracing::Level::INFO);
13540 assert_eq!(
13541 event.fields.get("op"),
13542 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
13543 );
13544 assert_eq!(
13545 event.fields.get("module_id"),
13546 Some(&"\"broca\"".to_string())
13547 );
13548 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
13549 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
13550 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
13551 }
13552
13553 #[tokio::test(flavor = "current_thread")]
13554 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
13555 let registry = Arc::new(Registry::default());
13556 let module_connection = ConnectionId::new(303);
13557 registry
13558 .register_with_control_ops(
13559 manifest("aft", PROTOCOL_VERSION),
13560 PROTOCOL_VERSION,
13561 module_connection,
13562 module_baseline_control_ops(),
13563 )
13564 .unwrap();
13565 let handler = ControlHandler::new(registry);
13566 let capture = EventCapture::default();
13567 let _subscriber =
13568 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13569 let (ctx, _rx) = route_ctx(module_connection);
13570 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13571
13572 let _ = handler
13573 .handle_client_control_request(
13574 &ctx,
13575 frame,
13576 ClientControlRequest::SupervisorSetEnabled {
13577 module_id: "broca".to_string(),
13578 enabled: false,
13579 },
13580 &mut None,
13581 )
13582 .await
13583 .unwrap();
13584
13585 let callers = capture
13586 .events()
13587 .into_iter()
13588 .filter(|event| {
13589 event.fields.get("message").map(String::as_str)
13590 == Some("supervisor request received")
13591 })
13592 .map(|event| event.fields.get("caller").cloned())
13593 .collect::<Vec<_>>();
13594 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13595 }
13596
13597 #[tokio::test(flavor = "current_thread")]
13598 async fn supervisor_list_does_not_log_request_received() {
13599 let handler = ControlHandler::new(Arc::new(Registry::default()));
13600 let capture = EventCapture::default();
13601 let _subscriber =
13602 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13603 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13604 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13605
13606 handler
13607 .handle_client_control_request(
13608 &ctx,
13609 frame,
13610 ClientControlRequest::SupervisorList {},
13611 &mut None,
13612 )
13613 .await
13614 .unwrap();
13615
13616 assert!(capture.events().into_iter().all(|event| {
13617 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13618 }));
13619 }
13620
13621 #[tokio::test(flavor = "current_thread")]
13622 async fn supervisor_rescan_preview_does_not_log_request_received() {
13623 let handler = ControlHandler::new(Arc::new(Registry::default()));
13624 let capture = EventCapture::default();
13625 let _subscriber =
13626 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13627 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13628 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13629
13630 handler
13631 .handle_client_control_request(
13632 &ctx,
13633 frame,
13634 ClientControlRequest::SupervisorRescan { preview: true },
13635 &mut None,
13636 )
13637 .await
13638 .unwrap();
13639
13640 assert!(capture.events().into_iter().all(|event| {
13641 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13642 }));
13643 }
13644
13645 struct ApproveOperator;
13646 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13647 fn prompt(
13648 &self,
13649 _: &str,
13650 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13651 ) -> crate::operator_confirm::ProviderResult {
13652 crate::operator_confirm::ProviderResult::Approved
13653 }
13654 }
13655
13656 #[tokio::test]
13657 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13658 assert_eq!(
13659 ControlHandler::default().privileged_operator_prompt,
13660 PrivilegedOperatorPrompt::Log
13661 );
13662 for (initial, changed, running) in [
13663 ("log", "enforce", PrivilegedOperatorPrompt::Log),
13664 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13665 ] {
13666 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13667 let path = dir.join("subc.jsonc");
13668 std::fs::write(
13669 &path,
13670 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13671 )
13672 .unwrap();
13673 let config = crate::daemon_config::load(&path).unwrap().unwrap();
13674 let registry = Arc::new(Registry::default());
13675 let handle = SupervisorHandle::new();
13676 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13677 .with_handle(handle.clone());
13678 let handler = ControlHandler::new(registry)
13679 .with_supervisor(handle)
13680 .with_privileged_operator_prompt(config.privileged_operator_prompt)
13681 .with_supervisor_rescan(supervisor, &path, config.port);
13682 handler.forwarding.operator_confirms().configure(
13683 Arc::new(ApproveOperator),
13684 crate::operator_confirm::OperatorLimits::default(),
13685 );
13686 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13687
13688 for (disk_mode, needs_restart) in [
13689 (Some(initial), false),
13690 (Some(changed), true),
13691 (Some(changed), true),
13692 (None, running == PrivilegedOperatorPrompt::Enforce),
13693 (Some(initial), false),
13694 ] {
13695 let mut doc = json!({"version":1});
13696 if let Some(mode) = disk_mode {
13697 doc["privileged_operator_prompt"] = json!(mode);
13698 }
13699 std::fs::write(&path, doc.to_string()).unwrap();
13700 for preview in [true, false] {
13701 let request = ClientControlRequest::SupervisorRescan { preview };
13702 let frame = Frame::build(
13703 FrameType::Request,
13704 control_flags(),
13705 0,
13706 0,
13707 1,
13708 serde_json::to_vec(&request).unwrap(),
13709 )
13710 .unwrap();
13711 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13712 assert_eq!(frames.len(), 1);
13713 let ClientControlResponse::SupervisorRescan { result } =
13714 serde_json::from_slice(&frames[0].body).unwrap()
13715 else {
13716 panic!("expected a rescan response");
13717 };
13718 let expected = if needs_restart {
13719 vec!["privileged_operator_prompt"]
13720 } else {
13721 Vec::new()
13722 };
13723 assert_eq!(result.restart_required, expected);
13724 assert_eq!(handler.privileged_operator_prompt, running);
13725 assert_eq!(
13726 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13727 running
13728 );
13729 }
13730 }
13731 }
13732 }
13733
13734 #[tokio::test]
13735 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13736 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13737 let path = dir.join("subc.jsonc");
13738 let registry = Arc::new(Registry::default());
13739 let handle = SupervisorHandle::new();
13740 let supervisor =
13741 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13742 let handler = ControlHandler::new(registry)
13743 .with_supervisor(handle)
13744 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13745 .with_supervisor_rescan(supervisor, &path, None);
13746 handler.forwarding.operator_confirms().configure(
13747 Arc::new(ApproveOperator),
13748 crate::operator_confirm::OperatorLimits::default(),
13749 );
13750 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13751 for preview in [true, false] {
13752 std::fs::write(
13753 &path,
13754 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13755 )
13756 .unwrap();
13757 let request = ClientControlRequest::SupervisorRescan { preview };
13758 let frame = Frame::build(
13759 FrameType::Request,
13760 control_flags(),
13761 0,
13762 0,
13763 1,
13764 serde_json::to_vec(&request).unwrap(),
13765 )
13766 .unwrap();
13767 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13768 assert_eq!(frames.len(), 1);
13769 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13770 assert_eq!(
13771 handler.privileged_operator_prompt,
13772 PrivilegedOperatorPrompt::Enforce
13773 );
13774 }
13775 }
13776
13777 #[tokio::test]
13782 async fn route_open_refusal_names_the_check_that_refused() {
13783 let handler = ControlHandler::new(Arc::new(Registry::default()));
13784 let capture = EventCapture::default();
13785 let _subscriber =
13786 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13787 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13788 let response = handler
13789 .handle_control_frame(
13790 &ctx,
13791 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13792 )
13793 .await
13794 .unwrap();
13795
13796 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13797 let event = capture
13798 .events()
13799 .into_iter()
13800 .find(|event| {
13801 event.target == "control"
13802 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13803 })
13804 .expect("route.open refusal event");
13805 assert_eq!(
13806 event.fields.get("reason"),
13807 Some(&"\"not_registered\"".to_string())
13808 );
13809 }
13810
13811 #[tokio::test]
13812 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13813 let registry = Arc::new(Registry::default());
13814 let supervisor_handle = SupervisorHandle::new();
13815 let supervisor =
13816 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13817 .with_handle(supervisor_handle.clone())
13818 .with_connection_file_path(std::env::temp_dir().join(format!(
13819 "subc-route-open-refusal-info-{}",
13820 std::process::id()
13821 )));
13822 let module = supervisor
13823 .supervise_configured(
13824 ModuleSpec {
13825 module_id: "warming".to_string(),
13826 program: fake_aft_stub_path(),
13827 args: Vec::new(),
13828 env: Vec::new(),
13829 reserved: false,
13830 reserved_prefixes: Vec::new(),
13831 protocol: ModuleProtocol::Subc,
13832 overlap: Default::default(),
13833 },
13834 true,
13835 )
13836 .unwrap();
13837 assert_eq!(module.state().unwrap(), ModuleState::Running);
13838
13839 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13840 assert!(handler
13841 .counters()
13842 .snapshot()
13843 .get("route_open_refused_by_code")
13844 .is_none());
13845 let capture = EventCapture::default();
13846 let _subscriber =
13847 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13848 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13849 let response = handler
13850 .handle_control_frame(
13851 &ctx,
13852 route_open_frame(394, "warming", unique_project_root("refusal-info")),
13853 )
13854 .await
13855 .unwrap();
13856 module.stop().await.unwrap();
13857
13858 assert_eq!(parse_error(&response[0])["code"], "module_warming");
13859 let event = capture
13860 .events()
13861 .into_iter()
13862 .find(|event| {
13863 event.target == "control"
13864 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13865 })
13866 .expect("route.open refusal event");
13867 assert_eq!(
13868 event.fields.get("module_id"),
13869 Some(&"\"warming\"".to_string())
13870 );
13871 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13872 assert_eq!(
13873 event.fields.get("reason"),
13874 Some(&"\"supervised_not_registered\"".to_string())
13875 );
13876 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13877 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13878 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13879 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13880 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13881 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13882 assert_eq!(
13883 handler.counters().snapshot()["route_open_refused_by_code"],
13884 json!({ "module_warming": 1 })
13885 );
13886 }
13887
13888 const OUTAGE_START: &str = "route.open refusing module: not serving";
13889 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13890
13891 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13892 capture
13893 .events()
13894 .into_iter()
13895 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13896 .collect()
13897 }
13898
13899 fn supervise_stub(
13900 registry: &Arc<Registry>,
13901 module_id: &str,
13902 enabled: bool,
13903 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13904 let supervisor_handle = SupervisorHandle::new();
13905 let supervisor =
13906 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13907 .with_handle(supervisor_handle.clone())
13908 .with_connection_file_path(std::env::temp_dir().join(format!(
13909 "subc-route-outage-{module_id}-{}",
13910 std::process::id()
13911 )));
13912 let module = supervisor
13913 .supervise_configured(
13914 ModuleSpec {
13915 module_id: module_id.to_string(),
13916 program: fake_aft_stub_path(),
13917 args: Vec::new(),
13918 env: Vec::new(),
13919 reserved: false,
13920 reserved_prefixes: Vec::new(),
13921 protocol: ModuleProtocol::Subc,
13922 overlap: Default::default(),
13923 },
13924 enabled,
13925 )
13926 .unwrap();
13927 (supervisor_handle, module)
13928 }
13929
13930 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13931 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13932 module_id: module_id.to_string(),
13933 drain_timeout_ms: Some(50),
13934 })
13935 .unwrap();
13936 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13937 }
13938
13939 #[test]
13943 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13944 let registry = Arc::new(Registry::default());
13945 let forwarding = Arc::new(ForwardingTable::default());
13946 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13947 let second = ControlHandler::with_forwarding(registry, forwarding);
13948 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13949 }
13950
13951 #[tokio::test(flavor = "current_thread")]
13955 async fn route_open_unknown_module_refusals_add_no_outage_state() {
13956 let handler = ControlHandler::new(Arc::new(Registry::default()));
13957 let capture = EventCapture::default();
13958 let _subscriber =
13959 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13960 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13961 for corr in 0..8 {
13962 let response = handler
13963 .handle_control_frame(
13964 &ctx,
13965 route_open_frame(
13966 380 + corr,
13967 &format!("nobody-{corr}"),
13968 unique_project_root("outage-unknown"),
13969 ),
13970 )
13971 .await
13972 .unwrap();
13973 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13974 }
13975
13976 assert_eq!(handler.route_outages.tracked_module_count(), 0);
13977 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13978 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13979 }
13980
13981 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13986 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13987 let registry = Arc::new(Registry::default());
13988 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
13989 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13990 let capture = EventCapture::default();
13991 let _subscriber =
13992 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13993 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13994 handler.route_outages.record_accepted("outage-restart");
13997
13998 let response = handler
13999 .handle_control_frame(
14000 &ctx,
14001 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
14002 )
14003 .await
14004 .unwrap();
14005 assert_eq!(response[0].header.ty, FrameType::Error);
14006 let starts = outage_lines(&capture, OUTAGE_START);
14007 assert_eq!(starts.len(), 1, "{starts:?}");
14008 assert_eq!(starts[0].level, tracing::Level::WARN);
14009 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
14010 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
14011 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
14012 handler.route_outages.record_accepted("outage-restart");
14013 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
14014
14015 let restart = handler
14016 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
14017 .await
14018 .unwrap();
14019 assert_eq!(
14020 restart[0].header.ty,
14021 FrameType::Response,
14022 "{:?}",
14023 parse_error(&restart[0])
14024 );
14025 handler
14026 .handle_control_frame(
14027 &ctx,
14028 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
14029 )
14030 .await
14031 .unwrap();
14032 module.stop().await.unwrap();
14033
14034 let starts = outage_lines(&capture, OUTAGE_START);
14035 assert_eq!(starts.len(), 2, "{starts:?}");
14036 assert_eq!(starts[1].level, tracing::Level::INFO);
14037 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
14038 }
14039
14040 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14044 async fn failed_operator_restart_leaves_no_operator_mark() {
14045 let registry = Arc::new(Registry::default());
14046 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
14047 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14048 let capture = EventCapture::default();
14049 let _subscriber =
14050 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14051 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
14052 handler.route_outages.record_accepted("outage-disabled");
14053
14054 let restart = handler
14055 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
14056 .await
14057 .unwrap();
14058 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
14059 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
14060
14061 handler
14062 .handle_control_frame(
14063 &ctx,
14064 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
14065 )
14066 .await
14067 .unwrap();
14068 let starts = outage_lines(&capture, OUTAGE_START);
14069 assert_eq!(starts.len(), 1, "{starts:?}");
14070 assert_eq!(starts[0].level, tracing::Level::WARN);
14071 }
14072
14073 #[tokio::test(flavor = "current_thread")]
14074 async fn route_open_unknown_module_escapes_target_module_id() {
14075 let handler = ControlHandler::new(Arc::new(Registry::default()));
14076 let capture = EventCapture::default();
14077 let _subscriber =
14078 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14079 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
14080 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14081 let response = handler
14082 .handle_control_frame(
14083 &ctx,
14084 route_open_frame(
14085 395,
14086 hostile_module_id,
14087 unique_project_root("hostile-target-module-id"),
14088 ),
14089 )
14090 .await
14091 .unwrap();
14092
14093 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14094 let event = capture
14095 .events()
14096 .into_iter()
14097 .find(|event| {
14098 event.target == "control"
14099 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14100 })
14101 .expect("route.open unknown-module refusal event");
14102 let logged = event.fields.get("module_id").expect("module_id field");
14103 assert!(!logged.bytes().any(|byte| byte < 0x20));
14104 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14105 }
14106
14107 #[tokio::test(flavor = "current_thread")]
14110 async fn route_open_refusal_carries_daemon_origin() {
14111 let handler = ControlHandler::new(Arc::new(Registry::default()));
14112 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14113 let response = handler
14114 .handle_control_frame(
14115 &ctx,
14116 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
14117 )
14118 .await
14119 .unwrap();
14120
14121 assert_eq!(response.len(), 1);
14122 assert_eq!(response[0].header.ty, FrameType::Error);
14123 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14124 assert!(
14125 response[0].header.flags.is_daemon_origin(),
14126 "route.open refusal must carry daemon origin: {:?}",
14127 response[0].header.flags
14128 );
14129 }
14130
14131 #[tokio::test(flavor = "current_thread")]
14135 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
14136 let registry = Arc::new(Registry::default());
14137 let forwarding = Arc::new(ForwardingTable::default());
14138 let handler =
14139 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14140 let module_connection = ConnectionId::new(95);
14141 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14142 hello_via_sink(
14143 &handler,
14144 &module_ctx,
14145 &mut module_rx,
14146 hello_frame("aft", PROTOCOL_VERSION, 395),
14147 )
14148 .await;
14149 forwarding
14150 .begin_module_drain("aft", RouteCloseReason::Restart)
14151 .unwrap()
14152 .expect("the module endpoint drains");
14153
14154 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
14155 let response = handler
14156 .handle_control_frame(
14157 &client_ctx,
14158 route_open_frame(396, "aft", unique_project_root("drain-origin")),
14159 )
14160 .await
14161 .unwrap();
14162
14163 assert_eq!(response.len(), 1);
14164 assert_eq!(response[0].header.ty, FrameType::Error);
14165 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14166 assert!(
14167 response[0].header.flags.is_daemon_origin(),
14168 "drain module_reloading must carry daemon origin: {:?}",
14169 response[0].header.flags
14170 );
14171 }
14172
14173 #[tokio::test(flavor = "current_thread")]
14177 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
14178 let registry = Arc::new(Registry::default());
14179 let forwarding = Arc::new(ForwardingTable::default());
14180 let handler =
14181 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14182 let module_connection = ConnectionId::new(95);
14183 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14184 hello_via_sink(
14185 &handler,
14186 &module_ctx,
14187 &mut module_rx,
14188 hello_frame("aft", PROTOCOL_VERSION, 395),
14189 )
14190 .await;
14191
14192 let client_connection = ConnectionId::new(96);
14193 let (client_ctx, _client_rx) = route_ctx(client_connection);
14194 let (route_task, bind) = relay_route_open(
14195 &handler,
14196 client_connection,
14197 &client_ctx.egress,
14198 &mut module_rx,
14199 396,
14200 "aft",
14201 "spoofed-reloading",
14202 )
14203 .await;
14204 let rejection = Frame::build(
14207 FrameType::Error,
14208 crate::router::daemon_error_flags(),
14209 0,
14210 0,
14211 bind.header.corr,
14212 serde_json::to_vec(&ErrorBody::new(
14213 "module_reloading",
14214 "module refused route.bind",
14215 ))
14216 .unwrap(),
14217 )
14218 .unwrap();
14219 handler
14220 .handle_control_frame(&module_ctx, rejection)
14221 .await
14222 .unwrap();
14223
14224 let response = route_task.await.unwrap();
14225 assert_eq!(response.len(), 1);
14226 assert_eq!(response[0].header.ty, FrameType::Error);
14227 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14228 assert!(
14229 !response[0].header.flags.is_daemon_origin(),
14230 "a module's own rejection must not carry daemon origin: {:?}",
14231 response[0].header.flags
14232 );
14233 }
14234
14235 #[tokio::test(flavor = "current_thread")]
14236 async fn route_open_module_rejection_uses_daemon_counter_key() {
14237 let registry = Arc::new(Registry::default());
14238 let forwarding = Arc::new(ForwardingTable::default());
14239 let handler =
14240 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14241 let module_connection = ConnectionId::new(95);
14242 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14243 hello_via_sink(
14244 &handler,
14245 &module_ctx,
14246 &mut module_rx,
14247 hello_frame("aft", PROTOCOL_VERSION, 395),
14248 )
14249 .await;
14250
14251 let client_connection = ConnectionId::new(96);
14252 let (client_ctx, _client_rx) = route_ctx(client_connection);
14253 let capture = EventCapture::default();
14254 let _subscriber =
14255 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14256 let (route_task, bind) = relay_route_open(
14257 &handler,
14258 client_connection,
14259 &client_ctx.egress,
14260 &mut module_rx,
14261 396,
14262 "aft",
14263 "hostile-module-code",
14264 )
14265 .await;
14266 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
14267 let rejection = Frame::build(
14268 FrameType::Error,
14269 control_flags(),
14270 0,
14271 0,
14272 bind.header.corr,
14273 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
14274 )
14275 .unwrap();
14276 handler
14277 .handle_control_frame(&module_ctx, rejection)
14278 .await
14279 .unwrap();
14280
14281 let response = route_task.await.unwrap();
14282 assert_eq!(parse_error(&response[0])["code"], hostile_code);
14283 let counters = handler.counters().snapshot();
14284 assert_eq!(
14285 counters["route_open_refused_by_code"],
14286 json!({ "module_rejected": 1 })
14287 );
14288 assert!(counters["route_open_refused_by_code"]
14289 .get(hostile_code)
14290 .is_none());
14291
14292 let event = capture
14293 .events()
14294 .into_iter()
14295 .find(|event| {
14296 event.target == "control"
14297 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
14298 })
14299 .expect("route.open module-rejection refusal event");
14300 let logged = event.fields.get("module_code").expect("module_code field");
14301 assert!(!logged.bytes().any(|byte| byte < 0x20));
14302 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14303 }
14304
14305 #[tokio::test]
14306 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
14307 let registry = Arc::new(Registry::default());
14308 let supervisor_handle = SupervisorHandle::new();
14309 let missing_program = std::env::temp_dir().join(format!(
14310 "subc-route-open-missing-program-{}",
14311 std::process::id()
14312 ));
14313 let supervisor =
14314 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14315 .with_handle(supervisor_handle.clone());
14316 let module = supervisor
14317 .supervise_configured(
14318 ModuleSpec {
14319 module_id: "failed".to_string(),
14320 program: missing_program,
14321 args: Vec::new(),
14322 env: Vec::new(),
14323 reserved: false,
14324 reserved_prefixes: Vec::new(),
14325 protocol: ModuleProtocol::Subc,
14326 overlap: Default::default(),
14327 },
14328 true,
14329 )
14330 .unwrap();
14331 assert_eq!(module.state().unwrap(), ModuleState::Failed);
14332
14333 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14334 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
14335 let response = handler
14336 .handle_control_frame(
14337 &ctx,
14338 route_open_frame(305, "failed", unique_project_root("failed")),
14339 )
14340 .await
14341 .unwrap();
14342
14343 assert_eq!(response[0].header.ty, FrameType::Error);
14344 let error = parse_error(&response[0]);
14345 assert_eq!(error["code"], "target_unavailable");
14346 assert!(error["message"]
14347 .as_str()
14348 .unwrap()
14349 .contains("state=failed, enabled=true, live=false"));
14350 }
14351
14352 #[tokio::test]
14353 async fn route_open_role_mismatch_remains_target_unavailable() {
14354 let registry = Arc::new(Registry::default());
14355 let handler = ControlHandler::new(Arc::clone(®istry));
14356 handler
14357 .handle_control(
14358 ConnectionId::new(41),
14359 non_routable_hello_frame_with_control_ops("health-only", 306, None),
14360 )
14361 .unwrap();
14362
14363 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
14364 let response = handler
14365 .handle_control_frame(
14366 &ctx,
14367 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
14368 )
14369 .await
14370 .unwrap();
14371
14372 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14373 assert!(parse_error(&response[0])["message"]
14374 .as_str()
14375 .unwrap()
14376 .contains("does not provide the requested target"));
14377 }
14378
14379 #[tokio::test]
14380 async fn route_open_inactive_registration_remains_target_unavailable() {
14381 let registry = Arc::new(Registry::default());
14382 let handler = ControlHandler::new(Arc::clone(®istry));
14383 handler
14384 .handle_control(
14385 ConnectionId::new(43),
14386 hello_frame("inactive", PROTOCOL_VERSION, 308),
14387 )
14388 .unwrap();
14389 assert!(registry
14390 .set_module_state_for_test("inactive", ChannelState::Closed)
14391 .unwrap());
14392
14393 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
14394 let response = handler
14395 .handle_control_frame(
14396 &ctx,
14397 route_open_frame(309, "inactive", unique_project_root("inactive")),
14398 )
14399 .await
14400 .unwrap();
14401
14402 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14403 assert!(parse_error(&response[0])["message"]
14404 .as_str()
14405 .unwrap()
14406 .contains("is not active"));
14407 }
14408
14409 #[tokio::test]
14410 async fn late_health_reply_is_recorded_through_the_module_response_path() {
14411 let registry = Arc::new(Registry::default());
14412 let forwarding = Arc::new(ForwardingTable::default());
14413 let supervisor_handle = SupervisorHandle::new();
14414 let supervisor =
14415 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
14416 .with_forwarding(Arc::clone(&forwarding))
14417 .with_handle(supervisor_handle.clone());
14418 let module = supervisor
14419 .supervise_configured(
14420 crate::ModuleSpec {
14421 module_id: "late-health-response".to_string(),
14422 program: PathBuf::from("disabled-module"),
14423 args: Vec::new(),
14424 env: Vec::new(),
14425 reserved: false,
14426 reserved_prefixes: Vec::new(),
14427 protocol: ModuleProtocol::Subc,
14428 overlap: Default::default(),
14429 },
14430 false,
14431 )
14432 .unwrap();
14433 let handler =
14434 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14435 .with_supervisor(supervisor_handle);
14436 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
14437 handler
14438 .handle_control_frame(
14439 &module_ctx,
14440 hello_frame_with_control_ops(
14441 "late-health-response",
14442 PROTOCOL_VERSION,
14443 7,
14444 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14445 ),
14446 )
14447 .await
14448 .unwrap();
14449 let probe_started_at = Instant::now() - Duration::from_millis(80);
14450 let pending = forwarding
14451 .begin_health_probe_rpc_for(
14452 "late-health-response",
14453 MODULE_CONTROL_OP_HEALTH_CHECK,
14454 probe_started_at,
14455 Instant::now() - Duration::from_millis(1),
14456 )
14457 .unwrap();
14458 assert!(forwarding
14459 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
14460 .unwrap());
14461
14462 let responses = handler
14463 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
14464 .await
14465 .unwrap();
14466
14467 assert!(responses.is_empty());
14468 let health = module.status().unwrap().health;
14469 assert_eq!(health.late_answer_count, 1);
14470 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
14471 }
14472
14473 #[tokio::test]
14474 async fn health_probe_timeout_and_module_death_are_typed() {
14475 let registry = Arc::new(Registry::default());
14476 let forwarding = Arc::new(ForwardingTable::default());
14477 let handler =
14478 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14479 .with_health_probe_timeout(Duration::from_millis(50));
14480 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14481 hello_via_sink(
14482 &handler,
14483 &module_ctx,
14484 &mut module_rx,
14485 hello_frame_with_control_ops(
14486 "aft",
14487 PROTOCOL_VERSION,
14488 7,
14489 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14490 ),
14491 )
14492 .await;
14493
14494 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14495 let responses = handler
14496 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
14497 .await
14498 .unwrap();
14499 assert_eq!(responses[0].header.ty, FrameType::Error);
14500 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
14501 let _ = module_rx.try_recv();
14502
14503 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
14504 let health_handler = handler.clone();
14505 let death_task = tokio::spawn(async move {
14506 health_handler
14507 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
14508 .await
14509 .unwrap()
14510 });
14511 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14512 .await
14513 .unwrap()
14514 .unwrap();
14515 handler
14516 .cleanup_connection(module_ctx.connection_id)
14517 .unwrap();
14518 let responses = death_task.await.unwrap();
14519 assert_eq!(responses[0].header.ty, FrameType::Error);
14520 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
14521 }
14522
14523 #[test]
14524 fn hello_requires_exact_protocol_version() {
14525 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
14526 let registry = Arc::new(Registry::default());
14527 let handler = ControlHandler::new(Arc::clone(®istry));
14528 let responses = handler
14529 .handle_control(
14530 ConnectionId::new(connection),
14531 hello_frame("aft", offered, 9),
14532 )
14533 .unwrap();
14534
14535 assert_eq!(responses.len(), 1);
14536 assert_eq!(responses[0].header.ty, FrameType::Error);
14537 let error = parse_error(&responses[0]);
14538 assert_eq!(error["code"], "version_unsupported");
14539 assert!(registry.get_module("aft").unwrap().is_none());
14540 assert_eq!(registry.active_registration_count().unwrap(), 0);
14541 }
14542 }
14543
14544 #[test]
14545 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
14546 let registry = Arc::new(Registry::default());
14547 let forwarding = Arc::new(ForwardingTable::default());
14548 let handler =
14549 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14550 let module_connection = ConnectionId::new(301);
14551 let registration = registry
14552 .register_with_control_ops(
14553 manifest("aft-push", PROTOCOL_VERSION),
14554 PROTOCOL_VERSION,
14555 module_connection,
14556 module_baseline_control_ops(),
14557 )
14558 .unwrap();
14559 let (module_tx, _module_rx) = mpsc::channel(8);
14560 let endpoint = forwarding
14561 .register_module_connection(
14562 module_connection,
14563 "aft-push".to_string(),
14564 PROTOCOL_VERSION,
14565 manifest_concurrency(®istration.manifest),
14566 FrameSink::new(module_tx),
14567 )
14568 .unwrap();
14569
14570 let unknown = Frame::build(
14572 FrameType::Push,
14573 control_flags(),
14574 0,
14575 0,
14576 5,
14577 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
14578 )
14579 .unwrap();
14580 let out = handler.handle_status_update(endpoint, unknown).unwrap();
14581 assert!(
14582 out.is_empty(),
14583 "unknown push op must be ignored, got {out:?}"
14584 );
14585
14586 let malformed = Frame::build(
14588 FrameType::Push,
14589 control_flags(),
14590 0,
14591 0,
14592 6,
14593 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
14594 )
14595 .unwrap();
14596 let out = handler.handle_status_update(endpoint, malformed).unwrap();
14597 assert_eq!(out.len(), 1);
14598 assert_eq!(out[0].header.ty, FrameType::Error);
14599 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
14600 }
14601
14602 #[test]
14603 fn hello_rejected_when_connection_already_owns_client_routes() {
14604 let registry = Arc::new(Registry::default());
14605 let forwarding = Arc::new(ForwardingTable::default());
14606 let handler =
14607 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14608 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
14610 let client_connection = ConnectionId::new(202);
14611
14612 let responses = handler
14615 .handle_control(
14616 client_connection,
14617 hello_frame("aft-second", PROTOCOL_VERSION, 9),
14618 )
14619 .unwrap();
14620 assert_eq!(responses[0].header.ty, FrameType::Error);
14621 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
14622 assert!(registry.get_module("aft-second").unwrap().is_none());
14623 }
14624
14625 #[tokio::test]
14626 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
14627 let registry = Arc::new(Registry::default());
14628 let forwarding = Arc::new(ForwardingTable::default());
14629 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
14630 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
14631 hello_via_sink(
14632 &handler,
14633 &module_ctx,
14634 &mut module_rx,
14635 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
14636 )
14637 .await;
14638 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
14639 let pending = forwarding
14640 .begin_route_bind_relay_for_test(
14641 client_ctx.connection_id,
14642 client_ctx.egress.clone(),
14643 2,
14644 "alpha",
14645 )
14646 .unwrap();
14647 forwarding
14648 .complete_pending_relay(
14649 module_ctx.connection_id,
14650 pending.corr,
14651 RouteBindRelayOutcome::Accepted,
14652 )
14653 .unwrap();
14654 client_rx.try_recv().unwrap();
14655 for module_id in ["beta", "alpha"] {
14656 let replies = handler
14657 .handle_control_frame(
14658 &module_ctx,
14659 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
14660 )
14661 .await
14662 .unwrap();
14663 assert_eq!(replies.len(), 1, "second HELLO must be refused");
14664 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
14665 }
14666 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
14667 assert!(registry.get_module("beta").unwrap().is_none());
14668 assert!(matches!(
14669 forwarding
14670 .lookup_data_route(
14671 client_ctx.connection_id,
14672 pending.client_channel,
14673 pending.client_epoch,
14674 )
14675 .unwrap(),
14676 DataRoute::Client(DataRouteState::Bound(_))
14677 ));
14678 assert!(handler
14679 .hello_launch_nonces
14680 .lock()
14681 .unwrap()
14682 .presented(module_ctx.connection_id, Some("alpha-nonce")));
14683 assert!(module_rx.try_recv().is_err());
14684 }
14685
14686 #[test]
14687 fn reserved_module_hello_requires_matching_launch_nonce() {
14688 let registry = Arc::new(Registry::default());
14689 let supervisor = SupervisorHandle::new();
14690 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
14693 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14694
14695 let no_nonce = handler
14697 .handle_control(
14698 ConnectionId::new(1),
14699 hello_frame("vault", PROTOCOL_VERSION, 1),
14700 )
14701 .unwrap();
14702 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
14703 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
14704 assert!(registry.get_module("vault").unwrap().is_none());
14705
14706 let wrong = handler
14708 .handle_control(
14709 ConnectionId::new(2),
14710 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
14711 )
14712 .unwrap();
14713 assert_eq!(wrong[0].header.ty, FrameType::Error);
14714 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14715 assert!(registry.get_module("vault").unwrap().is_none());
14716
14717 let ok = handler
14719 .handle_control(
14720 ConnectionId::new(3),
14721 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14722 )
14723 .unwrap();
14724 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14725 assert!(registry.get_module("vault").unwrap().is_some());
14726 }
14727
14728 #[test]
14729 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14730 let registry = Arc::new(Registry::default());
14731 let supervisor = SupervisorHandle::new();
14732 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14733 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14734 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14735
14736 let squat = handler
14737 .handle_control(
14738 ConnectionId::new(1),
14739 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14740 )
14741 .unwrap();
14742 assert_eq!(squat[0].header.ty, FrameType::Error);
14743 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14744 assert!(parse_error(&squat[0])["message"]
14745 .as_str()
14746 .unwrap()
14747 .contains("fed:"));
14748
14749 let accepted_peer = handler
14750 .handle_control(
14751 ConnectionId::new(2),
14752 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14753 )
14754 .unwrap();
14755 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14756
14757 let accepted_short = handler
14758 .handle_control(
14759 ConnectionId::new(3),
14760 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14761 )
14762 .unwrap();
14763 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14764
14765 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14766 let response = handler
14767 .handle_control(
14768 ConnectionId::new(conn),
14769 hello_frame(module_id, PROTOCOL_VERSION, conn),
14770 )
14771 .unwrap();
14772 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14773 }
14774 }
14775
14776 #[test]
14777 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14778 let registry = Arc::new(Registry::default());
14779 let supervisor = SupervisorHandle::new();
14780 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14781 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14782 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14783 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14784
14785 let owner_nonce = handler
14786 .handle_control(
14787 ConnectionId::new(1),
14788 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14789 )
14790 .unwrap();
14791 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14792 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14793 assert!(registry.get_module("fed:special").unwrap().is_none());
14794
14795 let exact_nonce = handler
14796 .handle_control(
14797 ConnectionId::new(2),
14798 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14799 )
14800 .unwrap();
14801 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14802 assert!(registry.get_module("fed:special").unwrap().is_some());
14803 }
14804
14805 #[test]
14806 fn non_reserved_module_ignores_launch_nonce() {
14807 let registry = Arc::new(Registry::default());
14808 let handler = ControlHandler::new(Arc::clone(®istry));
14811 let no_nonce = handler
14812 .handle_control(
14813 ConnectionId::new(1),
14814 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14815 )
14816 .unwrap();
14817 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14818 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14819
14820 let echoed_nonce = handler
14821 .handle_control(
14822 ConnectionId::new(2),
14823 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14824 )
14825 .unwrap();
14826 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14827 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14828 }
14829
14830 #[test]
14831 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14832 let handler = ControlHandler::default();
14833 let conn = ConnectionId::new(1);
14834 let malformed = Frame::build(
14835 FrameType::Hello,
14836 control_flags(),
14837 0,
14838 0,
14839 3,
14840 b"{not json".to_vec(),
14841 )
14842 .unwrap();
14843
14844 let error = handler.handle_control(conn, malformed).unwrap();
14845 assert_eq!(error[0].header.ty, FrameType::Error);
14846 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14847
14848 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14849 let pong = handler.handle_control(conn, ping).unwrap();
14850 assert_eq!(pong[0].header.ty, FrameType::Pong);
14851 assert_eq!(pong[0].header.corr, 4);
14852 }
14853
14854 #[test]
14855 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14856 let registry = Arc::new(Registry::default());
14857 let handler = ControlHandler::new(Arc::clone(®istry));
14858
14859 handler
14860 .handle_control(
14861 ConnectionId::new(1),
14862 hello_frame("aft", PROTOCOL_VERSION, 1),
14863 )
14864 .unwrap();
14865 let duplicate = handler
14866 .handle_control(
14867 ConnectionId::new(2),
14868 hello_frame("aft", PROTOCOL_VERSION, 2),
14869 )
14870 .unwrap();
14871
14872 assert_eq!(duplicate[0].header.ty, FrameType::Error);
14873 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14874 let registration = registry.get_module("aft").unwrap().unwrap();
14875 assert_eq!(registration.connection_id, ConnectionId::new(1));
14876 }
14877
14878 #[test]
14879 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14880 let registry = Arc::new(Registry::default());
14881 let forwarding = Arc::new(ForwardingTable::default());
14882 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14883 let handler =
14884 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14885 .with_process_liveness(process_liveness);
14886 let (ctx, route_channel, route_epoch) =
14887 bind_liveness_route(®istry, &forwarding, "aft-dead");
14888 let responses = handler
14889 .handle_route_poll(
14890 &ctx,
14891 route_poll_frame(41, PollKind::Liveness, route_channel),
14892 route_channel,
14893 route_epoch,
14894 PollKind::Liveness,
14895 )
14896 .unwrap();
14897
14898 assert_eq!(responses.len(), 1);
14899 assert_eq!(responses[0].header.ty, FrameType::Response);
14900 assert_route_poll_liveness(&responses[0], false);
14901 }
14902
14903 #[test]
14904 fn liveness_poll_without_process_source_uses_bound_route() {
14905 let registry = Arc::new(Registry::default());
14906 let forwarding = Arc::new(ForwardingTable::default());
14907 let handler =
14908 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14909 let (ctx, route_channel, route_epoch) =
14910 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
14911 let responses = handler
14912 .handle_route_poll(
14913 &ctx,
14914 route_poll_frame(42, PollKind::Liveness, route_channel),
14915 route_channel,
14916 route_epoch,
14917 PollKind::Liveness,
14918 )
14919 .unwrap();
14920
14921 assert_route_poll_liveness(&responses[0], true);
14922 }
14923
14924 #[test]
14925 fn liveness_poll_untracked_process_source_uses_bound_route() {
14926 let registry = Arc::new(Registry::default());
14927 let forwarding = Arc::new(ForwardingTable::default());
14928 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14929 let handler =
14930 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14931 .with_process_liveness(process_liveness);
14932 let (ctx, route_channel, route_epoch) =
14933 bind_liveness_route(®istry, &forwarding, "aft-untracked");
14934 let responses = handler
14935 .handle_route_poll(
14936 &ctx,
14937 route_poll_frame(43, PollKind::Liveness, route_channel),
14938 route_channel,
14939 route_epoch,
14940 PollKind::Liveness,
14941 )
14942 .unwrap();
14943
14944 assert_route_poll_liveness(&responses[0], true);
14945 }
14946
14947 #[tokio::test]
14948 async fn unknown_op_returns_unknown_control_op() {
14949 let handler = ControlHandler::default();
14950 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14951 let request = Frame::build(
14952 FrameType::Request,
14953 control_flags(),
14954 0,
14955 0,
14956 55,
14957 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14958 )
14959 .unwrap();
14960
14961 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14962
14963 assert_eq!(response.len(), 1);
14964 assert_eq!(response[0].header.ty, FrameType::Error);
14965 assert_eq!(response[0].header.corr, 55);
14966 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14967 }
14968
14969 #[tokio::test]
14970 async fn supervisor_provenance_rejects_unknown_exact_module() {
14971 let handler = ControlHandler::default();
14972 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14973 let request = Frame::build(
14974 FrameType::Request,
14975 control_flags(),
14976 0,
14977 0,
14978 57,
14979 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14980 )
14981 .unwrap();
14982
14983 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14984
14985 assert_eq!(response.len(), 1);
14986 assert_eq!(response[0].header.ty, FrameType::Error);
14987 assert_eq!(response[0].header.corr, 57);
14988 let error = parse_error(&response[0]);
14989 assert_eq!(error["code"], "unknown_module");
14990 assert_eq!(error["message"], "module_id 'missing' is not supervised");
14991 }
14992
14993 #[test]
14994 fn provenance_probe_override_keeps_handler_tests_deterministic() {
14995 let expected = subc_control::RunningImageAgreement::Unavailable {
14996 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14997 };
14998 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14999 assert_eq!(handler.provenance_probe_override, Some(expected));
15000 }
15001
15002 #[test]
15003 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
15004 let verdict = reload_verdict(
15005 std::path::Path::new("/bin/new"),
15006 Some(std::path::Path::new("/bin/old")),
15007 subc_control::RunningImageAgreement::Unavailable {
15008 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15009 },
15010 );
15011 assert!(matches!(
15012 verdict.path,
15013 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
15014 if configured == std::path::Path::new("/bin/new")
15015 && spawned_from == std::path::Path::new("/bin/old")
15016 ));
15017 }
15018
15019 #[test]
15020 fn launch_spec_without_running_digest_is_unavailable_not_match() {
15021 use subc_control::{
15022 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
15023 };
15024 let configured = crate::supervise::ModuleSpec {
15025 module_id: "legacy".into(),
15026 program: "/module".into(),
15027 args: Vec::new(),
15028 env: Vec::new(),
15029 reserved: false,
15030 reserved_prefixes: Vec::new(),
15031 protocol: subc_control::ModuleProtocol::Subc,
15032 overlap: Default::default(),
15033 };
15034 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
15035 assert_eq!(
15036 launch_spec_agreement(&configured, false, Some(&digest)),
15037 Agreement::Unavailable {
15038 reason: Reason::NotRunning
15039 }
15040 );
15041 assert_eq!(
15042 launch_spec_agreement(&configured, true, None),
15043 Agreement::Unavailable {
15044 reason: Reason::SpawnedSpecUnavailable
15045 }
15046 );
15047 assert_eq!(
15048 launch_spec_agreement(&configured, true, Some(&digest)),
15049 Agreement::Match
15050 );
15051 }
15052
15053 #[cfg(windows)]
15054 #[test]
15055 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
15056 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
15057 let configured = dir.join("module.exe");
15058 std::fs::write(&configured, b"executable path fixture").unwrap();
15059 let spawned = std::fs::canonicalize(&configured).unwrap();
15060 assert_ne!(
15061 configured, spawned,
15062 "fixture must exercise different path spellings"
15063 );
15064 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
15065 let lower_case = configured.to_string_lossy().to_lowercase();
15066 for spelling in [
15067 &configured,
15068 Path::new(&forward_slashes),
15069 Path::new(&lower_case),
15070 ] {
15071 let verdict = reload_verdict(
15072 spelling,
15073 Some(&spawned),
15074 subc_control::RunningImageAgreement::Unavailable {
15075 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15076 },
15077 );
15078 assert_eq!(
15079 verdict.path,
15080 ReloadPathAgreement::Match,
15081 "configured {spelling:?}, spawned {spawned:?}"
15082 );
15083 }
15084 }
15085
15086 #[cfg(windows)]
15087 #[test]
15088 fn windows_reload_path_keeps_different_existing_program_pending() {
15089 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
15090 let old = dir.join("old.exe");
15091 let new = dir.join("new.exe");
15092 std::fs::write(&old, b"same executable content").unwrap();
15093 std::fs::copy(&old, &new).unwrap();
15094 let spawned = std::fs::canonicalize(&old).unwrap();
15095 let image = subc_control::RunningImageAgreement::Unavailable {
15096 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15097 };
15098 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
15099 assert!(
15100 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
15101 if configured == new && spawned_from == spawned)
15102 );
15103 let missing = dir.join("missing.exe");
15104 assert!(matches!(
15105 reload_verdict(&missing, Some(&spawned), image).path,
15106 ReloadPathAgreement::Mismatch { .. }
15107 ));
15108 }
15109
15110 #[test]
15111 fn reload_verdict_detects_replaced_image_at_same_path() {
15112 let image = subc_control::RunningImageAgreement::Mismatch {
15113 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
15114 digest: "old".into(),
15115 },
15116 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
15117 digest: "new".into(),
15118 },
15119 };
15120 let verdict = reload_verdict(
15121 std::path::Path::new("/bin/same"),
15122 Some(std::path::Path::new("/bin/same")),
15123 image.clone(),
15124 );
15125 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15126 assert_eq!(verdict.image, image);
15127 }
15128
15129 #[test]
15130 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
15131 let image = subc_control::RunningImageAgreement::Unavailable {
15132 reason: subc_control::RunningImageUnavailableReason::NotRunning,
15133 };
15134 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
15135 assert_eq!(
15136 verdict.path,
15137 subc_control::ReloadPathAgreement::Unavailable {
15138 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
15139 }
15140 );
15141 assert_eq!(verdict.image, image);
15142
15143 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
15144 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15145 };
15146 let verdict = reload_verdict(
15147 std::path::Path::new("/bin/same"),
15148 Some(std::path::Path::new("/bin/same")),
15149 unconfirmed.clone(),
15150 );
15151 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15152 assert_eq!(verdict.image, unconfirmed);
15153 }
15154
15155 #[test]
15156 fn reload_verdict_preserves_each_image_unavailability_reason() {
15157 use subc_control::RunningImageUnavailableReason as Reason;
15158
15159 for reason in [
15160 Reason::NotRunning,
15161 Reason::UnsupportedPlatform,
15162 Reason::RunningExecutableUnreadable,
15163 Reason::SpawnedPathUnreadable,
15164 Reason::HashFailed,
15165 Reason::ProcessIdentityUnconfirmed,
15166 Reason::Unknown("future_probe_reason".to_string()),
15167 ] {
15168 let image = subc_control::RunningImageAgreement::Unavailable {
15169 reason: reason.clone(),
15170 };
15171 let verdict = reload_verdict(
15172 std::path::Path::new("/bin/same"),
15173 Some(std::path::Path::new("/bin/same")),
15174 image.clone(),
15175 );
15176 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15177 assert_eq!(verdict.image, image, "{reason:?}");
15178 }
15179 }
15180
15181 #[tokio::test]
15182 async fn malformed_control_bodies_return_invalid_control_body() {
15183 let handler = ControlHandler::default();
15184 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
15185
15186 for (corr, body) in [
15187 (56, br#"{"route_channel":1}"#.as_slice()),
15188 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
15189 (
15190 58,
15191 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
15192 ),
15193 ] {
15194 let request = Frame::build(
15195 FrameType::Request,
15196 control_flags(),
15197 0,
15198 0,
15199 corr,
15200 body.to_vec(),
15201 )
15202 .unwrap();
15203 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15204
15205 assert_eq!(response.len(), 1);
15206 assert_eq!(response[0].header.ty, FrameType::Error);
15207 assert_eq!(response[0].header.corr, corr);
15208 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
15209 }
15210 }
15211
15212 #[tokio::test]
15213 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
15214 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15215 let registry = Arc::new(Registry::default());
15216 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15217 let router = Router::with_control_handler(Arc::clone(&control));
15218 let connection = router.begin_connection();
15219 let (ctx, mut rx) = route_ctx(connection.id());
15220
15221 router
15222 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
15223 .await
15224 .unwrap();
15225 let response = rx.recv().await.unwrap();
15226 let ack = parse_ack(&response);
15227 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15228 let channel = 1;
15229
15230 let goodbye =
15231 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
15232 router.route_for_connection(&ctx, goodbye).await.unwrap();
15233 assert!(rx.try_recv().is_err());
15234 assert!(registry.get_module("aft").unwrap().is_none());
15235
15236 router
15237 .route_for_connection(&ctx, channel_request(channel, 13))
15238 .await
15239 .unwrap();
15240 let error_frame = rx.recv().await.unwrap();
15241 assert_eq!(error_frame.header.ty, FrameType::Error);
15242 assert_eq!(error_frame.header.channel, channel);
15243 let captured = crate::router::test_log::captured_logs(&logs);
15244 assert_eq!(
15245 captured
15246 .lines()
15247 .filter(|line| {
15248 line.contains("module_id=aft")
15249 && line.contains("reason=explicit_goodbye")
15250 && line.contains("module registration ended")
15251 })
15252 .count(),
15253 1,
15254 "unexpected GOODBYE registry log: {captured}"
15255 );
15256 }
15257
15258 #[tokio::test]
15259 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
15260 let registry = Arc::new(Registry::default());
15261 let handler = ControlHandler::new(registry).with_capability_config(
15262 [("prov".to_string(), true), ("cons".to_string(), true)],
15263 BTreeMap::new(),
15264 );
15265 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
15266 register_capability_manifest(
15267 &handler,
15268 &provider_ctx,
15269 &mut provider_rx,
15270 capability_manifest("prov", &["thing/v1"], &[]),
15271 1,
15272 )
15273 .await;
15274 let mut consumer = capability_manifest("cons", &[], &[]);
15275 consumer.capabilities.as_mut().unwrap().requires.push(
15276 subc_protocol::manifest::CapabilityRequirement {
15277 capability: "thing/v1".to_string(),
15278 need: subc_protocol::manifest::CapabilityNeed::Required,
15279 },
15280 );
15281 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
15282 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
15283 assert_eq!(
15284 handler.capability_evaluator.verdict("cons", "thing/v1"),
15285 Some(CapabilityVerdict::Provided)
15286 );
15287 let (mut client_rx, _) = open_route_for_capability_test(
15288 &handler,
15289 &provider_ctx,
15290 &mut provider_rx,
15291 703,
15292 3,
15293 "prov",
15294 None,
15295 )
15296 .await;
15297 let goodbye =
15298 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15299 handler
15300 .handle_control_frame(&provider_ctx, goodbye)
15301 .await
15302 .unwrap();
15303 assert_eq!(
15304 handler.capability_evaluator.verdict("cons", "thing/v1"),
15305 Some(CapabilityVerdict::NeverProvided)
15306 );
15307 let closed = client_rx
15308 .try_recv()
15309 .expect("GOODBYE pushes route.closed before route GOODBYE");
15310 assert!(
15311 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
15312 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
15313 );
15314 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
15315 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
15316 }
15317
15318 struct RouteClosedFixture {
15319 handler: ControlHandler,
15320 #[cfg(unix)]
15321 supervisor: Supervisor,
15322 module_ctx: RouteCtx,
15323 #[cfg(unix)]
15324 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15325 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15326 route: Arc<crate::forwarding::RouteBinding>,
15327 }
15328
15329 async fn route_closed_fixture() -> RouteClosedFixture {
15330 let registry = Arc::new(Registry::default());
15331 let handler = ControlHandler::new(Arc::clone(®istry));
15332 #[cfg(unix)]
15333 let supervisor = Supervisor::new(registry, RestartPolicy::default())
15334 .with_forwarding(Arc::clone(&handler.forwarding));
15335 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
15336 register_capability_manifest(
15337 &handler,
15338 &module_ctx,
15339 &mut module_rx,
15340 capability_manifest("prov", &[], &[]),
15341 1,
15342 )
15343 .await;
15344 let (client_rx, bind) = open_route_for_capability_test(
15345 &handler,
15346 &module_ctx,
15347 &mut module_rx,
15348 702,
15349 2,
15350 "prov",
15351 None,
15352 )
15353 .await;
15354 let ModuleControlRequest::RouteBind {
15355 route_channel: channel,
15356 epoch: route_epoch,
15357 ..
15358 } = bind
15359 else {
15360 panic!("expected route.bind");
15361 };
15362 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
15363 handler
15364 .forwarding
15365 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
15366 .unwrap()
15367 else {
15368 panic!("the acknowledged route must be bound");
15369 };
15370 RouteClosedFixture {
15371 handler,
15372 #[cfg(unix)]
15373 supervisor,
15374 module_ctx,
15375 #[cfg(unix)]
15376 module_rx,
15377 client_rx,
15378 route,
15379 }
15380 }
15381
15382 impl RouteClosedFixture {
15383 #[cfg(unix)]
15384 async fn cut(&mut self) {
15385 let notices = async {
15386 let mut draining = self.module_rx.recv().await.unwrap();
15387 assert!(matches!(
15388 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
15389 &draining.body
15390 )
15391 .unwrap(),
15392 subc_protocol::session::ModuleControlCommand::Draining {
15393 reason: RouteCloseReason::Restart,
15394 ..
15395 }
15396 ));
15397 draining.flushed.take().unwrap().send(()).unwrap();
15398 let mut closing = self.client_rx.recv().await.unwrap();
15399 assert!(matches!(
15400 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
15401 ClientControlPush::RouteClosing {
15402 reason: RouteCloseReason::Restart,
15403 ..
15404 }
15405 ));
15406 closing.flushed.take().unwrap().send(()).unwrap();
15407 };
15408 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
15409 result.unwrap();
15410 }
15411
15412 fn disconnect(&mut self) -> ClientControlPush {
15413 self.handler
15414 .cleanup_connection(self.module_ctx.connection_id)
15415 .unwrap();
15416 let closed = self.client_rx.try_recv().unwrap();
15417 assert_eq!(closed.header.ty, FrameType::Push);
15418 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
15419 assert!(matches!(
15420 &push,
15421 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
15422 if channels == &[self.route.client_channel]
15423 ));
15424 assert_eq!(
15425 self.client_rx.try_recv().unwrap().header.ty,
15426 FrameType::Goodbye
15427 );
15428 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
15429 push
15430 }
15431 }
15432
15433 #[cfg(unix)]
15434 #[tokio::test(start_paused = true)]
15435 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
15436 let mut fixture = route_closed_fixture().await;
15437 fixture.cut().await;
15438 assert!(matches!(
15439 fixture.disconnect(),
15440 ClientControlPush::RouteClosed {
15441 reason: RouteCloseReason::Restart,
15442 drained: true,
15443 ..
15444 }
15445 ));
15446 }
15447
15448 #[cfg(unix)]
15449 #[tokio::test(start_paused = true)]
15450 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
15451 let mut fixture = route_closed_fixture().await;
15452 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
15453 let started = tokio::time::Instant::now();
15454 fixture.cut().await;
15455 assert!(started.elapsed() >= Duration::from_secs(2));
15456 assert_eq!(fixture.route.flow.in_flight(), 1);
15457 assert!(matches!(
15458 fixture.disconnect(),
15459 ClientControlPush::RouteClosed {
15460 reason: RouteCloseReason::Restart,
15461 drained: false,
15462 ..
15463 }
15464 ));
15465 }
15466
15467 #[tokio::test]
15468 async fn module_connection_crash_reports_crash_and_not_drained() {
15469 let mut fixture = route_closed_fixture().await;
15470 assert!(matches!(
15474 fixture.disconnect(),
15475 ClientControlPush::RouteClosed {
15476 reason: RouteCloseReason::Crash,
15477 drained: false,
15478 ..
15479 }
15480 ));
15481 }
15482
15483 #[tokio::test]
15484 async fn dropping_router_connection_releases_registration() {
15485 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15486 let registry = Arc::new(Registry::default());
15487 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15488 let router = Router::with_control_handler(Arc::clone(&control));
15489 let connection = router.begin_connection();
15490 let connection_id = connection.id();
15491 let (ctx, mut rx) = route_ctx(connection_id);
15492
15493 router
15494 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
15495 .await
15496 .unwrap();
15497 let response = rx.recv().await.unwrap();
15498 let ack = parse_ack(&response);
15499 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15500 assert!(registry.get_module("aft").unwrap().is_some());
15501
15502 drop(connection);
15503
15504 assert!(registry.get_module("aft").unwrap().is_none());
15505 assert_eq!(registry.active_registration_count().unwrap(), 0);
15506
15507 control
15508 .cleanup_connection(ConnectionId::new(u64::MAX))
15509 .unwrap();
15510 let captured = crate::router::test_log::captured_logs(&logs);
15511 println!("captured registry lifecycle logs:\n{captured}");
15512 let events: Vec<_> = captured
15513 .lines()
15514 .filter(|line| {
15515 line.contains("module registered module_id=aft ")
15516 || line.contains("module registration ended")
15517 })
15518 .collect();
15519 assert_eq!(
15520 events.len(),
15521 2,
15522 "unexpected registry lifecycle logs: {captured}"
15523 );
15524 assert!(events[0].contains("module registered module_id=aft "));
15525 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
15526 assert!(events[1].contains(&format!(
15527 "module_id=aft connection_id={}",
15528 connection_id.get()
15529 )));
15530 assert!(events[1].contains("reason=connection_closed"));
15531 assert!(events[1].contains("module registration ended"));
15532 assert_eq!(
15533 captured
15534 .lines()
15535 .filter(|line| line.contains("module registered module_id=aft "))
15536 .count(),
15537 1,
15538 "legacy registration admission line must appear once: {captured}"
15539 );
15540 }
15541
15542 #[tokio::test]
15543 async fn hello_registration_keeps_legacy_module_registered_line_once() {
15544 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15545 let registry = Arc::new(Registry::default());
15546 let control = ControlHandler::new(Arc::clone(®istry));
15547 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
15548 hello_via_sink(
15549 &control,
15550 &ctx,
15551 &mut rx,
15552 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
15553 )
15554 .await;
15555
15556 let captured = crate::router::test_log::captured_logs(&logs);
15557 let admissions: Vec<_> = captured
15558 .lines()
15559 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
15560 .collect();
15561 assert_eq!(
15562 admissions.len(),
15563 1,
15564 "legacy admission line must remain exactly once: {captured}"
15565 );
15566 assert!(admissions[0].contains("routable_provider=true"));
15567 assert!(admissions[0].contains("connection_id=777"));
15568 }
15569
15570 fn capability_manifest(
15571 module_id: &str,
15572 provides: &[&str],
15573 must_never_reach: &[&str],
15574 ) -> ModuleManifest {
15575 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
15576 manifest.capabilities = Some(CapabilityDeclarations {
15577 provides: provides
15578 .iter()
15579 .map(|capability| (*capability).to_string())
15580 .collect(),
15581 requires: Vec::new(),
15582 must_never_reach: must_never_reach
15583 .iter()
15584 .map(|capability| (*capability).to_string())
15585 .collect(),
15586 });
15587 manifest
15588 }
15589
15590 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
15591 Frame::build(
15592 FrameType::Hello,
15593 control_flags(),
15594 0,
15595 0,
15596 corr,
15597 serde_json::to_vec(&ModuleHelloBody {
15598 protocol_ver: manifest.protocol_ver,
15599 manifest,
15600 control_ops: None,
15601 launch_nonce: None,
15602 })
15603 .expect("capability test HELLO serializes"),
15604 )
15605 .expect("capability test HELLO frame builds")
15606 }
15607
15608 fn catalog_update_with_capabilities_frame(
15609 corr: u64,
15610 capabilities: CapabilityDeclarations,
15611 ) -> Frame {
15612 Frame::build(
15613 FrameType::Request,
15614 control_flags(),
15615 0,
15616 0,
15617 corr,
15618 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15619 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
15620 capabilities: Some(capabilities),
15621 ready: None,
15622 })
15623 .expect("capability catalog.update serializes"),
15624 )
15625 .expect("capability catalog.update frame builds")
15626 }
15627
15628 async fn register_capability_manifest(
15629 handler: &ControlHandler,
15630 ctx: &RouteCtx,
15631 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15632 manifest: ModuleManifest,
15633 corr: u64,
15634 ) {
15635 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
15636 }
15637
15638 async fn open_route_for_capability_test(
15639 handler: &ControlHandler,
15640 target_ctx: &RouteCtx,
15641 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15642 client_connection_id: u64,
15643 corr: u64,
15644 target_module_id: &str,
15645 consumer_identity: Option<ConsumerIdentity>,
15646 ) -> (
15647 mpsc::Receiver<crate::router::OutboundFrame>,
15648 ModuleControlRequest,
15649 ) {
15650 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
15651 let route_handler = handler.clone();
15652 let target_module_id = target_module_id.to_string();
15653 let route_task = tokio::spawn(async move {
15654 route_handler
15655 .handle_control_frame(
15656 &client_ctx,
15657 route_open_frame_with_admission_facts(
15658 corr,
15659 &target_module_id,
15660 unique_project_root("admission-facts"),
15661 consumer_identity,
15662 None,
15663 ),
15664 )
15665 .await
15666 .expect("capability test route.open succeeds")
15667 });
15668 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
15669 .await
15670 .expect("capability test route.open must reach route.bind")
15671 .expect("target control receiver stays open");
15672 let bind_request: ModuleControlRequest =
15673 serde_json::from_slice(&bind.body).expect("route.bind decodes");
15674 handler
15675 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
15676 .await
15677 .expect("capability test route.bind ACK succeeds");
15678 assert!(route_task.await.expect("route.open task joins").is_empty());
15679 let opened = client_rx
15680 .recv()
15681 .await
15682 .expect("successful route.open publishes a response");
15683 assert!(matches!(
15684 serde_json::from_slice::<ClientControlResponse>(&opened.body),
15685 Ok(ClientControlResponse::RouteOpen { .. })
15686 ));
15687 (client_rx, bind_request)
15688 }
15689
15690 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
15691 assert_eq!(frame.header.ty, FrameType::Push);
15692 assert_eq!(frame.header.channel, 0);
15693 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
15694 .expect("route.closed control push decodes");
15695 let ClientControlPush::RouteClosed { channels, .. } = &push else {
15696 panic!("expected route.closed");
15697 };
15698 assert_eq!(channels.len(), 1, "exactly one violating route closed");
15699 let closed_channel = channels[0];
15700 let channels = channels.clone();
15701 assert_eq!(
15702 push,
15703 ClientControlPush::RouteClosed {
15704 module_id: target_module_id.to_string(),
15705 channels,
15706 reason: RouteCloseReason::CapabilityDenied,
15707 drained: false,
15708 abandoned: 0,
15709 excluded_subscriptions: 0,
15710 terminal: Some(false),
15711 }
15712 );
15713 closed_channel
15714 }
15715
15716 #[tokio::test]
15717 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15718 let registry = Arc::new(Registry::default());
15719 let forwarding = Arc::new(ForwardingTable::default());
15720 let supervisor = SupervisorHandle::new();
15721 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15722 let handler =
15723 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15724 .with_supervisor(supervisor);
15725 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15726 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15727 register_capability_manifest(
15728 &handler,
15729 &target_ctx,
15730 &mut target_rx,
15731 capability_manifest("target", &["credentials-provider/v1"], &[]),
15732 1,
15733 )
15734 .await;
15735 register_capability_manifest(
15736 &handler,
15737 &opener_ctx,
15738 &mut opener_rx,
15739 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15740 2,
15741 )
15742 .await;
15743
15744 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15745 let replies = handler
15746 .handle_control_frame(
15747 &client_ctx,
15748 route_open_frame_with_admission_facts(
15749 3,
15750 "target",
15751 unique_project_root("admission-facts"),
15752 Some(ConsumerIdentity {
15753 module_id: "opener".to_string(),
15754 launch_nonce: "opener-nonce".to_string(),
15755 }),
15756 None,
15757 ),
15758 )
15759 .await
15760 .expect("denied route.open returns a typed frame");
15761 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15762 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15763 assert!(
15764 target_rx.try_recv().is_err(),
15765 "forbidden route.open must not relay route.bind"
15766 );
15767 }
15768
15769 #[tokio::test]
15770 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15771 let registry = Arc::new(Registry::default());
15772 let forwarding = Arc::new(ForwardingTable::default());
15773 let supervisor = SupervisorHandle::new();
15774 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15775 let handler =
15776 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15777 .with_supervisor(supervisor);
15778 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15779 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15780 register_capability_manifest(
15781 &handler,
15782 &target_ctx,
15783 &mut target_rx,
15784 capability_manifest("target", &["credentials-provider/v1"], &[]),
15785 1,
15786 )
15787 .await;
15788 register_capability_manifest(
15789 &handler,
15790 &old_opener_ctx,
15791 &mut old_opener_rx,
15792 capability_manifest("opener", &[], &[]),
15793 2,
15794 )
15795 .await;
15796 let (mut client_rx, _) = open_route_for_capability_test(
15797 &handler,
15798 &target_ctx,
15799 &mut target_rx,
15800 712,
15801 3,
15802 "target",
15803 Some(ConsumerIdentity {
15804 module_id: "opener".to_string(),
15805 launch_nonce: "opener-nonce".to_string(),
15806 }),
15807 )
15808 .await;
15809 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15810
15811 handler
15812 .cleanup_connection(old_opener_ctx.connection_id)
15813 .expect("old opener registration cleans up");
15814 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15815 register_capability_manifest(
15816 &handler,
15817 &new_opener_ctx,
15818 &mut new_opener_rx,
15819 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15820 4,
15821 )
15822 .await;
15823
15824 let closed_channel = assert_capability_denied_push(
15825 client_rx
15826 .try_recv()
15827 .expect("HELLO deny addition must emit route.closed")
15828 .frame,
15829 "target",
15830 );
15831 let goodbye = client_rx
15832 .try_recv()
15833 .expect("HELLO deny addition must send the client a route GOODBYE")
15834 .frame;
15835 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15836 assert_eq!(goodbye.header.channel, closed_channel);
15837 assert_ne!(goodbye.header.channel, 0);
15838 assert_ne!(goodbye.header.epoch, 0);
15839 assert!(client_rx.try_recv().is_err());
15840 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15841 assert!(matches!(
15842 target_rx.try_recv(),
15843 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15844 ));
15845 }
15846
15847 #[tokio::test]
15848 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15849 let registry = Arc::new(Registry::default());
15850 let forwarding = Arc::new(ForwardingTable::default());
15851 let supervisor = SupervisorHandle::new();
15852 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15853 let handler =
15854 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15855 .with_supervisor(supervisor);
15856 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15857 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15858 register_capability_manifest(
15859 &handler,
15860 &target_ctx,
15861 &mut target_rx,
15862 capability_manifest("target", &[], &[]),
15863 1,
15864 )
15865 .await;
15866 register_capability_manifest(
15867 &handler,
15868 &opener_ctx,
15869 &mut opener_rx,
15870 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15871 2,
15872 )
15873 .await;
15874 let (mut client_rx, _) = open_route_for_capability_test(
15875 &handler,
15876 &target_ctx,
15877 &mut target_rx,
15878 722,
15879 3,
15880 "target",
15881 Some(ConsumerIdentity {
15882 module_id: "opener".to_string(),
15883 launch_nonce: "opener-nonce".to_string(),
15884 }),
15885 )
15886 .await;
15887 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15888
15889 let replies = handler
15890 .handle_control_frame(
15891 &target_ctx,
15892 catalog_update_with_capabilities_frame(
15893 4,
15894 CapabilityDeclarations {
15895 provides: vec!["credentials-provider/v1".to_string()],
15896 requires: Vec::new(),
15897 must_never_reach: Vec::new(),
15898 },
15899 ),
15900 )
15901 .await
15902 .expect("claim catalog.update succeeds");
15903 assert!(matches!(
15904 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15905 Ok(ModuleControlResponseToModule::CatalogUpdate {})
15906 ));
15907 let closed_channel = assert_capability_denied_push(
15908 client_rx
15909 .try_recv()
15910 .expect("claim addition must emit route.closed")
15911 .frame,
15912 "target",
15913 );
15914 let goodbye = client_rx
15915 .try_recv()
15916 .expect("claim addition must send the client a route GOODBYE")
15917 .frame;
15918 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15919 assert_eq!(goodbye.header.channel, closed_channel);
15920 assert_ne!(goodbye.header.channel, 0);
15921 assert_ne!(goodbye.header.epoch, 0);
15922 assert!(client_rx.try_recv().is_err());
15923 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15924 assert!(matches!(
15925 target_rx.try_recv(),
15926 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15927 ));
15928 }
15929
15930 #[tokio::test]
15931 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15932 let registry = Arc::new(Registry::default());
15933 let forwarding = Arc::new(ForwardingTable::default());
15934 let supervisor = SupervisorHandle::new();
15935 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15936 let handler =
15937 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15938 .with_supervisor(supervisor);
15939 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15940 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15941 register_capability_manifest(
15942 &handler,
15943 &target_ctx,
15944 &mut target_rx,
15945 capability_manifest("target", &["credentials-provider/v1"], &[]),
15946 1,
15947 )
15948 .await;
15949 register_capability_manifest(
15950 &handler,
15951 &opener_ctx,
15952 &mut opener_rx,
15953 capability_manifest("opener", &[], &[]),
15954 2,
15955 )
15956 .await;
15957 let (mut client_rx, _) = open_route_for_capability_test(
15958 &handler,
15959 &target_ctx,
15960 &mut target_rx,
15961 732,
15962 3,
15963 "target",
15964 Some(ConsumerIdentity {
15965 module_id: "opener".to_string(),
15966 launch_nonce: "opener-nonce".to_string(),
15967 }),
15968 )
15969 .await;
15970 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15971
15972 handler
15973 .handle_control_frame(
15974 &target_ctx,
15975 catalog_update_with_capabilities_frame(
15976 4,
15977 CapabilityDeclarations {
15978 provides: Vec::new(),
15979 requires: Vec::new(),
15980 must_never_reach: Vec::new(),
15981 },
15982 ),
15983 )
15984 .await
15985 .expect("claim removal catalog.update succeeds");
15986 assert_eq!(
15987 forwarding.active_binding_count().unwrap(),
15988 1,
15989 "removing an attested target claim must leave the route census unchanged"
15990 );
15991 assert!(
15992 client_rx.try_recv().is_err(),
15993 "claim removal must not emit route.closed or a client route GOODBYE"
15994 );
15995 assert!(
15996 target_rx.try_recv().is_err(),
15997 "claim removal must not send the target a route GOODBYE"
15998 );
15999 }
16000
16001 #[tokio::test]
16004 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
16005 let registry = Arc::new(Registry::default());
16006 let forwarding = Arc::new(ForwardingTable::default());
16007 let supervisor = SupervisorHandle::new();
16008 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16009 let handler =
16010 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16011 .with_supervisor(supervisor);
16012 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
16013 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
16014 register_capability_manifest(
16015 &handler,
16016 &target_ctx,
16017 &mut target_rx,
16018 capability_manifest("target", &["credentials-provider/v1"], &[]),
16019 1,
16020 )
16021 .await;
16022 register_capability_manifest(
16023 &handler,
16024 &opener_ctx,
16025 &mut opener_rx,
16026 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16027 2,
16028 )
16029 .await;
16030
16031 let (_client_rx, bind) = open_route_for_capability_test(
16032 &handler,
16033 &target_ctx,
16034 &mut target_rx,
16035 742,
16036 3,
16037 "target",
16038 None,
16039 )
16040 .await;
16041 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
16042 panic!("direct scope-honesty route must bind");
16043 };
16044 assert_eq!(principal, Some(Principal::Direct));
16045 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16046 }
16047
16048 #[tokio::test]
16051 async fn must_never_reach_self_route_is_capability_forbidden() {
16052 let registry = Arc::new(Registry::default());
16053 let forwarding = Arc::new(ForwardingTable::default());
16054 let supervisor = SupervisorHandle::new();
16055 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
16056 let handler =
16057 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16058 .with_supervisor(supervisor);
16059 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
16060 register_capability_manifest(
16061 &handler,
16062 &self_ctx,
16063 &mut self_rx,
16064 capability_manifest(
16065 "self-provider",
16066 &["credentials-provider/v1"],
16067 &["credentials-provider/v1"],
16068 ),
16069 1,
16070 )
16071 .await;
16072
16073 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
16074 let replies = handler
16075 .handle_control_frame(
16076 &client_ctx,
16077 route_open_frame_with_admission_facts(
16078 2,
16079 "self-provider",
16080 unique_project_root("admission-facts"),
16081 Some(ConsumerIdentity {
16082 module_id: "self-provider".to_string(),
16083 launch_nonce: "self-nonce".to_string(),
16084 }),
16085 None,
16086 ),
16087 )
16088 .await
16089 .expect("self-route refusal returns a typed frame");
16090 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16091 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16092 assert!(
16093 self_rx.try_recv().is_err(),
16094 "self denial must not relay route.bind"
16095 );
16096 }
16097
16098 #[test]
16099 fn unsupported_channel_zero_frame_returns_error() {
16100 let handler = ControlHandler::default();
16101 let request = Frame::build(
16102 FrameType::Request,
16103 control_flags(),
16104 0,
16105 0,
16106 21,
16107 b"opaque".to_vec(),
16108 )
16109 .unwrap();
16110
16111 let response = handler
16112 .handle_control(ConnectionId::new(1), request)
16113 .unwrap();
16114
16115 assert_eq!(response[0].header.ty, FrameType::Error);
16116 assert_eq!(
16117 parse_error(&response[0])["code"],
16118 "unsupported_control_frame"
16119 );
16120 }
16121
16122 mod swap {
16127 use super::*;
16128
16129 const INCUMBENT: ConnectionId = ConnectionId::new(30);
16130 const CANDIDATE: ConnectionId = ConnectionId::new(40);
16131
16132 struct Swap {
16133 registry: Arc<Registry>,
16134 forwarding: Arc<ForwardingTable>,
16135 handler: ControlHandler,
16136 incumbent_ctx: RouteCtx,
16137 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16138 candidate_ctx: RouteCtx,
16139 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16140 }
16141
16142 async fn swap_with_incumbent() -> Swap {
16143 let registry = Arc::new(Registry::default());
16144 let forwarding = Arc::new(ForwardingTable::default());
16145 let handler =
16146 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16147 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
16148 hello_via_sink(
16149 &handler,
16150 &incumbent_ctx,
16151 &mut incumbent_rx,
16152 hello_frame("aft", PROTOCOL_VERSION, 7),
16153 )
16154 .await;
16155 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
16156 Swap {
16157 registry,
16158 forwarding,
16159 handler,
16160 incumbent_ctx,
16161 incumbent_rx,
16162 candidate_ctx,
16163 candidate_rx,
16164 }
16165 }
16166
16167 fn register_candidate(swap: &Swap, ready: Option<bool>) {
16168 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
16169 candidate_manifest.ready = ready;
16170 let registration = swap
16171 .registry
16172 .register_candidate_with_control_ops(
16173 candidate_manifest,
16174 PROTOCOL_VERSION,
16175 CANDIDATE,
16176 module_baseline_control_ops(),
16177 )
16178 .unwrap();
16179 swap.forwarding
16180 .register_candidate_module_connection(
16181 CANDIDATE,
16182 "aft".to_string(),
16183 PROTOCOL_VERSION,
16184 manifest_concurrency(®istration.manifest),
16185 swap.candidate_ctx.egress.clone(),
16186 )
16187 .unwrap();
16188 }
16189
16190 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
16191 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
16192 swap.registry.promote_candidate("aft").unwrap().unwrap();
16193 cutover.incumbent.unwrap()
16194 }
16195
16196 #[tokio::test]
16205 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
16206 let mut swap = swap_with_incumbent().await;
16207 let hold = Arc::new(std::sync::Barrier::new(2));
16208 let handler = swap
16209 .handler
16210 .clone()
16211 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
16212 handler
16213 .supervisor
16214 .open_swap("aft", "candidate-nonce".to_string());
16215 let hello = {
16216 let handler = handler.clone();
16217 let candidate_ctx = swap.candidate_ctx.clone();
16218 std::thread::spawn(move || {
16219 handler
16220 .handle_hello(
16221 candidate_ctx.connection_id,
16222 Some(candidate_ctx.egress.clone()),
16223 hello_frame_with_nonce(
16224 "aft",
16225 PROTOCOL_VERSION,
16226 8,
16227 Some("candidate-nonce"),
16228 ),
16229 )
16230 .unwrap()
16231 })
16232 };
16233
16234 hold.wait();
16235 assert!(swap
16238 .registry
16239 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
16240 .unwrap()
16241 .is_some());
16242 assert!(swap
16243 .registry
16244 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16245 .unwrap()
16246 .is_none());
16247 assert!(swap
16248 .forwarding
16249 .module_endpoint_for_connection(CANDIDATE)
16250 .unwrap()
16251 .is_none());
16252
16253 hold.wait();
16254 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
16255 assert_eq!(
16256 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
16257 FrameType::HelloAck
16258 );
16259 let published = swap
16260 .registry
16261 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16262 .unwrap()
16263 .expect("candidate is published once its HELLO completes");
16264 assert_eq!(published.connection_id, CANDIDATE);
16265 assert!(swap
16266 .forwarding
16267 .module_endpoint_for_connection(CANDIDATE)
16268 .unwrap()
16269 .is_some());
16270 }
16271
16272 fn keyed_total(counters: &Value, key: &str) -> u64 {
16273 counters[key]
16274 .as_object()
16275 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
16276 .unwrap_or(0)
16277 }
16278
16279 #[tokio::test]
16280 async fn replacement_logs_old_end_and_new_admission_once() {
16281 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16282 let mut swap = swap_with_incumbent().await;
16283 swap.handler
16284 .supervisor
16285 .open_swap("aft", "candidate-nonce".to_string());
16286 hello_via_sink(
16287 &swap.handler,
16288 &swap.candidate_ctx,
16289 &mut swap.candidate_rx,
16290 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
16291 )
16292 .await;
16293 cutover(&swap);
16294 swap.handler.cleanup_connection(INCUMBENT).unwrap();
16295
16296 let captured = crate::router::test_log::captured_logs(&logs);
16297 println!("captured replacement registry logs:\n{captured}");
16298 let new_admissions: Vec<_> = captured
16299 .lines()
16300 .filter(|line| {
16301 line.contains("connection_id=40")
16302 && line.contains("swap candidate registered; not routable until cutover")
16303 })
16304 .collect();
16305 assert_eq!(
16306 new_admissions.len(),
16307 1,
16308 "unexpected admission logs: {captured}"
16309 );
16310 assert!(new_admissions[0].contains("module_id=aft"));
16311 assert!(new_admissions[0].contains("connection_id=40"));
16312 assert!(
16313 new_admissions[0].contains("swap candidate registered; not routable until cutover")
16314 );
16315 assert!(new_admissions[0].contains("ready=true"));
16316
16317 let old_admissions: Vec<_> = captured
16318 .lines()
16319 .filter(|line| {
16320 line.contains("module registered module_id=aft ")
16321 && line.contains("connection_id=30")
16322 })
16323 .collect();
16324 assert_eq!(
16325 old_admissions.len(),
16326 1,
16327 "unexpected incumbent admission: {captured}"
16328 );
16329
16330 let old_ends: Vec<_> = captured
16331 .lines()
16332 .filter(|line| {
16333 line.contains("connection_id=30") && line.contains("module registration ended")
16334 })
16335 .collect();
16336 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
16337 assert!(old_ends[0].contains("module_id=aft"));
16338 assert!(old_ends[0].contains("connection_id=30"));
16339 assert!(old_ends[0].contains("reason=replaced"));
16340 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
16341 assert_eq!(
16342 captured
16343 .lines()
16344 .filter(|line| line.contains("module registration promoted"))
16345 .count(),
16346 1,
16347 "unexpected promotion logs: {captured}"
16348 );
16349 }
16350
16351 #[tokio::test]
16357 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
16358 let mut swap = swap_with_incumbent().await;
16359 let handler = swap.handler.clone();
16360
16361 let cotenant = ConnectionId::new(31);
16363 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
16364 let (cotenant_task, cotenant_bind) = relay_route_open(
16365 &handler,
16366 cotenant,
16367 &cotenant_ctx.egress,
16368 &mut swap.incumbent_rx,
16369 100,
16370 "aft",
16371 "swap-cotenant",
16372 )
16373 .await;
16374 handler
16375 .handle_control_frame(
16376 &swap.incumbent_ctx,
16377 route_bind_ack(cotenant_bind.header.corr),
16378 )
16379 .await
16380 .unwrap();
16381 assert!(cotenant_task.await.unwrap().is_empty());
16382 let (cotenant_channel, cotenant_epoch) =
16383 published_route(&cotenant_rx.recv().await.unwrap());
16384
16385 let caller = ConnectionId::new(32);
16387 let (caller_ctx, mut caller_rx) = route_ctx(caller);
16388 let (caller_task, caller_bind) = relay_route_open(
16389 &handler,
16390 caller,
16391 &caller_ctx.egress,
16392 &mut swap.incumbent_rx,
16393 101,
16394 "aft",
16395 "swap-caller",
16396 )
16397 .await;
16398 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
16399
16400 register_candidate(&swap, None);
16401 cutover(&swap);
16402
16403 let ack = handler
16405 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
16406 .await;
16407 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
16408 if module_loop_error.is_some() {
16409 handler.cleanup_connection(INCUMBENT).unwrap();
16412 }
16413
16414 assert!(
16416 cotenant_rx.try_recv().is_err(),
16417 "the co-tenant route on the incumbent was torn down by one late ack: \
16418 {module_loop_error:?}"
16419 );
16420 assert!(matches!(
16421 swap.forwarding
16422 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16423 .unwrap(),
16424 DataRoute::Client(DataRouteState::Bound(_))
16425 ));
16426 assert_eq!(module_loop_error, None);
16427 assert!(swap
16428 .registry
16429 .get_module_by_connection(INCUMBENT)
16430 .unwrap()
16431 .is_some());
16432
16433 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
16435 .await
16436 .expect("the incumbent is told to drop the abandoned binding")
16437 .unwrap()
16438 .frame;
16439 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16440 assert_eq!(goodbye.header.channel, abandoned_channel);
16441 assert_eq!(goodbye.header.epoch, abandoned_epoch);
16442 assert!(swap.incumbent_rx.try_recv().is_err());
16443
16444 let response = caller_task.await.unwrap();
16446 assert_eq!(response.len(), 1);
16447 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
16448 assert!(caller_rx.try_recv().is_err());
16449
16450 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
16454 let counters = handler.counters().snapshot();
16455 assert_eq!(
16456 keyed_total(&counters, "route_open_accepted_by_principal"),
16457 1
16458 );
16459 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
16460 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
16461 }
16462
16463 #[tokio::test]
16467 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
16468 let mut swap = swap_with_incumbent().await;
16469 register_candidate(&swap, None);
16470 let incumbent = cutover(&swap);
16471 swap.forwarding
16472 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
16473 .unwrap()
16474 .expect("the incumbent is still registered");
16475
16476 let client = ConnectionId::new(33);
16477 let (client_ctx, mut client_rx) = route_ctx(client);
16478 let route_handler = swap.handler.clone();
16479 let open_ctx = RouteCtx {
16480 connection_id: client,
16481 egress: client_ctx.egress.clone(),
16482 };
16483 let mut route_task = tokio::spawn(async move {
16484 route_handler
16485 .handle_control_frame(
16486 &open_ctx,
16487 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
16488 )
16489 .await
16490 .unwrap()
16491 });
16492 let bind = tokio::select! {
16493 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
16494 response = &mut route_task => {
16495 let response = response.unwrap();
16496 panic!(
16497 "post-cutover route.open was refused instead of relayed to the candidate: {}",
16498 parse_error(&response[0])["code"]
16499 );
16500 }
16501 };
16502 swap.handler
16503 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
16504 .await
16505 .unwrap();
16506 assert!(route_task.await.unwrap().is_empty());
16507 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
16508 match swap
16509 .forwarding
16510 .lookup_data_route(client, channel, epoch)
16511 .unwrap()
16512 {
16513 DataRoute::Client(DataRouteState::Bound(route)) => {
16514 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
16515 }
16516 other => panic!("expected a bound route on the candidate, got {other:?}"),
16517 }
16518 assert!(swap.incumbent_rx.try_recv().is_err());
16519 }
16520
16521 #[tokio::test]
16526 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
16527 let swap = swap_with_incumbent().await;
16528 register_candidate(&swap, Some(false));
16529 let update = Frame::build(
16530 FrameType::Request,
16531 control_flags(),
16532 0,
16533 0,
16534 55,
16535 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16536 provides: manifest("aft", PROTOCOL_VERSION).provides,
16537 capabilities: None,
16538 ready: Some(true),
16539 })
16540 .unwrap(),
16541 )
16542 .unwrap();
16543
16544 let replies = swap
16545 .handler
16546 .handle_control_frame(&swap.candidate_ctx, update)
16547 .await
16548 .unwrap();
16549
16550 assert_eq!(replies.len(), 1);
16551 assert_eq!(
16552 replies[0].header.ty,
16553 FrameType::Response,
16554 "candidate catalog.update was refused: {:?}",
16555 serde_json::from_slice::<Value>(&replies[0].body).ok()
16556 );
16557 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
16558 assert_eq!(
16559 swap.registry
16560 .get_module("aft")
16561 .unwrap()
16562 .unwrap()
16563 .connection_id,
16564 INCUMBENT
16565 );
16566 }
16567 }
16568
16569 mod swap_admission {
16573 use super::*;
16574
16575 const INCUMBENT_NONCE: &str = "incumbent-nonce";
16576 const CANDIDATE_NONCE: &str = "candidate-nonce";
16577
16578 fn handler_with_incumbent(
16579 module_id: &str,
16580 reserved: bool,
16581 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
16582 let registry = Arc::new(Registry::default());
16583 let supervisor = SupervisorHandle::new();
16584 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
16585 if reserved {
16586 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
16587 }
16588 let handler =
16589 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
16590 let incumbent = handler
16591 .handle_control(
16592 ConnectionId::new(1),
16593 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
16594 )
16595 .unwrap();
16596 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
16597 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
16598 (registry, supervisor, handler)
16599 }
16600
16601 #[test]
16607 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
16608 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
16609
16610 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
16611 let replies = handler
16612 .handle_control(
16613 ConnectionId::new(connection),
16614 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
16615 )
16616 .unwrap();
16617 assert_eq!(replies[0].header.ty, FrameType::Error);
16618 assert_eq!(
16619 parse_error(&replies[0])["code"],
16620 "swap_token_invalid",
16621 "nonce {nonce:?}"
16622 );
16623 }
16624 assert!(registry.get_candidate("aft").unwrap().is_none());
16625 assert_eq!(
16626 registry.get_module("aft").unwrap().unwrap().connection_id,
16627 ConnectionId::new(1)
16628 );
16629
16630 let admitted = handler
16633 .handle_control(
16634 ConnectionId::new(4),
16635 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
16636 )
16637 .unwrap();
16638 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
16639 assert_eq!(
16640 registry
16641 .get_candidate("aft")
16642 .unwrap()
16643 .unwrap()
16644 .connection_id,
16645 ConnectionId::new(4)
16646 );
16647 assert_eq!(
16648 registry.get_module("aft").unwrap().unwrap().connection_id,
16649 ConnectionId::new(1),
16650 "the candidate must not take the active slot"
16651 );
16652 let replayed = handler
16653 .handle_control(
16654 ConnectionId::new(5),
16655 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
16656 )
16657 .unwrap();
16658 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
16659
16660 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
16664 let squatter = handler
16665 .handle_control(
16666 ConnectionId::new(6),
16667 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
16668 )
16669 .unwrap();
16670 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
16671 assert!(
16672 registry.get_module("aft").unwrap().is_none(),
16673 "a squatter took the active slot of an id being swapped"
16674 );
16675 }
16676
16677 #[test]
16682 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
16683 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
16684
16685 let replies = handler
16686 .handle_control(
16687 ConnectionId::new(2),
16688 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16689 )
16690 .unwrap();
16691
16692 assert_eq!(
16693 replies[0].header.ty,
16694 FrameType::HelloAck,
16695 "reserved candidate refused: {:?}",
16696 serde_json::from_slice::<Value>(&replies[0].body).ok()
16697 );
16698 assert_eq!(
16699 registry
16700 .get_candidate("vault")
16701 .unwrap()
16702 .unwrap()
16703 .connection_id,
16704 ConnectionId::new(2)
16705 );
16706 }
16707
16708 #[test]
16711 fn without_an_open_swap_the_ordinary_gates_decide() {
16712 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16713 supervisor.close_swap("vault");
16714
16715 let candidate = handler
16716 .handle_control(
16717 ConnectionId::new(2),
16718 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16719 )
16720 .unwrap();
16721 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16722 let duplicate = handler
16723 .handle_control(
16724 ConnectionId::new(3),
16725 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16726 )
16727 .unwrap();
16728 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16729 assert!(registry.get_candidate("vault").unwrap().is_none());
16730 }
16731 }
16732
16733 mod scopes {
16737 mod idle_tests {
16738 use super::*;
16739 include!("scope_idle_tests.rs");
16740 }
16741 mod route_open_timings {
16742 use super::*;
16743 include!("route_open_timing_tests.rs");
16744 }
16745 mod route_open_refusal_log {
16746 use super::*;
16747 include!("route_open_refusal_log_tests.rs");
16748 }
16749 mod activity {
16750 include!("activity_scope_tests.rs");
16751 }
16752 mod delta {
16753 include!("scope_handler_delta_tests.rs");
16754 }
16755 mod call_modes {
16756 include!("scope_call_mode_tests.rs");
16757 }
16758 #[tokio::test]
16759 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16760 let rig = rig().await;
16761 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16762 "s".into(),
16763 1,
16764 "tool".into(),
16765 ))
16766 .unwrap();
16767 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16768 let (direct, _rx) = wide_ctx(999);
16769 for (ctx, expected) in [
16770 (&rig.owner, FrameType::Response),
16771 (&direct, FrameType::Error),
16772 ] {
16773 let frame =
16774 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16775 .unwrap();
16776 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16777 assert_eq!(replies[0].header.ty, expected);
16778 if expected == FrameType::Response {
16779 let reply: subc_control::ScopeServedByReply =
16780 serde_json::from_slice(&replies[0].body).unwrap();
16781 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16782 } else {
16783 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16784 }
16785 }
16786 let reply = call(
16787 &rig.handler,
16788 &rig.owner,
16789 &ModuleControlRequestFromModule::ScopeDescribe {
16790 owner: Principal::Reserved {
16791 module_id: OWNER.into(),
16792 },
16793 scope_ref: "s".into(),
16794 },
16795 )
16796 .await;
16797 assert!(matches!(
16798 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16799 ModuleControlResponseToModule::ScopeDescribe { .. }
16800 ));
16801 }
16802
16803 use subc_protocol::scope::{
16804 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16805 ScopeStatus,
16806 };
16807
16808 use super::*;
16809
16810 const OWNER: &str = "prefrontal-core";
16811
16812 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16813 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16814 }
16815
16816 async fn call(
16817 handler: &ControlHandler,
16818 ctx: &RouteCtx,
16819 request: &ModuleControlRequestFromModule,
16820 ) -> Frame {
16821 let body = serde_json::to_vec(request).unwrap();
16822 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16823 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16824 assert_eq!(replies.len(), 1, "{replies:?}");
16825 replies.pop().unwrap()
16826 }
16827
16828 async fn sync(
16829 handler: &ControlHandler,
16830 ctx: &RouteCtx,
16831 generation: u64,
16832 scopes: Vec<ScopeRecord>,
16833 ) -> Result<ModuleControlResponseToModule, String> {
16834 let reply = call(
16835 handler,
16836 ctx,
16837 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16838 )
16839 .await;
16840 match reply.header.ty {
16841 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16842 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16843 }
16844 }
16845
16846 async fn describe(
16847 handler: &ControlHandler,
16848 ctx: &RouteCtx,
16849 owner: &str,
16850 scope_ref: &str,
16851 ) -> ModuleControlResponseToModule {
16852 let reply = call(
16853 handler,
16854 ctx,
16855 &ModuleControlRequestFromModule::ScopeDescribe {
16856 owner: Principal::Reserved {
16857 module_id: owner.to_string(),
16858 },
16859 scope_ref: scope_ref.to_string(),
16860 },
16861 )
16862 .await;
16863 assert_eq!(
16864 reply.header.ty,
16865 FrameType::Response,
16866 "{:?}",
16867 parse_error(&reply)
16868 );
16869 serde_json::from_slice(&reply.body).unwrap()
16870 }
16871
16872 #[tokio::test]
16873 async fn scope_describe_never_echoes_bind_proof_or_hash() {
16874 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16875 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16876 let mut rig = rig().await;
16877 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16878 .await;
16879 let _bound = rig
16880 .bound(
16881 Some(AFT),
16882 PLEXUS,
16883 Some(
16884 rig.selector("s", Some(1))
16885 .with_bind_proof(Some(PROOF.into())),
16886 ),
16887 )
16888 .await;
16889 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16890 let output = serde_json::to_string(&reply).unwrap();
16891 assert!(output.contains("\"status\":\"live\""));
16892 assert!(!output.contains(PROOF));
16893 assert!(!output.contains(HASH));
16894 }
16895
16896 async fn module(
16898 handler: &ControlHandler,
16899 connection: u64,
16900 module_id: &str,
16901 nonce: Option<&str>,
16902 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16903 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16904 hello_via_sink(
16905 handler,
16906 &ctx,
16907 &mut rx,
16908 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16909 )
16910 .await;
16911 (ctx, rx)
16912 }
16913
16914 #[tokio::test]
16917 async fn a_client_connection_cannot_sync_or_describe() {
16918 let handler = ControlHandler::new(Arc::new(Registry::default()));
16919 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16920 for request in [
16921 ModuleControlRequestFromModule::ScopeSync {
16922 generation: 1,
16923 scopes: vec![head("s", 1)],
16924 },
16925 ModuleControlRequestFromModule::ScopeDescribe {
16926 owner: Principal::Direct,
16927 scope_ref: "s".to_string(),
16928 },
16929 ] {
16930 let reply = call(&handler, &ctx, &request).await;
16931 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16932 }
16933 assert!(
16934 !handler
16935 .scopes
16936 .read()
16937 .unwrap()
16938 .describe(
16939 &Principal::Reserved {
16940 module_id: OWNER.to_string()
16941 },
16942 "s"
16943 )
16944 .owner_synced
16945 );
16946 }
16947
16948 #[tokio::test]
16951 async fn a_module_without_a_supervised_launch_cannot_sync() {
16952 let handler = ControlHandler::new(Arc::new(Registry::default()));
16953 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16954 assert_eq!(
16955 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16956 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16957 );
16958 }
16959
16960 #[tokio::test]
16961 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16962 let supervisor = SupervisorHandle::new();
16963 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16964 let handler = ControlHandler::new(Arc::new(Registry::default()))
16965 .with_supervisor(supervisor.clone());
16966 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16967 sync(&handler, &incumbent, 1, vec![head("s", 1)])
16968 .await
16969 .expect("the current launch syncs");
16970
16971 supervisor.open_swap(OWNER, "n2".to_string());
16974 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16975 assert_eq!(
16976 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16977 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16978 );
16979 sync(&handler, &incumbent, 2, vec![head("s", 1)])
16980 .await
16981 .expect("the serving owner syncs during the swap");
16982
16983 supervisor.close_swap(OWNER);
16986 assert_eq!(
16987 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16988 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16989 );
16990 sync(&handler, &incumbent, 3, vec![head("s", 1)])
16991 .await
16992 .expect("the serving owner syncs after the rollback");
16993 handler.cleanup_connection(candidate.connection_id).unwrap();
16994
16995 supervisor.open_swap(OWNER, "n3".to_string());
17000 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
17001 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
17002 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
17003 .await
17004 .expect("the promoted launch takes authority");
17005 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17006 panic!("unexpected reply {reply:?}");
17007 };
17008 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
17009 assert_eq!(
17010 sync(&handler, &incumbent, 4, Vec::new()).await,
17011 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17012 );
17013 }
17014
17015 #[tokio::test]
17018 async fn closing_the_authority_connection_frees_sync_authority() {
17019 let supervisor = SupervisorHandle::new();
17020 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17021 let handler = ControlHandler::new(Arc::new(Registry::default()))
17022 .with_supervisor(supervisor.clone());
17023 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17024 sync(&handler, &first, 10, vec![head("s", 1)])
17025 .await
17026 .unwrap();
17027 handler.cleanup_connection(first.connection_id).unwrap();
17028
17029 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
17030 sync(&handler, &second, 1, vec![head("s", 1)])
17031 .await
17032 .expect("the next connection takes the released authority");
17033 }
17034
17035 #[tokio::test]
17036 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
17037 let supervisor = SupervisorHandle::new();
17038 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17039 let handler =
17040 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
17041 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
17042 sync(&handler, &first, 10, vec![head("s", 1)])
17043 .await
17044 .unwrap();
17045 handler
17046 .handle_control_frame(
17047 &first,
17048 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
17049 )
17050 .await
17051 .unwrap();
17052 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
17053 sync(&handler, &second, 1, vec![head("s", 1)])
17054 .await
17055 .expect("GOODBYE releases authority even if the old socket remains open");
17056 }
17057
17058 #[tokio::test]
17059 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
17060 let registry = Arc::new(Registry::default());
17061 let supervisor_handle = SupervisorHandle::new();
17062 let supervisor =
17063 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17064 .with_handle(supervisor_handle.clone())
17065 .with_daemon_incarnation("incarnation-7".to_string());
17066 supervisor
17069 .supervise_configured(
17070 ModuleSpec {
17071 module_id: OWNER.to_string(),
17072 program: PathBuf::from("/nonexistent/prefrontal-core"),
17073 args: Vec::new(),
17074 env: Vec::new(),
17075 reserved: false,
17076 reserved_prefixes: Vec::new(),
17077 protocol: ModuleProtocol::Subc,
17078 overlap: Default::default(),
17079 },
17080 false,
17081 )
17082 .unwrap();
17083 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
17084 let handler =
17085 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17086 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
17087
17088 let ModuleControlResponseToModule::ScopeDescribe {
17090 status,
17091 daemon_incarnation,
17092 owner_synced,
17093 owner_configured,
17094 scope,
17095 ..
17096 } = describe(&handler, &reader, OWNER, "s").await
17097 else {
17098 panic!("not a describe reply");
17099 };
17100 assert_eq!(status, ScopeStatus::NotLive);
17101 assert_eq!(daemon_incarnation, "incarnation-7");
17102 assert!(!owner_synced);
17103 assert!(owner_configured);
17104 assert!(scope.is_none());
17105
17106 let ModuleControlResponseToModule::ScopeDescribe {
17109 status,
17110 owner_configured,
17111 ..
17112 } = describe(&handler, &reader, "ghost", "s").await
17113 else {
17114 panic!("not a describe reply");
17115 };
17116 assert_eq!(status, ScopeStatus::NotLive);
17117 assert!(!owner_configured);
17118
17119 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
17121 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
17122 let ModuleControlResponseToModule::ScopeDescribe {
17123 status,
17124 scope_epoch,
17125 owner_synced,
17126 scope,
17127 ..
17128 } = describe(&handler, &reader, OWNER, "s").await
17129 else {
17130 panic!("not a describe reply");
17131 };
17132 assert_eq!(status, ScopeStatus::Live);
17133 assert_eq!(scope_epoch, Some(4));
17134 assert!(owner_synced);
17135 let stamp = scope.expect("a live scope carries its stamp");
17136 assert!(
17137 stamp.owner_authorized,
17138 "prefrontal-core is the default authority"
17139 );
17140 assert_eq!(stamp.kind, ScopeKind::Head);
17141 }
17142
17143 #[tokio::test]
17144 async fn scope_authority_owners_decides_owner_authorized() {
17145 let supervisor = SupervisorHandle::new();
17146 supervisor.set_spawn_nonce("broca", "b1".to_string());
17147 let handler = ControlHandler::new(Arc::new(Registry::default()))
17148 .with_supervisor(supervisor)
17149 .with_scope_authority_owners(vec!["broca".to_string()]);
17150 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
17151 let mut gated = head("s", 1);
17152 gated.attributes.agent_id = Some("agent".to_string());
17153 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
17154 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
17155 describe(&handler, &broca, "broca", "s").await
17156 else {
17157 panic!("not a describe reply");
17158 };
17159 assert!(scope.unwrap().owner_authorized);
17160 }
17161
17162 #[tokio::test]
17166 async fn scope_ops_and_the_scopes_capability_are_advertised() {
17167 let handler = ControlHandler::new(Arc::new(Registry::default()));
17168 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
17169 let ack = hello_via_sink(
17170 &handler,
17171 &ctx,
17172 &mut rx,
17173 hello_frame("m", PROTOCOL_VERSION, 1),
17174 )
17175 .await;
17176 let ack = parse_ack(&ack);
17177 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
17178 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
17179 }
17180 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
17181
17182 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
17183 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
17184 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
17185 let reply = handler
17186 .handle_control_frame(&client, frame)
17187 .await
17188 .unwrap()
17189 .pop()
17190 .unwrap();
17191 let ClientControlResponse::ServerDescribe { capabilities, .. } =
17192 serde_json::from_slice(&reply.body).unwrap()
17193 else {
17194 panic!("not a server.describe reply");
17195 };
17196 assert!(
17197 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
17198 "{capabilities:?}"
17199 );
17200 }
17201
17202 const PLEXUS: &str = "plexus";
17205 const OTHER: &str = "other";
17206 const AFT: &str = "aft";
17207 const BROCA: &str = "broca";
17208 const MAGIC: &str = "magic-context";
17209
17210 fn nonce(module_id: &str) -> String {
17211 format!("nonce-{module_id}")
17212 }
17213
17214 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17215 let (tx, rx) = mpsc::channel(64);
17216 (
17217 RouteCtx {
17218 connection_id: ConnectionId::new(connection),
17219 egress: FrameSink::new(tx),
17220 },
17221 rx,
17222 )
17223 }
17224
17225 struct Rig {
17229 handler: ControlHandler,
17230 forwarding: Arc<ForwardingTable>,
17231 owner: RouteCtx,
17232 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17233 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
17234 generation: u64,
17235 next_connection: u64,
17236 _supervisor: Supervisor,
17237 }
17238
17239 async fn rig() -> Rig {
17240 rig_with_flow_support(true).await
17241 }
17242
17243 async fn rig_with_flow_support(flow_support: bool) -> Rig {
17244 rig_with_scope_capabilities(flow_support, false).await
17245 }
17246
17247 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
17248 let mut provides = Vec::new();
17249 if flow_support {
17250 provides.push("flow-scopes/v1");
17251 }
17252 if run_support {
17253 provides.push("agent-run-scopes/v1");
17254 }
17255 rig_with_target_capabilities(&provides).await
17256 }
17257
17258 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
17259 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
17260 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17261 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17264 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
17265 Frame::build(
17266 FrameType::Hello,
17267 control_flags(),
17268 0,
17269 0,
17270 connection,
17271 serde_json::to_vec(&body).unwrap(),
17272 )
17273 .unwrap()
17274 }
17275
17276 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
17277 let registry = Arc::new(Registry::default());
17278 let forwarding = Arc::new(ForwardingTable::default());
17279 let supervisor_handle = SupervisorHandle::new();
17280 let supervisor =
17281 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17282 .with_handle(supervisor_handle.clone());
17283 supervisor
17284 .supervise_configured(
17285 ModuleSpec {
17286 module_id: OWNER.to_string(),
17287 program: PathBuf::from("/nonexistent/prefrontal-core"),
17288 args: Vec::new(),
17289 env: Vec::new(),
17290 reserved: false,
17291 reserved_prefixes: Vec::new(),
17292 protocol: ModuleProtocol::Subc,
17293 overlap: Default::default(),
17294 },
17295 false,
17296 )
17297 .unwrap();
17298 for module_id in [OWNER, AFT, BROCA, MAGIC] {
17299 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
17300 }
17301 let handler =
17302 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17303 .with_supervisor(supervisor_handle);
17304 let (owner, mut owner_rx) = wide_ctx(1);
17305 hello_via_sink(
17306 &handler,
17307 &owner,
17308 &mut owner_rx,
17309 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
17310 )
17311 .await;
17312 let mut modules = BTreeMap::new();
17313 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
17314 let (ctx, mut rx) = wide_ctx(connection);
17315 let hello = hello_with_capabilities(module_id, connection, provides);
17316 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
17317 modules.insert(module_id.to_string(), (ctx, rx));
17318 }
17319 Rig {
17320 handler,
17321 forwarding,
17322 owner,
17323 _owner_rx: owner_rx,
17324 modules,
17325 generation: 0,
17326 next_connection: 100,
17327 _supervisor: supervisor,
17328 }
17329 }
17330
17331 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
17332 ScopeCarrier::new(Principal::Reserved {
17333 module_id: module_id.to_string(),
17334 })
17335 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
17336 }
17337
17338 fn session(scope_epoch: u64) -> ScopeRecord {
17341 let mut record = head("s", scope_epoch);
17342 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
17343 record.attributes.agent_id = Some("agent-1".to_string());
17344 record.attributes.delegates = true;
17345 record
17346 }
17347
17348 impl Rig {
17349 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
17350 self.generation += 1;
17351 sync(&self.handler, &self.owner, self.generation, scopes)
17352 .await
17353 .expect("the owner's sync is accepted");
17354 }
17355
17356 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17357 ScopeSelector::new(
17358 Principal::Reserved {
17359 module_id: OWNER.to_string(),
17360 },
17361 scope_ref.to_string(),
17362 scope_epoch,
17363 )
17364 }
17365
17366 fn open_frame(
17367 &mut self,
17368 opener: Option<&str>,
17369 target: &str,
17370 scope: Option<ScopeSelector>,
17371 ) -> (
17372 RouteCtx,
17373 mpsc::Receiver<crate::router::OutboundFrame>,
17374 Frame,
17375 ) {
17376 self.next_connection += 1;
17377 let (ctx, rx) = wide_ctx(self.next_connection);
17378 let root = unique_project_root("scoped-open");
17379 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
17380 target: RouteTarget::ToolProvider {
17381 module_id: target.to_string(),
17382 },
17383 identity: BindIdentity::new(
17384 root.path().to_path_buf(),
17385 "unit".to_string(),
17386 "session".to_string(),
17387 ),
17388 consumer_identity: opener.map(|module_id| ConsumerIdentity {
17389 module_id: module_id.to_string(),
17390 launch_nonce: nonce(module_id),
17391 }),
17392 consumer_capabilities: None,
17393 role_versions: None,
17394 admission_facts: None,
17395 scope,
17396 })
17397 .unwrap();
17398 let frame = Frame::build(
17399 FrameType::Request,
17400 control_flags(),
17401 0,
17402 0,
17403 self.next_connection,
17404 body,
17405 )
17406 .unwrap();
17407 (ctx, rx, frame)
17408 }
17409
17410 async fn refused(
17412 &mut self,
17413 opener: Option<&str>,
17414 target: &str,
17415 scope: Option<ScopeSelector>,
17416 ) -> String {
17417 self.refusal_body(opener, target, scope).await["code"]
17418 .as_str()
17419 .unwrap()
17420 .to_string()
17421 }
17422
17423 async fn refusal_body(
17424 &mut self,
17425 opener: Option<&str>,
17426 target: &str,
17427 scope: Option<ScopeSelector>,
17428 ) -> Value {
17429 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
17430 let replies = tokio::time::timeout(
17431 Duration::from_secs(2),
17432 self.handler.handle_control_frame(&ctx, frame),
17433 )
17434 .await
17435 .expect("the open must be refused before waiting for a bind ack")
17436 .unwrap();
17437 assert_eq!(replies.len(), 1, "{replies:?}");
17438 assert_eq!(replies[0].header.ty, FrameType::Error);
17439 let (_, module_rx) = self.modules.get_mut(target).unwrap();
17440 assert!(
17441 module_rx.try_recv().is_err(),
17442 "a refused open relays nothing"
17443 );
17444 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
17445 parse_error(&replies[0])
17446 }
17447
17448 async fn relayed(
17450 &mut self,
17451 opener: Option<&str>,
17452 target: &str,
17453 scope: Option<ScopeSelector>,
17454 ) -> Relayed {
17455 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
17456 let handler = self.handler.clone();
17457 let task_ctx = ctx.clone();
17458 let task = tokio::spawn(async move {
17459 handler
17460 .handle_control_frame(&task_ctx, frame)
17461 .await
17462 .unwrap()
17463 });
17464 let (_, module_rx) = self.modules.get_mut(target).unwrap();
17465 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
17466 .await
17467 .expect("the target receives the relayed route.bind")
17468 .unwrap()
17469 .frame;
17470 Relayed {
17471 target: target.to_string(),
17472 client: ctx,
17473 client_rx: rx,
17474 task,
17475 bind,
17476 }
17477 }
17478
17479 async fn ack(&self, relayed: &Relayed) {
17480 let (module, _) = &self.modules[&relayed.target];
17481 self.handler
17482 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
17483 .await
17484 .unwrap();
17485 }
17486
17487 async fn bound(
17489 &mut self,
17490 opener: Option<&str>,
17491 target: &str,
17492 scope: Option<ScopeSelector>,
17493 ) -> Bound {
17494 let relayed = self.relayed(opener, target, scope).await;
17495 self.ack(&relayed).await;
17496 let Relayed {
17497 target,
17498 client,
17499 mut client_rx,
17500 task,
17501 bind,
17502 } = relayed;
17503 assert!(
17504 task.await.unwrap().is_empty(),
17505 "the open is answered by commit"
17506 );
17507 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
17508 Bound {
17509 target,
17510 client,
17511 client_rx,
17512 channel,
17513 epoch,
17514 bind,
17515 }
17516 }
17517
17518 fn live(&self, route: &Bound) -> bool {
17519 matches!(
17520 self.forwarding
17521 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17522 .unwrap(),
17523 DataRoute::Client(DataRouteState::Bound(_))
17524 )
17525 }
17526 }
17527
17528 struct Relayed {
17529 target: String,
17530 client: RouteCtx,
17531 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17532 task: tokio::task::JoinHandle<Vec<Frame>>,
17533 bind: Frame,
17534 }
17535
17536 struct Bound {
17537 target: String,
17538 client: RouteCtx,
17539 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17540 channel: u16,
17541 epoch: u32,
17542 bind: Frame,
17543 }
17544
17545 impl Bound {
17546 fn closed_reason(&mut self) -> RouteCloseReason {
17549 let mut reason = None;
17550 let mut goodbye = false;
17551 while let Ok(outbound) = self.client_rx.try_recv() {
17552 let frame = outbound.frame;
17553 match frame.header.ty {
17554 FrameType::Goodbye => {
17555 assert_eq!(
17556 (frame.header.channel, frame.header.epoch),
17557 (self.channel, self.epoch)
17558 );
17559 goodbye = true;
17560 }
17561 FrameType::Push => {
17562 let ClientControlPush::RouteClosed {
17563 reason: r,
17564 module_id,
17565 ..
17566 } = serde_json::from_slice(&frame.body).unwrap()
17567 else {
17568 panic!("unexpected push");
17569 };
17570 assert_eq!(module_id, self.target);
17571 reason = Some(r);
17572 }
17573 other => panic!("unexpected frame {other:?}"),
17574 }
17575 }
17576 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
17577 reason.expect("the client is told why the route closed")
17578 }
17579
17580 fn untouched(&mut self) -> bool {
17581 self.client_rx.try_recv().is_err()
17582 }
17583
17584 fn stamp(&self) -> Option<ScopeStamp> {
17585 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
17586 ModuleControlRequest::RouteBind { scope, .. } => scope,
17587 other => panic!("expected a route.bind, got {other:?}"),
17588 }
17589 }
17590 }
17591
17592 #[tokio::test]
17593 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
17594 ) {
17595 let mut rig = rig().await;
17596 let mut record = session(1);
17597 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
17598 record.child_owners = vec![Principal::Reserved {
17599 module_id: MAGIC.to_string(),
17600 }];
17601 rig.sync(vec![record]).await;
17602 let scope = || Some(rig_selector("s", Some(1)));
17603
17604 rig.bound(Some(OWNER), PLEXUS, scope()).await;
17607 rig.bound(Some(AFT), OTHER, scope()).await;
17608 rig.bound(Some(BROCA), PLEXUS, scope()).await;
17609
17610 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
17614 assert_eq!(
17615 rig.refused(opener, target, scope()).await,
17616 error_codes::SCOPE_NOT_CARRIER,
17617 "{opener:?} -> {target}"
17618 );
17619 }
17620 }
17621
17622 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17623 ScopeSelector::new(
17624 Principal::Reserved {
17625 module_id: OWNER.to_string(),
17626 },
17627 scope_ref.to_string(),
17628 scope_epoch,
17629 )
17630 }
17631
17632 #[tokio::test]
17633 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
17634 let mut rig = rig().await;
17635 rig.sync(vec![session(1)]).await;
17636 for opener in [OWNER, AFT] {
17637 assert_eq!(
17638 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
17639 .await,
17640 error_codes::SCOPE_EPOCH_REQUIRED,
17641 "{opener}"
17642 );
17643 }
17644 }
17645
17646 #[tokio::test]
17647 async fn admission_separates_not_synced_not_live_and_ended() {
17648 let mut rig = rig().await;
17649 let code = rig
17651 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17652 .await;
17653 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
17654 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
17655
17656 let ghost = ScopeSelector::new(
17658 Principal::Reserved {
17659 module_id: "ghost".to_string(),
17660 },
17661 "s".to_string(),
17662 Some(1),
17663 );
17664 assert_eq!(
17665 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
17666 error_codes::SCOPE_NOT_LIVE
17667 );
17668
17669 rig.sync(vec![session(2)]).await;
17670 assert_eq!(
17671 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
17672 .await,
17673 error_codes::SCOPE_NOT_LIVE
17674 );
17675 for epoch in [1, 3] {
17676 assert_eq!(
17677 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
17678 .await,
17679 error_codes::SCOPE_ENDED,
17680 "epoch {epoch}"
17681 );
17682 }
17683 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
17685 .await;
17686 }
17687
17688 #[tokio::test]
17689 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
17690 let mut rig = rig().await;
17691 rig.sync(vec![session(1)]).await;
17692 let route = rig
17693 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17694 .await;
17695 let stamp = route.stamp().expect("a scoped bind carries the stamp");
17696 assert_eq!(stamp.scope_ref, "s");
17697 assert_eq!(stamp.scope_epoch, 1);
17698 assert_eq!(stamp.kind, ScopeKind::Head);
17699 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
17700 assert!(stamp.attributes.delegates);
17701 assert!(stamp.owner_authorized);
17702 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17703 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
17704
17705 let (broca, mut broca_rx) = wide_ctx(50);
17708 hello_via_sink(
17709 &rig.handler,
17710 &broca,
17711 &mut broca_rx,
17712 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17713 )
17714 .await;
17715 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17716 .await
17717 .unwrap();
17718 let own = ScopeSelector::new(
17719 Principal::Reserved {
17720 module_id: BROCA.to_string(),
17721 },
17722 "b".to_string(),
17723 Some(1),
17724 );
17725 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17726 assert!(!route.stamp().unwrap().owner_authorized);
17727 }
17728
17729 #[tokio::test]
17730 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17731 let mut rig = rig().await;
17732 let mut record = head("s", 1);
17733 record.carriers = vec![carrier(AFT, None)];
17734 let flow_id = "Flow:run-7/step_2!~";
17735 record.attributes.flow_id = Some(flow_id.to_string());
17736 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17737 .await
17738 .unwrap();
17739 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17740 panic!("not a sync reply");
17741 };
17742 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17743 let route = rig
17744 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17745 .await;
17746 assert!(rig.live(&route), "the stamped bind committed");
17747 let stamp = route.stamp().expect("a flow scope carries a stamp");
17748 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17749 assert_eq!(stamp.attributes.agent_id, None);
17750 assert!(!stamp.attributes.delegates);
17751 assert!(stamp.owner_authorized);
17752 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17753 assert_eq!(unscoped.stamp(), None);
17754 }
17755
17756 #[tokio::test]
17757 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17758 let mut rig = rig_with_flow_support(false).await;
17759 let mut record = session(1);
17760 record.attributes.flow_id = Some("flow:7".to_string());
17761 rig.sync(vec![record]).await;
17762 for opener in [OWNER, AFT] {
17763 let body = rig
17764 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17765 .await;
17766 assert_eq!(body["code"], "target_flow_unsupported");
17767 let message = body["message"].as_str().unwrap();
17768 for required in [PLEXUS, "flow-scopes/v1"] {
17769 assert!(message.contains(required), "{message}");
17770 }
17771 }
17772 }
17773
17774 #[tokio::test]
17775 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17776 let mut rig = rig_with_flow_support(true).await;
17777 let mut record = session(1);
17778 record.attributes.flow_id = Some("flow:7".to_string());
17779 rig.sync(vec![record]).await;
17780 for opener in [OWNER, AFT] {
17781 let route = rig
17782 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17783 .await;
17784 assert!(rig.live(&route));
17785 assert_eq!(
17786 route.stamp().unwrap().attributes.flow_id.as_deref(),
17787 Some("flow:7")
17788 );
17789 }
17790 }
17791
17792 #[tokio::test]
17793 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17794 use std::future::Future;
17795
17796 let mut rig = rig_with_flow_support(true).await;
17797 let mut record = session(1);
17798 record.attributes.flow_id = Some("flow:7".to_string());
17799 rig.sync(vec![record]).await;
17800 let (client, mut client_rx, frame) =
17801 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17802 for _ in 0..64 {
17805 client.egress.try_send(route_bind_ack(1)).unwrap();
17806 }
17807 let handler = rig.handler.clone();
17808 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17809 std::future::poll_fn(|cx| {
17810 assert!(open.as_mut().poll(cx).is_pending());
17811 std::task::Poll::Ready(())
17812 })
17813 .await;
17814 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17815
17816 let old_connection = rig.modules[PLEXUS].0.connection_id;
17817 rig.handler.cleanup_connection(old_connection).unwrap();
17818 let (replacement, mut replacement_rx) = wide_ctx(200);
17819 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17820 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17821 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17822 let hello = Frame::build(
17823 FrameType::Hello,
17824 control_flags(),
17825 0,
17826 0,
17827 200,
17828 serde_json::to_vec(&body).unwrap(),
17829 )
17830 .unwrap();
17831 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17832
17833 client_rx.try_recv().unwrap();
17834 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17835 .await
17836 .expect("the replacement is refused without waiting for a bind ack")
17837 .unwrap();
17838 assert_eq!(replies.len(), 1);
17839 let body = parse_error(&replies[0]);
17840 assert_eq!(body["code"], "target_flow_unsupported");
17841 for required in [PLEXUS, "flow-scopes/v1"] {
17842 assert!(body["message"].as_str().unwrap().contains(required));
17843 }
17844 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17845 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17846 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17847 assert!(rig
17848 .handler
17849 .registry
17850 .get_module_by_connection(replacement.connection_id)
17851 .unwrap()
17852 .is_some());
17853 }
17854
17855 #[tokio::test]
17856 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17857 use std::future::Future;
17858 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17859 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17860 const NEXT_HASH: &str =
17861 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17862 let mut rig = rig().await;
17863 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17864 record.carriers.push(ScopeCarrier::new(Principal::Direct));
17865 rig.sync(vec![record.clone()]).await;
17866 let (client, mut rx, frame) = rig.open_frame(
17867 None,
17868 PLEXUS,
17869 Some(
17870 rig.selector("s", Some(1))
17871 .with_bind_proof(Some(PROOF.into())),
17872 ),
17873 );
17874 for _ in 0..64 {
17877 client.egress.try_send(route_bind_ack(1)).unwrap();
17878 }
17879 let handler = rig.handler.clone();
17880 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17881 std::future::poll_fn(|cx| {
17882 assert!(open.as_mut().poll(cx).is_pending());
17883 std::task::Poll::Ready(())
17884 })
17885 .await;
17886 record.bind_proof_sha256 = Some(NEXT_HASH.into());
17887 rig.sync(vec![record]).await;
17888 rx.try_recv().unwrap();
17889 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17890 .await
17891 .unwrap()
17892 .unwrap();
17893 assert_eq!(
17894 parse_error(&replies[0])["code"],
17895 error_codes::SCOPE_BIND_PROOF_MISMATCH
17896 );
17897 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17898 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17899 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17900 }
17901
17902 #[tokio::test]
17903 async fn scope_bind_proof_reserved_openers_ignore_proof() {
17904 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17905 let mut rig = rig().await;
17906 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17907 .await;
17908 for opener in [OWNER, AFT] {
17909 for proof in [None, Some("malformed+proof=")] {
17910 let route = rig
17911 .bound(
17912 Some(opener),
17913 PLEXUS,
17914 Some(
17915 rig.selector("s", Some(1))
17916 .with_bind_proof(proof.map(str::to_string)),
17917 ),
17918 )
17919 .await;
17920 assert!(rig.live(&route));
17921 }
17922 }
17923 }
17924
17925 #[tokio::test]
17926 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17927 use std::future::Future;
17928 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17929 const NEXT_HASH: &str =
17930 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17931 let mut rig = rig().await;
17932 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17933 .await;
17934 let (client, mut rx, frame) =
17935 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17936 for _ in 0..64 {
17937 client.egress.try_send(route_bind_ack(1)).unwrap();
17938 }
17939 let handler = rig.handler.clone();
17940 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17941 std::future::poll_fn(|cx| {
17942 assert!(open.as_mut().poll(cx).is_pending());
17943 std::task::Poll::Ready(())
17944 })
17945 .await;
17946 rig.sync(vec![
17947 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17948 ])
17949 .await;
17950 rx.try_recv().unwrap();
17951 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17952 let bind = std::future::poll_fn(|cx| {
17953 assert!(open.as_mut().poll(cx).is_pending());
17954 module_rx.poll_recv(cx)
17955 })
17956 .await
17957 .unwrap()
17958 .frame;
17959 handler
17960 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17961 .await
17962 .unwrap();
17963 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17964 .await
17965 .unwrap()
17966 .unwrap();
17967 assert_eq!(replies.len(), 1);
17968 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17971 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17972 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17973 let route = rig
17974 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17975 .await;
17976 assert!(rig.live(&route));
17977 }
17978
17979 #[tokio::test]
17980 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17981 {
17982 let mut rig = rig_with_flow_support(false).await;
17983 rig.sync(vec![session(1)]).await;
17984 let route = rig
17985 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17986 .await;
17987 assert!(rig.live(&route));
17988 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17989 let mut record = session(1);
17990 record.attributes.flow_id = Some("flow:7".to_string());
17991 rig.sync(vec![record]).await;
17992 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17993 assert!(rig.live(&unscoped));
17994 assert_eq!(unscoped.stamp(), None);
17995 }
17996
17997 #[tokio::test]
17998 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17999 let mut rig = rig().await;
18000 let mut record = session(1);
18001 record.attributes.flow_id = Some("flow:7".to_string());
18002 rig.sync(vec![record.clone()]).await;
18003 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18004 let mut owner_route = rig
18005 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18006 .await;
18007 let mut carrier_route = rig
18008 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
18009 .await;
18010 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18011 rig.sync(vec![record.clone()]).await;
18012 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18013 assert!(rig.live(&owner_route) && owner_route.untouched());
18014 assert!(rig.live(&carrier_route) && carrier_route.untouched());
18015
18016 record.attributes.flow_id = Some("flow:8".to_string());
18017 rig.sync(vec![record]).await;
18018 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
18019 let before = before.unwrap();
18020 assert_eq!(after.scope_epoch, before.scope_epoch);
18021 assert!(after.version > before.version);
18022 for route in [&mut owner_route, &mut carrier_route] {
18023 assert!(!rig.live(route));
18024 assert_eq!(
18025 route.closed_reason(),
18026 RouteCloseReason::ScopeDelegationChanged
18027 );
18028 }
18029 assert!(rig.live(&unscoped) && unscoped.untouched());
18030 for target in [PLEXUS, OTHER] {
18033 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
18034 let goodbye = module_rx
18035 .try_recv()
18036 .expect("the provider sees the drain")
18037 .frame;
18038 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18039 assert!(module_rx.try_recv().is_err());
18040 }
18041 let rebound = rig
18042 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18043 .await;
18044 assert_eq!(
18045 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
18046 Some("flow:8")
18047 );
18048 }
18049
18050 #[tokio::test]
18055 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
18056 let mut rig = rig().await;
18057 rig.sync(vec![session(1)]).await;
18058 let mut cotenant = rig
18059 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18060 .await;
18061
18062 let mut changed = session(1);
18063 changed.child_owners.push(Principal::Reserved {
18064 module_id: MAGIC.to_string(),
18065 });
18066 let mut ended = None;
18067 for (code, next) in [
18068 (error_codes::SCOPE_CHANGED, vec![changed]),
18069 (error_codes::SCOPE_ENDED, Vec::new()),
18070 ] {
18071 let relayed = rig
18072 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18073 .await;
18074 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
18075 ended = Some(next.is_empty());
18076 rig.sync(next).await;
18077 rig.ack(&relayed).await;
18078 let replies = relayed.task.await.unwrap();
18079 assert_eq!(replies.len(), 1, "{replies:?}");
18080 assert_eq!(parse_error(&replies[0])["code"], code);
18081 assert_eq!(
18082 subc_protocol::error_codes::is_retryable_route_open(code),
18083 code == error_codes::SCOPE_CHANGED
18084 );
18085 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18086 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18088 let mut goodbyes = Vec::new();
18091 while let Ok(outbound) = plexus_rx.try_recv() {
18092 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
18093 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
18094 }
18095 assert!(
18096 goodbyes.contains(&(bind_channel, bind_epoch)),
18097 "{goodbyes:?}"
18098 );
18099 assert!(rig
18100 .handler
18101 .registry
18102 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
18103 .unwrap()
18104 .is_some());
18105 }
18106 assert_eq!(ended, Some(true));
18107 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
18110 }
18111
18112 #[tokio::test]
18115 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
18116 struct Case {
18117 name: &'static str,
18118 change: fn(&mut ScopeRecord),
18119 closed: [Option<RouteCloseReason>; 4],
18122 }
18123 use RouteCloseReason::*;
18124 let cases = [
18125 Case {
18126 name: "a carrier entry removed",
18127 change: |r| {
18128 r.carriers.retain(|c| {
18129 c.principal
18130 != Principal::Reserved {
18131 module_id: AFT.to_string(),
18132 }
18133 })
18134 },
18135 closed: [None, Some(ScopeCarrierRemoved), None, None],
18136 },
18137 Case {
18138 name: "a target removed from a carrier",
18139 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
18140 closed: [None, None, None, Some(ScopeCarrierRemoved)],
18141 },
18142 Case {
18143 name: "a bare carrier narrowed to targets",
18144 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
18145 closed: [None, Some(ScopeCarrierRemoved), None, None],
18146 },
18147 Case {
18148 name: "delegates turned off",
18149 change: |r| r.attributes.delegates = false,
18150 closed: [Some(ScopeDelegationChanged); 4],
18151 },
18152 Case {
18153 name: "agent_id changed",
18154 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
18155 closed: [Some(ScopeDelegationChanged); 4],
18156 },
18157 Case {
18158 name: "a carrier added, child owners changed, the record re-sent",
18159 change: |r| {
18160 r.carriers.push(carrier(MAGIC, None));
18161 r.child_owners.push(Principal::Reserved {
18162 module_id: MAGIC.to_string(),
18163 });
18164 },
18165 closed: [None; 4],
18166 },
18167 Case {
18168 name: "a target added",
18169 change: |r| {
18170 r.carriers[1]
18171 .targets
18172 .as_mut()
18173 .unwrap()
18174 .push("third".to_string())
18175 },
18176 closed: [None; 4],
18177 },
18178 Case {
18179 name: "delegates turned on",
18180 change: |r| r.attributes.delegates = true,
18181 closed: [None; 4],
18182 },
18183 ];
18184 for case in cases {
18185 let mut rig = rig().await;
18186 rig.sync(vec![session(1)]).await;
18187 let scope = || Some(rig_selector("s", Some(1)));
18188 let mut routes = [
18189 rig.bound(Some(OWNER), PLEXUS, scope()).await,
18190 rig.bound(Some(AFT), PLEXUS, scope()).await,
18191 rig.bound(Some(BROCA), PLEXUS, scope()).await,
18192 rig.bound(Some(BROCA), OTHER, scope()).await,
18193 ];
18194 let mut record = session(1);
18195 (case.change)(&mut record);
18196 rig.sync(vec![record]).await;
18197 for (index, expected) in case.closed.iter().enumerate() {
18198 let route = &mut routes[index];
18199 match expected {
18200 Some(reason) => {
18201 assert!(!rig.live(route), "{}: route {index} still live", case.name);
18202 assert_eq!(
18203 route.closed_reason(),
18204 *reason,
18205 "{}: route {index}",
18206 case.name
18207 );
18208 }
18209 None => {
18210 assert!(rig.live(route), "{}: route {index} closed", case.name);
18211 assert!(
18212 route.untouched(),
18213 "{}: route {index} was told something",
18214 case.name
18215 );
18216 }
18217 }
18218 }
18219 }
18220 }
18221
18222 #[tokio::test]
18223 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
18224 for next in [Vec::new(), vec![session(2)]] {
18226 let mut rig = rig().await;
18227 rig.sync(vec![session(1)]).await;
18228 let mut route = rig
18229 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18230 .await;
18231 rig.sync(next).await;
18232 assert!(!rig.live(&route));
18233 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
18234 }
18235
18236 let mut rig = rig().await;
18239 let mut child = session(1);
18240 child.scope_ref = "child".to_string();
18241 child.kind = ScopeKind::Worker;
18242 child.parent = Some(ScopeParent::new(
18243 Principal::Reserved {
18244 module_id: OWNER.to_string(),
18245 },
18246 "s".to_string(),
18247 1,
18248 ));
18249 rig.sync(vec![session(1), child.clone()]).await;
18250 let mut child_route = rig
18251 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
18252 .await;
18253 assert_eq!(
18254 child_route.stamp().unwrap().parent_state,
18255 Some(ParentState::Linked)
18256 );
18257 rig.sync(vec![child]).await;
18258 assert!(!rig.live(&child_route));
18259 assert_eq!(
18260 child_route.closed_reason(),
18261 RouteCloseReason::ScopeParentEnded
18262 );
18263 }
18264
18265 #[tokio::test]
18266 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
18267 ) {
18268 let mut rig = rig().await;
18269 rig.sync(vec![session(1)]).await;
18270 let mut route = rig
18271 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18272 .await;
18273 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18274 rig.sync(vec![session(1)]).await;
18275 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18276 assert!(rig.live(&route) && route.untouched());
18277
18278 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
18282 .forwarding
18283 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18284 .unwrap()
18285 else {
18286 panic!("the route is bound");
18287 };
18288 binding.flow.acquire_tagged(9, false).await.unwrap();
18289 let mut widened = session(1);
18290 widened.carriers.push(carrier(MAGIC, None));
18291 rig.sync(vec![widened]).await;
18292 assert!(rig.live(&route) && route.untouched());
18293 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18294 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
18295 assert_eq!(binding.flow.in_flight(), 1);
18298 binding
18299 .flow
18300 .acquire_tagged(10, false)
18301 .await
18302 .expect("the flow is still open");
18303 }
18304
18305 #[tokio::test]
18308 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
18309 let mut rig = rig().await;
18310 rig.sync(vec![session(1)]).await;
18311 let mut on_incumbent = rig
18312 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18313 .await;
18314
18315 let (candidate, _candidate_rx) = wide_ctx(9);
18318 let registration = rig
18319 .handler
18320 .registry
18321 .register_candidate_with_control_ops(
18322 manifest(PLEXUS, PROTOCOL_VERSION),
18323 PROTOCOL_VERSION,
18324 candidate.connection_id,
18325 module_baseline_control_ops(),
18326 )
18327 .unwrap();
18328 rig.forwarding
18329 .register_candidate_module_connection(
18330 candidate.connection_id,
18331 PLEXUS.to_string(),
18332 PROTOCOL_VERSION,
18333 manifest_concurrency(®istration.manifest),
18334 candidate.egress.clone(),
18335 )
18336 .unwrap();
18337 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
18338 rig.handler
18339 .registry
18340 .promote_candidate(PLEXUS)
18341 .unwrap()
18342 .unwrap();
18343 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
18344
18345 rig.sync(Vec::new()).await;
18346 assert!(!rig.live(&on_incumbent));
18347 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
18348 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18349 let goodbye = incumbent_rx
18350 .try_recv()
18351 .expect("the superseded endpoint is told")
18352 .frame;
18353 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18354 }
18355 }
18356}
18357
18358#[cfg(test)]
18359mod concurrency_default_exposure_tests {
18360 use super::*;
18361
18362 fn hello_body(role_json: &str) -> Vec<u8> {
18363 format!(
18364 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
18365 )
18366 .into_bytes()
18367 }
18368
18369 fn manifest_from(body: &[u8]) -> ModuleManifest {
18370 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
18371 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
18372 .expect("manifest parses")
18373 }
18374
18375 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
18376
18377 #[test]
18378 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
18379 let body = hello_body(&format!(
18380 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
18381 ));
18382 let manifest = manifest_from(&body);
18383 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18386 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
18387 }
18388
18389 #[test]
18390 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
18391 let body = hello_body(&format!(
18392 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
18393 ));
18394 let manifest = manifest_from(&body);
18395 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18396 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18397 }
18398
18399 #[test]
18400 fn non_management_roles_are_never_reported() {
18401 let body = hello_body(
18402 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
18403 );
18404 let manifest = manifest_from(&body);
18405 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18406 }
18407}