1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_confirm_operations,
25 validate_hello_event_declarations, validate_hello_flow_operations,
26 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
27 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
28 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
29 },
30 peer::{
31 encode_route_bind, ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY,
32 OPERATOR_APP_CAPABILITY,
33 },
34 scope::{
35 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
36 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
37 SCOPE_SYNC_OP,
38 },
39 session::{
40 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
41 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
42 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
43 ROLE_VERSIONS_FIELD,
44 },
45 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
46 Priority, RouteTarget, PROTOCOL_VERSION,
47};
48use tokio::time::{timeout_at, Instant};
49use tracing::{debug, info, warn};
50
51use crate::{
52 caller_identity::CallerIdentity,
53 capability_requirements::{
54 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
55 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
56 RequirementStatus, RuntimeModule,
57 },
58 child_roster::DaemonShutdownFlag,
59 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
60 forwarding::{
61 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
62 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
63 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
64 },
65 observability::{
66 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
67 },
68 privileged_prompt::{prompt_text, PrivilegedOperation},
69 provenance::{
70 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
71 },
72 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
73 router::{daemon_error_flags, RouteCtx, RouterError},
74 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
75 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
76 stderr_tail::{CaptureState, TailEntry},
77 supervise::{
78 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
79 SupervisorHandle, SwapHelloAdmission,
80 },
81 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
82};
83
84pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
90
91const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
92const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
93const CAP_PING_PONG: &str = "ping_pong_v1";
94const CAP_SESSION_ATTACH: &str = "session_attach_v1";
95const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
96
97const SUBC_CONTROL_OPS: &[&str] = &[
98 ops::ACTIVITY_RECORD,
99 ops::ACTIVITY_QUERY,
100 ops::ACTIVITY_SUBSCRIBE,
101 ops::SERVER_DESCRIBE,
102 ops::CATALOG_LIST,
103 ops::ROUTE_OPEN,
104 ops::ROUTE_POLL,
105 ops::ROUTE_CLOSING,
106 ops::ROUTE_CLOSED,
107 ops::SUPERVISOR_LIST,
108 ops::SUPERVISOR_RESTART,
109 ops::SUPERVISOR_SWAP,
110 ops::SUPERVISOR_RELOAD,
111 ops::SUPERVISOR_RESCAN,
112 ops::SUPERVISOR_RELEASE_RESERVED,
113 ops::SUPERVISOR_SET_ENABLED,
114 ops::SUPERVISOR_HEALTH_PROBE,
115 ops::SUPERVISOR_HEALTH,
116 ops::SUPERVISOR_STDERR_TAIL,
117 ops::SUPERVISOR_TERMINALS,
118 ops::SUPERVISOR_ROUTES,
119 ops::SUPERVISOR_PROVENANCE,
120 ops::SUPERVISOR_SPAWN_SNAPSHOT,
121 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
122];
123
124const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
131
132mod rescan_checked;
133use rescan_checked::{RescanMode, RescanPlan};
134
135const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
136 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
137 "supervisor.live_roots",
138 SCOPE_SYNC_OP,
139 SCOPE_APPLY_OP,
140 SCOPE_DESCRIBE_OP,
141 "operator.confirm",
142];
143
144const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
148
149const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
150
151pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
159
160pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
175
176pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
194
195const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
196const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
197
198fn reload_verdict(
199 configured: &Path,
200 spawned_from: Option<&Path>,
201 image: subc_control::RunningImageAgreement,
202) -> PendingReloadVerdict {
203 let path = match spawned_from {
204 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
205 ReloadPathAgreement::Match
206 }
207 Some(spawned_from) => ReloadPathAgreement::Mismatch {
208 configured: configured.to_path_buf(),
209 spawned_from: spawned_from.to_path_buf(),
210 },
211 None => ReloadPathAgreement::Unavailable {
212 reason: if matches!(
213 image,
214 subc_control::RunningImageAgreement::Unavailable {
215 reason: subc_control::RunningImageUnavailableReason::NotRunning
216 }
217 ) {
218 ReloadPathUnavailableReason::NotRunning
219 } else {
220 ReloadPathUnavailableReason::SpawnedPathUnavailable
221 },
222 },
223 };
224 PendingReloadVerdict {
225 path,
226 image,
227 spec: None,
228 }
229}
230
231fn launch_spec_agreement(
232 configured: &crate::supervise::ModuleSpec,
233 running: bool,
234 spawned: Option<&crate::supervise::LaunchSpecDigest>,
235) -> subc_control::LaunchSpecAgreement {
236 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
237 if !running {
238 return Agreement::Unavailable {
239 reason: Reason::NotRunning,
240 };
241 }
242 match spawned {
243 None => Agreement::Unavailable {
244 reason: Reason::SpawnedSpecUnavailable,
245 },
246 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
247 Agreement::Match
248 }
249 Some(_) => Agreement::Mismatch,
250 }
251}
252
253fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
254 if configured == spawned_from {
255 return true;
256 }
257 #[cfg(windows)]
258 {
259 match (
265 std::fs::canonicalize(configured),
266 std::fs::canonicalize(spawned_from),
267 ) {
268 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
269 _ => false,
270 }
271 }
272 #[cfg(not(windows))]
273 {
274 false
275 }
276}
277
278#[derive(Clone)]
279struct DaemonProvenanceFacts {
280 build: DaemonBuildProvenance,
281 pid: Option<u32>,
282 started_at_ms: Option<u64>,
283 start_clock: Option<crate::clock::StartClock>,
284 executable_path: Option<PathBuf>,
285 executable_identity: Option<SpawnedFileIdentity>,
286 process_start_time: Option<u64>,
287 probe: ExecutableIdentityProbe,
288}
289
290impl Default for DaemonProvenanceFacts {
291 fn default() -> Self {
292 Self {
293 build: DaemonBuildProvenance {
294 build_git_sha: None,
295 build_lock_digest: None,
296 },
297 pid: None,
298 started_at_ms: None,
299 start_clock: None,
300 executable_path: None,
301 executable_identity: None,
302 process_start_time: None,
303 probe: ExecutableIdentityProbe::default(),
304 }
305 }
306}
307
308#[derive(Debug, Clone)]
309struct SupervisorRescanContext {
310 supervisor: Supervisor,
311 config_path: PathBuf,
312 configured_port: Option<u16>,
313 storage_config: Option<crate::daemon_config::StorageConfig>,
314 admission_facts_carrier_module_id: Option<String>,
315 admission_facts_targets: Option<Vec<String>>,
316 scope_authority_owners: Vec<String>,
317 privileged_operator_prompt: PrivilegedOperatorPrompt,
318 generation: Arc<std::sync::atomic::AtomicU64>,
324}
325
326const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
347 "reloading",
348 "supervisor_not_live",
349 "registration_not_active",
350 "no_forwarding_connection",
351 "relay_send_failed",
352];
353
354#[derive(Clone)]
356pub struct ControlHandler {
357 registry: Arc<Registry>,
358 activity: Option<crate::activity::ActivityLedger>,
359 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
360 forwarding: Arc<ForwardingTable>,
361 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
362 supervisor: SupervisorHandle,
363 subc_capabilities: Arc<[String]>,
364 route_bind_relay_timeout: Duration,
368 route_bind_relay_timeouts: BTreeMap<String, Duration>,
372 route_bind_breakers: RouteBindBreakers,
375 route_bind_concurrency: RouteBindConcurrency,
378 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
383 route_bind_breaker_threshold: u32,
385 route_bind_breaker_cooldown: Duration,
387 health_probe_timeout: Duration,
388 storage_config: Option<crate::daemon_config::StorageConfig>,
391 machine_id: Option<crate::machine_id::MachineId>,
395 admission_facts_carrier_module_id: Option<String>,
396 admission_facts_targets: Option<Vec<String>>,
397 scopes: Arc<RwLock<ScopeTable>>,
401 scope_deadline_added: Arc<tokio::sync::Notify>,
405 scope_authority_owners: Vec<String>,
408 privileged_operator_prompt: PrivilegedOperatorPrompt,
410 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
413 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
414 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
415 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
416 pub(crate) operator_apps:
417 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
418 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
423 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
429 #[allow(dead_code)]
433 #[cfg(windows)]
434 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
435 #[allow(dead_code)]
436 #[cfg(windows)]
437 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
438 #[allow(dead_code)]
442 #[cfg(windows)]
443 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
444 #[cfg(all(test, windows))]
447 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
448 #[cfg(test)]
449 caller_lookup: Option<TestCallerLookup>,
450 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
453 rescan: Option<SupervisorRescanContext>,
454 connected_clients: ConnectedClients,
455 counters: DaemonCounters,
456 local_endpoint_unavailable: Option<String>,
459 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
460 daemon_shutdown: DaemonShutdownFlag,
465 daemon_provenance: DaemonProvenanceFacts,
466 #[cfg(test)]
467 control_dispatch_delay: Option<Duration>,
468 #[cfg(test)]
469 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
470 #[cfg(test)]
474 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
475 #[cfg(test)]
481 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
482}
483
484impl fmt::Debug for ControlHandler {
485 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
486 f.debug_struct("ControlHandler")
487 .field("registry", &self.registry)
488 .field("forwarding", &self.forwarding)
489 .field("process_liveness", &self.process_liveness.is_some())
490 .field("supervisor", &self.supervisor)
491 .field("subc_capabilities", &self.subc_capabilities)
492 .finish()
493 }
494}
495
496struct RouteOpenRequest {
497 target: RouteTarget,
498 identity: BindIdentity,
499 consumer_identity: Option<ConsumerIdentity>,
500 consumer_capabilities: Option<Vec<String>>,
501 role_versions: Option<BTreeMap<String, String>>,
502 admission_facts: Option<serde_json::Value>,
503 scope: Option<ScopeSelector>,
504}
505
506#[derive(Clone, Copy)]
507enum RouteOpenPhase {
508 Admission,
509 ProjectRoot,
510 EgressReservation,
511 BindSend,
512 BindAckWait,
513}
514
515struct RouteOpenTimings {
519 module_id: String,
520 principal: String,
521 admission: Duration,
522 project_root: Duration,
523 egress_reservation: Duration,
524 bind_send: Duration,
525 bind_ack_wait: Duration,
526 phase: Option<RouteOpenPhase>,
527 phase_started_at: Instant,
528}
529
530impl RouteOpenTimings {
531 fn new() -> Self {
532 Self {
533 module_id: String::new(),
534 principal: "unresolved".to_string(),
536 admission: Duration::ZERO,
537 project_root: Duration::ZERO,
538 egress_reservation: Duration::ZERO,
539 bind_send: Duration::ZERO,
540 bind_ack_wait: Duration::ZERO,
541 phase: Some(RouteOpenPhase::Admission),
542 phase_started_at: Instant::now(),
543 }
544 }
545
546 fn finish(&mut self) {
547 let elapsed = self.phase_started_at.elapsed();
548 match self.phase.take() {
549 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
550 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
551 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
552 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
553 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
554 None => {}
555 }
556 }
557
558 fn enter(&mut self, phase: RouteOpenPhase) {
559 self.finish();
560 self.phase = Some(phase);
561 self.phase_started_at = Instant::now();
562 }
563}
564
565struct RouteBindReservationGuard {
566 forwarding: Arc<ForwardingTable>,
567 endpoint: ModuleEndpointId,
568 relay_corr: u64,
569 armed: bool,
570}
571
572struct ModuleControlRpcGuard {
573 forwarding: Arc<ForwardingTable>,
574 endpoint: ModuleEndpointId,
575 corr: u64,
576 armed: bool,
577}
578
579impl ModuleControlRpcGuard {
580 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
581 Self {
582 forwarding,
583 endpoint,
584 corr,
585 armed: true,
586 }
587 }
588
589 fn disarm(&mut self) {
590 self.armed = false;
591 }
592}
593
594impl Drop for ModuleControlRpcGuard {
595 fn drop(&mut self) {
596 if self.armed {
597 let _ = self
598 .forwarding
599 .cancel_module_control_rpc(self.endpoint, self.corr);
600 }
601 }
602}
603
604impl RouteBindReservationGuard {
605 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
606 Self {
607 forwarding,
608 endpoint,
609 relay_corr,
610 armed: true,
611 }
612 }
613
614 fn release_and_disarm(&mut self) {
615 if !self.armed {
616 return;
617 }
618 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
619 self.endpoint,
620 self.relay_corr,
621 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
622 ) {
623 send_goodbye_target_best_effort(
624 &self.forwarding.counters(),
625 &target,
626 "canceled route.bind",
627 );
628 }
629 self.armed = false;
630 }
631
632 fn disarm(&mut self) {
633 self.armed = false;
634 }
635}
636
637impl Drop for RouteBindReservationGuard {
638 fn drop(&mut self) {
639 self.release_and_disarm();
640 }
641}
642
643#[derive(Debug, Clone, Default)]
673pub(crate) struct RouteBindBreakers {
674 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
675}
676
677#[derive(Debug, Clone, Default)]
678pub(crate) struct RouteBindConcurrency {
679 modules: Arc<Mutex<HashMap<String, usize>>>,
680}
681
682struct RouteBindConcurrencyGuard {
683 concurrency: RouteBindConcurrency,
684 module_id: String,
685}
686
687impl RouteBindConcurrency {
688 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
691 let mut modules = self
692 .modules
693 .lock()
694 .expect("route.bind concurrency mutex poisoned");
695 let in_flight = modules.entry(module_id.to_string()).or_default();
696 if *in_flight >= limit {
697 return Err(*in_flight);
698 }
699 *in_flight += 1;
700 Ok(RouteBindConcurrencyGuard {
701 concurrency: self.clone(),
702 module_id: module_id.to_string(),
703 })
704 }
705}
706
707impl Drop for RouteBindConcurrencyGuard {
708 fn drop(&mut self) {
709 let mut modules = self
710 .concurrency
711 .modules
712 .lock()
713 .expect("route.bind concurrency mutex poisoned");
714 let remove = {
715 let in_flight = modules
716 .get_mut(&self.module_id)
717 .expect("admitted route.bind has a concurrency entry");
718 *in_flight -= 1;
719 *in_flight == 0
720 };
721 if remove {
722 modules.remove(&self.module_id);
723 }
724 }
725}
726
727#[derive(Debug, Default)]
728struct ModuleBreakerState {
729 consecutive_timeouts: u32,
731 cooldown_until: Option<Instant>,
734 probe_in_flight: Option<Arc<()>>,
739}
740
741enum RouteBindAdmission<'a> {
743 Admitted {
744 guard: RouteBindBreakerGuard<'a>,
745 probe: bool,
748 },
749 Refused {
750 consecutive_timeouts: u32,
751 retry_in: Duration,
755 probe_in_flight: bool,
756 },
757}
758
759struct RouteBindBreakerGuard<'a> {
766 breakers: RouteBindBreakers,
767 module_id: &'a str,
768 probe_token: Option<Arc<()>>,
769 settled: bool,
770}
771
772impl RouteBindBreakerGuard<'_> {
773 fn record_accepted(&mut self) -> bool {
777 self.settled = true;
778 self.breakers.record_accepted(self.module_id)
779 }
780
781 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
784 self.settled = true;
785 self.breakers.record_timeout(
786 self.module_id,
787 self.probe_token.as_ref(),
788 threshold,
789 cooldown,
790 )
791 }
792
793 fn record_inconclusive(&mut self) {
802 self.settled = true;
803 self.breakers
804 .record_inconclusive(self.module_id, self.probe_token.as_ref());
805 }
806}
807
808impl Drop for RouteBindBreakerGuard<'_> {
809 fn drop(&mut self) {
810 if !self.settled {
811 self.breakers
812 .record_inconclusive(self.module_id, self.probe_token.as_ref());
813 }
814 }
815}
816
817struct BreakerOpened {
821 consecutive_timeouts: u32,
822 reopened_after_probe: bool,
825}
826
827impl RouteBindBreakers {
828 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
829 self.modules
830 .lock()
831 .expect("route.bind breaker mutex poisoned")
832 }
833
834 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
837 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
838 probe: probe_token.is_some(),
839 guard: RouteBindBreakerGuard {
840 breakers: self.clone(),
841 module_id,
842 probe_token,
843 settled: false,
844 },
845 };
846
847 let mut modules = self.lock();
848 let Some(state) = modules.get_mut(module_id) else {
849 return admitted(None);
850 };
851 let Some(cooldown_until) = state.cooldown_until else {
852 return admitted(None);
853 };
854 if state.probe_in_flight.is_some() {
855 return RouteBindAdmission::Refused {
856 consecutive_timeouts: state.consecutive_timeouts,
857 retry_in: Duration::ZERO,
858 probe_in_flight: true,
859 };
860 }
861 let now = Instant::now();
862 if now < cooldown_until {
863 return RouteBindAdmission::Refused {
864 consecutive_timeouts: state.consecutive_timeouts,
865 retry_in: cooldown_until - now,
866 probe_in_flight: false,
867 };
868 }
869 let token = Arc::new(());
870 state.probe_in_flight = Some(Arc::clone(&token));
871 admitted(Some(token))
872 }
873
874 fn record_accepted(&self, module_id: &str) -> bool {
875 self.lock()
876 .remove(module_id)
877 .is_some_and(|state| state.cooldown_until.is_some())
878 }
879
880 fn record_timeout(
881 &self,
882 module_id: &str,
883 probe_token: Option<&Arc<()>>,
884 threshold: u32,
885 cooldown: Duration,
886 ) -> Option<BreakerOpened> {
887 let mut modules = self.lock();
888 let state = modules.entry(module_id.to_string()).or_default();
889 let was_open = state.cooldown_until.is_some();
890 let was_probe = Self::owns_probe(state, probe_token);
891 if was_probe {
892 state.probe_in_flight = None;
893 }
894 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
895 if state.consecutive_timeouts < threshold {
896 return None;
897 }
898 state.cooldown_until = Some(Instant::now() + cooldown);
899 Some(BreakerOpened {
900 consecutive_timeouts: state.consecutive_timeouts,
901 reopened_after_probe: was_open && was_probe,
902 })
903 }
904
905 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
906 state
910 .probe_in_flight
911 .as_ref()
912 .zip(token)
913 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
914 }
915
916 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
917 if let Some(state) = self.lock().get_mut(module_id) {
918 if Self::owns_probe(state, probe_token) {
919 state.probe_in_flight = None;
920 }
921 }
922 }
923
924 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
939 self.lock()
940 .remove(module_id)
941 .map(|state| state.consecutive_timeouts)
942 .filter(|discarded| *discarded > 0)
943 }
944
945 fn open_snapshot(&self) -> Option<serde_json::Value> {
952 let now = Instant::now();
953 let modules = self.lock();
954 let open = modules
955 .iter()
956 .filter_map(|(module_id, state)| {
957 let cooldown_until = state.cooldown_until?;
958 Some((
959 module_id.clone(),
960 serde_json::json!({
961 "consecutive_timeouts": state.consecutive_timeouts,
962 "cooldown_remaining_ms":
963 cooldown_until.saturating_duration_since(now).as_millis() as u64,
964 "probe_in_flight": state.probe_in_flight.is_some(),
965 }),
966 ))
967 })
968 .collect::<serde_json::Map<String, serde_json::Value>>();
969 (!open.is_empty()).then_some(serde_json::Value::Object(open))
970 }
971}
972
973impl ControlHandler {
974 pub fn new(registry: Arc<Registry>) -> Self {
975 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
976 }
977
978 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
979 let counters = forwarding.counters();
980 let route_bind_breakers = forwarding.route_bind_breakers();
984 let route_bind_concurrency = forwarding.route_bind_concurrency();
985 let route_outages = forwarding.route_outages();
986 Self {
987 registry,
988 activity: None,
989 activity_streams: Arc::new(Mutex::new(HashMap::new())),
990 forwarding,
991 process_liveness: None,
992 supervisor: SupervisorHandle::new(),
993 subc_capabilities: Arc::from([
994 CAP_MANIFEST_REGISTRATION.to_string(),
995 CAP_CHANNEL_LIFECYCLE.to_string(),
996 CAP_PING_PONG.to_string(),
997 CAP_SESSION_ATTACH.to_string(),
998 CAP_ADMISSION_FACTS_RELAY.to_string(),
999 CAP_SCOPES_V1.to_string(),
1000 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1001 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1002 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1003 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1004 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1005 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1006 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1007 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1008 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1009 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1010 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1011 ]),
1012 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1013 route_bind_relay_timeouts: BTreeMap::new(),
1014 route_bind_breakers,
1015 route_bind_concurrency,
1016 route_outages,
1017 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1018 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1019 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1020 storage_config: None,
1021 machine_id: None,
1022 admission_facts_carrier_module_id: None,
1023 admission_facts_targets: None,
1024 scopes: Arc::new(RwLock::new(ScopeTable::new(
1025 crate::daemon_config::default_scope_authority_owners(),
1026 ))),
1027 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1028 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1029 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1030 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1031 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1032 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1033 operator_app_policy: Default::default(),
1034 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1035 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1036 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1037 #[cfg(windows)]
1038 windows_verify: None,
1039 #[cfg(windows)]
1040 windows_flash: None,
1041 #[cfg(windows)]
1042 windows_track: None,
1043 #[cfg(all(test, windows))]
1044 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1045 #[cfg(test)]
1046 caller_lookup: None,
1047 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1048 rescan: None,
1049 connected_clients: ConnectedClients::new(),
1050 local_endpoint_unavailable: None,
1051 counters,
1052 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1053 daemon_shutdown: DaemonShutdownFlag::default(),
1054 daemon_provenance: DaemonProvenanceFacts::default(),
1055 #[cfg(test)]
1056 control_dispatch_delay: None,
1057 #[cfg(test)]
1058 provenance_probe_override: None,
1059 #[cfg(test)]
1060 route_bind_relay_sent_hold: None,
1061 #[cfg(test)]
1062 hello_registry_to_forwarding_hold: None,
1063 }
1064 }
1065
1066 pub fn with_storage_config(
1069 mut self,
1070 storage_config: Option<crate::daemon_config::StorageConfig>,
1071 ) -> Self {
1072 self.storage_config = storage_config;
1073 self
1074 }
1075
1076 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1079 self.activity = Some(crate::activity::ActivityLedger::open(
1080 run_dir.join("activity.jsonl"),
1081 incarnation,
1082 ));
1083 self.supervisor.set_activity(self.activity.clone());
1084 self
1085 }
1086
1087 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1090 self.machine_id = machine_id;
1091 self
1092 }
1093
1094 pub fn with_admission_facts_config(
1098 mut self,
1099 carrier_module_id: Option<String>,
1100 targets: Option<Vec<String>>,
1101 ) -> Self {
1102 self.admission_facts_carrier_module_id = carrier_module_id;
1103 self.admission_facts_targets = targets;
1104 self
1105 }
1106
1107 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1111 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1112 self.scope_authority_owners = owners;
1113 self
1114 }
1115
1116 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1118 self.privileged_operator_prompt = mode;
1119 self
1120 }
1121
1122 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1125 self.route_bind_relay_timeout = timeout;
1126 self
1127 }
1128
1129 pub fn with_route_bind_relay_timeouts(
1135 mut self,
1136 timeouts: impl IntoIterator<Item = (String, Duration)>,
1137 ) -> Self {
1138 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1139 self
1140 }
1141
1142 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1148 self.route_bind_relay_timeouts
1149 .get(module_id)
1150 .copied()
1151 .unwrap_or(self.route_bind_relay_timeout)
1152 }
1153
1154 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1162 self.route_bind_breaker_threshold = threshold.max(1);
1163 self.route_bind_breaker_cooldown = cooldown;
1164 self
1165 }
1166
1167 #[cfg(test)]
1168 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1169 self.health_probe_timeout = timeout;
1170 self
1171 }
1172
1173 #[cfg(test)]
1174 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1175 self.control_dispatch_delay = Some(delay);
1176 self
1177 }
1178
1179 #[cfg(test)]
1183 pub(crate) fn with_route_bind_relay_sent_hold(
1184 mut self,
1185 hold: Arc<tokio::sync::Barrier>,
1186 ) -> Self {
1187 self.route_bind_relay_sent_hold = Some(hold);
1188 self
1189 }
1190
1191 #[cfg(test)]
1196 pub(crate) fn with_hello_registry_to_forwarding_hold(
1197 mut self,
1198 hold: Arc<std::sync::Barrier>,
1199 ) -> Self {
1200 self.hello_registry_to_forwarding_hold = Some(hold);
1201 self
1202 }
1203
1204 #[cfg(test)]
1205 fn hold_between_registry_and_forwarding(&self) {
1206 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1207 hold.wait();
1208 hold.wait();
1209 }
1210 }
1211
1212 pub fn with_process_liveness(
1213 mut self,
1214 process_liveness: Arc<dyn ModuleProcessLiveness>,
1215 ) -> Self {
1216 self.process_liveness = Some(process_liveness);
1217 self
1218 }
1219
1220 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1221 self.supervisor = supervisor;
1222 self.supervisor.set_activity(self.activity.clone());
1223 self
1224 }
1225
1226 pub fn with_daemon_provenance(
1227 mut self,
1228 pid: u32,
1229 started_at_ms: u64,
1230 executable_path: Option<PathBuf>,
1231 build_git_sha: Option<String>,
1232 build_lock_digest: Option<String>,
1233 ) -> Self {
1234 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1235 let process_start_time = process_start_time(pid);
1236 self.daemon_provenance = DaemonProvenanceFacts {
1237 build: DaemonBuildProvenance {
1238 build_git_sha,
1239 build_lock_digest,
1240 },
1241 pid: Some(pid),
1242 started_at_ms: Some(started_at_ms),
1243 start_clock: None,
1244 executable_path,
1245 executable_identity,
1246 process_start_time,
1247 probe: ExecutableIdentityProbe::default(),
1248 };
1249 self
1250 }
1251
1252 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1253 self.daemon_provenance.start_clock = Some(clock);
1254 self
1255 }
1256
1257 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1259 self.scopes
1260 .write()
1261 .unwrap_or_else(|poisoned| poisoned.into_inner())
1262 .set_wall_clock(Arc::new(clock));
1263 self
1264 }
1265
1266 #[cfg(test)]
1267 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1268 self.provenance_probe_override = Some(result);
1269 self
1270 }
1271
1272 pub fn with_capability_config(
1276 self,
1277 modules: impl IntoIterator<Item = (String, bool)>,
1278 reserved_capabilities: BTreeMap<String, String>,
1279 ) -> Self {
1280 self.capability_evaluator
1281 .configure(modules, reserved_capabilities);
1282 self
1283 }
1284
1285 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1289 self.daemon_shutdown = flag;
1290 self
1291 }
1292
1293 pub fn with_supervisor_rescan(
1294 mut self,
1295 supervisor: Supervisor,
1296 config_path: impl Into<PathBuf>,
1297 configured_port: Option<u16>,
1298 ) -> Self {
1299 self.rescan = Some(SupervisorRescanContext {
1300 supervisor,
1301 config_path: config_path.into(),
1302 configured_port,
1303 storage_config: self.storage_config.clone(),
1304 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1305 admission_facts_targets: self.admission_facts_targets.clone(),
1306 scope_authority_owners: self.scope_authority_owners.clone(),
1307 privileged_operator_prompt: self.privileged_operator_prompt,
1308 generation: Arc::default(),
1309 });
1310 self
1311 }
1312
1313 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1314 self.connected_clients = connected_clients;
1315 self
1316 }
1317
1318 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1321 self.local_endpoint_unavailable = reason;
1322 self
1323 }
1324
1325 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1326 Arc::clone(&self.forwarding)
1327 }
1328
1329 pub(crate) fn counters(&self) -> DaemonCounters {
1330 self.counters.clone()
1331 }
1332
1333 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1336 tokio::spawn(async move {
1337 loop {
1338 self.capability_evaluator
1339 .wait_for_change_or_deadline()
1340 .await;
1341 self.refresh_capability_requirements();
1342 }
1343 });
1344 }
1345
1346 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1352 tokio::spawn(async move {
1353 loop {
1354 let has_deadlines = self
1355 .scopes
1356 .read()
1357 .unwrap_or_else(|poisoned| poisoned.into_inner())
1358 .has_deadlines();
1359 if !has_deadlines {
1360 self.scope_deadline_added.notified().await;
1361 continue;
1362 }
1363 tokio::time::sleep(Duration::from_secs(1)).await;
1364 if let Err(error) = self.sweep_expired_scopes() {
1365 warn!(%error, "scope expiry sweep failed");
1366 }
1367 }
1368 });
1369 }
1370
1371 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1372 let mut table = self
1373 .scopes
1374 .write()
1375 .unwrap_or_else(|poisoned| poisoned.into_inner());
1376 let swept = table.sweep_expired();
1377 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1378 drop(table);
1379 self.close_scope_drained_routes(drained);
1380 Ok(())
1381 }
1382
1383 fn publish_scope_changes(
1384 &self,
1385 changes: &[crate::scopes::ScopeTagChange],
1386 expired: &[crate::scopes::ScopeExpired],
1387 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1388 if changes.is_empty() {
1389 return Ok(Vec::new());
1390 }
1391 let drained = self
1392 .forwarding
1393 .publish_scope_changes(changes)
1394 .map_err(RouterError::Forwarding)?;
1395 let mut counts = HashMap::new();
1396 for route in &drained {
1397 *counts
1398 .entry((
1399 route.scope.owner.as_str(),
1400 route.scope.scope_ref.as_str(),
1401 route.scope.tag.scope_epoch,
1402 ))
1403 .or_insert(0usize) += 1;
1404 }
1405 for scope in expired {
1406 let routes_closed = counts
1407 .get(&(
1408 scope.owner.as_str(),
1409 scope.scope_ref.as_str(),
1410 scope.scope_epoch,
1411 ))
1412 .copied()
1413 .unwrap_or(0);
1414 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1415 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1416 routes_closed, "scope expired");
1417 }
1418 Ok(drained)
1419 }
1420
1421 fn runtime_capability_snapshot(
1422 &self,
1423 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1424 let runtime = self
1425 .supervisor
1426 .list()
1427 .into_iter()
1428 .map(|module| {
1429 let status = module.status().map_err(|err| {
1430 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1431 })?;
1432 Ok(RuntimeModule {
1433 module_id: status.module_id,
1434 state: status.state,
1435 enabled: status.enabled,
1436 })
1437 })
1438 .collect::<Result<Vec<_>, RouterError>>()?;
1439 let registrations = self.registry.list_registered_modules().map_err(|err| {
1445 RouterError::backend(
1446 0,
1447 0,
1448 format!("failed to list capability registrations: {err}"),
1449 )
1450 })?;
1451 let registrations = registrations
1452 .into_iter()
1453 .map(|registration| RegisteredModule {
1454 module_id: registration.manifest.module_id,
1455 module_version: registration.manifest.module_version,
1456 capabilities: registration.manifest.capabilities,
1457 })
1458 .collect();
1459 Ok((runtime, registrations))
1460 }
1461
1462 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1469 let cached_registration = RegisteredModule {
1470 module_id: registration.manifest.module_id.clone(),
1471 module_version: registration.manifest.module_version.clone(),
1472 capabilities: registration.manifest.capabilities.clone(),
1473 };
1474 let change = self.capability_evaluator.record_hello(&cached_registration);
1475 log_manifest_change(&cached_registration.module_id, &change);
1476 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1477 self.enforce_capability_denies();
1478 }
1479 self.refresh_capability_requirements();
1480 }
1481
1482 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1486 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1487 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1488 self.supervisor.set_swap_promotion_observer(observer);
1489 }
1490
1491 pub fn refresh_capability_requirements(&self) {
1492 match self.runtime_capability_snapshot() {
1493 Ok((runtime, registrations)) => {
1494 log_requirement_events(
1495 self.capability_evaluator
1496 .evaluate_now(&runtime, ®istrations),
1497 &self.daemon_shutdown,
1498 );
1499 }
1500 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1501 }
1502 }
1503
1504 fn enforce_capability_denies(&self) {
1508 let registrations = match self.registry.list_registered_modules() {
1509 Ok(snapshot) => snapshot,
1510 Err(err) => {
1511 warn!(error = %err, "failed to read registrations for capability deny census");
1512 return;
1513 }
1514 };
1515 let manifests = registrations
1516 .into_iter()
1517 .map(|registration| {
1518 (
1519 registration.manifest.module_id.clone(),
1520 registration.manifest,
1521 )
1522 })
1523 .collect::<BTreeMap<_, _>>();
1524 let census = match self.forwarding.route_census(None) {
1525 Ok(census) => census,
1526 Err(err) => {
1527 warn!(error = %err, "failed to read route census for capability deny enforcement");
1528 return;
1529 }
1530 };
1531
1532 for (target_module_id, routes) in census {
1533 let Some(target_manifest) = manifests.get(&target_module_id) else {
1534 continue;
1535 };
1536 let mut closed_routes = Vec::new();
1537 let mut goodbyes = Vec::new();
1538 for route in routes {
1539 let Principal::Reserved {
1540 module_id: opening_module_id,
1541 } = &route.principal
1542 else {
1543 continue;
1544 };
1545 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1546 continue;
1547 };
1548 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1549 continue;
1550 };
1551
1552 match self.forwarding.release_client_route(
1553 route.goodbye_target.connection_id,
1554 route.goodbye_target.channel,
1555 route.goodbye_target.epoch,
1556 ) {
1557 Ok(RouteRelease::Removed(module_goodbye)) => {
1558 warn!(
1559 opening_module_id,
1560 target_module_id,
1561 capability,
1562 "force-closing route because an attested capability deny edge now matches"
1563 );
1564 goodbyes.push(module_goodbye);
1565 goodbyes.push(route.goodbye_target.clone());
1566 closed_routes.push(route);
1567 }
1568 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1569 Err(err) => warn!(
1570 opening_module_id,
1571 target_module_id,
1572 capability,
1573 error = %err,
1574 "failed to force-close capability-denied route"
1575 ),
1576 }
1577 }
1578
1579 if closed_routes.is_empty() {
1580 continue;
1581 }
1582 send_route_control_pushes(
1583 &self.forwarding,
1584 closed_routes,
1585 ClientControlPush::RouteClosed {
1586 module_id: target_module_id,
1587 channels: Vec::new(),
1588 reason: RouteCloseReason::CapabilityDenied,
1589 drained: false,
1590 abandoned: 0,
1591 excluded_subscriptions: 0,
1592 terminal: Some(false),
1593 },
1594 );
1595 self.emit_route_goodbyes(goodbyes);
1598 }
1599 }
1600
1601 fn not_ready_reason(
1607 &self,
1608 registration: &crate::registry::ModuleRegistration,
1609 ) -> Option<NotReadyReason> {
1610 if !registration.ready {
1611 return Some(NotReadyReason {
1612 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1613 capability: None,
1614 });
1615 }
1616 self.first_unprovided_required_capability(registration)
1617 .map(|capability| NotReadyReason {
1618 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1619 capability: Some(capability),
1620 })
1621 }
1622
1623 fn first_unprovided_required_capability(
1641 &self,
1642 registration: &crate::registry::ModuleRegistration,
1643 ) -> Option<String> {
1644 let required = registration
1645 .manifest
1646 .capabilities
1647 .iter()
1648 .flat_map(|declarations| declarations.requires.iter())
1649 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1650 .map(|requirement| requirement.capability.as_str())
1651 .collect::<BTreeSet<_>>();
1652 if required.is_empty() {
1653 return None;
1654 }
1655 let module_id = registration.manifest.module_id.as_str();
1656 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1657 if required
1658 .iter()
1659 .any(|capability| verdict(capability).is_none())
1660 {
1661 self.refresh_capability_requirements();
1662 }
1663 required
1664 .into_iter()
1665 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1666 .map(str::to_string)
1667 }
1668
1669 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1670 self.capability_evaluator
1671 .statuses()
1672 .into_iter()
1673 .map(capability_requirement_status)
1674 .collect()
1675 }
1676
1677 fn deregister_connection(
1684 &self,
1685 connection_id: ConnectionId,
1686 reason: RegistrationEndReason,
1687 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1688 self.registry
1689 .deregister_connection_with_reason(connection_id, reason)
1690 }
1691
1692 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1693 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1694 return None;
1695 }
1696 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1697 parse_client_control_request(&frame.body)
1698 else {
1699 return None;
1700 };
1701 Some(target_module_id(&target).to_string())
1702 }
1703
1704 pub(crate) fn route_open_capacity_refusal(
1705 &self,
1706 ctx: &RouteCtx,
1707 frame: &Frame,
1708 target_module_id: &str,
1709 in_flight: usize,
1710 limit: usize,
1711 ) -> Result<Frame, RouterError> {
1712 let caller = self.route_open_caller_from_frame(frame);
1715 self.route_open_admission_refusal_frame(
1716 ctx,
1717 &caller,
1718 frame,
1719 target_module_id,
1720 "open_admission_full",
1721 (in_flight, limit),
1722 format!(
1723 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1724 ),
1725 )
1726 }
1727
1728 fn route_open_target_capacity_refusal(
1729 &self,
1730 ctx: &RouteCtx,
1731 caller: &RouteOpenCaller,
1732 frame: &Frame,
1733 target_module_id: &str,
1734 in_flight: usize,
1735 ) -> Result<Frame, RouterError> {
1736 self.route_open_admission_refusal_frame(
1737 ctx,
1738 caller,
1739 frame,
1740 target_module_id,
1741 "target_binds_full",
1742 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1743 format!(
1744 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1745 ),
1746 )
1747 }
1748
1749 #[allow(clippy::too_many_arguments)]
1757 fn route_open_admission_refusal_frame(
1758 &self,
1759 ctx: &RouteCtx,
1760 caller: &RouteOpenCaller,
1761 frame: &Frame,
1762 target_module_id: &str,
1763 reason: &'static str,
1764 (in_flight, limit): (usize, usize),
1765 message: impl Into<String>,
1766 ) -> Result<Frame, RouterError> {
1767 let code = error_codes::TARGET_UNAVAILABLE;
1768 self.counters.increment_route_open_refused(code);
1769 info!(
1770 target: "control",
1771 code,
1772 reason,
1773 module_id = ?target_module_id,
1774 connection_id = ctx.connection_id.get(),
1775 principal = %caller.principal,
1776 scope = %caller.scope,
1777 harness = %caller.harness,
1778 in_flight,
1779 limit,
1780 "route.open refused"
1781 );
1782 control_error_frame(frame, code, message.into())
1783 }
1784
1785 #[cfg(test)]
1791 pub fn handle_control(
1792 &self,
1793 connection_id: ConnectionId,
1794 frame: Frame,
1795 ) -> Result<Vec<Frame>, RouterError> {
1796 match frame.header.ty {
1797 FrameType::Ping => Ok(vec![pong(&frame)?]),
1798 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1799 FrameType::Goodbye => self.handle_goodbye(connection_id),
1800 ty => Ok(vec![control_error_frame(
1801 &frame,
1802 "unsupported_control_frame",
1803 format!("unsupported channel-0 frame {ty:?}"),
1804 )?]),
1805 }
1806 }
1807
1808 pub async fn handle_control_frame(
1809 &self,
1810 ctx: &RouteCtx,
1811 frame: Frame,
1812 ) -> Result<Vec<Frame>, RouterError> {
1813 self.handle_control_frame_timed(ctx, frame, None).await
1814 }
1815
1816 #[cfg(any(test, feature = "test-support"))]
1817 pub fn with_test_operator_app_policy(
1818 mut self,
1819 policy: crate::operator_apps::TestOperatorAppPolicy,
1820 ) -> Self {
1821 self.operator_app_policy = policy.into();
1822 self
1823 }
1824
1825 #[cfg(all(windows, any(test, feature = "test-support")))]
1829 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1830 self.windows_verify = Some(verify);
1831 self
1832 }
1833
1834 #[cfg(all(windows, any(test, feature = "test-support")))]
1836 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1837 self.windows_flash = Some(flash);
1838 self
1839 }
1840
1841 #[cfg(all(windows, any(test, feature = "test-support")))]
1845 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1846 self.windows_track = Some(track);
1847 self
1848 }
1849
1850 pub(crate) fn record_captured_peer(
1851 &self,
1852 connection: ConnectionId,
1853 peer: subc_os::local_peer::CapturedPeer,
1854 ) {
1855 self.record_observed_peer(connection, peer.observation().clone());
1856 #[cfg(windows)]
1857 if let Some(stamp) = self.stamp_windows_launch(&peer) {
1858 self.operator_apps
1859 .lock()
1860 .unwrap_or_else(|p| p.into_inner())
1861 .insert(connection, stamp);
1862 }
1863 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1864 self.operator_apps
1865 .lock()
1866 .unwrap_or_else(|p| p.into_inner())
1867 .insert(connection, stamp);
1868 }
1869 }
1870
1871 pub(crate) fn record_observed_peer(
1872 &self,
1873 connection: ConnectionId,
1874 peer: subc_os::local_peer::ObservedPeer,
1875 ) {
1876 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1877 let peer = match peer {
1878 OsPeer::Unobserved => ObservedPeer::unobserved(),
1879 OsPeer::Observed(peer) => {
1880 let observed = match peer.user {
1881 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1882 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1883 };
1884 observed
1885 .with_exe(
1886 peer.exe
1887 .and_then(|path| path.into_os_string().into_string().ok()),
1888 )
1889 .with_signing_id(peer.signing_id)
1890 }
1891 };
1892 self.observed_peers
1893 .lock()
1894 .unwrap_or_else(|p| p.into_inner())
1895 .insert(connection, peer);
1896 }
1897
1898 pub(crate) fn record_caller_endpoints(
1899 &self,
1900 connection: ConnectionId,
1901 endpoints: Option<(SocketAddr, SocketAddr)>,
1902 ) {
1903 let mut callers = self
1904 .caller_endpoints
1905 .lock()
1906 .unwrap_or_else(|p| p.into_inner());
1907 callers.remove(&connection);
1908 if let Some(endpoints) = endpoints {
1909 callers.insert(connection, endpoints);
1910 }
1911 }
1912
1913 #[cfg(test)]
1914 pub(crate) fn with_caller_lookup_for_test(
1915 mut self,
1916 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1917 ) -> Self {
1918 self.caller_lookup = Some(Arc::new(lookup));
1919 self
1920 }
1921
1922 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1923 let endpoints = self
1924 .caller_endpoints
1925 .lock()
1926 .unwrap_or_else(|p| p.into_inner())
1927 .get(&connection)
1928 .copied();
1929 #[cfg(test)]
1930 let lookup = self.caller_lookup.clone();
1931 tokio::task::spawn_blocking(move || {
1932 #[cfg(test)]
1933 if let Some(lookup) = lookup {
1934 return lookup(endpoints);
1935 }
1936 endpoints
1937 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1938 .unwrap_or_default()
1939 })
1940 .await
1941 .unwrap_or_default()
1942 }
1943
1944 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1945 frame.header.channel == 0
1946 && frame.header.ty == FrameType::Request
1947 && self
1948 .forwarding
1949 .module_endpoint_for_connection(ctx.connection_id)
1950 .is_ok_and(|endpoint| endpoint.is_none())
1951 && parse_client_control_request(&frame.body)
1952 .ok()
1953 .is_some_and(|request| privileged_operation(&request).is_some())
1954 }
1955
1956 pub(crate) async fn handle_control_frame_timed(
1957 &self,
1958 ctx: &RouteCtx,
1959 frame: Frame,
1960 dispatch_started_at: Option<StdInstant>,
1961 ) -> Result<Vec<Frame>, RouterError> {
1962 match frame.header.ty {
1963 FrameType::Ping => Ok(vec![pong(&frame)?]),
1964 FrameType::Hello => {
1965 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1966 }
1967 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1968 FrameType::Cancel => {
1969 if self
1972 .forwarding
1973 .operator_confirms()
1974 .cancel(ctx.connection_id, frame.header.corr)
1975 || self
1976 .supervisor
1977 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1978 || self.cancel_activity_subscription(ctx, &frame)?
1979 {
1980 Ok(Vec::new())
1981 } else {
1982 Ok(vec![control_error_frame(
1983 &frame,
1984 "unknown_subscription",
1985 "no subscription has this correlation id",
1986 )?])
1987 }
1988 }
1989 FrameType::Request => {
1990 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1993 if op
1994 .as_ref()
1995 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1996 {
1997 return self.handle_activity_record(ctx, frame).await;
1998 }
1999 if op.as_ref().is_some_and(|probe| {
2000 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2001 }) {
2002 return self.handle_activity_read(ctx, frame).await;
2003 }
2004 if op
2005 .as_ref()
2006 .is_some_and(|probe| probe.op == "operator.confirm")
2007 {
2008 return self.handle_operator_confirm(ctx, frame);
2009 }
2010 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2011 return self.handle_app_open(ctx, frame).await;
2012 }
2013 if op
2014 .as_ref()
2015 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2016 {
2017 return self.handle_scope_served_by(ctx.connection_id, frame);
2018 }
2019 if self
2020 .forwarding
2021 .module_endpoint_for_connection(ctx.connection_id)
2022 .map_err(RouterError::Forwarding)?
2023 .is_some()
2024 {
2025 if !is_known_module_request_op(&frame.body) {
2026 return Ok(vec![control_error_frame(
2027 &frame,
2028 "unsupported_control_frame",
2029 "module-originated channel-0 REQUEST is not supported",
2030 )?]);
2031 }
2032 let request = match parse_module_control_request_from_module(&frame.body) {
2033 Ok(request) => request,
2034 Err((err, ControlRequestBodyError::UnknownOp)) => {
2035 return Ok(vec![control_error_frame(
2036 &frame,
2037 "unsupported_control_frame",
2038 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2039 )?])
2040 }
2041 Err((err, ControlRequestBodyError::InvalidBody)) => {
2042 return Ok(vec![control_error_frame(
2043 &frame,
2044 "invalid_control_body",
2045 format!("malformed module control body: {err}"),
2046 )?])
2047 }
2048 };
2049 let op = module_control_request_op(&request);
2050 let corr = frame.header.corr;
2051 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2052 let result =
2053 self.handle_module_control_request(ctx.connection_id, frame, request);
2054 log_slow_control_dispatch(
2055 dispatch_started_at,
2056 op,
2057 ctx.connection_id,
2058 corr,
2059 None,
2060 );
2061 return result;
2062 }
2063
2064 if is_known_module_request_op(&frame.body) {
2065 return Ok(vec![control_error_frame(
2066 &frame,
2067 "not_registered",
2068 "catalog.update requires an active module registration owned by this connection",
2069 )?]);
2070 }
2071
2072 let request = match parse_client_control_request(&frame.body) {
2073 Ok(request) => request,
2074 Err((err, ControlRequestBodyError::UnknownOp)) => {
2075 return Ok(vec![control_error_frame(
2076 &frame,
2077 "unknown_control_op",
2078 format!("unknown client control op: {err}"),
2079 )?])
2080 }
2081 Err((err, ControlRequestBodyError::InvalidBody)) => {
2082 return Ok(vec![control_error_frame(
2083 &frame,
2084 "invalid_control_body",
2085 format!("malformed client control body: {err}"),
2086 )?])
2087 }
2088 };
2089 let op = client_control_request_op(&request);
2090 let corr = frame.header.corr;
2091 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2092 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2093 #[cfg(test)]
2094 if let Some(delay) = self.control_dispatch_delay {
2095 tokio::time::sleep(delay).await;
2096 }
2097 let result = self
2098 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2099 .await;
2100 if let Some(timings) = route_open_timings.as_mut() {
2101 timings.finish();
2102 }
2103 log_slow_control_dispatch(
2104 dispatch_started_at,
2105 op,
2106 ctx.connection_id,
2107 corr,
2108 route_open_timings.as_ref(),
2109 );
2110 result
2111 }
2112 FrameType::Push => {
2113 let Some(endpoint) = self
2114 .forwarding
2115 .module_endpoint_for_connection(ctx.connection_id)
2116 .map_err(RouterError::Forwarding)?
2117 else {
2118 return Ok(vec![control_error_frame(
2119 &frame,
2120 "unsupported_control_frame",
2121 "client-originated channel-0 PUSH is not supported",
2122 )?]);
2123 };
2124 self.handle_status_update(endpoint, frame)
2125 }
2126 FrameType::Response | FrameType::Error
2127 if self
2128 .forwarding
2129 .module_endpoint_for_connection(ctx.connection_id)
2130 .map_err(RouterError::Forwarding)?
2131 .is_some() =>
2132 {
2133 self.handle_module_relay_response(ctx.connection_id, frame)
2134 }
2135 ty => Ok(vec![control_error_frame(
2136 &frame,
2137 "unsupported_control_frame",
2138 format!("unsupported channel-0 frame {ty:?}"),
2139 )?]),
2140 }
2141 }
2142
2143 pub fn cleanup_connection(
2144 &self,
2145 connection_id: ConnectionId,
2146 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2147 self.caller_endpoints
2148 .lock()
2149 .unwrap_or_else(|p| p.into_inner())
2150 .remove(&connection_id);
2151 self.build_class_warned
2152 .lock()
2153 .unwrap_or_else(|p| p.into_inner())
2154 .remove(&connection_id);
2155 self.observed_peers
2156 .lock()
2157 .unwrap_or_else(|p| p.into_inner())
2158 .remove(&connection_id);
2159 self.operator_apps
2160 .lock()
2161 .unwrap_or_else(|p| p.into_inner())
2162 .remove(&connection_id);
2163 self.forwarding
2164 .operator_confirms()
2165 .connection_closed(connection_id);
2166 self.cleanup_connection_with_end_reason(
2167 connection_id,
2168 RegistrationEndReason::ConnectionClosed,
2169 )
2170 }
2171
2172 fn cleanup_connection_with_end_reason(
2173 &self,
2174 connection_id: ConnectionId,
2175 requested_reason: RegistrationEndReason,
2176 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2177 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2178 self.registry
2179 .get_module_by_connection(connection_id)?
2180 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2181 .and_then(|module| module.registration_end_reason().ok().flatten())
2182 .unwrap_or(requested_reason)
2183 } else {
2184 requested_reason
2185 };
2186 let crash_closed = self
2187 .registry
2188 .get_module_by_connection(connection_id)?
2189 .and_then(|registration| {
2190 self.forwarding
2191 .module_endpoint_for_connection(connection_id)
2192 .ok()
2193 .flatten()
2194 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2195 .map(|routes| (registration.manifest.module_id, routes))
2196 });
2197 let crash_closed = crash_closed.map(|(module_id, routes)| {
2198 let terminal = match self.supervisor.get(&module_id) {
2199 None => false,
2200 Some(module) => match module.will_recover_after_connection_loss() {
2201 Ok(will_recover) => !will_recover,
2202 Err(err) => {
2203 warn!(
2204 %module_id,
2205 error = %err,
2206 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2207 );
2208 false
2209 }
2210 },
2211 };
2212 let (reason, drained) = match self
2216 .forwarding
2217 .daemon_drain_result_for_connection(connection_id)
2218 {
2219 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2220 Ok(None) => (RouteCloseReason::Crash, false),
2221 Err(err) => {
2222 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2223 (RouteCloseReason::Crash, false)
2224 }
2225 };
2226 (module_id, routes, reason, drained, terminal)
2227 });
2228 let registrations = self.deregister_connection(connection_id, end_reason);
2229 let cleanup = if crash_closed.is_some() {
2230 self.forwarding.cleanup_connection_counted(connection_id)
2231 } else {
2232 self.forwarding
2238 .cleanup_connection(connection_id)
2239 .map(|released| crate::forwarding::ConnectionCleanup {
2240 released,
2241 abandoned_relays: 0,
2242 })
2243 };
2244 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2249 let abandoned = cleanup
2250 .as_ref()
2251 .map_or(0, |cleanup| cleanup.abandoned_relays);
2252 send_route_control_pushes(
2253 &self.forwarding,
2254 routes,
2255 ClientControlPush::RouteClosed {
2256 module_id,
2257 channels: Vec::new(),
2258 reason,
2259 drained,
2260 abandoned,
2261 excluded_subscriptions: 0,
2262 terminal: Some(terminal),
2263 },
2264 );
2265 }
2266 if let Ok(cleanup) = cleanup {
2267 self.emit_route_goodbyes(cleanup.released);
2268 }
2269 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2273 crate::supervise::notify_registration_release();
2274 self.capability_evaluator.wake_deadline_loop();
2275 self.refresh_capability_requirements();
2276 }
2277 self.supervisor.remove_spawn_subscribers(connection_id);
2278 self.activity_streams
2279 .lock()
2280 .unwrap_or_else(|p| p.into_inner())
2281 .retain(|(id, _), task| {
2282 if *id == connection_id {
2283 task.abort();
2284 false
2285 } else {
2286 true
2287 }
2288 });
2289 self.hello_launch_nonces
2292 .lock()
2293 .unwrap_or_else(|poisoned| poisoned.into_inner())
2294 .forget(connection_id);
2295 self.scopes
2296 .write()
2297 .unwrap_or_else(|poisoned| poisoned.into_inner())
2298 .release_connection(connection_id);
2299 registrations
2300 }
2301
2302 pub(crate) fn handle_route_goodbye(
2303 &self,
2304 connection_id: ConnectionId,
2305 route_channel: u16,
2306 route_epoch: u32,
2307 ) -> Result<bool, RouterError> {
2308 debug!(
2309 connection_id = connection_id.get(),
2310 route_channel, route_epoch, "handling route GOODBYE"
2311 );
2312 let RouteRelease::Removed(released_route) = self
2313 .forwarding
2314 .release_client_route(connection_id, route_channel, route_epoch)
2315 .map_err(RouterError::Forwarding)?
2316 else {
2317 return Ok(false);
2318 };
2319 self.emit_route_goodbyes(vec![released_route]);
2320 Ok(true)
2321 }
2322
2323 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2324 for released in released_routes {
2325 let frame = match Frame::build_with_version(
2326 released.negotiated_ver,
2327 FrameType::Goodbye,
2328 control_flags(),
2329 released.channel,
2330 released.epoch,
2331 0,
2332 Vec::new(),
2333 ) {
2334 Ok(frame) => frame,
2335 Err(err) => {
2336 warn!(
2337 route_channel = released.channel,
2338 error = %err,
2339 "failed to build route GOODBYE frame"
2340 );
2341 continue;
2342 }
2343 };
2344 if !released.close_on_delivery_failure() {
2345 crate::forwarding::send_module_route_goodbye(
2346 &self.counters,
2347 &released.sink,
2348 frame,
2349 released.module_id.as_deref(),
2350 "client route released",
2351 );
2352 continue;
2353 }
2354 if let Err(err) = released.sink.try_send(frame) {
2355 warn!(
2356 target_connection_id = released.connection_id.get(),
2357 route_channel = released.channel,
2358 error = %err,
2359 "route GOODBYE was not delivered to client; closing target connection"
2360 );
2361 if self
2362 .forwarding
2363 .escalate_client_delivery_failure(
2364 released.connection_id,
2365 released.channel,
2366 released.epoch,
2367 CloseReason::new(
2368 "route_goodbye_delivery_failed",
2369 format!(
2370 "failed to enqueue route GOODBYE for channel {}: {err}",
2371 released.channel
2372 ),
2373 ),
2374 crate::forwarding::UndeliveredFrame {
2375 module_id: released.module_id.as_deref(),
2376 sink: &released.sink,
2377 },
2378 )
2379 .unwrap_or(false)
2380 {
2381 self.counters.increment_goodbye_relay_client_failed();
2382 }
2383 }
2384 }
2385 }
2386
2387 fn send_abandoned_route_bind_goodbye(
2399 &self,
2400 module_sink: &crate::FrameSink,
2401 negotiated_ver: u8,
2402 module_channel: u16,
2403 module_epoch: u32,
2404 ) {
2405 let frame = match Frame::build_with_version(
2406 negotiated_ver,
2407 FrameType::Goodbye,
2408 control_flags(),
2409 module_channel,
2410 module_epoch,
2411 0,
2412 Vec::new(),
2413 ) {
2414 Ok(frame) => frame,
2415 Err(err) => {
2416 warn!(
2417 route_channel = module_channel,
2418 error = %err,
2419 "failed to build GOODBYE for abandoned route.bind"
2420 );
2421 return;
2422 }
2423 };
2424 crate::forwarding::send_module_route_goodbye(
2425 &self.counters,
2426 module_sink,
2427 frame,
2428 None,
2429 "abandoned route.bind",
2430 );
2431 }
2432
2433 fn handle_hello(
2434 &self,
2435 connection_id: ConnectionId,
2436 sink: Option<crate::FrameSink>,
2437 frame: Frame,
2438 ) -> Result<Vec<Frame>, RouterError> {
2439 debug!(
2440 connection_id = connection_id.get(),
2441 corr = frame.header.corr,
2442 "handling HELLO"
2443 );
2444 if self
2448 .registry
2449 .get_module_by_connection(connection_id)
2450 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2451 .is_some()
2452 {
2453 return Ok(vec![control_error_frame(
2454 &frame,
2455 "invalid_hello",
2456 "connection is already registered as a module",
2457 )?]);
2458 }
2459 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2460 Ok(value) => value,
2461 Err(err) => {
2462 return Ok(vec![control_error_frame(
2463 &frame,
2464 "invalid_hello",
2465 format!("malformed HELLO body: {err}"),
2466 )?])
2467 }
2468 };
2469 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2470 return Ok(vec![control_error_frame(
2471 &frame,
2472 "invalid_capability_grammar",
2473 err.to_string(),
2474 )?]);
2475 }
2476 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2477 return Ok(vec![control_error_frame(
2478 &frame,
2479 "invalid_manifest",
2480 err.to_string(),
2481 )?]);
2482 }
2483 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2484 return Ok(vec![control_error_frame(
2485 &frame,
2486 "invalid_event_declaration",
2487 err.to_string(),
2488 )?]);
2489 }
2490 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2491 return Ok(vec![control_error_frame(
2492 &frame,
2493 "invalid_manifest",
2494 err.to_string(),
2495 )?]);
2496 }
2497 if let Some(provenance) = hello_value
2498 .get("manifest")
2499 .and_then(|manifest| manifest.get("provenance"))
2500 {
2501 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2502 return Ok(vec![control_error_frame(
2503 &frame,
2504 "invalid_manifest",
2505 format!("malformed manifest provenance: {err}"),
2506 )?]);
2507 }
2508 }
2509 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2510 let module_id = hello_value["manifest"]["module_id"]
2511 .as_str()
2512 .unwrap_or("<unknown>");
2513 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2514 }
2515 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2516 let module_id = hello_value["manifest"]["module_id"]
2517 .as_str()
2518 .unwrap_or("<unknown>");
2519 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2520 }
2521 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2522 let module_id = hello_value["manifest"]["module_id"]
2523 .as_str()
2524 .unwrap_or("<unknown>");
2525 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2526 }
2527 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2528 Ok(hello) => hello,
2529 Err(err) => {
2530 return Ok(vec![control_error_frame(
2531 &frame,
2532 "invalid_hello",
2533 format!("malformed HELLO body: {err}"),
2534 )?])
2535 }
2536 };
2537
2538 if hello.protocol_ver != hello.manifest.protocol_ver {
2539 return Ok(vec![control_error_frame(
2540 &frame,
2541 "invalid_manifest",
2542 format!(
2543 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2544 hello.protocol_ver, hello.manifest.protocol_ver
2545 ),
2546 )?]);
2547 }
2548
2549 if hello.manifest.module_id.trim().is_empty() {
2550 return Ok(vec![control_error_frame(
2551 &frame,
2552 "invalid_manifest",
2553 "manifest module_id must not be empty",
2554 )?]);
2555 }
2556
2557 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2558 Ok(negotiated_ver) => negotiated_ver,
2559 Err(message) => {
2560 return Ok(vec![control_error_frame(
2561 &frame,
2562 "version_unsupported",
2563 message,
2564 )?])
2565 }
2566 };
2567
2568 let swap_admission = self
2576 .supervisor
2577 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2578 if swap_admission == SwapHelloAdmission::Refused {
2579 warn!(
2580 module_id = %hello.manifest.module_id,
2581 connection_id = connection_id.get(),
2582 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2583 );
2584 return Ok(vec![control_error_frame(
2585 &frame,
2586 "swap_token_invalid",
2587 format!(
2588 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2589 hello.manifest.module_id
2590 ),
2591 )?]);
2592 }
2593 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2594
2595 let nonce_verified = if swap_candidate {
2600 true
2601 } else {
2602 match self.supervisor.reserved_hello_verification(
2603 &hello.manifest.module_id,
2604 hello.launch_nonce.as_deref(),
2605 ) {
2606 Ok(verified) => verified,
2607 Err(rejection) => {
2608 let message = match rejection {
2609 ReservedHelloRejection::Exact { module_id } => format!(
2610 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2611 ),
2612 ReservedHelloRejection::Prefix {
2613 prefix,
2614 owner_module_id,
2615 } => format!(
2616 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2617 hello.manifest.module_id
2618 ),
2619 };
2620 return Ok(vec![control_error_frame(
2621 &frame,
2622 "reserved_module",
2623 message,
2624 )?]);
2625 }
2626 }
2627 };
2628
2629 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2630 &hello.manifest.module_id,
2631 hello.manifest.capabilities.as_ref(),
2632 );
2633 if let Some(refusal) = reserved_capability_refusals.first() {
2634 let capability = refusal.capability.clone();
2635 let bound_module = refusal.claimants[0].clone();
2636 log_duplicate_claim_events(reserved_capability_refusals);
2637 return Ok(vec![control_error_frame(
2638 &frame,
2639 "reserved_capability",
2640 format!(
2641 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2642 capability, bound_module, hello.manifest.module_id
2643 ),
2644 )?]);
2645 }
2646
2647 if self
2650 .forwarding
2651 .connection_has_client_routes(connection_id)
2652 .map_err(RouterError::Forwarding)?
2653 {
2654 return Ok(vec![control_error_frame(
2655 &frame,
2656 "invalid_hello",
2657 "connection has open client routes and cannot also register as a module",
2658 )?]);
2659 }
2660
2661 self.hello_launch_nonces
2666 .lock()
2667 .unwrap_or_else(|poisoned| poisoned.into_inner())
2668 .record(connection_id, hello.launch_nonce.as_deref());
2669 let control_ops = effective_module_control_ops(hello.control_ops);
2670 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2673 if swap_candidate {
2674 return self.register_swap_candidate(
2675 connection_id,
2676 sink,
2677 &frame,
2678 hello.manifest,
2679 negotiated_ver,
2680 control_ops,
2681 hello_ack,
2682 );
2683 }
2684 let registration = match self
2691 .registry
2692 .register_unpublished_verified_with_control_ops(
2693 hello.manifest,
2694 negotiated_ver,
2695 connection_id,
2696 control_ops,
2697 nonce_verified,
2698 ) {
2699 Ok(registration) => registration,
2700 Err(RegistryError::DuplicateModuleId { module_id }) => {
2701 return Ok(vec![control_error_frame(
2702 &frame,
2703 "duplicate_module_id",
2704 format!(
2705 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2706 ),
2707 )?])
2708 }
2709 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2710 return Ok(vec![control_error_frame(
2711 &frame,
2712 "invalid_module_id",
2713 err.to_string(),
2714 )?])
2715 }
2716 Err(err) => {
2717 return Ok(vec![control_error_frame(
2718 &frame,
2719 "registry_error",
2720 err.to_string(),
2721 )?])
2722 }
2723 };
2724 #[cfg(test)]
2725 self.hold_between_registry_and_forwarding();
2726
2727 let reply = if let Some(sink) = sink {
2728 let concurrency = manifest_concurrency(®istration.manifest);
2744 if let Err(err) = self.forwarding.register_module_connection_acked(
2745 connection_id,
2746 registration.manifest.module_id.clone(),
2747 negotiated_ver,
2748 concurrency,
2749 sink,
2750 hello_ack,
2751 ) {
2752 if matches!(
2757 self.deregister_connection(
2758 connection_id,
2759 RegistrationEndReason::RegistrationFailed,
2760 ),
2761 Ok(r) if !r.is_empty()
2762 ) {
2763 crate::supervise::notify_registration_release();
2764 }
2765 return Ok(vec![control_error_frame(
2766 &frame,
2767 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2768 "invalid_hello"
2769 } else {
2770 forwarding_error_code(&err)
2771 },
2772 err.to_string(),
2773 )?]);
2774 }
2775 Vec::new()
2776 } else {
2777 vec![hello_ack]
2780 };
2781
2782 if !self
2787 .registry
2788 .publish_registration(connection_id)
2789 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2790 {
2791 debug!(
2792 module_id = %registration.manifest.module_id,
2793 connection_id = connection_id.get(),
2794 "registration was removed before it could be published"
2795 );
2796 }
2797
2798 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2807 info!(
2808 module_id = %registration.manifest.module_id,
2809 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2810 );
2811 }
2812
2813 self.apply_registration_capabilities(®istration);
2814
2815 self.supervisor
2816 .record_activity_registration(®istration.manifest.module_id);
2817
2818 info!(
2819 module_id = %registration.manifest.module_id,
2820 module_version = %registration.manifest.module_version,
2821 negotiated_ver,
2822 routable_provider = manifest_provides_routable_role(®istration.manifest),
2823 connection_id = connection_id.get(),
2824 "module registered"
2825 );
2826
2827 Ok(reply)
2828 }
2829
2830 #[allow(clippy::too_many_arguments)]
2847 fn register_swap_candidate(
2848 &self,
2849 connection_id: ConnectionId,
2850 sink: Option<crate::FrameSink>,
2851 frame: &Frame,
2852 manifest: ModuleManifest,
2853 negotiated_ver: u8,
2854 control_ops: Vec<String>,
2855 hello_ack: Frame,
2856 ) -> Result<Vec<Frame>, RouterError> {
2857 let module_id = manifest.module_id.clone();
2858 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2859 manifest,
2860 negotiated_ver,
2861 connection_id,
2862 control_ops,
2863 ) {
2864 Ok(registration) => registration,
2865 Err(RegistryError::DuplicateModuleId { module_id }) => {
2866 return Ok(vec![control_error_frame(
2867 frame,
2868 "duplicate_module_id",
2869 format!(
2870 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2871 ),
2872 )?])
2873 }
2874 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2875 return Ok(vec![control_error_frame(
2876 frame,
2877 "invalid_module_id",
2878 err.to_string(),
2879 )?])
2880 }
2881 Err(err) => {
2882 return Ok(vec![control_error_frame(
2883 frame,
2884 "registry_error",
2885 err.to_string(),
2886 )?])
2887 }
2888 };
2889 #[cfg(test)]
2890 self.hold_between_registry_and_forwarding();
2891 let reply = if let Some(sink) = sink {
2892 let concurrency = manifest_concurrency(®istration.manifest);
2896 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2897 connection_id,
2898 module_id.clone(),
2899 negotiated_ver,
2900 concurrency,
2901 sink,
2902 hello_ack,
2903 ) {
2904 if matches!(
2905 self.deregister_connection(
2906 connection_id,
2907 RegistrationEndReason::RegistrationFailed,
2908 ),
2909 Ok(r) if !r.is_empty()
2910 ) {
2911 crate::supervise::notify_registration_release();
2912 }
2913 return Ok(vec![control_error_frame(
2914 frame,
2915 forwarding_error_code(&err),
2916 err.to_string(),
2917 )?]);
2918 }
2919 Vec::new()
2920 } else {
2921 vec![hello_ack]
2922 };
2923 if !self
2924 .registry
2925 .publish_registration(connection_id)
2926 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2927 {
2928 debug!(
2929 module_id = %module_id,
2930 connection_id = connection_id.get(),
2931 "swap candidate registration was removed before it could be published"
2932 );
2933 }
2934 self.supervisor.mark_swap_candidate_admitted(&module_id);
2935 info!(
2936 module_id = %module_id,
2937 module_version = %registration.manifest.module_version,
2938 negotiated_ver,
2939 ready = registration.ready,
2940 connection_id = connection_id.get(),
2941 "swap candidate registered; not routable until cutover"
2942 );
2943 Ok(reply)
2944 }
2945
2946 fn build_hello_ack(
2947 &self,
2948 frame: &Frame,
2949 negotiated_ver: u8,
2950 module_id: &str,
2951 ) -> Result<Frame, RouterError> {
2952 let ack = ModuleHelloAckBody {
2953 negotiated_ver,
2954 subc_ops: module_subc_ops(),
2955 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2956 storage: self
2957 .storage_config
2958 .as_ref()
2959 .map(|cfg| cfg.descriptor_for(module_id)),
2960 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2961 };
2962 let body = serde_json::to_vec(&ack).map_err(|err| {
2963 RouterError::backend(
2964 0,
2965 frame.header.corr,
2966 format!("failed to encode HELLO_ACK: {err}"),
2967 )
2968 })?;
2969
2970 Frame::build_with_version(
2971 negotiated_ver,
2972 FrameType::HelloAck,
2973 control_flags(),
2974 0,
2975 0,
2976 frame.header.corr,
2977 body,
2978 )
2979 .map_err(RouterError::FrameBuild)
2980 }
2981
2982 async fn handle_client_control_request(
2983 &self,
2984 ctx: &RouteCtx,
2985 frame: Frame,
2986 request: ClientControlRequest,
2987 route_open_timings: &mut Option<RouteOpenTimings>,
2988 ) -> Result<Vec<Frame>, RouterError> {
2989 if let Some(operation) = privileged_operation(&request) {
2990 let caller = self.resolve_caller(ctx.connection_id).await;
2991 let op = client_control_request_op(&request);
2992 let text = prompt_text(operation, &caller);
2993 match self.privileged_operator_prompt {
2994 PrivilegedOperatorPrompt::Log => {
2995 info!(target: "subc_daemon::privileged_prompt", operation = op,
2996 target = ?operation, caller = %caller.description(), summary = %text,
2997 "privileged request would prompt the operator");
2998 }
2999 PrivilegedOperatorPrompt::Enforce => {
3000 let key =
3001 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3002 let stamp = self
3003 .operator_apps
3004 .lock()
3005 .unwrap_or_else(|p| p.into_inner())
3006 .get(&ctx.connection_id)
3007 .cloned();
3008 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3009 &self.operator_app_policy,
3010 &self.launch_table,
3011 stamp.as_ref(),
3012 )
3013 .then(|| self.shortcut_context(stamp, None, None));
3014 let outcome = self
3015 .forwarding
3016 .operator_confirms()
3017 .approve_privileged_with_shortcut(
3018 ctx,
3019 &frame,
3020 op,
3021 text,
3022 caller.description(),
3023 key,
3024 shortcut,
3025 )
3026 .await;
3027 if outcome.code != "confirmed" {
3028 return Ok(vec![outcome.frame(&frame)]);
3029 }
3030 }
3031 }
3032 }
3033 match request {
3034 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3035 ClientControlRequest::CatalogList { module_id } => {
3036 self.handle_catalog_list(frame, module_id)
3037 }
3038 ClientControlRequest::RouteOpen {
3039 target,
3040 identity,
3041 consumer_identity,
3042 consumer_capabilities,
3043 role_versions,
3044 admission_facts,
3045 scope,
3046 } => {
3047 self.handle_route_open(
3048 ctx,
3049 frame,
3050 RouteOpenRequest {
3051 target,
3052 identity,
3053 consumer_identity,
3054 consumer_capabilities,
3055 role_versions,
3056 admission_facts,
3057 scope,
3058 },
3059 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3060 )
3061 .await
3062 }
3063 ClientControlRequest::RoutePoll {
3064 route_channel,
3065 route_epoch,
3066 kind,
3067 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3068 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
3069 ClientControlRequest::SupervisorSpawnSnapshot {} => {
3070 self.handle_supervisor_spawn_snapshot(frame)
3071 }
3072 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3073 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3074 }
3075 ClientControlRequest::SupervisorRestart {
3076 module_id,
3077 drain_timeout_ms,
3078 } => {
3079 self.log_supervisor_request_received(
3080 ctx,
3081 frame.header.corr,
3082 ops::SUPERVISOR_RESTART,
3083 Some(&module_id),
3084 None,
3085 )?;
3086 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3087 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3088 .await
3089 }
3090 ClientControlRequest::SupervisorSwap {
3091 module_id,
3092 ready_timeout_ms,
3093 } => {
3094 self.log_supervisor_request_received(
3095 ctx,
3096 frame.header.corr,
3097 ops::SUPERVISOR_SWAP,
3098 Some(&module_id),
3099 None,
3100 )?;
3101 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3102 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3103 .await
3104 }
3105 ClientControlRequest::SupervisorReload { module_id } => {
3106 self.log_supervisor_request_received(
3107 ctx,
3108 frame.header.corr,
3109 ops::SUPERVISOR_RELOAD,
3110 Some(&module_id),
3111 None,
3112 )?;
3113 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3114 self.handle_supervisor_reload(frame, module_id, requester)
3115 .await
3116 }
3117 ClientControlRequest::SupervisorRescan { preview } => {
3118 if !preview {
3119 self.log_supervisor_request_received(
3120 ctx,
3121 frame.header.corr,
3122 ops::SUPERVISOR_RESCAN,
3123 None,
3124 None,
3125 )?;
3126 }
3127 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3128 self.handle_supervisor_rescan(
3129 frame,
3130 if preview {
3131 RescanMode::Preview
3132 } else {
3133 RescanMode::Execute
3134 },
3135 requester,
3136 )
3137 .await
3138 }
3139 ClientControlRequest::SupervisorRescanChecked {
3140 preview,
3141 candidate,
3142 expect,
3143 } => {
3144 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
3145 .await
3146 }
3147 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3148 self.log_supervisor_request_received(
3149 ctx,
3150 frame.header.corr,
3151 ops::SUPERVISOR_RELEASE_RESERVED,
3152 Some(&module_id),
3153 None,
3154 )?;
3155 self.handle_supervisor_release_reserved(frame, module_id)
3156 .await
3157 }
3158 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3159 self.log_supervisor_request_received(
3160 ctx,
3161 frame.header.corr,
3162 ops::SUPERVISOR_SET_ENABLED,
3163 Some(&module_id),
3164 Some(enabled),
3165 )?;
3166 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3167 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3168 .await
3169 }
3170 ClientControlRequest::SupervisorHealthProbe { module_id } => {
3171 self.handle_supervisor_health_probe(frame, module_id).await
3172 }
3173 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3174 ClientControlRequest::SupervisorRoutes { module_id } => {
3175 self.handle_supervisor_routes(frame, module_id)
3176 }
3177 ClientControlRequest::SupervisorProvenance { module_id } => {
3178 self.handle_supervisor_provenance(frame, module_id).await
3179 }
3180 ClientControlRequest::SupervisorStderrTail {
3181 module_id,
3182 max_lines,
3183 max_bytes,
3184 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3185 ClientControlRequest::SupervisorTerminals { module_id } => {
3186 self.handle_supervisor_terminals(frame, module_id).await
3187 }
3188 }
3189 }
3190
3191 fn log_supervisor_request_received(
3192 &self,
3193 ctx: &RouteCtx,
3194 corr: u64,
3195 op: &'static str,
3196 module_id: Option<&str>,
3197 enabled: Option<bool>,
3198 ) -> Result<(), RouterError> {
3199 let caller = self.supervisor_requester(ctx, corr)?;
3200 let caller = principal_label(&caller);
3201 info!(
3204 target: "control",
3205 op,
3206 module_id,
3207 enabled,
3208 connection_id = ctx.connection_id.get(),
3209 caller = %caller,
3210 "supervisor request received"
3211 );
3212 Ok(())
3213 }
3214
3215 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3216 Ok(self
3217 .registry
3218 .get_module_by_connection(ctx.connection_id)
3219 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3220 .map(|registration| Principal::Reserved {
3221 module_id: registration.manifest.module_id,
3222 })
3223 .unwrap_or(Principal::Direct))
3224 }
3225
3226 fn handle_module_control_request(
3227 &self,
3228 connection_id: ConnectionId,
3229 frame: Frame,
3230 request: ModuleControlRequestFromModule,
3231 ) -> Result<Vec<Frame>, RouterError> {
3232 match request {
3233 ModuleControlRequestFromModule::CatalogUpdate {
3234 provides,
3235 capabilities,
3236 ready,
3237 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3238 ModuleControlRequestFromModule::LiveRoots {} => {
3239 let registered = self
3240 .registry
3241 .get_module_by_connection(connection_id)
3242 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3243 let Some(registration) = registered else {
3244 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3245 };
3246 let response = self
3247 .forwarding
3248 .live_roots(®istration.manifest.module_id)
3249 .map_err(RouterError::Forwarding)?;
3250 Ok(vec![control_response_body_frame(
3251 &frame,
3252 &response,
3253 "ModuleControlResponseToModule::LiveRoots",
3254 )?])
3255 }
3256 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3257 self.handle_scope_sync(connection_id, frame, generation, scopes)
3258 }
3259 ModuleControlRequestFromModule::ScopeApply {
3260 generation,
3261 upsert,
3262 end,
3263 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3264 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3265 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3266 }
3267 }
3268 }
3269
3270 fn handle_scope_served_by(
3271 &self,
3272 connection_id: ConnectionId,
3273 frame: Frame,
3274 ) -> Result<Vec<Frame>, RouterError> {
3275 let registered = self
3276 .registry
3277 .get_module_by_connection(connection_id)
3278 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3279 let Some(registered) = registered else {
3280 return Ok(vec![control_error_frame(
3281 &frame,
3282 "not_registered",
3283 "scope.served_by requires an active module registration owned by this connection",
3284 )?]);
3285 };
3286 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3287 {
3288 Ok(request) => request,
3289 Err(err) => {
3290 return Ok(vec![control_error_frame(
3291 &frame,
3292 "invalid_control_body",
3293 format!("malformed scope.served_by body: {err}"),
3294 )?])
3295 }
3296 };
3297 let answer = self
3298 .scopes
3299 .read()
3300 .unwrap_or_else(|poisoned| poisoned.into_inner())
3301 .served_by(
3302 ®istered.manifest.module_id,
3303 &request.scope_ref,
3304 request.scope_epoch,
3305 &request.module_id,
3306 );
3307 let reply = subc_control::ScopeServedByReply::new(
3308 answer,
3309 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3310 );
3311 Ok(vec![control_response_body_frame(
3312 &frame,
3313 &reply,
3314 "ScopeServedByReply",
3315 )?])
3316 }
3317
3318 fn handle_operator_confirm(
3319 &self,
3320 ctx: &RouteCtx,
3321 frame: Frame,
3322 ) -> Result<Vec<Frame>, RouterError> {
3323 use crate::operator_confirm::{audit, Outcome};
3324 let registration = self
3325 .registry
3326 .get_module_by_connection(ctx.connection_id)
3327 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3328 let Some(registration) = registration else {
3329 let outcome = Outcome::refusal("not_registered");
3330 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3331 return Ok(vec![outcome.frame(&frame)]);
3332 };
3333 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3334 Ok(request) => request,
3335 Err(_) => {
3336 let outcome = Outcome::refusal("invalid_control_body");
3337 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3338 return Ok(vec![outcome.frame(&frame)]);
3339 }
3340 };
3341 let declared = matches!(
3342 request.kind,
3343 Some(subc_protocol::session::OperatorConfirmKind::Operation)
3344 ) && request.operation.as_ref().is_some_and(|operation| {
3345 registration
3346 .manifest
3347 .confirm_operations
3348 .as_ref()
3349 .is_some_and(|operations| operations.contains(operation))
3350 });
3351 let module_id = registration.manifest.module_id;
3352 let nonce = self
3356 .hello_launch_nonces
3357 .lock()
3358 .unwrap_or_else(|p| p.into_inner())
3359 .nonce(ctx.connection_id)
3360 .map(str::to_owned);
3361 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3362 self.supervisor
3363 .spawned_consumer_authorized(&module_id, nonce)
3364 });
3365 let confirms = self.forwarding.operator_confirms();
3366 self.forwarding
3367 .with_operator_route(
3368 ctx.connection_id,
3369 request.route_channel,
3370 request.route_epoch,
3371 |binding| {
3372 let shortcut = binding
3373 .filter(|binding| {
3374 nonce_proven
3375 && crate::operator_confirm::valid_summary(&request.summary)
3376 && declared
3377 && crate::operator_confirm::ShortcutContext::route_eligible(
3378 &self.operator_app_policy,
3379 &self.launch_table,
3380 binding,
3381 )
3382 })
3383 .map(|binding| {
3384 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3385 &self.operator_app_policy,
3386 binding,
3387 );
3388 let stamp = if phone.is_some() {
3389 None
3390 } else {
3391 binding.operator_app.clone()
3392 };
3393 self.shortcut_context(
3394 stamp,
3395 phone,
3396 Some((request.route_channel, request.route_epoch)),
3397 )
3398 });
3399 confirms.admit_with_shortcut(
3400 ctx,
3401 frame,
3402 module_id,
3403 nonce_proven,
3404 request,
3405 binding,
3406 shortcut,
3407 )
3408 },
3409 )
3410 .map_err(RouterError::Forwarding)
3411 }
3412
3413 fn shortcut_context(
3414 &self,
3415 stamp: Option<crate::operator_apps::OperatorAppStamp>,
3416 phone_fingerprint: Option<String>,
3417 route: Option<(u16, u32)>,
3418 ) -> crate::operator_confirm::ShortcutContext {
3419 crate::operator_confirm::ShortcutContext {
3420 ledger: self.activity.clone(),
3421 forwarding: self.forwarding.clone(),
3422 launches: self.launch_table.clone(),
3423 stamp,
3424 phone_fingerprint,
3425 route,
3426 connections: self.operator_apps.clone(),
3427 #[cfg(test)]
3428 hooks: Default::default(),
3429 }
3430 }
3431
3432 #[cfg(any(test, feature = "test-support"))]
3434 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3435 self.operator_app_policy.phone_enabled = enabled;
3436 self
3437 }
3438
3439 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3442 let mut scopes = ScopeRecords::from(scopes);
3443 for (index, record) in scopes.records.iter_mut().enumerate() {
3444 if let Some(refusal) = Self::canonicalize_workdir(record) {
3445 scopes.refusals.insert(index, refusal);
3446 }
3447 }
3448 scopes
3449 }
3450
3451 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3452 let workdir = record.workdir.as_mut()?;
3453 let path = std::path::Path::new(workdir);
3454 if !path.is_absolute() {
3455 return Some((
3456 error_codes::INVALID_WORKDIR,
3457 "workdir must be absolute".to_string(),
3458 ));
3459 }
3460 match ProjectRootId::from_path_allowing_missing(path) {
3461 Ok(root) => match root.as_path().to_str() {
3462 Some(canonical) => {
3463 *workdir = canonical.to_string();
3464 None
3465 }
3466 None => Some((
3467 error_codes::INVALID_WORKDIR,
3468 "canonical workdir is not UTF-8".to_string(),
3469 )),
3470 },
3471 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3472 }
3473 }
3474
3475 fn handle_scope_sync(
3479 &self,
3480 connection_id: ConnectionId,
3481 frame: Frame,
3482 generation: u64,
3483 scopes: Vec<ScopeRecord>,
3484 ) -> Result<Vec<Frame>, RouterError> {
3485 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3486 }
3487
3488 fn handle_scope_change(
3489 &self,
3490 connection_id: ConnectionId,
3491 frame: Frame,
3492 generation: u64,
3493 scopes: Vec<ScopeRecord>,
3494 end: Option<Vec<ScopeEnd>>,
3495 ) -> Result<Vec<Frame>, RouterError> {
3496 let is_apply = end.is_some();
3497 let Some(registration) = self
3498 .registry
3499 .get_module_by_connection(connection_id)
3500 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3501 else {
3502 return Ok(vec![control_error_frame(
3503 &frame,
3504 "not_registered",
3505 "scope.sync and scope.apply require an active module registration owned by this connection",
3506 )?]);
3507 };
3508 let owner = registration.manifest.module_id;
3509 let scopes = Self::prepare_scope_records(scopes);
3510 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3511 let is_current_launch = |connection: ConnectionId| {
3512 self.hello_launch_nonces
3513 .lock()
3514 .unwrap_or_else(|poisoned| poisoned.into_inner())
3515 .presented(connection, current_nonce.as_deref())
3516 };
3517 let mut table = self
3522 .scopes
3523 .write()
3524 .unwrap_or_else(|poisoned| poisoned.into_inner());
3525 let outcome = match end {
3526 Some(end) => table.apply(
3527 &owner,
3528 connection_id,
3529 is_current_launch,
3530 generation,
3531 scopes,
3532 end,
3533 ),
3534 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3535 };
3536 let drained = match &outcome {
3537 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3538 Err(_) => Vec::new(),
3539 };
3540 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3541 drop(table);
3542 if has_deadlines {
3543 self.scope_deadline_added.notify_one();
3544 }
3545 match outcome {
3546 Ok(applied) => {
3547 let counts = ScopeOutcomeCounts::of(&applied.results);
3548 info!(
3549 owner = %owner,
3550 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3551 generation,
3552 records = applied.results.len(),
3553 created = counts.created,
3554 replaced = counts.replaced,
3555 updated = counts.updated,
3556 unchanged = counts.unchanged,
3557 refused = counts.refused,
3558 ended = applied.ended.len(),
3559 tag_changes = applied.tag_changes.len(),
3560 routes_closed = drained.len(),
3561 "scope change accepted"
3562 );
3563 for refused in applied
3569 .results
3570 .iter()
3571 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3572 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3573 {
3574 warn!(
3575 owner = %owner,
3576 generation,
3577 scope_ref = %refused.scope_ref,
3578 scope_epoch = refused.scope_epoch,
3579 code = refused.code.as_deref().unwrap_or(""),
3580 "scope record refused"
3581 );
3582 }
3583 self.close_scope_drained_routes(drained);
3584 let response = if is_apply {
3585 ModuleControlResponseToModule::ScopeApply {
3586 generation,
3587 results: applied.results,
3588 end_results: applied.end_results,
3589 ended: applied.ended,
3590 }
3591 } else {
3592 ModuleControlResponseToModule::ScopeSync {
3593 generation,
3594 results: applied.results,
3595 ended: applied.ended,
3596 }
3597 };
3598 Ok(vec![control_response_body_frame(
3599 &frame,
3600 &response,
3601 "scope change response",
3602 )?])
3603 }
3604 Err(refusal) => {
3605 info!(
3606 owner = %owner,
3607 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3608 generation,
3609 code = refusal.code,
3610 "scope change refused"
3611 );
3612 Ok(vec![control_error_frame(
3613 &frame,
3614 refusal.code,
3615 refusal.message,
3616 )?])
3617 }
3618 }
3619 }
3620
3621 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3627 if drained.is_empty() {
3628 return;
3629 }
3630 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3631 BTreeMap::new();
3632 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3633 for route in drained {
3634 warn!(
3635 module_id = %route.module_id,
3636 reason = ?route.reason,
3637 client_connection_id = route.client.connection_id.get(),
3638 route_channel = route.client.channel,
3639 "closing route because its scope changed"
3640 );
3641 pushes
3642 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3643 .or_insert_with(|| (route.reason, Vec::new()))
3644 .1
3645 .push(EndpointRoute {
3646 goodbye_target: route.client.clone(),
3647 principal: Principal::Unverified,
3648 bound_at: Instant::now(),
3649 draining: false,
3650 drain_reason: None,
3651 });
3652 goodbyes.push(route.module);
3653 goodbyes.push(route.client);
3654 }
3655 for ((module_id, _), (reason, routes)) in pushes {
3656 send_route_control_pushes(
3657 &self.forwarding,
3658 routes,
3659 ClientControlPush::RouteClosed {
3660 module_id,
3661 channels: Vec::new(),
3662 reason,
3663 drained: false,
3664 abandoned: 0,
3665 excluded_subscriptions: 0,
3666 terminal: Some(false),
3667 },
3668 );
3669 }
3670 self.emit_route_goodbyes(goodbyes);
3671 }
3672
3673 fn handle_scope_describe(
3676 &self,
3677 connection_id: ConnectionId,
3678 frame: Frame,
3679 owner: Principal,
3680 scope_ref: String,
3681 ) -> Result<Vec<Frame>, RouterError> {
3682 let registered = self
3683 .registry
3684 .get_module_by_connection(connection_id)
3685 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3686 let Some(registered) = registered else {
3687 return Ok(vec![control_error_frame(
3688 &frame,
3689 "not_registered",
3690 "scope.describe requires an active module registration owned by this connection",
3691 )?]);
3692 };
3693 let mut description = self
3694 .scopes
3695 .read()
3696 .unwrap_or_else(|poisoned| poisoned.into_inner())
3697 .describe(&owner, &scope_ref);
3698 if !registered
3701 .manifest
3702 .capabilities
3703 .as_ref()
3704 .is_some_and(|caps| {
3705 caps.provides
3706 .iter()
3707 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3708 })
3709 {
3710 if let Some(stamp) = description.stamp.as_mut() {
3711 stamp.attributes.on_behalf_of_agent = None;
3712 }
3713 }
3714 if !registered
3715 .manifest
3716 .capabilities
3717 .as_ref()
3718 .is_some_and(|caps| {
3719 caps.provides
3720 .iter()
3721 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3722 })
3723 {
3724 if let Some(stamp) = description.stamp.as_mut() {
3725 stamp.attributes.flow_version = None;
3726 stamp.attributes.flow_code_hash = None;
3727 stamp.attributes.flow_install_id = None;
3728 }
3729 }
3730 let owner_configured = match &owner {
3731 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3736 _ => false,
3737 };
3738 let response = ModuleControlResponseToModule::ScopeDescribe {
3739 status: description.status,
3740 scope_epoch: description.scope_epoch,
3741 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3742 owner_synced: description.owner_synced,
3743 owner_configured,
3744 scope: description.stamp,
3745 };
3746 Ok(vec![control_response_body_frame(
3747 &frame,
3748 &response,
3749 "ModuleControlResponseToModule::ScopeDescribe",
3750 )?])
3751 }
3752
3753 fn handle_catalog_update(
3754 &self,
3755 connection_id: ConnectionId,
3756 frame: Frame,
3757 provides: Vec<ProviderRole>,
3758 capabilities: Option<CapabilityDeclarations>,
3759 ready: Option<bool>,
3760 ) -> Result<Vec<Frame>, RouterError> {
3761 self.refresh_capability_requirements();
3762 let Some(registration) = self
3763 .registry
3764 .get_module_by_connection(connection_id)
3765 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3766 else {
3767 return Ok(vec![control_error_frame(
3768 &frame,
3769 "not_registered",
3770 "catalog.update requires an active module registration owned by this connection",
3771 )?]);
3772 };
3773
3774 if let Some(message) =
3775 catalog_update_frozen_field_message(®istration.manifest, &provides)
3776 {
3777 return Ok(vec![control_error_frame(
3778 &frame,
3779 "catalog_update_frozen_field",
3780 message,
3781 )?]);
3782 }
3783
3784 let mut candidate = registration.manifest.clone();
3785 candidate.provides = provides.clone();
3786 candidate.capabilities = capabilities
3787 .clone()
3788 .or_else(|| registration.manifest.capabilities.clone());
3789 if let Some(rich_ui) = &candidate.rich_ui {
3790 if let Err(err) = rich_ui.validate() {
3791 return Ok(vec![control_error_frame(
3792 &frame,
3793 "invalid_manifest",
3794 err.to_string(),
3795 )?]);
3796 }
3797 }
3798 if let Err(err) = candidate.validate_capability_grammar() {
3799 return Ok(vec![control_error_frame(
3800 &frame,
3801 "invalid_capability_grammar",
3802 err.to_string(),
3803 )?]);
3804 }
3805
3806 let mut conflicts = self
3809 .capability_evaluator
3810 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3811 if let Some(conflict) = conflicts.first() {
3812 let message = format!(
3813 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3814 conflict.capability, conflict.claimants[0], candidate.module_id
3815 );
3816 for conflict in &mut conflicts {
3817 conflict.source = DuplicateClaimSource::CatalogUpdate;
3818 }
3819 log_duplicate_claim_events(conflicts);
3820 return Ok(vec![control_error_frame(
3821 &frame,
3822 "reserved_capability",
3823 message,
3824 )?]);
3825 }
3826
3827 let updated = self
3828 .registry
3829 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3830 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3831 if updated.is_none() {
3832 return Ok(vec![control_error_frame(
3833 &frame,
3834 "not_registered",
3835 "catalog.update requires an active module registration owned by this connection",
3836 )?]);
3837 }
3838 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3839 log_duplicate_claim_events(
3840 self.capability_evaluator
3841 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3842 );
3843 }
3844 if capability_census_trigger(
3845 registration.manifest.capabilities.as_ref(),
3846 updated
3847 .as_ref()
3848 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3849 ) {
3850 self.enforce_capability_denies();
3851 }
3852 self.refresh_capability_requirements();
3853
3854 let response = ModuleControlResponseToModule::CatalogUpdate {};
3855 control_response_body_frame(
3856 &frame,
3857 &response,
3858 "ModuleControlResponseToModule::CatalogUpdate",
3859 )
3860 .map(|frame| vec![frame])
3861 }
3862
3863 pub(crate) fn with_activity_ledger_config(
3864 mut self,
3865 run_dir: &Path,
3866 incarnation: String,
3867 config: crate::activity::ActivityConfig,
3868 ) -> Self {
3869 self.activity = Some(crate::activity::ActivityLedger::open_config(
3870 run_dir.join("activity.jsonl"),
3871 incarnation,
3872 config,
3873 ));
3874 self.supervisor.set_activity(self.activity.clone());
3875 self
3876 }
3877
3878 pub(crate) async fn configure_activity(
3879 &self,
3880 config: crate::activity::ActivityConfig,
3881 ) -> Result<(), &'static str> {
3882 match &self.activity {
3883 Some(ledger) => ledger.configure(config).await,
3884 None => Ok(()),
3885 }
3886 }
3887
3888 fn cancel_activity_subscription(
3889 &self,
3890 ctx: &RouteCtx,
3891 frame: &Frame,
3892 ) -> Result<bool, RouterError> {
3893 let task = self
3894 .activity_streams
3895 .lock()
3896 .unwrap_or_else(|p| p.into_inner())
3897 .remove(&(ctx.connection_id, frame.header.corr));
3898 let Some(task) = task else {
3899 return Ok(false);
3900 };
3901 task.abort();
3902 let end = Frame::build_with_version(
3903 response_version(frame),
3904 FrameType::StreamEnd,
3905 control_flags(),
3906 0,
3907 0,
3908 frame.header.corr,
3909 Vec::new(),
3910 )
3911 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3912 let sink = ctx.egress.clone();
3913 tokio::spawn(async move {
3914 let _ = sink.send(end).await;
3915 });
3916 Ok(true)
3917 }
3918
3919 async fn handle_activity_read(
3920 &self,
3921 ctx: &RouteCtx,
3922 frame: Frame,
3923 ) -> Result<Vec<Frame>, RouterError> {
3924 use subc_control::{
3925 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3926 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3927 };
3928 #[derive(Deserialize)]
3929 struct Identity {
3930 #[serde(default)]
3931 consumer_identity: Option<ConsumerIdentity>,
3932 }
3933 let principal = match self
3937 .forwarding
3938 .module_id_for_connection(ctx.connection_id)
3939 .map_err(RouterError::Forwarding)?
3940 {
3941 Some(module_id) => Ok(Principal::Reserved { module_id }),
3942 None => {
3943 let identity: Identity = match serde_json::from_slice(&frame.body) {
3944 Ok(identity) => identity,
3945 Err(_) => {
3946 return Ok(vec![control_error_frame(
3947 &frame,
3948 ACTIVITY_MALFORMED,
3949 "malformed activity read",
3950 )?])
3951 }
3952 };
3953 self.route_open_principal(&frame, identity.consumer_identity)?
3954 }
3955 };
3956 let admitted = matches!(principal, Ok(Principal::Direct))
3957 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3958 if !admitted {
3959 return Ok(vec![control_error_frame(
3960 &frame,
3961 ACTIVITY_READ_FORBIDDEN,
3962 "activity reads require an operator or trusted mirror principal",
3963 )?]);
3964 }
3965 let Some(ledger) = &self.activity else {
3966 return Ok(vec![control_error_frame(
3967 &frame,
3968 ACTIVITY_UNAVAILABLE,
3969 "activity ledger unavailable",
3970 )?]);
3971 };
3972 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3973 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3974 {
3975 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3976 Ok(request) => ledger.query(request).await,
3977 Err(_) => Err(ACTIVITY_MALFORMED),
3978 };
3979 return match result {
3980 Ok(reply) => Ok(vec![control_response_body_frame(
3981 &frame,
3982 &reply,
3983 ops::ACTIVITY_QUERY,
3984 )?]),
3985 Err(code) => Ok(vec![control_error_frame(
3986 &frame,
3987 code,
3988 "activity query refused",
3989 )?]),
3990 };
3991 }
3992 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3993 Ok(request) => request,
3994 Err(_) => {
3995 return Ok(vec![control_error_frame(
3996 &frame,
3997 ACTIVITY_MALFORMED,
3998 "malformed activity subscription",
3999 )?])
4000 }
4001 };
4002 let mut subscription = match ledger.subscribe(request.since).await {
4003 Ok(subscription) => subscription,
4004 Err(code) => {
4005 return Ok(vec![control_error_frame(
4006 &frame,
4007 code,
4008 "activity subscription refused",
4009 )?])
4010 }
4011 };
4012 let key = (ctx.connection_id, frame.header.corr);
4013 let sink = ctx.egress.clone();
4014 let mut streams = self
4015 .activity_streams
4016 .lock()
4017 .unwrap_or_else(|p| p.into_inner());
4018 streams.retain(|_, task| !task.is_finished());
4019 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4020 return Ok(vec![control_error_frame(
4021 &frame,
4022 "duplicate_subscription",
4023 "activity subscription correlation id already in use",
4024 )?]);
4025 }
4026 let task = tokio::spawn(async move {
4027 loop {
4028 let entry = if let Some(entry) = subscription.replay.pop_front() {
4029 Some(entry)
4030 } else {
4031 subscription.receiver.recv().await
4032 };
4033 let Some(entry) = entry else {
4034 break;
4035 };
4036 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4037 return;
4038 };
4039 let Ok(next) = Frame::build_with_version(
4040 response_version(&frame),
4041 FrameType::StreamData,
4042 control_flags(),
4043 0,
4044 0,
4045 frame.header.corr,
4046 body,
4047 ) else {
4048 return;
4049 };
4050 if sink.send(next).await.is_err() {
4051 return;
4052 }
4053 }
4054 if let Some(code) = subscription.terminal_code() {
4057 if let Ok(end) =
4058 control_error_frame(&frame, code, "activity subscriber fell behind")
4059 {
4060 let _ = sink.send(end).await;
4061 }
4062 }
4063 });
4064 streams.insert(key, task.abort_handle());
4065 Ok(Vec::new())
4066 }
4067
4068 async fn handle_activity_record(
4069 &self,
4070 ctx: &RouteCtx,
4071 frame: Frame,
4072 ) -> Result<Vec<Frame>, RouterError> {
4073 use subc_control::{
4074 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4075 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4076 };
4077 let result = async {
4078 let module = self
4079 .forwarding
4080 .module_id_for_connection(ctx.connection_id)
4081 .ok()
4082 .flatten()
4083 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4084 let request: ActivityRecordRequest =
4085 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4086 crate::activity::validate(&module, &request)?;
4087 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4088 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4089 let at_ms = std::time::SystemTime::now()
4090 .duration_since(std::time::UNIX_EPOCH)
4091 .unwrap_or_default()
4092 .as_millis() as u64;
4093 ledger.record(module, request, actor, at_ms).await
4094 }
4095 .await;
4096 match result {
4097 Ok(entry) => Ok(vec![control_response_body_frame(
4098 &frame,
4099 &ActivityRecordResponse { seq: entry.seq },
4100 "activity.record",
4101 )?]),
4102 Err(code) => Ok(vec![control_error_frame(
4103 &frame,
4104 code,
4105 "activity recording refused",
4106 )?]),
4107 }
4108 }
4109
4110 fn activity_actor(
4111 &self,
4112 connection: ConnectionId,
4113 module: &str,
4114 on: Option<&subc_control::ActivityOn>,
4115 ) -> Result<subc_control::ActivityActor, &'static str> {
4116 use crate::forwarding::{DataRoute, DataRouteState};
4117 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4118 let principal = |id: &str| Principal::Reserved {
4119 module_id: id.to_owned(),
4120 };
4121 let mut actor = ActivityActor {
4122 principal: principal(module),
4123 scope_owner: None,
4124 agent_id: None,
4125 run_id: None,
4126 flow_id: None,
4127 project_root: None,
4128 workdir: None,
4129 project_id: None,
4130 };
4131 let scope = match on {
4132 None => return Ok(actor),
4133 Some(ActivityOn::Scope(scope)) => {
4134 let selector = subc_protocol::scope::ScopeSelector::new(
4135 principal(module),
4136 scope.scope_ref.clone(),
4137 Some(scope.epoch),
4138 );
4139 self.scopes
4140 .read()
4141 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4142 .attribute(&principal(module), module, &selector, true)
4143 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4144 }
4145 Some(ActivityOn::Route(route)) => {
4146 let DataRoute::Module(DataRouteState::Bound(binding)) = self
4147 .forwarding
4148 .lookup_data_route(connection, route.channel, route.epoch)
4149 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4150 else {
4151 return Err(ACTIVITY_ACTOR_UNKNOWN);
4152 };
4153 if binding.module_id != module {
4154 return Err(ACTIVITY_ACTOR_UNKNOWN);
4155 }
4156 actor.principal = binding.principal.clone();
4157 actor.project_root = binding
4158 .project_root
4159 .as_ref()
4160 .map(|root| root.as_path().to_path_buf());
4161 let Some(bound) = &binding.scope else {
4162 return Ok(actor);
4163 };
4164 let owner = principal(&bound.owner);
4165 let selector = subc_protocol::scope::ScopeSelector::new(
4166 owner.clone(),
4167 bound.scope_ref.clone(),
4168 Some(bound.tag.scope_epoch),
4169 );
4170 let admitted = self
4171 .scopes
4172 .read()
4173 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4174 .attribute(&owner, module, &selector, true)
4175 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4176 if admitted.tag != bound.tag {
4177 return Err(ACTIVITY_ACTOR_UNKNOWN);
4178 }
4179 admitted
4180 }
4181 };
4182 actor.scope_owner = Some(scope.stamp.owner);
4183 actor.agent_id = scope.stamp.attributes.agent_id;
4184 actor.run_id = scope.stamp.attributes.run_id;
4185 actor.flow_id = scope.stamp.attributes.flow_id;
4186 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4187 actor.project_id = scope.stamp.project_id;
4188 Ok(actor)
4189 }
4190
4191 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4192 self.refresh_capability_requirements();
4193 let mut counters = self.counters.snapshot();
4199 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4200 obj.insert(
4201 "activity_unparseable_lines".into(),
4202 ledger.read_errors().into(),
4203 );
4204 obj.insert(
4205 "activity_lifecycle_append_failures".into(),
4206 ledger.lifecycle_failures().into(),
4207 );
4208 obj.insert(
4209 "activity_started_digest_skips".into(),
4210 ledger.lifecycle_digest_skips().into(),
4211 );
4212 }
4213 if let (Ok((connections_with_routes, max)), Some(obj)) = (
4214 self.forwarding.client_route_concentration(),
4215 counters.as_object_mut(),
4216 ) {
4217 obj.insert(
4218 "client_connections_with_routes".into(),
4219 connections_with_routes.into(),
4220 );
4221 obj.insert("max_routes_on_one_connection".into(), max.into());
4222 }
4223 if let (Some(open_breakers), Some(obj)) = (
4229 self.route_bind_breakers.open_snapshot(),
4230 counters.as_object_mut(),
4231 ) {
4232 obj.insert("route_bind_breakers_open".into(), open_breakers);
4233 }
4234 if let (Some(reason), Some(obj)) = (
4239 self.local_endpoint_unavailable.clone(),
4240 counters.as_object_mut(),
4241 ) {
4242 obj.insert("local_endpoint_unavailable".into(), reason.into());
4243 }
4244 let response = ClientControlResponse::ServerDescribe {
4245 protocol_ver: PROTOCOL_VERSION,
4246 subc_ops: subc_ops(),
4247 capabilities: self.describe_capabilities(),
4248 connected_clients: self.connected_clients.count(),
4249 counters: Some(counters),
4250 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4251 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4252 capability_requirements: self.capability_requirement_statuses(),
4253 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4254 config_path: self
4255 .rescan
4256 .as_ref()
4257 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4258 };
4259 Ok(vec![control_response_body_frame(
4260 &frame,
4261 &response,
4262 "ClientControlResponse::ServerDescribe",
4263 )?])
4264 }
4265
4266 fn handle_catalog_list(
4267 &self,
4268 frame: Frame,
4269 module_id: Option<String>,
4270 ) -> Result<Vec<Frame>, RouterError> {
4271 let (generation, modules) = self
4272 .registry
4273 .list_modules_with_nonce_verification()
4274 .map_err(|err| {
4275 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4276 })?;
4277 let entries = modules
4278 .into_iter()
4279 .filter(|(registration, _)| {
4280 module_id
4281 .as_deref()
4282 .map(|wanted| registration.manifest.module_id == wanted)
4283 .unwrap_or(true)
4284 })
4285 .map(|(registration, nonce_verified)| {
4286 let not_ready = self.not_ready_reason(®istration);
4287 let roles = registration.manifest.provides;
4288 CatalogEntry::new(
4289 registration.manifest.module_id,
4290 roles,
4291 registration.control_ops,
4292 )
4293 .with_ready(not_ready.is_none())
4294 .with_not_ready(not_ready)
4295 .with_module_version(Some(registration.manifest.module_version))
4296 .with_capabilities(registration.manifest.capabilities)
4297 .with_self_signals(registration.manifest.self_signals)
4298 .with_events(registration.manifest.events)
4299 .with_operation_delivery(registration.manifest.operation_delivery)
4300 .with_operation_payload(registration.manifest.operation_payload)
4301 .with_flow_operations(registration.manifest.flow_operations)
4302 .with_confirm_operations(registration.manifest.confirm_operations)
4303 .with_rich_ui(registration.manifest.rich_ui)
4304 .with_role_declarations(registration.manifest.role_declarations)
4305 .with_nonce_verified(nonce_verified)
4306 })
4307 .collect();
4308 let response = ClientControlResponse::CatalogList {
4309 generation,
4310 modules: entries,
4311 subc_ops: subc_ops(),
4312 };
4313 Ok(vec![control_response_body_frame(
4314 &frame,
4315 &response,
4316 "ClientControlResponse::CatalogList",
4317 )?])
4318 }
4319
4320 fn attest_route_open_consumer(
4325 &self,
4326 consumer_identity: Option<&ConsumerIdentity>,
4327 ) -> Option<Principal> {
4328 let Some(consumer_identity) = consumer_identity else {
4329 return Some(Principal::Direct);
4330 };
4331 self.supervisor
4332 .spawned_consumer_authorized(
4333 &consumer_identity.module_id,
4334 &consumer_identity.launch_nonce,
4335 )
4336 .then(|| Principal::Reserved {
4337 module_id: consumer_identity.module_id.clone(),
4338 })
4339 }
4340
4341 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4345 let Ok(ClientControlRequest::RouteOpen {
4346 identity,
4347 consumer_identity,
4348 scope,
4349 ..
4350 }) = parse_client_control_request(&frame.body)
4351 else {
4352 return RouteOpenCaller::unknown();
4353 };
4354 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4355 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4356 }
4357
4358 fn route_open_principal(
4359 &self,
4360 frame: &Frame,
4361 consumer_identity: Option<ConsumerIdentity>,
4362 ) -> Result<Result<Principal, Frame>, RouterError> {
4363 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4364 self.route_open_principal_attested(frame, consumer_identity, attested)
4365 }
4366
4367 fn route_open_principal_attested(
4371 &self,
4372 frame: &Frame,
4373 consumer_identity: Option<ConsumerIdentity>,
4374 attested: Option<Principal>,
4375 ) -> Result<Result<Principal, Frame>, RouterError> {
4376 if let Some(principal) = attested {
4377 return Ok(Ok(principal));
4378 }
4379 let Some(consumer_identity) = consumer_identity else {
4380 return Ok(Ok(Principal::Direct));
4381 };
4382
4383 Ok(Err(control_error_frame(
4384 frame,
4385 "bad_consumer_identity",
4386 format!(
4387 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4388 consumer_identity.module_id
4389 ),
4390 )?))
4391 }
4392
4393 #[allow(clippy::too_many_arguments)]
4400 fn route_open_refusal_frame(
4401 &self,
4402 ctx: &RouteCtx,
4403 caller: &RouteOpenCaller,
4404 frame: &Frame,
4405 module_id: &str,
4406 reason: &'static str,
4407 code: &'static str,
4408 message: impl Into<String>,
4409 ) -> Result<Frame, RouterError> {
4410 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4411 control_error_frame(frame, code, message.into())
4412 }
4413
4414 #[allow(clippy::too_many_arguments)]
4423 fn route_bind_rejection(
4424 &self,
4425 ctx: &RouteCtx,
4426 caller: &RouteOpenCaller,
4427 frame: &Frame,
4428 target_module_id: &str,
4429 body: ErrorBody,
4430 daemon_authored: bool,
4431 ) -> Result<Vec<Frame>, RouterError> {
4432 if daemon_authored {
4438 let scope_code = match body.code.as_str() {
4439 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4440 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4441 _ => None,
4442 };
4443 if let Some(code) = scope_code {
4444 self.observe_route_open_refusal(
4445 ctx,
4446 caller,
4447 target_module_id,
4448 "scope_changed_before_commit",
4449 code,
4450 );
4451 return Ok(vec![control_error_body_frame(frame, body)?]);
4452 }
4453 }
4454 self.counters
4455 .increment_route_open_refused("module_rejected");
4456 info!(
4457 target: "control",
4458 code = "module_rejected",
4459 module_code = ?body.code,
4460 module_id = ?target_module_id,
4461 connection_id = ctx.connection_id.get(),
4462 principal = %caller.principal,
4463 scope = %caller.scope,
4464 harness = %caller.harness,
4465 "route.open refused"
4466 );
4467 let refusal = if daemon_authored {
4468 control_error_body_frame(frame, body)?
4469 } else {
4470 control_relayed_error_body_frame(frame, body)?
4471 };
4472 Ok(vec![refusal])
4473 }
4474
4475 #[allow(clippy::too_many_arguments)]
4493 fn route_open_breaker_refusal_frame(
4494 &self,
4495 ctx: &RouteCtx,
4496 caller: &RouteOpenCaller,
4497 frame: &Frame,
4498 module_id: &str,
4499 consecutive_timeouts: u32,
4500 retry_in: Duration,
4501 probe_in_flight: bool,
4502 ) -> Result<Frame, RouterError> {
4503 self.counters
4504 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4505 debug!(
4506 target: "control",
4507 code = "module_timeout",
4508 module_id = ?module_id,
4509 connection_id = ctx.connection_id.get(),
4510 principal = %caller.principal,
4511 scope = %caller.scope,
4512 harness = %caller.harness,
4513 consecutive_timeouts,
4514 retry_in_ms = retry_in.as_millis() as u64,
4515 probe_in_flight,
4516 "route.open refused by open bind-relay breaker"
4517 );
4518 let detail = if probe_in_flight {
4525 "one test route is already being tried; retry once it settles".to_string()
4526 } else {
4527 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4528 };
4529 control_error_frame(
4530 frame,
4531 "module_timeout",
4532 format!(
4533 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4534 timed out in a row); {detail}; its established routes are unaffected"
4535 ),
4536 )
4537 }
4538
4539 fn observe_route_open_refusal(
4552 &self,
4553 ctx: &RouteCtx,
4554 caller: &RouteOpenCaller,
4555 module_id: &str,
4556 reason: &'static str,
4557 code: &'static str,
4558 ) {
4559 self.counters.increment_route_open_refused(code);
4560 info!(
4561 target: "control",
4562 code,
4563 reason,
4564 module_id = ?module_id,
4565 connection_id = ctx.connection_id.get(),
4566 principal = %caller.principal,
4567 scope = %caller.scope,
4568 harness = %caller.harness,
4569 "route.open refused"
4570 );
4571 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4572 self.route_outages.record_not_serving(module_id, reason);
4573 }
4574 }
4575
4576 fn observe_route_open_accept(
4621 &self,
4622 ctx: &RouteCtx,
4623 module_id: &str,
4624 principal: &str,
4625 scope: &str,
4626 ) {
4627 self.route_outages.record_accepted(module_id);
4628 self.counters.increment_route_open_accepted(principal);
4629 info!(
4630 target: "control",
4631 principal,
4632 module_id,
4633 connection_id = ctx.connection_id.get(),
4634 scope = %scope,
4635 "route.open accepted"
4636 );
4637 }
4638
4639 fn supervised_absent_route_open_refusal_frame(
4640 &self,
4641 ctx: &RouteCtx,
4642 caller: &RouteOpenCaller,
4643 frame: &Frame,
4644 module_id: &str,
4645 code: &'static str,
4646 status: &crate::supervise::ModuleStatus,
4647 ) -> Result<Frame, RouterError> {
4648 self.counters.increment_route_open_refused(code);
4649 info!(
4650 target: "control",
4651 code,
4652 reason = "supervised_not_registered",
4653 module_id = ?module_id,
4654 connection_id = ctx.connection_id.get(),
4655 principal = %caller.principal,
4656 scope = %caller.scope,
4657 harness = %caller.harness,
4658 state = %status.state,
4659 enabled = status.enabled,
4660 live = status.live,
4661 "route.open refused"
4662 );
4663 self.route_outages
4667 .record_not_serving(module_id, "supervised_not_registered");
4668 let message = if let Some(pid) = status.waiting_for_previous_pid {
4669 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4670 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4671 })
4672 } else {
4673 format!(
4674 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4675 status.state, status.enabled, status.live
4676 )
4677 };
4678 control_error_frame(frame, code, message)
4679 }
4680
4681 async fn handle_route_open(
4682 &self,
4683 ctx: &RouteCtx,
4684 frame: Frame,
4685 request: RouteOpenRequest,
4686 timings: &mut RouteOpenTimings,
4687 ) -> Result<Vec<Frame>, RouterError> {
4688 let RouteOpenRequest {
4689 target,
4690 mut identity,
4691 consumer_identity,
4692 consumer_capabilities,
4693 role_versions,
4694 admission_facts,
4695 scope,
4696 } = request;
4697 let target_module_id = target_module_id(&target).to_string();
4698 timings.module_id.clone_from(&target_module_id);
4699 if self
4700 .registry
4701 .get_module_by_connection(ctx.connection_id)
4702 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4703 .is_some()
4704 {
4705 return Ok(vec![control_error_frame(
4706 &frame,
4707 "invalid_request",
4708 "module connections cannot open client routes",
4709 )?]);
4710 }
4711 debug!(
4712 connection_id = ctx.connection_id.get(),
4713 corr = frame.header.corr,
4714 module_id = %target_module_id,
4715 "handling route.open"
4716 );
4717
4718 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4723 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4724
4725 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4731 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4732 self.observe_route_open_refusal(
4733 ctx,
4734 &caller,
4735 &target_module_id,
4736 "invalid_role_versions",
4737 error_codes::INVALID_REQUEST,
4738 );
4739 return Ok(vec![control_error_body_frame(
4740 &frame,
4741 ErrorBody {
4742 code: error_codes::INVALID_REQUEST.to_string(),
4743 message: error.to_string(),
4744 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4745 },
4746 )?]);
4747 }
4748
4749 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4750 if status.waiting_for_previous_pid.is_some() {
4751 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4752 ctx,
4753 &caller,
4754 &frame,
4755 &target_module_id,
4756 "module_warming",
4757 &status,
4758 )?]);
4759 }
4760 }
4761
4762 let Some(registration) = self
4781 .registry
4782 .get_module(&target_module_id)
4783 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4784 else {
4785 if let Some((status, warming)) =
4786 self.supervisor_status(&target_module_id, frame.header.corr)?
4787 {
4788 if status.protocol == ModuleProtocol::None {
4795 return Ok(vec![self.route_open_refusal_frame(
4796 ctx,
4797 &caller,
4798 &frame,
4799 &target_module_id,
4800 "protocol_none",
4801 error_codes::MODULE_NO_PROTOCOL,
4802 format!(
4803 "module_id '{target_module_id}' is declared protocol: none; \
4804 it speaks no subc wire and serves no routes"
4805 ),
4806 )?]);
4807 }
4808 let code = if warming {
4809 "module_warming"
4810 } else {
4811 "target_unavailable"
4812 };
4813 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4814 ctx,
4815 &caller,
4816 &frame,
4817 &target_module_id,
4818 code,
4819 &status,
4820 )?]);
4821 }
4822 if let Some(removed_ago_ms) =
4823 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4824 {
4825 return Ok(vec![self.route_open_refusal_frame(
4826 ctx,
4827 &caller,
4828 &frame,
4829 &target_module_id,
4830 "removed",
4831 error_codes::MODULE_REMOVED,
4832 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4833 )?]);
4834 }
4835 return Ok(vec![self.route_open_refusal_frame(
4836 ctx,
4837 &caller,
4838 &frame,
4839 &target_module_id,
4840 "not_registered",
4841 error_codes::UNKNOWN_MODULE,
4842 format!("module_id '{target_module_id}' is not registered"),
4843 )?]);
4844 };
4845
4846 if !registration.ready {
4850 self.counters
4851 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4852 info!(
4853 target: "control",
4854 code = error_codes::MODULE_WARMING,
4855 module_id = ?target_module_id,
4856 connection_id = ctx.connection_id.get(),
4857 principal = %caller.principal,
4858 scope = %caller.scope,
4859 harness = %caller.harness,
4860 reason = "declared_not_ready",
4861 "route.open refused"
4862 );
4863 self.route_outages
4866 .record_not_serving(&target_module_id, "declared_not_ready");
4867 return Ok(vec![control_error_body_frame(
4868 &frame,
4869 ErrorBody {
4870 code: error_codes::MODULE_WARMING.to_string(),
4871 message: format!(
4872 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4873 ),
4874 detail: Some(serde_json::json!({
4875 "reason": "declared_not_ready"
4876 })),
4877 },
4878 )?]);
4879 }
4880
4881 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4900 self.counters
4901 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4902 info!(
4903 target: "control",
4904 code = error_codes::MODULE_WARMING,
4905 module_id = ?target_module_id,
4906 connection_id = ctx.connection_id.get(),
4907 principal = %caller.principal,
4908 scope = %caller.scope,
4909 harness = %caller.harness,
4910 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4911 capability = %capability,
4912 "route.open refused"
4913 );
4914 return Ok(vec![control_error_body_frame(
4915 &frame,
4916 ErrorBody {
4917 code: error_codes::MODULE_WARMING.to_string(),
4918 message: format!(
4919 "module_id '{target_module_id}' requires capability '{capability}', \
4920 which no registered module provides; retry"
4921 ),
4922 detail: Some(serde_json::json!({
4923 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4924 "capability": capability,
4925 })),
4926 },
4927 )?]);
4928 }
4929
4930 if !target_has_required_role(&target, ®istration.manifest.provides) {
4931 return Ok(vec![self.route_open_refusal_frame(
4932 ctx,
4933 &caller,
4934 &frame,
4935 &target_module_id,
4936 "role_not_provided",
4937 "target_unavailable",
4938 format!("module_id '{target_module_id}' does not provide the requested target"),
4939 )?]);
4940 }
4941
4942 if registration.state != ChannelState::Active {
4943 return Ok(vec![self.route_open_refusal_frame(
4944 ctx,
4945 &caller,
4946 &frame,
4947 &target_module_id,
4948 "registration_not_active",
4949 "target_unavailable",
4950 format!("module_id '{target_module_id}' is not active"),
4951 )?]);
4952 }
4953
4954 if self
4955 .forwarding
4956 .module_is_draining(&target_module_id)
4957 .map_err(RouterError::Forwarding)?
4958 {
4959 return Ok(vec![self.route_open_refusal_frame(
4960 ctx,
4961 &caller,
4962 &frame,
4963 &target_module_id,
4964 "reloading",
4965 "module_reloading",
4966 format!("module_id '{target_module_id}' is reloading"),
4967 )?]);
4968 }
4969
4970 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4971 process_liveness.process_live(&target_module_id) == Some(false)
4972 }) {
4973 if process_liveness.process_replacing(&target_module_id) {
4980 return Ok(vec![self.route_open_refusal_frame(
4981 ctx,
4982 &caller,
4983 &frame,
4984 &target_module_id,
4985 "reloading",
4986 "module_reloading",
4987 format!("module_id '{target_module_id}' is reloading"),
4988 )?]);
4989 }
4990 return Ok(vec![self.route_open_refusal_frame(
4991 ctx,
4992 &caller,
4993 &frame,
4994 &target_module_id,
4995 "supervisor_not_live",
4996 "target_unavailable",
4997 format!("module_id '{target_module_id}' is not live"),
4998 )?]);
4999 }
5000
5001 if !self
5002 .forwarding
5003 .has_live_module_connection(&target_module_id)
5004 .map_err(RouterError::Forwarding)?
5005 {
5006 return Ok(vec![self.route_open_refusal_frame(
5007 ctx,
5008 &caller,
5009 &frame,
5010 &target_module_id,
5011 "no_forwarding_connection",
5012 "target_unavailable",
5013 format!("module_id '{target_module_id}' has no live forwarding connection"),
5014 )?]);
5015 }
5016
5017 if let Some(error) =
5018 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5019 {
5020 self.observe_route_open_refusal(
5021 ctx,
5022 &caller,
5023 &target_module_id,
5024 "op_not_allowed",
5025 "op_not_allowed",
5026 );
5027 return Ok(vec![error]);
5028 }
5029
5030 let principal =
5031 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5032 Ok(principal) => principal,
5033 Err(error) => {
5034 self.observe_route_open_refusal(
5035 ctx,
5036 &caller,
5037 &target_module_id,
5038 "bad_consumer_identity",
5039 "bad_consumer_identity",
5040 );
5041 return Ok(vec![error]);
5042 }
5043 };
5044 timings.principal = principal_label(&principal);
5045
5046 let build_class = identity.build_class.take().filter(|build_class| {
5049 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5050 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5051 } else {
5052 if self
5053 .build_class_warned
5054 .lock()
5055 .unwrap_or_else(|p| p.into_inner())
5056 .insert(ctx.connection_id)
5057 {
5058 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5059 "ignoring build_class from a non-Callosum opener");
5060 }
5061 false
5062 }
5063 });
5064
5065 if let Principal::Reserved {
5069 module_id: opening_module_id,
5070 } = &principal
5071 {
5072 if let Some(opening_registration) = self
5073 .registry
5074 .get_module(opening_module_id)
5075 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5076 {
5077 if let Some(capability) =
5078 denied_capability(&opening_registration.manifest, ®istration.manifest)
5079 {
5080 warn!(
5081 opening_module_id,
5082 target_module_id,
5083 capability,
5084 "refusing route.open because an attested capability deny edge matches"
5085 );
5086 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5087 &frame,
5088 &target_module_id,
5089 "capability_deny_edge",
5090 "capability_forbidden",
5091 format!(
5092 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5093 ),
5094 )?]);
5095 }
5096 }
5097 }
5098
5099 if admission_facts.is_some() {
5100 let carrier_matches = matches!(
5101 &principal,
5102 Principal::Reserved { module_id }
5103 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5104 );
5105 if !carrier_matches {
5106 return Ok(vec![self.route_open_refusal_frame(
5107 ctx,
5108 &caller,
5109 &frame,
5110 &target_module_id,
5111 "admission_facts_carrier_not_permitted",
5112 "admission_facts_not_permitted",
5113 "admission facts may only be carried by the configured reserved module",
5114 )?]);
5115 }
5116
5117 let target_allowed = self
5118 .admission_facts_targets
5119 .as_ref()
5120 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5121 if !target_allowed {
5122 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5123 &frame,
5124 &target_module_id,
5125 "admission_facts_target_not_listed",
5126 "admission_facts_target_not_allowed",
5127 format!(
5128 "admission facts are not permitted for target module_id '{target_module_id}'"
5129 ),
5130 )?]);
5131 }
5132
5133 }
5137
5138 let (bound_scope, mut scope_stamp) = match &scope {
5144 None => (None, None),
5145 Some(selector) => {
5146 let owner_configured = match &selector.owner {
5147 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5153 _ => false,
5154 };
5155 let mut scopes = self
5156 .scopes
5157 .write()
5158 .unwrap_or_else(|poisoned| poisoned.into_inner());
5159 let admitted = scopes
5160 .admit(&principal, &target_module_id, selector, owner_configured)
5161 .and_then(|admission| {
5162 crate::scopes::check_target_flow_support(
5163 &admission.stamp,
5164 &target_module_id,
5165 registration.manifest.capabilities.as_ref(),
5166 )?;
5167 crate::scopes::check_target_agent_run_support(
5168 &admission.stamp,
5169 &target_module_id,
5170 registration.manifest.capabilities.as_ref(),
5171 )?;
5172 let bound = scopes.reserve_bind(
5173 &admission,
5174 &principal,
5175 ctx.connection_id,
5176 selector.hold_presence,
5177 );
5178 Ok((admission, bound))
5179 });
5180 match admitted {
5181 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5182 Err(refusal) => {
5183 return Ok(vec![self.route_open_refusal_frame(
5184 ctx,
5185 &caller,
5186 &frame,
5187 &target_module_id,
5188 refusal.code,
5189 refusal.code,
5190 refusal.message,
5191 )?]);
5192 }
5193 }
5194 }
5195 };
5196 if let Some(stamp) = &scope_stamp {
5199 caller.scope = route_open_scope_label(stamp);
5200 }
5201
5202 timings.enter(RouteOpenPhase::ProjectRoot);
5224 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
5225 Ok(project_root) => project_root,
5226 Err(err) => {
5227 return Ok(vec![control_error_frame(
5228 &frame,
5229 "invalid_project_root",
5230 err.to_string(),
5231 )?])
5232 }
5233 };
5234 timings.enter(RouteOpenPhase::Admission);
5235 identity.project_root = project_root.as_path().to_path_buf();
5236
5237 let _concurrency_guard = match self
5248 .route_bind_concurrency
5249 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5250 {
5251 Ok(guard) => guard,
5252 Err(in_flight) => {
5253 return Ok(vec![self.route_open_target_capacity_refusal(
5254 ctx,
5255 &caller,
5256 &frame,
5257 &target_module_id,
5258 in_flight,
5259 )?]);
5260 }
5261 };
5262
5263 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5266 RouteBindAdmission::Admitted { guard, probe } => {
5267 if probe {
5268 info!(
5269 module_id = %target_module_id,
5270 connection_id = ctx.connection_id.get(),
5271 "route.bind breaker half-open: admitting one probe"
5272 );
5273 }
5274 guard
5275 }
5276 RouteBindAdmission::Refused {
5277 consecutive_timeouts,
5278 retry_in,
5279 probe_in_flight,
5280 } => {
5281 return Ok(vec![self.route_open_breaker_refusal_frame(
5282 ctx,
5283 &caller,
5284 &frame,
5285 &target_module_id,
5286 consecutive_timeouts,
5287 retry_in,
5288 probe_in_flight,
5289 )?]);
5290 }
5291 };
5292
5293 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5298 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5299 let operator_app = self
5300 .operator_apps
5301 .lock()
5302 .unwrap_or_else(|p| p.into_inner())
5303 .get(&ctx.connection_id)
5304 .cloned();
5305 timings.enter(RouteOpenPhase::EgressReservation);
5306 let pending = match self
5307 .forwarding
5308 .begin_route_bind_relay_for(
5309 ctx.connection_id,
5310 ctx.egress.clone(),
5311 response_version(&frame),
5312 frame.header.corr,
5313 &target_module_id,
5314 principal.clone(),
5315 bound_scope,
5316 Some(project_root),
5317 identity.harness.clone(),
5318 build_class,
5319 operator_app.clone(),
5320 relay_deadline,
5321 )
5322 .await
5323 {
5324 Ok(pending) => pending,
5325 Err(err) => {
5326 return Ok(vec![self.route_open_refusal_frame(
5327 ctx,
5328 &caller,
5329 &frame,
5330 &target_module_id,
5331 "relay_reservation_failed",
5332 forwarding_error_code(&err),
5333 err.to_string(),
5334 )?])
5335 }
5336 };
5337 timings.enter(RouteOpenPhase::Admission);
5338 let crate::forwarding::PendingRouteBindRelay {
5339 endpoint,
5340 module_sink,
5341 negotiated_ver,
5342 client_channel,
5343 client_epoch,
5344 module_channel,
5345 module_epoch,
5346 corr: relay_corr,
5347 receiver,
5348 } = pending;
5349 let mut reservation =
5350 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5351
5352 let relay_registration = self
5357 .registry
5358 .get_module_by_connection(endpoint.connection_id)
5359 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5360 let relay_capabilities = relay_registration
5361 .as_ref()
5362 .and_then(|registration| registration.manifest.capabilities.as_ref());
5363 if let Some(selector) = &scope {
5364 if let Err(refusal) = self
5367 .scopes
5368 .read()
5369 .unwrap_or_else(|poisoned| poisoned.into_inner())
5370 .check_bind_proof(&principal, selector)
5371 {
5372 reservation.release_and_disarm();
5373 return Ok(vec![self.route_open_refusal_frame(
5374 ctx,
5375 &caller,
5376 &frame,
5377 &target_module_id,
5378 refusal.code,
5379 refusal.code,
5380 refusal.message,
5381 )?]);
5382 }
5383 }
5384 if let Some(stamp) = scope_stamp
5385 .as_ref()
5386 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5387 {
5388 if let Err(refusal) = crate::scopes::check_target_flow_support(
5389 stamp,
5390 &target_module_id,
5391 relay_capabilities,
5392 )
5393 .and_then(|()| {
5394 crate::scopes::check_target_agent_run_support(
5395 stamp,
5396 &target_module_id,
5397 relay_capabilities,
5398 )
5399 }) {
5400 reservation.release_and_disarm();
5401 return Ok(vec![self.route_open_refusal_frame(
5402 ctx,
5403 &caller,
5404 &frame,
5405 &target_module_id,
5406 refusal.code,
5407 refusal.code,
5408 refusal.message,
5409 )?]);
5410 }
5411 }
5412
5413 if !relay_capabilities.is_some_and(|caps| {
5416 caps.provides
5417 .iter()
5418 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5419 }) {
5420 if let Some(stamp) = scope_stamp.as_mut() {
5421 stamp.workdir = None;
5422 stamp.project_id = None;
5423 }
5424 }
5425 let worker_run = scope_stamp
5430 .as_ref()
5431 .is_some_and(crate::scopes::is_worker_run)
5432 && relay_capabilities.is_some_and(|caps| {
5433 caps.provides
5434 .iter()
5435 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5436 });
5437 if !worker_run
5438 && !relay_capabilities.is_some_and(|caps| {
5439 caps.provides
5440 .iter()
5441 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5442 })
5443 {
5444 if let Some(stamp) = scope_stamp.as_mut() {
5445 stamp.attributes.on_behalf_of_agent = None;
5446 }
5447 }
5448 if !relay_capabilities.is_some_and(|caps| {
5449 caps.provides
5450 .iter()
5451 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5452 }) {
5453 if let Some(stamp) = scope_stamp.as_mut() {
5454 stamp.attributes.flow_version = None;
5455 stamp.attributes.flow_code_hash = None;
5456 stamp.attributes.flow_install_id = None;
5457 }
5458 }
5459 let call_mode = if worker_run
5460 || relay_capabilities
5461 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5462 {
5463 Some(derive_call_mode(scope_stamp.as_ref()))
5464 } else {
5465 if let Some(stamp) = scope_stamp.as_mut() {
5466 stamp.expires_at_ms = None;
5467 }
5468 None
5469 };
5470
5471 debug!(
5472 connection_id = ctx.connection_id.get(),
5473 client_channel,
5474 client_epoch,
5475 module_channel,
5476 module_epoch,
5477 "reserved route handle pair"
5478 );
5479 let route_principal_label = principal_label(&principal);
5482 let relay = ModuleControlRequest::RouteBind {
5483 route_channel: module_channel,
5484 epoch: module_epoch,
5485 target,
5486 identity,
5487 principal: Some(principal),
5488 consumer_capabilities,
5489 role_versions,
5490 admission_facts,
5491 scope: scope_stamp,
5492 call_mode,
5493 };
5494 let peer = RouteBindPeer::new()
5495 .with_observed_peer(
5496 relay_capabilities
5497 .is_some_and(|caps| {
5498 caps.provides
5499 .iter()
5500 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5501 })
5502 .then(|| {
5503 self.observed_peers
5504 .lock()
5505 .unwrap_or_else(|p| p.into_inner())
5506 .get(&ctx.connection_id)
5507 .cloned()
5508 .unwrap_or_else(ObservedPeer::unobserved)
5509 }),
5510 )
5511 .with_operator_app(
5512 operator_app
5513 .filter(|stamp| {
5514 stamp.delivery_enabled(&self.operator_app_policy)
5515 && relay_capabilities.is_some_and(|caps| {
5516 caps.provides
5517 .iter()
5518 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5519 })
5520 })
5521 .map(|stamp| stamp.app),
5522 );
5523 let relay_body = encode_route_bind(&relay, &peer).map_err(|err| {
5524 RouterError::backend(
5525 0,
5526 frame.header.corr,
5527 format!("failed to encode route.bind request: {err}"),
5528 )
5529 })?;
5530 let relay_frame = Frame::build_with_version(
5531 negotiated_ver,
5532 FrameType::Request,
5533 control_flags(),
5534 0,
5535 0,
5536 relay_corr,
5537 relay_body,
5538 )
5539 .map_err(RouterError::FrameBuild)?;
5540
5541 timings.enter(RouteOpenPhase::BindSend);
5542 let sent = module_sink.send(relay_frame).await;
5543 timings.enter(RouteOpenPhase::Admission);
5544 if let Err(err) = sent {
5545 reservation.release_and_disarm();
5546 return Ok(vec![self.route_open_refusal_frame(
5547 ctx,
5548 &caller,
5549 &frame,
5550 &target_module_id,
5551 "relay_send_failed",
5552 "target_unavailable",
5553 err.to_string(),
5554 )?]);
5555 }
5556
5557 #[cfg(test)]
5558 if let Some(hold) = &self.route_bind_relay_sent_hold {
5559 hold.wait().await;
5563 hold.wait().await;
5564 }
5565
5566 if self
5571 .forwarding
5572 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5573 .map_err(RouterError::Forwarding)?
5574 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5575 {
5576 self.send_abandoned_route_bind_goodbye(
5577 &module_sink,
5578 negotiated_ver,
5579 module_channel,
5580 module_epoch,
5581 );
5582 }
5583
5584 timings.enter(RouteOpenPhase::BindAckWait);
5585 let outcome = timeout_at(relay_deadline, receiver).await;
5586 timings.enter(RouteOpenPhase::Admission);
5587 match outcome {
5588 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5589 reservation.disarm();
5590 if breaker.record_accepted() {
5591 info!(
5592 module_id = %target_module_id,
5593 "route.bind breaker closed: the probe was accepted"
5594 );
5595 }
5596 self.observe_route_open_accept(
5597 ctx,
5598 &target_module_id,
5599 &route_principal_label,
5600 &caller.scope,
5601 );
5602 Ok(Vec::new())
5603 }
5604 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5605 reservation.release_and_disarm();
5606 breaker.record_inconclusive();
5610 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5611 }
5612 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5613 reservation.release_and_disarm();
5614 breaker.record_inconclusive();
5615 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5616 }
5617 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5618 reservation.release_and_disarm();
5619 breaker.record_inconclusive();
5620 tracing::warn!(
5627 module_id = %target_module_id,
5628 "route.bind relay abandoned: {message}"
5629 );
5630 Ok(vec![self.route_open_refusal_frame(
5631 ctx,
5632 &caller,
5633 &frame,
5634 &target_module_id,
5635 "relay_abandoned",
5636 "target_unavailable",
5637 message,
5638 )?])
5639 }
5640 Ok(Err(_)) => {
5641 reservation.release_and_disarm();
5642 breaker.record_inconclusive();
5643 Ok(vec![self.route_open_refusal_frame(
5644 ctx,
5645 &caller,
5646 &frame,
5647 &target_module_id,
5648 "relay_waiter_canceled",
5649 "target_unavailable",
5650 "route.bind relay waiter was canceled before the module responded",
5651 )?])
5652 }
5653 Err(_) => {
5654 reservation.release_and_disarm();
5655 if let Some(opened) = breaker.record_timeout(
5659 self.route_bind_breaker_threshold,
5660 self.route_bind_breaker_cooldown,
5661 ) {
5662 warn!(
5663 module_id = %target_module_id,
5664 consecutive_timeouts = opened.consecutive_timeouts,
5665 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5666 reopened_after_probe = opened.reopened_after_probe,
5667 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5668 );
5669 }
5670 tracing::warn!(
5678 module_id = %target_module_id,
5679 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5680 "route.bind relay timed out: module did not ack within budget"
5681 );
5682 Ok(vec![self.route_open_refusal_frame(
5683 ctx,
5684 &caller,
5685 &frame,
5686 &target_module_id,
5687 "relay_timed_out",
5688 "module_timeout",
5689 format!(
5690 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5691 route_bind_relay_timeout
5692 ),
5693 )?])
5694 }
5695 }
5696 }
5697
5698 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5699 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5700 snapshot: self.supervisor.spawn_snapshot(),
5701 };
5702 Ok(vec![control_response_body_frame(
5703 &frame,
5704 &response,
5705 "ClientControlResponse::SupervisorSpawnSnapshot",
5706 )?])
5707 }
5708
5709 fn handle_supervisor_spawn_subscribe(
5710 &self,
5711 ctx: &RouteCtx,
5712 frame: Frame,
5713 since: Option<SpawnCursor>,
5714 ) -> Result<Vec<Frame>, RouterError> {
5715 match self.supervisor.subscribe_spawns(
5716 ctx.connection_id,
5717 frame.header.corr,
5718 response_version(&frame),
5719 since,
5720 ctx.egress.clone(),
5721 ) {
5722 Ok(()) => Ok(Vec::new()),
5723 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5724 Ok(vec![control_error_body_frame(
5725 &frame,
5726 ErrorBody {
5727 code: "spawn_cursor_incarnation_mismatch".to_string(),
5728 message: "spawn cursor belongs to a different daemon incarnation"
5729 .to_string(),
5730 detail: Some(serde_json::json!({
5731 "current_daemon_incarnation": current
5732 })),
5733 },
5734 )?])
5735 }
5736 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5737 &frame,
5738 ErrorBody {
5739 code: "spawn_cursor_too_old".to_string(),
5740 message: "spawn cursor predates the retained event ring".to_string(),
5741 detail: Some(serde_json::json!({
5742 "oldest_retained_cursor": oldest
5743 })),
5744 },
5745 )?]),
5746 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5747 0,
5748 frame.header.corr,
5749 format!("failed to open supervisor spawn subscription: {error}"),
5750 )),
5751 }
5752 }
5753
5754 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5755 let generation = self
5756 .registry
5757 .generation()
5758 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5759 let mut modules = Vec::new();
5760 for module in self.supervisor.list() {
5761 let status = module.status_for_control("list").map_err(|err| {
5762 RouterError::backend(
5763 0,
5764 frame.header.corr,
5765 format!("failed to read supervisor status: {err}"),
5766 )
5767 })?;
5768 let (configured, _) = module.configuration().map_err(|err| {
5769 RouterError::backend(
5770 0,
5771 frame.header.corr,
5772 format!("failed to read module configuration: {err}"),
5773 )
5774 })?;
5775 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5776 RouterError::backend(
5777 0,
5778 frame.header.corr,
5779 format!("failed to read retry time: {err}"),
5780 )
5781 })?;
5782 let image = module.running_image_agreement().await;
5784 let resources = Some(module.child_resource_usage());
5787 let mut verdict =
5788 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5789 verdict.spec = Some(launch_spec_agreement(
5790 &configured,
5791 status.process_alive,
5792 status.launch_spec_digest.as_ref(),
5793 ));
5794 let pending_reload = Some(verdict);
5795 let configured_enabled = self
5796 .capability_evaluator
5797 .configured_enabled(&status.module_id);
5798 modules.push(
5801 SupervisorEntry::new(
5802 status.module_id,
5803 status.state.to_string(),
5804 status.enabled,
5805 status.live,
5806 status.health.status,
5807 )
5808 .with_configured_enabled(configured_enabled)
5809 .with_next_retry_at_ms(next_retry_at_ms)
5810 .with_launch_nonce_env(Some(!cfg!(unix)))
5811 .with_launch_nonce_source(status.launch_nonce_source)
5812 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5813 .with_protocol(status.protocol)
5814 .with_pending_reload(pending_reload)
5815 .with_last_probe_ms(status.health.last_probe_ms)
5816 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5817 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5818 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5819 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5820 .with_restart_count(Some(status.restart_count))
5821 .with_max_restarts(Some(status.max_restarts))
5822 .with_lifetime_restarts(Some(status.lifetime_restarts))
5823 .with_spawn_generation(Some(status.spawn_generation))
5824 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5825 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5826 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5827 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5828 .with_resources(resources),
5829 );
5830 }
5831 let response = ClientControlResponse::SupervisorList {
5832 generation,
5833 modules,
5834 };
5835 Ok(vec![control_response_body_frame(
5836 &frame,
5837 &response,
5838 "ClientControlResponse::SupervisorList",
5839 )?])
5840 }
5841
5842 fn handle_supervisor_stderr_tail(
5843 &self,
5844 frame: Frame,
5845 module_id: String,
5846 max_lines: Option<u32>,
5847 max_bytes: Option<u32>,
5848 ) -> Result<Vec<Frame>, RouterError> {
5849 let Some(module) = self.supervisor.get(&module_id) else {
5850 return Ok(vec![control_error_frame(
5851 &frame,
5852 "unknown_module",
5853 format!("module_id '{module_id}' is not supervised"),
5854 )?]);
5855 };
5856
5857 let snapshot = module.stderr_tail(
5858 max_lines.map(|value| value as usize),
5859 max_bytes.map(|value| value as usize),
5860 );
5861
5862 let response = ClientControlResponse::SupervisorStderrTail {
5863 module_id,
5864 tail: StderrTail {
5865 capture: match snapshot.capture {
5866 CaptureState::Captured => StderrCaptureState::Captured,
5867 CaptureState::Incomplete { reason } => {
5868 StderrCaptureState::Incomplete { reason }
5869 }
5870 CaptureState::NotCaptured { reason } => {
5871 StderrCaptureState::NotCaptured { reason }
5872 }
5873 },
5874 entries: snapshot
5875 .entries
5876 .into_iter()
5877 .map(|entry| match entry {
5878 TailEntry::Line {
5879 text,
5880 truncated,
5881 at_ms,
5882 } => StderrTailEntry::Line {
5883 text,
5884 truncated,
5885 at_ms,
5886 },
5887 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5888 })
5889 .collect(),
5890 dropped_lines: snapshot.dropped_lines,
5891 },
5892 };
5893 Ok(vec![control_response_body_frame(
5894 &frame,
5895 &response,
5896 "ClientControlResponse::SupervisorStderrTail",
5897 )?])
5898 }
5899
5900 async fn handle_supervisor_terminals(
5901 &self,
5902 frame: Frame,
5903 module_id: String,
5904 ) -> Result<Vec<Frame>, RouterError> {
5905 let Some(module) = self.supervisor.get(&module_id) else {
5906 return Ok(vec![control_error_frame(
5907 &frame,
5908 "unknown_module",
5909 format!("module_id '{module_id}' is not supervised"),
5910 )?]);
5911 };
5912
5913 let terminals = module
5916 .read_durable_terminal_history()
5917 .await
5918 .map_err(|error| {
5919 RouterError::backend(
5920 0,
5921 frame.header.corr,
5922 format!("failed to read terminal history: {error}"),
5923 )
5924 })?;
5925 let response = ClientControlResponse::SupervisorTerminals {
5926 module_id,
5927 terminals,
5928 };
5929 Ok(vec![control_response_body_frame(
5930 &frame,
5931 &response,
5932 "ClientControlResponse::SupervisorTerminals",
5933 )?])
5934 }
5935
5936 fn handle_supervisor_routes(
5937 &self,
5938 frame: Frame,
5939 module_id: Option<String>,
5940 ) -> Result<Vec<Frame>, RouterError> {
5941 let modules = self
5942 .forwarding
5943 .route_census(module_id.as_deref())
5944 .map_err(RouterError::Forwarding)?
5945 .into_iter()
5946 .map(|(module_id, routes)| SupervisorRouteModule {
5947 module_id,
5948 routes: routes
5949 .into_iter()
5950 .map(|route| SupervisorRoute {
5951 consumer: match route.principal {
5952 Principal::Reserved { module_id } => {
5953 SupervisorRouteConsumer::Reserved { module_id }
5954 }
5955 Principal::Direct | Principal::Unverified => {
5956 SupervisorRouteConsumer::Direct {
5957 connection_id: route.goodbye_target.connection_id.get(),
5958 }
5959 }
5960 },
5961 age_ms: Instant::now()
5962 .saturating_duration_since(route.bound_at)
5963 .as_millis()
5964 .try_into()
5965 .unwrap_or(u64::MAX),
5966 draining: route.draining,
5967 drain_reason: route.drain_reason,
5968 })
5969 .collect(),
5970 })
5971 .collect();
5972 let response = ClientControlResponse::SupervisorRoutes { modules };
5973 Ok(vec![control_response_body_frame(
5974 &frame,
5975 &response,
5976 "ClientControlResponse::SupervisorRoutes",
5977 )?])
5978 }
5979
5980 async fn handle_supervisor_provenance(
5981 &self,
5982 frame: Frame,
5983 module_id: Option<String>,
5984 ) -> Result<Vec<Frame>, RouterError> {
5985 let mut selected = if let Some(module_id) = module_id {
5986 let Some(module) = self.supervisor.get(&module_id) else {
5987 return Ok(vec![control_error_frame(
5988 &frame,
5989 "unknown_module",
5990 format!("module_id '{module_id}' is not supervised"),
5991 )?]);
5992 };
5993 vec![module]
5994 } else {
5995 self.supervisor.list()
5996 };
5997
5998 let mut modules = Vec::with_capacity(selected.len());
5999 for module in selected.drain(..) {
6000 let (status, observed_image) = module
6001 .status_and_running_image_agreement()
6002 .await
6003 .map_err(|err| {
6004 RouterError::backend(
6005 0,
6006 frame.header.corr,
6007 format!("failed to read supervisor status: {err}"),
6008 )
6009 })?;
6010 let module_declared = self
6011 .registry
6012 .get_module(&status.module_id)
6013 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6014 .and_then(|registration| registration.manifest.provenance)
6015 .map(|build| ModuleDeclaredProvenance::Reported { build })
6016 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6017 #[cfg(test)]
6018 let running_image = match &self.provenance_probe_override {
6019 Some(result) => result.clone(),
6020 None => observed_image,
6021 };
6022 #[cfg(not(test))]
6023 let running_image = observed_image;
6024 modules.push(SupervisorModuleProvenance {
6025 module_id: status.module_id,
6026 module_declared,
6027 daemon_observed: SupervisorObservedProcess {
6028 launch_nonce_source: status.launch_nonce_source,
6029 pid: status.pid,
6030 spawned_at_ms: status.spawned_at_ms,
6031 spawned_from: status.spawned_from,
6032 running_image,
6033 },
6034 });
6035 }
6036 let daemon = SupervisorDaemonProvenance {
6037 daemon_build: self.daemon_provenance.build.clone(),
6038 daemon_observed: DaemonObservedProcess {
6039 pid: self.daemon_provenance.pid,
6040 started_at_ms: self
6041 .daemon_provenance
6042 .start_clock
6043 .map(|clock| clock.started_at_ms())
6044 .or(self.daemon_provenance.started_at_ms),
6045 running_image: self
6046 .daemon_provenance
6047 .probe
6048 .observe(
6049 self.daemon_provenance.pid,
6050 self.daemon_provenance.executable_path.as_deref(),
6051 self.daemon_provenance.executable_identity.clone(),
6052 self.daemon_provenance.process_start_time,
6053 )
6054 .await,
6055 },
6056 };
6057 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6058 Ok(vec![control_response_body_frame(
6059 &frame,
6060 &response,
6061 "ClientControlResponse::SupervisorProvenance",
6062 )?])
6063 }
6064
6065 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6066 self.refresh_capability_requirements();
6067 let generation = self
6068 .registry
6069 .generation()
6070 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6071 let modules = self
6072 .supervisor
6073 .list()
6074 .into_iter()
6075 .map(|module| {
6076 let status = module.status_for_control("health").map_err(|err| {
6077 RouterError::backend(
6078 0,
6079 frame.header.corr,
6080 format!("failed to read supervisor health: {err}"),
6081 )
6082 })?;
6083 let module_id = status.module_id;
6084 let capability_detail = self
6085 .capability_evaluator
6086 .required_problem_detail(&module_id);
6087 Ok(SupervisorHealthEntry {
6088 module_id,
6089 status: status.health.status,
6090 detail: append_capability_problem_detail(
6091 status.health.detail,
6092 capability_detail,
6093 ),
6094 metrics: status.health.metrics,
6095 consecutive_failures: status.health.consecutive_failures,
6096 late_answer_count: status.health.late_answer_count,
6097 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6098 last_action: status.health.last_action,
6099 last_action_ms: status.health.last_action_ms,
6100 last_probe_ms: status.health.last_probe_ms,
6101 })
6102 })
6103 .collect::<Result<Vec<_>, RouterError>>()?;
6104 let response = ClientControlResponse::SupervisorHealth {
6105 generation,
6106 modules,
6107 };
6108 Ok(vec![control_response_body_frame(
6109 &frame,
6110 &response,
6111 "ClientControlResponse::SupervisorHealth",
6112 )?])
6113 }
6114
6115 async fn handle_supervisor_restart(
6116 &self,
6117 frame: Frame,
6118 module_id: String,
6119 drain_timeout_ms: Option<u64>,
6120 requester: Principal,
6121 ) -> Result<Vec<Frame>, RouterError> {
6122 let operation_lock = self.supervisor.operation_lock();
6123 let _operation_guard = operation_lock.lock().await;
6124 let Some(module) = self.supervisor.get(&module_id) else {
6125 return Ok(vec![control_error_frame(
6126 &frame,
6127 "unknown_module",
6128 format!("module_id '{module_id}' is not supervised"),
6129 )?]);
6130 };
6131
6132 self.route_outages.mark_operator_action(&module_id);
6133 if let Err(err) = module
6134 .restart_requested(drain_timeout_ms, Some(requester))
6135 .await
6136 {
6137 self.route_outages
6138 .operator_action_ended_unrefused(&module_id);
6139 let (code, message) = match err {
6140 crate::supervise::SuperviseError::Disabled { .. } => {
6141 ("module_disabled", err.to_string())
6142 }
6143 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6144 ("swap_in_progress", err.to_string())
6145 }
6146 _ => (
6147 "target_unavailable",
6148 format!("failed to restart module_id '{module_id}': {err}"),
6149 ),
6150 };
6151 return Ok(vec![control_error_frame(&frame, code, message)?]);
6152 }
6153
6154 let response = ClientControlResponse::SupervisorAck {
6155 module_id,
6156 applied: true,
6157 };
6158 Ok(vec![control_response_body_frame(
6159 &frame,
6160 &response,
6161 "ClientControlResponse::SupervisorAck",
6162 )?])
6163 }
6164
6165 async fn handle_supervisor_swap(
6169 &self,
6170 frame: Frame,
6171 module_id: String,
6172 ready_timeout_ms: Option<u64>,
6173 requester: Principal,
6174 ) -> Result<Vec<Frame>, RouterError> {
6175 let module = {
6182 let operation_lock = self.supervisor.operation_lock();
6183 let _operation_guard = operation_lock.lock().await;
6184 self.supervisor.get(&module_id)
6185 };
6186 let Some(module) = module else {
6187 return Ok(vec![control_error_frame(
6188 &frame,
6189 "unknown_module",
6190 format!("module_id '{module_id}' is not supervised"),
6191 )?]);
6192 };
6193
6194 self.route_outages.mark_operator_action(&module_id);
6195 if let Err(err) = module
6196 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6197 .await
6198 {
6199 self.route_outages
6200 .operator_action_ended_unrefused(&module_id);
6201 use crate::supervise::SuperviseError;
6202 let message = err.to_string();
6203 let error = match err {
6204 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6205 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6206 code: "swap_refused".to_string(),
6207 message,
6208 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6209 },
6210 SuperviseError::SwapFailed {
6211 arm,
6212 candidate_exit,
6213 ..
6214 } => ErrorBody {
6215 code: "swap_failed".to_string(),
6216 message,
6217 detail: Some(serde_json::json!({
6218 "arm": arm.as_str(),
6219 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6220 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6221 })),
6222 },
6223 _ => ErrorBody::new(
6224 "target_unavailable",
6225 format!("failed to swap module_id '{module_id}': {message}"),
6226 ),
6227 };
6228 return Ok(vec![control_error_body_frame(&frame, error)?]);
6229 }
6230 self.route_outages
6234 .operator_action_ended_unrefused(&module_id);
6235
6236 let response = ClientControlResponse::SupervisorAck {
6237 module_id,
6238 applied: true,
6239 };
6240 Ok(vec![control_response_body_frame(
6241 &frame,
6242 &response,
6243 "ClientControlResponse::SupervisorAck",
6244 )?])
6245 }
6246
6247 async fn handle_supervisor_reload(
6248 &self,
6249 frame: Frame,
6250 module_id: String,
6251 requester: Principal,
6252 ) -> Result<Vec<Frame>, RouterError> {
6253 let operation_lock = self.supervisor.operation_lock();
6254 let _operation_guard = operation_lock.lock().await;
6255 let Some(module) = self.supervisor.get(&module_id) else {
6256 return Ok(vec![control_error_frame(
6257 &frame,
6258 "unknown_module",
6259 format!("module_id '{module_id}' is not supervised"),
6260 )?]);
6261 };
6262
6263 self.route_outages.mark_operator_action(&module_id);
6264 if let Err(err) = module.reload_requested(Some(requester)).await {
6265 self.route_outages
6266 .operator_action_ended_unrefused(&module_id);
6267 let (code, message) = match err {
6268 crate::supervise::SuperviseError::Disabled { .. } => {
6269 ("module_disabled", err.to_string())
6270 }
6271 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6272 ("swap_in_progress", err.to_string())
6273 }
6274 _ => (
6275 "reload_failed",
6276 format!("failed to reload module_id '{module_id}': {err}"),
6277 ),
6278 };
6279 return Ok(vec![control_error_frame(&frame, code, message)?]);
6280 }
6281
6282 let response = ClientControlResponse::SupervisorAck {
6283 module_id,
6284 applied: true,
6285 };
6286 Ok(vec![control_response_body_frame(
6287 &frame,
6288 &response,
6289 "ClientControlResponse::SupervisorAck",
6290 )?])
6291 }
6292
6293 async fn handle_supervisor_rescan(
6297 &self,
6298 frame: Frame,
6299 mode: RescanMode,
6300 requester: Principal,
6301 ) -> Result<Vec<Frame>, RouterError> {
6302 let preview = mode.previews();
6303 let Some(context) = self.rescan.clone() else {
6304 return Ok(vec![control_error_frame(
6305 &frame,
6306 "rescan_unavailable",
6307 "the daemon was not started with a reloadable config path".to_string(),
6308 )?]);
6309 };
6310
6311 let operation_lock = self.supervisor.operation_lock();
6312 let _operation_guard = operation_lock.lock().await;
6313 let loaded = match self.load_rescan_config(&mode, &context) {
6317 Ok(config) => config,
6318 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6319 };
6320 let Some((config, activity_config)) = loaded else {
6328 return Ok(vec![control_error_frame(
6329 &frame,
6330 "invalid_daemon_config",
6331 format!(
6332 "daemon config not found at {}; refusing to rescan (an absent config would \
6333 retire every supervised module)",
6334 context.config_path.display()
6335 ),
6336 )?]);
6337 };
6338 let (
6339 configured_port,
6340 storage_config,
6341 admission_facts_carrier_module_id,
6342 admission_facts_targets,
6343 scope_authority_owners,
6344 privileged_operator_prompt,
6345 modules,
6346 reserved_capabilities,
6347 ) = (
6348 config.port,
6349 config.storage,
6350 config.admission_facts_carrier_module_id,
6351 config.admission_facts_targets,
6352 config.scope_authority_owners,
6353 config.privileged_operator_prompt,
6354 config.modules,
6355 config.reserved_capabilities,
6356 );
6357
6358 let mut restart_required = Vec::new();
6367 for section in RestartRequiredSection::ALL {
6368 let changed = match section {
6369 RestartRequiredSection::Port => configured_port != context.configured_port,
6370 RestartRequiredSection::Storage => storage_config != context.storage_config,
6371 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6372 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6373 }
6374 RestartRequiredSection::AdmissionFactsTargets => {
6375 admission_facts_targets != context.admission_facts_targets
6376 }
6377 RestartRequiredSection::ScopeAuthorityOwners => {
6378 scope_authority_owners != context.scope_authority_owners
6379 }
6380 RestartRequiredSection::PrivilegedOperatorPrompt => {
6381 privileged_operator_prompt != context.privileged_operator_prompt
6382 }
6383 };
6384 if changed {
6385 restart_required.push(section.label().to_string());
6386 }
6387 }
6388 if !restart_required.is_empty() {
6389 warn!(
6390 config_path = %context.config_path.display(),
6391 sections = %restart_required.join(", "),
6392 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6393 );
6394 }
6395
6396 for configured in &modules {
6397 if let Err(err) = validate_spec(&configured.module_spec()) {
6398 return Ok(vec![control_error_frame(
6399 &frame,
6400 "invalid_daemon_config",
6401 format!("supervisor rescan rejected daemon config: {err}"),
6402 )?]);
6403 }
6404 }
6405
6406 let configured_capabilities = modules
6407 .iter()
6408 .map(|module| (module.module_id.clone(), module.enabled))
6409 .collect::<Vec<_>>();
6410 let preview_capability_warnings = if preview {
6411 let (_, registrations) = self.runtime_capability_snapshot()?;
6412 let current_modules = self
6413 .supervisor
6414 .list()
6415 .into_iter()
6416 .map(|module| module.module_id().to_string())
6417 .collect::<BTreeSet<_>>();
6418 let resulting_modules = configured_capabilities.clone();
6419 let removed = current_modules
6420 .into_iter()
6421 .filter(|module_id| {
6422 !resulting_modules
6423 .iter()
6424 .any(|(configured_id, _)| configured_id == module_id)
6425 })
6426 .collect::<Vec<_>>();
6427 let disabled = match &mode {
6430 RescanMode::Candidate(_) => resulting_modules
6431 .iter()
6432 .filter(|(_, enabled)| !enabled)
6433 .map(|(module_id, _)| module_id.clone())
6434 .collect::<Vec<_>>(),
6435 _ => Vec::new(),
6436 };
6437 self.capability_evaluator.preview_warnings(
6438 resulting_modules,
6439 &removed,
6440 &disabled,
6441 ®istrations,
6442 )
6443 } else {
6444 Vec::new()
6445 };
6446 let reconciled = match &mode {
6447 RescanMode::Candidate(candidate) => self
6448 .preview_candidate(&context, modules, candidate)
6449 .map(|result| (result, Vec::new())),
6450 _ => {
6451 self.reconcile_supervised_modules(
6452 &context.supervisor,
6453 modules,
6454 preview,
6455 requester.clone(),
6456 )
6457 .await
6458 }
6459 };
6460 let (result, changed) = match reconciled {
6461 Ok(result) => result,
6462 Err(message) => {
6463 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6464 }
6465 };
6466 if !preview {
6467 if let Some(ledger) = &self.activity {
6468 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6469 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6470 }
6471 if let Err(code) = self.configure_activity(activity_config).await {
6472 warn!(
6473 code,
6474 "activity retention could not be applied during rescan"
6475 );
6476 }
6477 self.capability_evaluator
6478 .configure(configured_capabilities, reserved_capabilities);
6479 self.capability_evaluator.wake_deadline_loop();
6480 self.refresh_capability_requirements();
6481 }
6482 let mut result = result;
6483 result.restart_required = restart_required;
6484 result.capability_warnings = preview_capability_warnings;
6485 let response = ClientControlResponse::SupervisorRescan { result };
6486 Ok(vec![control_response_body_frame(
6487 &frame,
6488 &response,
6489 "ClientControlResponse::SupervisorRescan",
6490 )?])
6491 }
6492
6493 async fn handle_supervisor_release_reserved(
6494 &self,
6495 frame: Frame,
6496 module_id: String,
6497 ) -> Result<Vec<Frame>, RouterError> {
6498 let Some(context) = self.rescan.clone() else {
6499 return Ok(vec![control_error_frame(
6500 &frame,
6501 "release_unavailable",
6502 "reserved-id release requires a daemon started with a reloadable config path",
6503 )?]);
6504 };
6505 let operation_lock = self.supervisor.operation_lock();
6506 let _operation_guard = operation_lock.lock().await;
6507 let loaded = match crate::daemon_config::load(&context.config_path) {
6508 Ok(Some(config)) => config,
6509 Ok(None) => {
6510 return Ok(vec![control_error_frame(
6511 &frame,
6512 "invalid_daemon_config",
6513 format!(
6514 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6515 context.config_path.display()
6516 ),
6517 )?])
6518 }
6519 Err(err) => {
6520 return Ok(vec![control_error_frame(
6521 &frame,
6522 "invalid_daemon_config",
6523 format!("unable to verify reserved-id release against daemon config: {err}"),
6524 )?])
6525 }
6526 };
6527 if loaded
6528 .modules
6529 .iter()
6530 .any(|configured| configured.module_id == module_id)
6531 {
6532 return Ok(vec![control_error_frame(
6533 &frame,
6534 "reserved_module_configured",
6535 format!(
6536 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6537 ),
6538 )?]);
6539 }
6540 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6541 return Ok(vec![control_error_frame(
6542 &frame,
6543 "reserved_gate_not_retained",
6544 format!(
6545 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6546 ),
6547 )?]);
6548 }
6549
6550 let response = ClientControlResponse::SupervisorAck {
6551 module_id,
6552 applied: true,
6553 };
6554 Ok(vec![control_response_body_frame(
6555 &frame,
6556 &response,
6557 "ClientControlResponse::SupervisorAck",
6558 )?])
6559 }
6560
6561 fn plan_rescan(
6565 &self,
6566 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6567 ) -> Result<RescanPlan, String> {
6568 let mut current = BTreeMap::new();
6569 for module in self.supervisor.list() {
6570 let (spec, health) = module.configuration().map_err(|err| {
6571 format!(
6572 "failed to read configuration for module_id '{}': {err}",
6573 module.module_id()
6574 )
6575 })?;
6576 let enabled = module
6577 .status()
6578 .map_err(|err| {
6579 format!(
6580 "failed to read status for module_id '{}': {err}",
6581 module.module_id()
6582 )
6583 })?
6584 .enabled;
6585 current.insert(
6586 module.module_id().to_string(),
6587 (module, spec, health, enabled),
6588 );
6589 }
6590 let configured = configured_modules
6591 .into_iter()
6592 .map(|module| (module.module_id.clone(), module))
6593 .collect::<BTreeMap<_, _>>();
6594
6595 let added = configured
6596 .keys()
6597 .filter(|module_id| !current.contains_key(*module_id))
6598 .cloned()
6599 .collect::<Vec<_>>();
6600 let removed = current
6601 .keys()
6602 .filter(|module_id| !configured.contains_key(*module_id))
6603 .cloned()
6604 .collect::<Vec<_>>();
6605 let mut changed_pending_reload = Vec::new();
6606 let mut configuration_changes = BTreeSet::new();
6607 let mut enabled_changes = BTreeSet::new();
6608 let mut unchanged = 0_u32;
6609
6610 for (module_id, configured_module) in &configured {
6611 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6612 else {
6613 continue;
6614 };
6615 let launch_changed = *current_spec != configured_module.module_spec();
6619 let configuration_changed =
6620 launch_changed || *current_health != configured_module.health;
6621 let enabled_changed = *current_enabled != configured_module.enabled;
6622 if configuration_changed {
6623 configuration_changes.insert(module_id.clone());
6624 }
6625 if launch_changed {
6626 changed_pending_reload.push(module_id.clone());
6627 }
6628 if enabled_changed {
6629 enabled_changes.insert(module_id.clone());
6630 }
6631 if !configuration_changed && !enabled_changed {
6632 unchanged = unchanged.saturating_add(1);
6633 }
6634 }
6635 Ok(RescanPlan {
6636 current,
6637 configured,
6638 added,
6639 removed,
6640 changed_pending_reload,
6641 configuration_changes,
6642 enabled_changes,
6643 unchanged,
6644 })
6645 }
6646
6647 async fn reconcile_supervised_modules(
6656 &self,
6657 supervisor: &Supervisor,
6658 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6659 preview: bool,
6660 requester: Principal,
6661 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6662 let plan = self.plan_rescan(configured_modules)?;
6663
6664 if preview {
6670 return Ok((plan.preview_result(), Vec::new()));
6671 }
6672 let changed = plan
6673 .configuration_changes
6674 .union(&plan.enabled_changes)
6675 .cloned()
6676 .collect::<Vec<_>>();
6677 let RescanPlan {
6678 current,
6679 configured,
6680 added,
6681 removed,
6682 changed_pending_reload,
6683 configuration_changes,
6684 enabled_changes,
6685 unchanged,
6686 } = plan;
6687 if !(added.is_empty()
6692 && removed.is_empty()
6693 && configuration_changes.is_empty()
6694 && enabled_changes.is_empty())
6695 {
6696 if let Some(context) = &self.rescan {
6697 context
6698 .generation
6699 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6700 }
6701 }
6702
6703 for module_id in &removed {
6704 let module = ¤t
6705 .get(module_id)
6706 .expect("removed module came from current supervisor state")
6707 .0;
6708 module
6709 .retire_requested(Some(requester.clone()))
6710 .await
6711 .map_err(|err| {
6712 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6713 })?;
6714 self.supervisor.record_rescan_removal(module_id);
6748 self.supervisor.retire(module_id);
6749 self.route_outages.forget(module_id);
6750 }
6751
6752 for module_id in configured.keys() {
6753 let Some((module, _, _, _)) = current.get(module_id) else {
6754 continue;
6755 };
6756 let configured_module = configured
6757 .get(module_id)
6758 .expect("configured module id came from configured map");
6759 if configuration_changes.contains(module_id) {
6760 module
6761 .update_configuration(
6762 configured_module.module_spec(),
6763 configured_module.health.clone(),
6764 configured_module.drain_timeout_ms,
6765 )
6766 .await
6767 .map_err(|err| {
6768 format!(
6769 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6770 )
6771 })?;
6772 }
6773 if enabled_changes.contains(module_id) {
6774 self.route_outages.mark_operator_action(module_id);
6777 module
6778 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6779 .await
6780 .map_err(|err| {
6781 self.route_outages.operator_action_ended_unrefused(module_id);
6782 format!(
6783 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6784 configured_module.enabled
6785 )
6786 })?;
6787 }
6788 }
6789
6790 for module_id in &added {
6791 let configured_module = configured
6792 .get(module_id)
6793 .expect("added module id came from configured map");
6794 supervisor
6795 .supervise_configured_requested(
6796 configured_module.module_spec(),
6797 configured_module.enabled,
6798 configured_module.health.clone(),
6799 configured_module.drain_timeout_ms,
6800 configured_module.restart,
6801 configured_module.register_timeout,
6802 Some(requester.clone()),
6803 )
6804 .map_err(|err| {
6805 format!("failed to add module_id '{module_id}' during rescan: {err}")
6806 })?;
6807 }
6808
6809 Ok((
6810 SupervisorRescanResult {
6811 added,
6812 removed,
6813 changed_pending_reload,
6814 enabled_changes: enabled_changes.iter().cloned().collect(),
6815 unchanged,
6816 preview: false,
6817 restart_required: Vec::new(),
6820 capability_warnings: Vec::new(),
6821 module_actions: Vec::new(),
6822 earlier_pending: Vec::new(),
6823 expect: None,
6824 },
6825 changed,
6826 ))
6827 }
6828
6829 async fn handle_supervisor_set_enabled(
6830 &self,
6831 frame: Frame,
6832 module_id: String,
6833 enabled: bool,
6834 requester: Principal,
6835 ) -> Result<Vec<Frame>, RouterError> {
6836 let operation_lock = self.supervisor.operation_lock();
6837 let _operation_guard = operation_lock.lock().await;
6838 let Some(module) = self.supervisor.get(&module_id) else {
6839 return Ok(vec![control_error_frame(
6840 &frame,
6841 "unknown_module",
6842 format!("module_id '{module_id}' is not supervised"),
6843 )?]);
6844 };
6845
6846 self.route_outages.mark_operator_action(&module_id);
6849 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6850 Ok(applied) => applied,
6851 Err(err) => {
6852 self.route_outages
6853 .operator_action_ended_unrefused(&module_id);
6854 return Ok(vec![control_error_frame(
6855 &frame,
6856 "target_unavailable",
6857 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6858 )?]);
6859 }
6860 };
6861 if !applied {
6862 self.route_outages
6865 .operator_action_ended_unrefused(&module_id);
6866 }
6867
6868 self.capability_evaluator.wake_deadline_loop();
6869 self.refresh_capability_requirements();
6870 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6871 Ok(vec![control_response_body_frame(
6872 &frame,
6873 &response,
6874 "ClientControlResponse::SupervisorAck",
6875 )?])
6876 }
6877
6878 async fn handle_supervisor_health_probe(
6879 &self,
6880 frame: Frame,
6881 module_id: String,
6882 ) -> Result<Vec<Frame>, RouterError> {
6883 self.refresh_capability_requirements();
6884 let Some(registration) = self
6885 .registry
6886 .get_module(&module_id)
6887 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6888 else {
6889 return Ok(vec![control_error_frame(
6890 &frame,
6891 "unknown_module",
6892 format!("module_id '{module_id}' is not registered"),
6893 )?]);
6894 };
6895
6896 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6909 {
6910 return Ok(vec![control_error_frame(
6911 &frame,
6912 "health_not_advertised",
6913 format!("module_id '{module_id}' did not advertise health.check"),
6914 )?]);
6915 }
6916
6917 let deadline = Instant::now() + self.health_probe_timeout;
6918 let pending = match self.forwarding.begin_module_control_rpc_for(
6919 &module_id,
6920 MODULE_CONTROL_OP_HEALTH_CHECK,
6921 deadline,
6922 ) {
6923 Ok(pending) => pending,
6924 Err(err) => {
6925 return Ok(vec![control_error_frame(
6926 &frame,
6927 forwarding_error_code(&err),
6928 err.to_string(),
6929 )?])
6930 }
6931 };
6932
6933 let PendingModuleControlRpc {
6934 endpoint,
6935 module_sink,
6936 negotiated_ver,
6937 corr: probe_corr,
6938 receiver,
6939 } = pending;
6940 let mut guard =
6941 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6942 let probe_body =
6943 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6944 RouterError::backend(
6945 0,
6946 frame.header.corr,
6947 format!("failed to encode health.check request: {err}"),
6948 )
6949 })?;
6950 let probe_frame = Frame::build_with_version(
6951 negotiated_ver,
6952 FrameType::Request,
6953 control_flags(),
6954 0,
6955 0,
6956 probe_corr,
6957 probe_body,
6958 )
6959 .map_err(RouterError::FrameBuild)?;
6960
6961 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6964 Ok(Ok(())) => {}
6965 Ok(Err(err)) => {
6966 return Ok(vec![control_error_frame(
6967 &frame,
6968 "target_unavailable",
6969 err.to_string(),
6970 )?]);
6971 }
6972 Err(_) => {
6973 return Ok(vec![control_error_frame(
6974 &frame,
6975 "module_timeout",
6976 format!(
6977 "module_id '{module_id}' did not answer health.check within {:?}",
6978 self.health_probe_timeout
6979 ),
6980 )?]);
6981 }
6982 }
6983
6984 match timeout_at(deadline, receiver).await {
6985 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6986 guard.disarm();
6987 let Some(report) = response.health_report() else {
6988 return Ok(vec![control_error_frame(
6989 &frame,
6990 "invalid_control_body",
6991 "health.check RPC returned a non-health response",
6992 )?]);
6993 };
6994 let HealthReport {
6999 status,
7000 detail,
7001 metrics,
7002 } = report;
7003 let capability_detail = self
7004 .capability_evaluator
7005 .required_problem_detail(&module_id);
7006 let response = ClientControlResponse::SupervisorHealthProbe {
7007 module_id,
7008 status,
7009 detail: append_capability_problem_detail(detail, capability_detail),
7010 metrics,
7011 };
7012 Ok(vec![control_response_body_frame(
7013 &frame,
7014 &response,
7015 "ClientControlResponse::SupervisorHealthProbe",
7016 )?])
7017 }
7018 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7019 guard.disarm();
7020 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7023 }
7024 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7025 guard.disarm();
7026 Ok(vec![control_error_frame(
7027 &frame,
7028 "target_unavailable",
7029 message,
7030 )?])
7031 }
7032 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7033 guard.disarm();
7034 Ok(vec![control_error_frame(
7035 &frame,
7036 "invalid_control_body",
7037 message,
7038 )?])
7039 }
7040 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7041 guard.disarm();
7042 Ok(vec![control_error_frame(
7043 &frame,
7044 "invalid_control_body",
7045 format!("expected module-control op '{expected}', got '{actual}'"),
7046 )?])
7047 }
7048 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7049 guard.disarm();
7050 Ok(vec![control_error_frame(
7051 &frame,
7052 "module_timeout",
7053 format!(
7054 "module_id '{module_id}' answered health.check after {:?}",
7055 self.health_probe_timeout
7056 ),
7057 )?])
7058 }
7059 Ok(Err(_)) => Ok(vec![control_error_frame(
7060 &frame,
7061 "target_unavailable",
7062 "health.check waiter was canceled before the module responded",
7063 )?]),
7064 Err(_) => Ok(vec![control_error_frame(
7065 &frame,
7066 "module_timeout",
7067 format!(
7068 "module_id '{module_id}' did not answer health.check within {:?}",
7069 self.health_probe_timeout
7070 ),
7071 )?]),
7072 }
7073 }
7074
7075 fn supervisor_status(
7076 &self,
7077 module_id: &str,
7078 corr: u64,
7079 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7080 self.supervisor
7081 .get(module_id)
7082 .map(|module| {
7083 let warming = module.is_warming_for_control("status").map_err(|err| {
7084 RouterError::backend(
7085 0,
7086 corr,
7087 format!(
7088 "failed to read supervisor warming state for module_id '{module_id}': {err}"
7089 ),
7090 )
7091 })?;
7092 module.status_for_control("status").map_err(|err| {
7093 RouterError::backend(
7094 0,
7095 corr,
7096 format!(
7097 "failed to read supervisor status for module_id '{module_id}': {err}"
7098 ),
7099 )
7100 }).map(|status| (status, warming))
7101 })
7102 .transpose()
7103 }
7104
7105 fn guard_module_control_op(
7106 &self,
7107 frame: &Frame,
7108 module_id: &str,
7109 op: &str,
7110 ) -> Result<Option<Frame>, RouterError> {
7111 if self.module_grants_op(module_id, op, frame.header.corr)? {
7112 return Ok(None);
7113 }
7114
7115 Ok(Some(control_error_frame(
7116 frame,
7117 "op_not_allowed",
7118 format!("module_id '{module_id}' did not grant control op '{op}'"),
7119 )?))
7120 }
7121
7122 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7123 let Some(registration) = self
7124 .registry
7125 .get_module(module_id)
7126 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7127 else {
7128 return Ok(false);
7129 };
7130 Ok(module_registration_grants_op(®istration.control_ops, op))
7131 }
7132
7133 fn handle_status_update(
7134 &self,
7135 endpoint: ModuleEndpointId,
7136 frame: Frame,
7137 ) -> Result<Vec<Frame>, RouterError> {
7138 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7139 Ok(update) => update,
7140 Err(err) => {
7141 if is_known_module_push_op(&frame.body) {
7146 return Ok(vec![control_error_frame(
7147 &frame,
7148 "invalid_control_body",
7149 format!("malformed module control push body: {err}"),
7150 )?]);
7151 }
7152 return Ok(Vec::new());
7153 }
7154 };
7155
7156 match update {
7157 ModuleControlPush::RouteStatus {
7158 route_channel,
7159 route_epoch,
7160 status,
7161 } => {
7162 self.forwarding
7163 .cache_status(endpoint, route_channel, route_epoch, status)
7164 .map_err(RouterError::Forwarding)?;
7165 }
7166 }
7167 Ok(Vec::new())
7168 }
7169
7170 fn handle_route_poll(
7171 &self,
7172 ctx: &RouteCtx,
7173 frame: Frame,
7174 route_channel: u16,
7175 route_epoch: u32,
7176 kind: PollKind,
7177 ) -> Result<Vec<Frame>, RouterError> {
7178 let snapshot = self
7179 .forwarding
7180 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7181 .map_err(RouterError::Forwarding)?;
7182 let response = match (kind, snapshot) {
7183 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7184 ClientControlResponse::RoutePoll {
7185 route_channel,
7186 route_epoch,
7187 status,
7188 live: None,
7189 }
7190 }
7191 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7192 route_channel,
7193 route_epoch,
7194 status: None,
7195 live: None,
7196 },
7197 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7198 let live = self
7224 .process_liveness
7225 .as_ref()
7226 .and_then(|source| source.process_live(&module_id))
7227 .unwrap_or(true);
7228 ClientControlResponse::RoutePoll {
7229 route_channel,
7230 route_epoch,
7231 status: None,
7232 live: Some(live),
7233 }
7234 }
7235 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7236 route_channel,
7237 route_epoch,
7238 status: None,
7239 live: Some(false),
7240 },
7241 };
7242
7243 Ok(vec![control_response_body_frame(
7244 &frame,
7245 &response,
7246 "ClientControlResponse::RoutePoll",
7247 )?])
7248 }
7249
7250 pub(crate) fn observe_module_control_completion(
7251 &self,
7252 completion: ModuleControlRpcCompletion,
7253 ) -> bool {
7254 match completion {
7255 ModuleControlRpcCompletion::Unknown => false,
7256 ModuleControlRpcCompletion::Settled => true,
7257 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7258 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7259 info!(
7260 module_id = %module_id,
7261 latency_ms,
7262 "late health.check answer proves the module is alive"
7263 );
7264 match self
7265 .supervisor
7266 .record_late_health_answer(&module_id, latency_ms)
7267 {
7268 Ok(true) => {}
7269 Ok(false) => debug!(
7270 module_id = %module_id,
7271 latency_ms,
7272 "late health.check answer has no active supervisor snapshot"
7273 ),
7274 Err(err) => warn!(
7275 module_id = %module_id,
7276 latency_ms,
7277 error = %err,
7278 "failed to record late health.check answer"
7279 ),
7280 }
7281 true
7282 }
7283 }
7284 }
7285
7286 fn refuse_to_end_module_connection_for_a_client(
7307 &self,
7308 module_connection_id: ConnectionId,
7309 corr: u64,
7310 err: ForwardingError,
7311 ) -> Result<(), RouterError> {
7312 if let ForwardingError::ConnectionClosing { connection_id } = err {
7313 if connection_id != module_connection_id {
7314 warn!(
7315 module_connection_id = module_connection_id.get(),
7316 client_connection_id = connection_id.get(),
7317 corr,
7318 "dropping a route.bind response for a closing client; the module connection keeps serving"
7319 );
7320 return Ok(());
7321 }
7322 }
7323 Err(RouterError::Forwarding(err))
7324 }
7325
7326 fn handle_module_relay_response(
7327 &self,
7328 connection_id: ConnectionId,
7329 frame: Frame,
7330 ) -> Result<Vec<Frame>, RouterError> {
7331 let mut secondary_error = None;
7332 let outcome = match frame.header.ty {
7333 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7334 Ok(probe) if probe.op == "route.bind" => {
7335 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7336 Ok(ModuleControlResponse::RouteBindAck {}) => {
7337 RouteBindRelayOutcome::Accepted
7338 }
7339 Ok(other) => {
7340 let message =
7341 format!("route.bind response carried unexpected body: {other:?}");
7342 secondary_error = Some(control_error_frame(
7343 &frame,
7344 "invalid_control_body",
7345 message.clone(),
7346 )?);
7347 RouteBindRelayOutcome::ModuleGone(message)
7348 }
7349 Err(err) => {
7350 let message = format!("malformed route.bind response body: {err}");
7351 secondary_error = Some(control_error_frame(
7352 &frame,
7353 "invalid_control_body",
7354 message.clone(),
7355 )?);
7356 RouteBindRelayOutcome::ModuleGone(message)
7357 }
7358 }
7359 }
7360 Ok(probe) => {
7361 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7362 {
7363 Ok(response) => ModuleControlRpcOutcome::Response(response),
7364 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7365 "malformed {} response body: {err}",
7366 probe.op
7367 )),
7368 };
7369 let completion = self
7370 .forwarding
7371 .complete_module_control_rpc(
7372 connection_id,
7373 frame.header.corr,
7374 Some(&probe.op),
7375 outcome,
7376 )
7377 .map_err(RouterError::Forwarding)?;
7378 if !self.observe_module_control_completion(completion) {
7379 debug!(
7380 connection_id = connection_id.get(),
7381 corr = frame.header.corr,
7382 op = %probe.op,
7383 "dropping late or unknown module-control RPC response"
7384 );
7385 }
7386 return Ok(Vec::new());
7387 }
7388 Err(err) => {
7389 if let Some(expected_op) = self
7390 .forwarding
7391 .pending_module_control_op(connection_id, frame.header.corr)
7392 .map_err(RouterError::Forwarding)?
7393 {
7394 let completion = self
7395 .forwarding
7396 .complete_module_control_rpc(
7397 connection_id,
7398 frame.header.corr,
7399 None,
7400 ModuleControlRpcOutcome::MalformedResponse(format!(
7401 "malformed {expected_op} response body: {err}"
7402 )),
7403 )
7404 .map_err(RouterError::Forwarding)?;
7405 if !self.observe_module_control_completion(completion) {
7406 debug!(
7407 connection_id = connection_id.get(),
7408 corr = frame.header.corr,
7409 "dropping late malformed module-control RPC response"
7410 );
7411 }
7412 return Ok(Vec::new());
7413 }
7414 let message = format!("malformed route.bind response body: {err}");
7415 secondary_error = Some(control_error_frame(
7416 &frame,
7417 "invalid_control_body",
7418 message.clone(),
7419 )?);
7420 RouteBindRelayOutcome::ModuleGone(message)
7421 }
7422 },
7423 FrameType::Error => {
7424 if self
7425 .forwarding
7426 .pending_module_control_op(connection_id, frame.header.corr)
7427 .map_err(RouterError::Forwarding)?
7428 .is_some()
7429 {
7430 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7431 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7432 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7433 "malformed module-control ERROR body: {err}"
7434 )),
7435 };
7436 let completion = self
7437 .forwarding
7438 .complete_module_control_rpc(
7439 connection_id,
7440 frame.header.corr,
7441 None,
7442 outcome,
7443 )
7444 .map_err(RouterError::Forwarding)?;
7445 if !self.observe_module_control_completion(completion) {
7446 debug!(
7447 connection_id = connection_id.get(),
7448 corr = frame.header.corr,
7449 "dropping late or unknown module-control RPC error"
7450 );
7451 }
7452 return Ok(Vec::new());
7453 }
7454 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7455 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7456 Err(err) => {
7457 let message = format!("malformed route.bind ERROR body: {err}");
7458 secondary_error = Some(control_error_frame(
7459 &frame,
7460 "invalid_control_body",
7461 message.clone(),
7462 )?);
7463 RouteBindRelayOutcome::ModuleGone(message)
7464 }
7465 }
7466 }
7467 ty => {
7468 return Ok(vec![control_error_frame(
7469 &frame,
7470 "unsupported_control_frame",
7471 format!("unsupported module channel-0 frame {ty:?}"),
7472 )?])
7473 }
7474 };
7475
7476 let settled =
7477 self.forwarding
7478 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7479 let completion = match settled {
7480 Ok(completion) => completion,
7481 Err(err) => {
7482 self.refuse_to_end_module_connection_for_a_client(
7483 connection_id,
7484 frame.header.corr,
7485 err,
7486 )?;
7487 return Ok(secondary_error.into_iter().collect());
7488 }
7489 };
7490 if let Some(target) = completion.abandoned.as_ref() {
7491 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7492 }
7493 if !completion.settled {
7494 debug!(
7495 connection_id = connection_id.get(),
7496 corr = frame.header.corr,
7497 frame_type = ?frame.header.ty,
7498 "dropping late or unknown route.bind relay response"
7499 );
7500 }
7501 Ok(secondary_error.into_iter().collect())
7502 }
7503
7504 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7505 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7506 self.cleanup_connection_with_end_reason(
7510 connection_id,
7511 RegistrationEndReason::ExplicitGoodbye,
7512 )
7513 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7514 Ok(Vec::new())
7515 }
7516}
7517
7518impl Default for ControlHandler {
7519 fn default() -> Self {
7520 Self::new(Arc::new(Registry::default()))
7521 }
7522}
7523
7524impl crate::supervise::SwapPromotionObserver for ControlHandler {
7525 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7526 self.apply_registration_capabilities(registration);
7527 }
7528}
7529
7530fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7531 CapabilityRequirementStatus {
7532 consumer: status.consumer,
7533 capability: status.capability,
7534 need: match status.need {
7535 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7536 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7537 },
7538 verdict: status.verdict.as_str().to_string(),
7539 episode_seq: status.episode_seq,
7540 config_satisfiable: status.config_satisfiable,
7541 runtime_available: status.runtime_available,
7542 detail: status.detail,
7543 }
7544}
7545
7546fn append_capability_problem_detail(
7547 detail: Option<String>,
7548 capability_detail: Option<String>,
7549) -> Option<String> {
7550 match (detail, capability_detail) {
7551 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7552 (Some(detail), None) => Some(detail),
7553 (None, Some(capability_detail)) => Some(capability_detail),
7554 (None, None) => None,
7555 }
7556}
7557
7558fn subc_ops() -> Vec<String> {
7559 SUBC_CONTROL_OPS
7560 .iter()
7561 .map(|op| (*op).to_string())
7562 .collect()
7563}
7564
7565fn module_subc_ops() -> Vec<String> {
7566 SUBC_CONTROL_OPS
7567 .iter()
7568 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7569 .map(|op| (*op).to_string())
7570 .collect()
7571}
7572
7573#[cfg(test)]
7574fn module_baseline_control_ops() -> Vec<String> {
7575 MODULE_BASELINE_CONTROL_OPS
7576 .iter()
7577 .map(|op| (*op).to_string())
7578 .collect()
7579}
7580
7581fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7582 let mut seen = HashSet::new();
7583 let mut effective = Vec::new();
7584 for op in MODULE_BASELINE_CONTROL_OPS {
7585 if seen.insert((*op).to_string()) {
7586 effective.push((*op).to_string());
7587 }
7588 }
7589 for op in declared.unwrap_or_default() {
7590 if seen.insert(op.clone()) {
7591 effective.push(op);
7592 }
7593 }
7594 effective
7595}
7596
7597fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7598 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7599}
7600
7601fn target_module_id(target: &RouteTarget) -> &str {
7602 match target {
7603 RouteTarget::ToolProvider { module_id }
7604 | RouteTarget::ManagementSurface { module_id }
7605 | RouteTarget::InternalService { module_id, .. } => module_id,
7606 }
7607}
7608
7609fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7610 roles.iter().any(|role| match (target, role) {
7611 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7612 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7613 (
7614 RouteTarget::InternalService { service_id, .. },
7615 ProviderRole::InternalService {
7616 service_id: provided,
7617 ..
7618 },
7619 ) => service_id == provided,
7620 _ => false,
7621 })
7622}
7623
7624fn is_routable_role(role: &ProviderRole) -> bool {
7625 matches!(
7626 role,
7627 ProviderRole::ToolProvider { .. }
7628 | ProviderRole::ManagementSurface { .. }
7629 | ProviderRole::InternalService { .. }
7630 )
7631}
7632
7633#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7634enum ControlRequestBodyError {
7635 UnknownOp,
7636 InvalidBody,
7637}
7638
7639#[derive(Debug, Deserialize)]
7640struct ControlOpProbe {
7641 op: String,
7642}
7643
7644const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7647
7648fn is_known_module_push_op(body: &[u8]) -> bool {
7649 serde_json::from_slice::<ControlOpProbe>(body)
7650 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7651 .unwrap_or(false)
7652}
7653
7654fn is_known_module_request_op(body: &[u8]) -> bool {
7655 serde_json::from_slice::<ControlOpProbe>(body)
7656 .map(|probe| is_module_to_subc_op(&probe.op))
7657 .unwrap_or(false)
7658}
7659
7660fn is_module_to_subc_op(op: &str) -> bool {
7661 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7662}
7663
7664fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7665 debug!(
7666 op = %op,
7667 connection_id = connection_id.get(),
7668 corr,
7669 "control dispatch"
7670 );
7671}
7672
7673fn log_slow_control_dispatch(
7674 dispatch_started_at: Option<StdInstant>,
7675 op: &'static str,
7676 connection_id: ConnectionId,
7677 corr: u64,
7678 route_open_timings: Option<&RouteOpenTimings>,
7679) {
7680 let Some(dispatch_started_at) = dispatch_started_at else {
7681 return;
7682 };
7683 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7685 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7686 if let Some(timings) = route_open_timings {
7687 warn!(
7688 op = %op,
7689 connection_id = connection_id.get(),
7690 corr,
7691 elapsed_ms = elapsed.as_millis() as u64,
7692 module_id = ?timings.module_id,
7693 principal = %timings.principal,
7694 admission_ms = timings.admission.as_millis() as u64,
7695 project_root_ms = timings.project_root.as_millis() as u64,
7696 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7697 bind_send_ms = timings.bind_send.as_millis() as u64,
7698 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7699 "slow control dispatch"
7700 );
7701 return;
7702 }
7703 warn!(
7704 op = %op,
7705 connection_id = connection_id.get(),
7706 corr,
7707 elapsed_ms = elapsed.as_millis() as u64,
7708 "slow control dispatch"
7709 );
7710 }
7711}
7712
7713#[cfg(test)]
7715type TestCallerLookup =
7716 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7717
7718fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7720 match request {
7721 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7722 Some(PrivilegedOperation::Restart { module_id })
7723 }
7724 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7725 Some(PrivilegedOperation::Swap { module_id })
7726 }
7727 ClientControlRequest::SupervisorReload { module_id } => {
7728 Some(PrivilegedOperation::Reload { module_id })
7729 }
7730 ClientControlRequest::SupervisorRescan { preview } => {
7731 if *preview {
7732 None
7733 } else {
7734 Some(PrivilegedOperation::Rescan)
7735 }
7736 }
7737 ClientControlRequest::SupervisorRescanChecked {
7742 preview,
7743 candidate,
7744 expect,
7745 } => (!*preview && candidate.is_none() && expect.is_some())
7746 .then_some(PrivilegedOperation::Rescan),
7747 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7748 Some(PrivilegedOperation::SetEnabled {
7749 module_id,
7750 enabled: *enabled,
7751 })
7752 }
7753 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7754 Some(PrivilegedOperation::ReleaseReserved { module_id })
7755 }
7756 ClientControlRequest::ServerDescribe {}
7757 | ClientControlRequest::CatalogList { .. }
7758 | ClientControlRequest::RouteOpen { .. }
7759 | ClientControlRequest::RoutePoll { .. }
7760 | ClientControlRequest::SupervisorList {}
7761 | ClientControlRequest::SupervisorHealth {}
7762 | ClientControlRequest::SupervisorHealthProbe { .. }
7763 | ClientControlRequest::SupervisorRoutes { .. }
7764 | ClientControlRequest::SupervisorProvenance { .. }
7765 | ClientControlRequest::SupervisorStderrTail { .. }
7766 | ClientControlRequest::SupervisorTerminals { .. }
7767 | ClientControlRequest::SupervisorSpawnSnapshot {}
7768 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7769 }
7770}
7771
7772fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7773 match request {
7774 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7775 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7776 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7777 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7778 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7779 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7780 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7781 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7782 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7783 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7784 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7785 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7786 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7787 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7788 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7789 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7790 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7791 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7792 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7793 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7794 }
7795}
7796
7797struct RouteOpenCaller {
7805 principal: String,
7809 scope: String,
7812 harness: String,
7815}
7816
7817impl RouteOpenCaller {
7818 fn new(
7819 principal: Option<&Principal>,
7820 scope: Option<&ScopeSelector>,
7821 identity: &BindIdentity,
7822 ) -> Self {
7823 Self {
7824 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7825 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7826 harness: bounded_log_text(&identity.harness),
7827 }
7828 }
7829
7830 fn unknown() -> Self {
7833 Self {
7834 principal: "unknown".to_string(),
7835 scope: "unknown".to_string(),
7836 harness: "unknown".to_string(),
7837 }
7838 }
7839}
7840
7841const MAX_LOGGED_TEXT_CHARS: usize = 64;
7844
7845fn bounded_log_text(value: &str) -> String {
7849 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7850 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7851 bounded.push('…');
7852 }
7853 format!("{bounded:?}")
7854}
7855
7856fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7862 use subc_protocol::scope::ScopeKind;
7863 let kind = match stamp.kind {
7864 ScopeKind::Head => "head",
7865 ScopeKind::Worker => "worker",
7866 ScopeKind::Ephemeral => "ephemeral",
7867 };
7868 format!(
7869 "{kind}/{}/{}@{}",
7870 principal_label(&stamp.owner),
7871 bounded_log_text(&stamp.scope_ref),
7872 stamp.scope_epoch
7873 )
7874}
7875
7876fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7883 let epoch = selector
7884 .scope_epoch
7885 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7886 format!(
7887 "requested/{}/{}@{epoch}",
7888 bounded_log_text(&principal_label(&selector.owner)),
7889 bounded_log_text(&selector.scope_ref),
7890 )
7891}
7892
7893fn principal_label(principal: &Principal) -> String {
7894 match principal {
7895 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7896 Principal::Direct => "direct".to_string(),
7897 other => format!("{other:?}"),
7898 }
7899}
7900
7901fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7902 match request {
7903 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7904 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7905 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7906 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7907 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7908 }
7909}
7910
7911fn parse_client_control_request(
7912 body: &[u8],
7913) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7914 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7915 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7916 Ok(probe)
7917 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7918 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7919 {
7920 ControlRequestBodyError::InvalidBody
7921 }
7922 Ok(_) => ControlRequestBodyError::UnknownOp,
7923 Err(_) => ControlRequestBodyError::InvalidBody,
7924 };
7925 (err, classification)
7926 })
7927}
7928
7929fn parse_module_control_request_from_module(
7930 body: &[u8],
7931) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7932 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7933 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7934 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7935 Ok(_) => ControlRequestBodyError::UnknownOp,
7936 Err(_) => ControlRequestBodyError::InvalidBody,
7937 };
7938 (err, classification)
7939 })
7940}
7941
7942#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7943enum ProviderRoleKind {
7944 ToolProvider,
7945 PipelineStage,
7946 ManagementSurface,
7947 InternalService,
7948}
7949
7950fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7951 match role {
7952 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7953 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7954 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7955 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7956 }
7957}
7958
7959fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7960 roles.iter().map(provider_role_kind).collect()
7961}
7962
7963const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7966
7967#[derive(Debug, Default, PartialEq, Eq)]
7969struct ScopeOutcomeCounts {
7970 created: usize,
7971 replaced: usize,
7972 updated: usize,
7973 unchanged: usize,
7974 refused: usize,
7975}
7976
7977impl ScopeOutcomeCounts {
7978 fn of(results: &[ScopeRecordResult]) -> Self {
7979 let mut counts = Self::default();
7980 for result in results {
7981 let slot = match result.outcome {
7982 ScopeRecordOutcome::Created => &mut counts.created,
7983 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7984 ScopeRecordOutcome::Updated => &mut counts.updated,
7985 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7986 ScopeRecordOutcome::Refused => &mut counts.refused,
7987 };
7988 *slot += 1;
7989 }
7990 counts
7991 }
7992}
7993
7994#[cfg(test)]
7995mod scope_outcome_count_tests {
7996 use super::*;
7997
7998 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7999 ScopeRecordResult {
8000 scope_ref: "r".to_string(),
8001 scope_epoch: 1,
8002 outcome,
8003 code: None,
8004 message: None,
8005 version: None,
8006 parent_state: None,
8007 }
8008 }
8009
8010 #[test]
8013 fn every_outcome_is_counted_in_its_own_field() {
8014 let results = [
8015 result(ScopeRecordOutcome::Created),
8016 result(ScopeRecordOutcome::Created),
8017 result(ScopeRecordOutcome::Replaced),
8018 result(ScopeRecordOutcome::Updated),
8019 result(ScopeRecordOutcome::Unchanged),
8020 result(ScopeRecordOutcome::Refused),
8021 result(ScopeRecordOutcome::Refused),
8022 result(ScopeRecordOutcome::Refused),
8023 ];
8024 assert_eq!(
8025 ScopeOutcomeCounts::of(&results),
8026 ScopeOutcomeCounts {
8027 created: 2,
8028 replaced: 1,
8029 updated: 1,
8030 unchanged: 1,
8031 refused: 3,
8032 }
8033 );
8034 }
8035}
8036
8037fn capability_census_trigger(
8041 old: Option<&CapabilityDeclarations>,
8042 new: Option<&CapabilityDeclarations>,
8043) -> bool {
8044 let old_provides = old
8045 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8046 .unwrap_or_default();
8047 let old_denies = old
8048 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8049 .unwrap_or_default();
8050 let new = new.cloned().unwrap_or(CapabilityDeclarations {
8051 provides: Vec::new(),
8052 requires: Vec::new(),
8053 must_never_reach: Vec::new(),
8054 });
8055
8056 new.provides
8057 .iter()
8058 .any(|capability| !old_provides.contains(capability))
8059 || new
8060 .must_never_reach
8061 .iter()
8062 .any(|capability| !old_denies.contains(capability))
8063}
8064
8065fn denied_capability<'a>(
8068 opening_manifest: &'a ModuleManifest,
8069 target_manifest: &ModuleManifest,
8070) -> Option<&'a str> {
8071 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
8072 let target_capabilities = target_manifest.capabilities.as_ref()?;
8073 opening_capabilities
8074 .must_never_reach
8075 .iter()
8076 .find(|denied| {
8077 target_capabilities
8078 .provides
8079 .iter()
8080 .any(|provided| provided == *denied)
8081 })
8082 .map(String::as_str)
8083}
8084
8085fn catalog_update_frozen_field_message(
8086 registered: &ModuleManifest,
8087 provides: &[ProviderRole],
8088) -> Option<String> {
8089 let old_has_provides = !registered.provides.is_empty();
8090 let new_has_provides = !provides.is_empty();
8091 if old_has_provides != new_has_provides {
8092 return Some(format!(
8093 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8094 registered.module_id
8095 ));
8096 }
8097
8098 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
8099 return Some(format!(
8100 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8101 registered.module_id
8102 ));
8103 }
8104
8105 let registered_concurrency = manifest_concurrency(registered);
8106 let mut candidate = registered.clone();
8107 candidate.provides = provides.to_vec();
8108 let candidate_concurrency = manifest_concurrency(&candidate);
8109 if candidate_concurrency != registered_concurrency {
8110 return Some(format!(
8111 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8112 registered.module_id, registered_concurrency, candidate_concurrency
8113 ));
8114 }
8115
8116 None
8119}
8120
8121fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8122 manifest.provides.iter().any(is_routable_role)
8123}
8124
8125fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8131 manifest
8132 .provides
8133 .iter()
8134 .find_map(|provider| match provider {
8135 ProviderRole::ToolProvider { concurrency, .. }
8136 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8137 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8138 })
8139 .unwrap_or(Concurrency::ModuleManaged)
8140}
8141
8142fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8149 let has_management_surface = manifest
8150 .provides
8151 .iter()
8152 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8153 if !has_management_surface {
8154 return false;
8155 }
8156 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8157 return false;
8158 };
8159 let Some(provides) = raw
8160 .get("manifest")
8161 .and_then(|manifest| manifest.get("provides"))
8162 .and_then(serde_json::Value::as_array)
8163 else {
8164 return false;
8165 };
8166 provides.iter().any(|role| {
8171 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8172 && role.get("concurrency").is_none()
8173 })
8174}
8175
8176fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8177 if peer_version != PROTOCOL_VERSION {
8178 return Err(format!(
8179 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8180 ));
8181 }
8182 Ok(PROTOCOL_VERSION)
8183}
8184
8185fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8186 Frame::build_with_version(
8187 response_version(frame),
8188 FrameType::Pong,
8189 frame.header.flags,
8190 0,
8191 0,
8192 frame.header.corr,
8193 Vec::new(),
8194 )
8195 .map_err(RouterError::FrameBuild)
8196}
8197
8198pub(crate) fn control_error_frame(
8199 frame: &Frame,
8200 code: &'static str,
8201 message: impl Into<String>,
8202) -> Result<Frame, RouterError> {
8203 control_error_body_frame(
8204 frame,
8205 ErrorBody {
8206 code: code.to_string(),
8207 message: message.into(),
8208 detail: None,
8209 },
8210 )
8211}
8212
8213fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8221 control_error_frame_with_flags(frame, error, daemon_error_flags())
8222}
8223
8224fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8230 control_error_frame_with_flags(frame, error, control_flags())
8231}
8232
8233fn control_error_frame_with_flags(
8234 frame: &Frame,
8235 error: ErrorBody,
8236 flags: Flags,
8237) -> Result<Frame, RouterError> {
8238 let body = serde_json::to_vec(&error).map_err(|err| {
8239 RouterError::backend(
8240 0,
8241 frame.header.corr,
8242 format!("failed to encode control ERROR: {err}"),
8243 )
8244 })?;
8245
8246 Frame::build_with_version(
8247 response_version(frame),
8248 FrameType::Error,
8249 flags,
8250 0,
8251 0,
8252 frame.header.corr,
8253 body,
8254 )
8255 .map_err(RouterError::FrameBuild)
8256}
8257
8258pub(crate) fn control_response_body_frame<T: Serialize>(
8259 frame: &Frame,
8260 reply: &T,
8261 label: &'static str,
8262) -> Result<Frame, RouterError> {
8263 let body = serde_json::to_vec(reply).map_err(|err| {
8264 RouterError::backend(
8265 0,
8266 frame.header.corr,
8267 format!("failed to encode {label}: {err}"),
8268 )
8269 })?;
8270
8271 Frame::build_with_version(
8272 response_version(frame),
8273 FrameType::Response,
8274 control_flags(),
8275 0,
8276 0,
8277 frame.header.corr,
8278 body,
8279 )
8280 .map_err(RouterError::FrameBuild)
8281}
8282
8283fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8313 match err {
8314 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8315 ForwardingError::NoModuleConnection => "target_unavailable",
8316 ForwardingError::ModuleReloading { .. } => "module_reloading",
8317 ForwardingError::ClientRouteChannelExhausted { .. }
8318 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8319 ForwardingError::StaleModuleEndpoint
8320 | ForwardingError::UnknownReservation { .. }
8321 | ForwardingError::ConnectionClosing { .. }
8322 | ForwardingError::ClientEgressClosed { .. }
8323 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8324 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8327 ForwardingError::RelayCorrelationExhausted
8328 | ForwardingError::RouteOpenBuild(_)
8329 | ForwardingError::Poisoned => "forwarding_error",
8330 }
8331}
8332
8333fn response_version(frame: &Frame) -> u8 {
8334 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8335 frame.header.ver
8336 } else {
8337 PROTOCOL_VERSION
8338 }
8339}
8340
8341fn control_flags() -> Flags {
8342 Flags::new(false, Priority::Passive, false)
8343}
8344
8345fn send_goodbye_target_best_effort(
8350 counters: &DaemonCounters,
8351 target: &GoodbyeTarget,
8352 context: &'static str,
8353) {
8354 let Ok(frame) = Frame::build_with_version(
8355 target.negotiated_ver,
8356 FrameType::Goodbye,
8357 control_flags(),
8358 target.channel,
8359 target.epoch,
8360 0,
8361 Vec::new(),
8362 ) else {
8363 return;
8364 };
8365 crate::forwarding::send_module_route_goodbye(
8366 counters,
8367 &target.sink,
8368 frame,
8369 target.module_id.as_deref(),
8370 context,
8371 );
8372}
8373
8374pub(crate) fn send_route_control_pushes(
8375 forwarding: &ForwardingTable,
8376 routes: Vec<EndpointRoute>,
8377 push: ClientControlPush,
8378) {
8379 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8380 for route in routes {
8381 let target = route.goodbye_target;
8382 if let Some((existing, channels)) = targets
8383 .iter_mut()
8384 .find(|(existing, _)| existing.connection_id == target.connection_id)
8385 {
8386 debug_assert_eq!(
8387 existing.negotiated_ver, target.negotiated_ver,
8388 "one connection cannot negotiate multiple frame versions"
8389 );
8390 if !channels.contains(&target.channel) {
8391 channels.push(target.channel);
8392 }
8393 continue;
8394 }
8395 let channel = target.channel;
8396 targets.push((target, vec![channel]));
8397 }
8398 for (target, mut channels) in targets {
8399 channels.sort_unstable();
8400 let mut push = push.clone();
8401 match &mut push {
8402 ClientControlPush::RouteClosing {
8403 channels: covered, ..
8404 }
8405 | ClientControlPush::RouteClosed {
8406 channels: covered, ..
8407 } => *covered = channels,
8408 }
8409 let body = match serde_json::to_vec(&push) {
8410 Ok(body) => body,
8411 Err(err) => {
8412 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8413 continue;
8414 }
8415 };
8416 let frame = match Frame::build_with_version(
8417 target.negotiated_ver,
8418 FrameType::Push,
8419 control_flags(),
8420 0,
8421 0,
8422 0,
8423 body.clone(),
8424 ) {
8425 Ok(frame) => frame,
8426 Err(err) => {
8427 warn!(
8428 route_channel = target.channel,
8429 error = %err,
8430 "failed to build route lifecycle control PUSH frame"
8431 );
8432 continue;
8433 }
8434 };
8435 if let Err(err) = target.sink.try_send(frame) {
8436 if target.close_on_delivery_failure() {
8437 warn!(
8438 target_connection_id = target.connection_id.get(),
8439 route_channel = target.channel,
8440 error = %err,
8441 "route lifecycle control PUSH was not delivered to client; closing target connection"
8442 );
8443 let _ = forwarding.escalate_client_delivery_failure(
8444 target.connection_id,
8445 target.channel,
8446 target.epoch,
8447 CloseReason::new(
8448 "route_lifecycle_push_delivery_failed",
8449 format!(
8450 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8451 target.channel
8452 ),
8453 ),
8454 crate::forwarding::UndeliveredFrame {
8455 module_id: target.module_id.as_deref(),
8456 sink: &target.sink,
8457 },
8458 );
8459 }
8460 }
8461 }
8462}
8463
8464#[cfg(test)]
8465mod tests {
8466 mod peer_manifest {
8467 include!("peer_manifest_tests.rs");
8468 }
8469 mod activity_read {
8470 include!("activity_read_tests.rs");
8471 }
8472
8473 mod lifecycle_activity {
8474 include!("control_lifecycle_tests.rs");
8475 }
8476 #[cfg(unix)]
8477 #[tokio::test]
8478 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8479 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8480 let path = dir.join("subc.jsonc");
8481 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8482 "program":"/bin/sleep","args":["60"],"protocol":"none",
8483 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8484 }}}).to_string()).unwrap();
8485 let mut configured = crate::daemon_config::load(&path)
8486 .unwrap()
8487 .unwrap()
8488 .modules
8489 .pop()
8490 .unwrap();
8491 let registry = std::sync::Arc::new(crate::Registry::default());
8492 let handle = crate::SupervisorHandle::new();
8493 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8494 .with_handle(handle.clone());
8495 let module = supervisor
8496 .supervise_configured_with_health(
8497 configured.module_spec(),
8498 true,
8499 configured.health.clone(),
8500 None,
8501 configured.restart,
8502 )
8503 .unwrap();
8504 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8505 let before = module.status().unwrap().pid;
8506 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8507 configured.health.cadence = std::time::Duration::from_secs(3600);
8508 let health_only = handler
8509 .reconcile_supervised_modules(
8510 &supervisor,
8511 vec![configured.clone()],
8512 false,
8513 Principal::Direct,
8514 )
8515 .await
8516 .unwrap();
8517 assert!(
8518 health_only.0.changed_pending_reload.is_empty(),
8519 "health policy is already applied live"
8520 );
8521 assert_eq!(module.status().unwrap().pid, before);
8522 assert_eq!(
8523 module.configuration().unwrap().1.http,
8524 configured.health.http
8525 );
8526 configured.args = vec!["61".into()];
8527 let launch = handler
8528 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8529 .await
8530 .unwrap();
8531 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8532 assert_eq!(
8533 module.status().unwrap().pid,
8534 before,
8535 "a launch edit is stored until reload"
8536 );
8537 module.drain().await.unwrap();
8538 }
8539 use cortexkit_test_support::ScratchDir;
8540 use std::{
8541 collections::BTreeMap,
8542 fmt,
8543 path::PathBuf,
8544 sync::{Arc, Mutex},
8545 time::Duration,
8546 };
8547
8548 use serde_json::{json, Value};
8549 use subc_protocol::{
8550 manifest::{
8551 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8552 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8553 },
8554 session::HealthStatus,
8555 FrameType,
8556 };
8557
8558 use super::*;
8559 use crate::{
8560 forwarding::{DataRoute, DataRouteState},
8561 registry::ChannelState,
8562 router::FrameSink,
8563 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8564 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8565 RouteCtx, Router,
8566 };
8567 use tokio::{
8568 sync::mpsc,
8569 time::{sleep, Instant},
8570 };
8571 use tracing::{
8572 field::{Field, Visit},
8573 Event, Subscriber,
8574 };
8575 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8576
8577 fn fake_aft_stub_path() -> PathBuf {
8596 let mut path = std::env::current_exe().expect("current_exe available in tests");
8597 path.pop(); path.pop(); path.push(if cfg!(windows) {
8600 "fake-aft-stub.exe"
8601 } else {
8602 "fake-aft-stub"
8603 });
8604 assert!(
8605 path.exists(),
8606 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8607 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8608 path.display()
8609 );
8610 path
8611 }
8612
8613 fn client_retries(code: &str) -> bool {
8618 subc_protocol::error_codes::is_retryable_route_open(code)
8619 }
8620
8621 #[test]
8632 fn retryability_of_forwarding_codes_matches_the_failure() {
8633 let transient = [
8636 ForwardingError::NoModuleConnection,
8637 ForwardingError::ModuleReloading {
8638 module_id: "m".into(),
8639 },
8640 ForwardingError::StaleModuleEndpoint,
8641 ForwardingError::UnknownReservation {
8642 client_channel: 1,
8643 module_channel: 1,
8644 },
8645 ForwardingError::ConnectionClosing {
8646 connection_id: ConnectionId::new(1),
8647 },
8648 ForwardingError::ClientEgressClosed {
8649 connection_id: ConnectionId::new(1),
8650 },
8651 ForwardingError::ModuleEgressUnavailable {
8652 connection_id: ConnectionId::new(1),
8653 },
8654 ];
8655 for err in transient {
8656 let code = forwarding_error_code(&err);
8657 assert!(
8658 client_retries(code),
8659 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8660 );
8661 }
8662
8663 let permanent = [
8668 ForwardingError::ConnectionRoleConflict {
8669 connection_id: ConnectionId::new(1),
8670 },
8671 ForwardingError::ClientRouteChannelExhausted {
8672 connection_id: ConnectionId::new(1),
8673 },
8674 ForwardingError::ModuleRouteChannelExhausted {
8675 endpoint: ModuleEndpointId {
8676 connection_id: ConnectionId::new(1),
8677 generation: 1,
8678 },
8679 },
8680 ForwardingError::RelayCorrelationExhausted,
8681 ForwardingError::RouteOpenBuild("x".into()),
8682 ForwardingError::Poisoned,
8683 ];
8684 for err in permanent {
8685 let code = forwarding_error_code(&err);
8686 assert!(
8687 !client_retries(code),
8688 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8689 );
8690 }
8691 }
8692
8693 #[tokio::test]
8721 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8722 let handler = ControlHandler::default();
8723 let frame =
8724 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8725
8726 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8730 assert_eq!(
8731 stamped,
8732 Principal::Direct,
8733 "a caller that proved nothing must not be stamped as a supervised module"
8734 );
8735
8736 let forged = handler
8741 .route_open_principal(
8742 &frame,
8743 Some(ConsumerIdentity {
8744 module_id: "aft".to_string(),
8745 launch_nonce: "not-a-real-nonce".to_string(),
8746 }),
8747 )
8748 .unwrap();
8749 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8750 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8751 }
8752
8753 #[test]
8770 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8771 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8772 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8773 let ClientControlRequest::RouteOpen {
8774 consumer_identity, ..
8775 } = parsed
8776 else {
8777 panic!("route.open body must parse as RouteOpen");
8778 };
8779 assert_eq!(
8780 consumer_identity,
8781 Some(ConsumerIdentity {
8782 module_id: "aft".to_string(),
8783 launch_nonce: "n".to_string(),
8784 }),
8785 "the wire field name must reach the value route_open_principal reads"
8786 );
8787 }
8788
8789 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8790 ModuleManifest::builder(module_id, "0.1.0")
8791 .protocol_ver(protocol_ver)
8792 .provides(vec![ProviderRole::ToolProvider {
8793 tools: vec![Tool {
8794 name: "read".to_string(),
8795 description: None,
8796 execution_mode: ExecutionMode::Pure,
8797 schema: json!({"type": "object"}),
8798 }],
8799 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8800 concurrency: Concurrency::ModuleManaged,
8801 emits_push: true,
8802 sub_supervises: true,
8803 }])
8804 .build()
8805 }
8806
8807 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8808 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8809 }
8810
8811 fn hello_frame_with_control_ops(
8812 module_id: &str,
8813 protocol_ver: u8,
8814 corr: u64,
8815 control_ops: Option<Vec<String>>,
8816 ) -> Frame {
8817 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8818 }
8819
8820 fn hello_frame_with_nonce(
8821 module_id: &str,
8822 protocol_ver: u8,
8823 corr: u64,
8824 launch_nonce: Option<&str>,
8825 ) -> Frame {
8826 hello_frame_full(
8827 module_id,
8828 protocol_ver,
8829 corr,
8830 None,
8831 launch_nonce.map(ToOwned::to_owned),
8832 )
8833 }
8834
8835 fn hello_frame_full(
8836 module_id: &str,
8837 protocol_ver: u8,
8838 corr: u64,
8839 control_ops: Option<Vec<String>>,
8840 launch_nonce: Option<String>,
8841 ) -> Frame {
8842 let body = serde_json::to_vec(&ModuleHelloBody {
8843 manifest: manifest(module_id, protocol_ver),
8844 protocol_ver,
8845 control_ops,
8846 launch_nonce,
8847 })
8848 .unwrap();
8849 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8850 }
8851
8852 fn hello_frame_with_rich_ui(
8853 module_id: &str,
8854 protocol_ver: u8,
8855 corr: u64,
8856 launch_nonce: Option<&str>,
8857 rich_ui: Value,
8858 ) -> Frame {
8859 let mut body = serde_json::to_value(ModuleHelloBody {
8860 manifest: manifest(module_id, protocol_ver),
8861 protocol_ver,
8862 control_ops: None,
8863 launch_nonce: launch_nonce.map(str::to_string),
8864 })
8865 .unwrap();
8866 body["manifest"]["rich_ui"] = rich_ui;
8867 Frame::build(
8868 FrameType::Hello,
8869 control_flags(),
8870 0,
8871 0,
8872 corr,
8873 serde_json::to_vec(&body).unwrap(),
8874 )
8875 .unwrap()
8876 }
8877
8878 fn non_routable_hello_frame_with_control_ops(
8879 module_id: &str,
8880 corr: u64,
8881 control_ops: Option<Vec<String>>,
8882 ) -> Frame {
8883 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8884 manifest.provides.clear();
8885 let body = serde_json::to_vec(&ModuleHelloBody {
8886 manifest,
8887 protocol_ver: PROTOCOL_VERSION,
8888 control_ops,
8889 launch_nonce: None,
8890 })
8891 .unwrap();
8892 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8893 }
8894
8895 fn capability_grammar_hello_frame(
8896 capabilities: Value,
8897 runtime_computed: Option<Value>,
8898 corr: u64,
8899 ) -> Frame {
8900 let mut body = serde_json::to_value(ModuleHelloBody {
8901 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8902 protocol_ver: PROTOCOL_VERSION,
8903 control_ops: None,
8904 launch_nonce: None,
8905 })
8906 .expect("HELLO body serializes");
8907 body["manifest"]["capabilities"] = capabilities;
8908 if let Some(runtime_computed) = runtime_computed {
8909 body["runtime_computed"] = runtime_computed;
8910 }
8911 Frame::build(
8912 FrameType::Hello,
8913 control_flags(),
8914 0,
8915 0,
8916 corr,
8917 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8918 )
8919 .expect("HELLO frame builds")
8920 }
8921
8922 fn channel_request(channel: u16, corr: u64) -> Frame {
8923 Frame::build(
8924 FrameType::Request,
8925 Flags::new(true, Priority::Interactive, false),
8926 channel,
8927 0,
8928 corr,
8929 b"opaque".to_vec(),
8930 )
8931 .unwrap()
8932 }
8933
8934 fn route_ctx(
8935 connection_id: ConnectionId,
8936 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8937 let (tx, rx) = mpsc::channel(8);
8938 (
8939 RouteCtx {
8940 connection_id,
8941 egress: FrameSink::new(tx),
8942 },
8943 rx,
8944 )
8945 }
8946
8947 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8948 serde_json::from_slice(&frame.body).unwrap()
8949 }
8950
8951 async fn hello_via_sink(
8956 handler: &ControlHandler,
8957 ctx: &RouteCtx,
8958 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8959 hello: Frame,
8960 ) -> Frame {
8961 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8962 assert!(
8963 replies.is_empty(),
8964 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8965 );
8966 let ack = rx
8967 .try_recv()
8968 .expect("HELLO_ACK is queued on the module sink")
8969 .frame;
8970 assert_eq!(ack.header.ty, FrameType::HelloAck);
8971 ack
8972 }
8973
8974 fn parse_error(frame: &Frame) -> Value {
8975 serde_json::from_slice(&frame.body).unwrap()
8976 }
8977
8978 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8979 serde_json::from_slice(&frame.body).unwrap()
8980 }
8981
8982 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8983 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8984 route_channel,
8985 route_epoch: 0,
8986 kind,
8987 })
8988 .unwrap();
8989 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8990 }
8991
8992 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8993 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8994 module_id: module_id.to_string(),
8995 })
8996 .unwrap();
8997 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8998 }
8999
9000 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9001 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9002 }
9003
9004 fn route_open_frame_with_consumer_capabilities(
9005 corr: u64,
9006 module_id: &str,
9007 project_root: ScratchDir,
9008 consumer_capabilities: Option<Vec<String>>,
9009 ) -> Frame {
9010 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9011 target: RouteTarget::ToolProvider {
9012 module_id: module_id.to_string(),
9013 },
9014 identity: BindIdentity::new(
9015 project_root.path().to_path_buf(),
9016 "unit".to_string(),
9017 "session".to_string(),
9018 ),
9019 consumer_identity: None,
9020 consumer_capabilities,
9021 role_versions: None,
9022 admission_facts: None,
9023 scope: None,
9024 })
9025 .unwrap();
9026 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9027 }
9028
9029 fn route_open_frame_with_role_versions(
9030 corr: u64,
9031 module_id: &str,
9032 project_root: ScratchDir,
9033 role_versions: Option<BTreeMap<String, String>>,
9034 ) -> Frame {
9035 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9036 target: RouteTarget::ToolProvider {
9037 module_id: module_id.to_string(),
9038 },
9039 identity: BindIdentity::new(
9040 project_root.path().to_path_buf(),
9041 "unit".to_string(),
9042 format!("session-{corr}"),
9043 ),
9044 consumer_identity: None,
9045 consumer_capabilities: None,
9046 role_versions,
9047 admission_facts: None,
9048 scope: None,
9049 })
9050 .unwrap();
9051 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9052 }
9053
9054 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9055 entries
9056 .iter()
9057 .map(|(role, version)| (role.to_string(), version.to_string()))
9058 .collect()
9059 }
9060
9061 fn route_open_frame_with_admission_facts(
9062 corr: u64,
9063 module_id: &str,
9064 project_root: ScratchDir,
9065 consumer_identity: Option<subc_control::ConsumerIdentity>,
9066 facts: Option<Value>,
9067 ) -> Frame {
9068 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9069 target: RouteTarget::ToolProvider {
9070 module_id: module_id.to_string(),
9071 },
9072 identity: BindIdentity::new(
9073 project_root.path().to_path_buf(),
9074 "unit".to_string(),
9075 format!("session-{corr}"),
9076 ),
9077 consumer_identity,
9078 consumer_capabilities: None,
9079 role_versions: None,
9080 admission_facts: facts,
9081 scope: None,
9082 })
9083 .unwrap();
9084 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9085 }
9086
9087 #[derive(Clone, Default)]
9088 struct EventCapture {
9089 events: Arc<Mutex<Vec<CapturedEvent>>>,
9090 }
9091
9092 #[derive(Clone, Debug)]
9093 struct CapturedEvent {
9094 target: String,
9095 level: tracing::Level,
9096 fields: BTreeMap<String, String>,
9097 }
9098
9099 impl EventCapture {
9100 fn events(&self) -> Vec<CapturedEvent> {
9101 self.events.lock().unwrap().clone()
9102 }
9103 }
9104
9105 impl<S> Layer<S> for EventCapture
9106 where
9107 S: Subscriber,
9108 {
9109 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9110 let mut visitor = EventFieldVisitor::default();
9111 event.record(&mut visitor);
9112 self.events.lock().unwrap().push(CapturedEvent {
9113 target: event.metadata().target().to_string(),
9114 level: *event.metadata().level(),
9115 fields: visitor.fields,
9116 });
9117 }
9118 }
9119
9120 #[derive(Default)]
9121 struct EventFieldVisitor {
9122 fields: BTreeMap<String, String>,
9123 }
9124
9125 impl Visit for EventFieldVisitor {
9126 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9127 self.fields
9128 .insert(field.name().to_string(), format!("{value:?}"));
9129 }
9130 }
9131
9132 fn health_response(corr: u64, status: HealthStatus) -> Frame {
9133 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9134 status,
9135 detail: Some("warming".to_string()),
9136 metrics: Some(json!({"queue_depth": 3})),
9137 })
9138 .unwrap();
9139 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9140 }
9141
9142 fn route_bind_ack(corr: u64) -> Frame {
9143 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9144 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9145 }
9146
9147 fn unique_project_root(label: &str) -> ScratchDir {
9148 ScratchDir::new(label)
9149 }
9150
9151 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9152 match parse_route_poll(frame) {
9153 ClientControlResponse::RoutePoll {
9154 status: None,
9155 live: Some(live),
9156 ..
9157 } => assert_eq!(live, expected_live),
9158 other => panic!("unexpected route.poll response: {other:?}"),
9159 }
9160 }
9161
9162 fn bind_liveness_route(
9163 registry: &Registry,
9164 forwarding: &ForwardingTable,
9165 module_id: &str,
9166 ) -> (RouteCtx, u16, u32) {
9167 let module_connection = ConnectionId::new(101);
9168 let client_connection = ConnectionId::new(202);
9169 let registration = registry
9170 .register_with_control_ops(
9171 manifest(module_id, PROTOCOL_VERSION),
9172 PROTOCOL_VERSION,
9173 module_connection,
9174 module_baseline_control_ops(),
9175 )
9176 .unwrap();
9177 let (module_tx, _module_rx) = mpsc::channel(8);
9178 let endpoint = forwarding
9179 .register_module_connection(
9180 module_connection,
9181 module_id.to_string(),
9182 PROTOCOL_VERSION,
9183 manifest_concurrency(®istration.manifest),
9184 FrameSink::new(module_tx),
9185 )
9186 .unwrap();
9187 let (client_ctx, _client_rx) = route_ctx(client_connection);
9188 let pending = forwarding
9189 .begin_route_bind_relay_for_test(
9190 client_connection,
9191 client_ctx.egress.clone(),
9192 1,
9193 module_id,
9194 )
9195 .unwrap();
9196 assert_eq!(pending.endpoint, endpoint);
9197 let route_channel = pending.client_channel;
9198 let route_epoch = pending.client_epoch;
9199 forwarding
9200 .complete_pending_relay(
9201 module_connection,
9202 pending.corr,
9203 RouteBindRelayOutcome::Accepted,
9204 )
9205 .unwrap();
9206 (client_ctx, route_channel, route_epoch)
9207 }
9208
9209 struct FakeProcessLiveness {
9210 live: Option<bool>,
9211 }
9212
9213 impl ModuleProcessLiveness for FakeProcessLiveness {
9214 fn process_live(&self, _module_id: &str) -> Option<bool> {
9215 self.live
9216 }
9217 }
9218
9219 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9220 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9221 {
9222 let registry = Arc::new(Registry::default());
9223 let supervisor_handle = SupervisorHandle::new();
9224 let supervisor = Supervisor::new_for_test(
9225 Arc::clone(®istry),
9226 RestartPolicy::new(1, Duration::from_millis(10)),
9227 )
9228 .with_handle(supervisor_handle.clone());
9229 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9230 let module = supervisor
9231 .spawn(ModuleSpec {
9232 module_id: "stderr-tail-wire".to_string(),
9233 program: fake_aft_stub_path(),
9234 args: Vec::new(),
9235 env: vec![
9236 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9237 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9238 ],
9239 reserved: false,
9240 reserved_prefixes: Vec::new(),
9241 protocol: ModuleProtocol::Subc,
9242 overlap: Default::default(),
9243 })
9244 .unwrap();
9245
9246 let deadline = Instant::now() + Duration::from_secs(5);
9247 loop {
9248 let tail = module.stderr_tail(None, None);
9249 if tail
9250 .entries
9251 .iter()
9252 .any(|entry| matches!(entry, TailEntry::ProcessStart))
9253 && tail.entries.iter().any(|entry| {
9254 matches!(
9255 entry,
9256 TailEntry::Line {
9257 truncated: true,
9258 ..
9259 }
9260 )
9261 })
9262 {
9263 break;
9264 }
9265 assert!(
9266 Instant::now() < deadline,
9267 "module did not produce a truncated line and restart boundary: {tail:?}"
9268 );
9269 sleep(Duration::from_millis(10)).await;
9270 }
9271
9272 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9273 let request = ClientControlRequest::SupervisorStderrTail {
9274 module_id: "stderr-tail-wire".to_string(),
9275 max_lines: None,
9276 max_bytes: None,
9277 };
9278 let frame = Frame::build(
9279 FrameType::Request,
9280 control_flags(),
9281 0,
9282 0,
9283 1,
9284 serde_json::to_vec(&request).unwrap(),
9285 )
9286 .unwrap();
9287 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9288 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9289 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9290 serde_json::from_slice(&responses[0].body).unwrap()
9291 else {
9292 panic!("expected supervisor.stderr_tail response");
9293 };
9294
9295 assert!(
9296 tail.entries
9297 .iter()
9298 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9299 "the control response lost the restart boundary"
9300 );
9301 let Some(StderrTailEntry::Line {
9302 text,
9303 truncated,
9304 at_ms,
9305 }) = tail.entries.iter().find(|entry| {
9306 matches!(
9307 entry,
9308 StderrTailEntry::Line {
9309 truncated: true,
9310 ..
9311 }
9312 )
9313 })
9314 else {
9315 panic!("the control response lost the truncated line");
9316 };
9317 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9318 assert!(*truncated);
9319 assert!(
9320 at_ms.is_some(),
9321 "the control response lost the line's capture time"
9322 );
9323 }
9324
9325 #[tokio::test(flavor = "current_thread")]
9330 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9331 let dir = ScratchDir::new("terminals-off-worker");
9332 let journal_path = dir.join("terminals.jsonl");
9333 let registry = Arc::new(Registry::default());
9334 let supervisor_handle = SupervisorHandle::new();
9335 let supervisor =
9336 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9337 .with_handle(supervisor_handle.clone())
9338 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9339 let module = supervisor
9340 .spawn(ModuleSpec {
9341 module_id: "terminal-off-worker".to_string(),
9342 program: fake_aft_stub_path(),
9343 args: Vec::new(),
9344 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9345 reserved: false,
9346 reserved_prefixes: Vec::new(),
9347 protocol: ModuleProtocol::Subc,
9348 overlap: Default::default(),
9349 })
9350 .unwrap();
9351 let deadline = Instant::now() + Duration::from_secs(5);
9352 while module.terminal_history().entries.len() != 2 {
9353 assert!(Instant::now() < deadline, "module did not record two exits");
9354 sleep(Duration::from_millis(10)).await;
9355 }
9356
9357 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9358 let handler =
9359 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
9360 let frame = Frame::build(
9361 FrameType::Request,
9362 control_flags(),
9363 0,
9364 0,
9365 1,
9366 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9367 module_id: "terminal-off-worker".to_string(),
9368 })
9369 .unwrap(),
9370 )
9371 .unwrap();
9372 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9373 let spawned_at = std::time::Instant::now();
9374 let read = tokio::spawn({
9375 let handler = Arc::clone(&handler);
9376 async move { handler.handle_control_frame(&ctx, frame).await }
9377 });
9378 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9381 .await
9382 .unwrap()
9383 .expect("the history read reached its pause");
9384 let elapsed = spawned_at.elapsed();
9385 assert!(
9386 elapsed < Duration::from_secs(2) && !read.is_finished(),
9387 "this task could not run while the history read was paused \
9388 (resumed after {elapsed:?}, read finished: {})",
9389 read.is_finished()
9390 );
9391
9392 drop(release);
9393 let responses = read.await.unwrap().unwrap();
9394 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9395 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9396 panic!("expected supervisor.terminals response");
9397 };
9398 assert_eq!(terminals.entries.len(), 2);
9399 assert_eq!(terminals.journal_skipped_lines, 0);
9400 assert_eq!(terminals.journal_read_errors, 0);
9401 }
9402
9403 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9404 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9405 let registry = Arc::new(Registry::default());
9406 let supervisor_handle = SupervisorHandle::new();
9407 let supervisor =
9408 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9409 .with_handle(supervisor_handle.clone());
9410 let module = supervisor
9411 .spawn(ModuleSpec {
9412 module_id: "terminal-golden".to_string(),
9413 program: fake_aft_stub_path(),
9414 args: Vec::new(),
9415 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9416 reserved: false,
9417 reserved_prefixes: Vec::new(),
9418 protocol: ModuleProtocol::Subc,
9419 overlap: Default::default(),
9420 })
9421 .unwrap();
9422
9423 let deadline = Instant::now() + Duration::from_secs(5);
9424 while module.terminal_history().entries.len() != 2 {
9425 assert!(
9426 Instant::now() < deadline,
9427 "module did not retain two terminal exits: {:?}",
9428 module.terminal_history()
9429 );
9430 sleep(Duration::from_millis(10)).await;
9431 }
9432
9433 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9434 let request = ClientControlRequest::SupervisorTerminals {
9435 module_id: "terminal-golden".to_string(),
9436 };
9437 let frame = Frame::build(
9438 FrameType::Request,
9439 control_flags(),
9440 0,
9441 0,
9442 1,
9443 serde_json::to_vec(&request).unwrap(),
9444 )
9445 .unwrap();
9446 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9447 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9448 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9449 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9450 panic!("expected supervisor.terminals response");
9451 };
9452 assert_eq!(terminals.entries.len(), 2);
9453 assert_eq!(terminals.dropped, 0);
9454
9455 let mut rendered = serde_json::to_value(response).unwrap();
9456 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9459 for (index, entry) in rendered["entries"]
9460 .as_array_mut()
9461 .expect("terminal response entries array")
9462 .iter_mut()
9463 .enumerate()
9464 {
9465 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9466 }
9467
9468 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9469 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9470 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9471 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9472 std::fs::write(&golden_path, &serialized).unwrap();
9473 }
9474 let expected: Value =
9475 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9476 assert_eq!(rendered, expected);
9477 }
9478
9479 #[test]
9480 fn hello_registers_manifest_and_returns_ack() {
9481 let registry = Arc::new(Registry::default());
9482 let handler = ControlHandler::new(Arc::clone(®istry));
9483 let conn = ConnectionId::new(1);
9484
9485 let responses = handler
9486 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9487 .unwrap();
9488
9489 assert_eq!(responses.len(), 1);
9490 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9491 assert_eq!(responses[0].header.channel, 0);
9492 assert_eq!(responses[0].header.corr, 7);
9493 let ack = parse_ack(&responses[0]);
9494 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9495 assert!(ack
9496 .subc_capabilities
9497 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9498 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9499 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9500 assert!(ack
9501 .subc_ops
9502 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9503 assert!(ack
9504 .subc_ops
9505 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9506
9507 let registration = registry.get_module("aft").unwrap().unwrap();
9508 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9509 assert_eq!(registration.state, ChannelState::Active);
9510 assert_eq!(registration.connection_id, conn);
9511 assert_eq!(registration.control_ops, module_baseline_control_ops());
9512 }
9513
9514 #[test]
9515 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9516 let invalid_identifiers = [
9517 ("case_change", "credentials-Provider/v1"),
9518 ("leading_zero", "credentials-provider/v01"),
9519 ("trailing_hyphen", "credentials-provider-/v1"),
9520 ("consecutive_hyphens", "credentials--provider/v1"),
9521 ("uppercase", "Credentials-provider/v1"),
9522 ("missing_v", "credentials-provider/1"),
9523 ("whitespace", "credentials provider/v1"),
9524 ("zero_version", "credentials-provider/v0"),
9525 ("out_of_range_version", "credentials-provider/v4294967296"),
9526 (
9527 "overlength_name",
9528 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9529 ),
9530 ];
9531 let mut cases = invalid_identifiers
9532 .into_iter()
9533 .map(|(name, identifier)| {
9534 (
9535 format!("identifier_{name}"),
9536 "capabilities.provides[0]".to_string(),
9537 identifier.to_string(),
9538 json!({ "provides": [identifier] }),
9539 None,
9540 )
9541 })
9542 .collect::<Vec<_>>();
9543 cases.extend([
9544 (
9545 "unknown_need".to_string(),
9546 "capabilities.requires[0].need".to_string(),
9547 "deferred".to_string(),
9548 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9549 None,
9550 ),
9551 (
9552 "duplicate_provides".to_string(),
9553 "capabilities.provides[1]".to_string(),
9554 "credentials-provider/v1".to_string(),
9555 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9556 None,
9557 ),
9558 (
9559 "duplicate_must_never_reach".to_string(),
9560 "capabilities.must_never_reach[1]".to_string(),
9561 "credentials-provider/v1".to_string(),
9562 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9563 None,
9564 ),
9565 (
9566 "duplicate_requires_same_need".to_string(),
9567 "capabilities.requires[1]".to_string(),
9568 "credentials-provider/v1".to_string(),
9569 json!({ "requires": [
9570 { "capability": "credentials-provider/v1", "need": "required" },
9571 { "capability": "credentials-provider/v1", "need": "required" }
9572 ] }),
9573 None,
9574 ),
9575 (
9576 "duplicate_requires_conflicting_need".to_string(),
9577 "capabilities.requires[1]".to_string(),
9578 "credentials-provider/v1".to_string(),
9579 json!({ "requires": [
9580 { "capability": "credentials-provider/v1", "need": "required" },
9581 { "capability": "credentials-provider/v1", "need": "optional" }
9582 ] }),
9583 None,
9584 ),
9585 (
9586 "capabilities_root_pointer".to_string(),
9587 "runtime_computed[0]".to_string(),
9588 "/capabilities".to_string(),
9589 json!({}),
9590 Some(json!(["/capabilities"])),
9591 ),
9592 (
9593 "capabilities_descendant_pointer".to_string(),
9594 "runtime_computed[0]".to_string(),
9595 "/capabilities/provides".to_string(),
9596 json!({}),
9597 Some(json!(["/capabilities/provides"])),
9598 ),
9599 (
9600 "malformed_pointer_without_leading_slash".to_string(),
9601 "runtime_computed[0]".to_string(),
9602 "capabilities".to_string(),
9603 json!({}),
9604 Some(json!(["capabilities"])),
9605 ),
9606 (
9607 "malformed_pointer_escape".to_string(),
9608 "runtime_computed[0]".to_string(),
9609 "/roles/~2/tools".to_string(),
9610 json!({}),
9611 Some(json!(["/roles/~2/tools"])),
9612 ),
9613 (
9614 "unknown_capabilities_field".to_string(),
9615 "capabilities.future".to_string(),
9616 "<array>".to_string(),
9617 json!({ "future": [] }),
9618 None,
9619 ),
9620 ]);
9621
9622 for (index, (name, field, value, capabilities, runtime_computed)) in
9623 cases.into_iter().enumerate()
9624 {
9625 let registry = Arc::new(Registry::default());
9626 let handler = ControlHandler::new(Arc::clone(®istry));
9627 let response = handler
9628 .handle_control(
9629 ConnectionId::new((index + 1) as u64),
9630 capability_grammar_hello_frame(
9631 capabilities,
9632 runtime_computed,
9633 index as u64 + 1,
9634 ),
9635 )
9636 .expect("invalid HELLO returns a refusal");
9637
9638 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9639 let error = parse_error(&response[0]);
9640 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9641 let message = error["message"]
9642 .as_str()
9643 .expect("error message is a string");
9644 assert!(
9645 message.contains(&field),
9646 "{name}: field missing from {message}"
9647 );
9648 assert!(
9649 message.contains(&value),
9650 "{name}: value missing from {message}"
9651 );
9652 assert_eq!(
9653 registry
9654 .active_registration_count()
9655 .expect("registry reads"),
9656 0,
9657 "{name}: refused HELLO must not create a catalog entry"
9658 );
9659 }
9660 }
9661
9662 #[test]
9663 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9664 let registry = Arc::new(Registry::default());
9665 let handler = ControlHandler::new(Arc::clone(®istry));
9666 let capabilities = json!({
9667 "provides": ["credentials-provider/v1"],
9668 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9669 "must_never_reach": ["federation-transport/v1"]
9670 });
9671 let response = handler
9672 .handle_control(
9673 ConnectionId::new(99),
9674 capability_grammar_hello_frame(
9675 capabilities.clone(),
9676 Some(json!(["/roles/0/tools"])),
9677 99,
9678 ),
9679 )
9680 .expect("valid HELLO registers");
9681 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9682
9683 let request = Frame::build(
9684 FrameType::Request,
9685 control_flags(),
9686 0,
9687 0,
9688 100,
9689 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9690 .expect("catalog request serializes"),
9691 )
9692 .expect("catalog request frame builds");
9693 let response = handler
9694 .handle_catalog_list(request, None)
9695 .expect("catalog list succeeds");
9696 let ClientControlResponse::CatalogList { modules, .. } =
9697 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9698 else {
9699 panic!("catalog request must return catalog.list");
9700 };
9701 assert_eq!(modules.len(), 1);
9702 assert_eq!(
9703 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9704 capabilities
9705 );
9706 }
9707
9708 #[test]
9709 fn catalog_list_mirrors_management_operation_description() {
9710 let registry = Arc::new(Registry::default());
9711 let handler = ControlHandler::new(Arc::clone(®istry));
9712 let description = "List managed records and return their identifiers and metadata.";
9713 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9714 manifest.provides = vec![ProviderRole::ManagementSurface {
9715 operations: vec![ManagementOperation {
9716 name: "records.list".to_string(),
9717 kind: ManagementOperationKind::Query,
9718 description: Some(description.to_string()),
9719 }],
9720 config_schema: json!({"type": "object"}),
9721 observability: vec![ObservabilitySurface {
9722 name: "records.stats".to_string(),
9723 kind: ObservabilityKind::Snapshot,
9724 }],
9725 identity_scope: vec![IdentityScope::Project],
9726 concurrency: Concurrency::ModuleManaged,
9727 }];
9728 registry
9729 .register_with_control_ops(
9730 manifest,
9731 PROTOCOL_VERSION,
9732 ConnectionId::new(99),
9733 Vec::new(),
9734 )
9735 .expect("described management manifest registers");
9736
9737 let request = Frame::build(
9738 FrameType::Request,
9739 control_flags(),
9740 0,
9741 0,
9742 100,
9743 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9744 .expect("catalog request serializes"),
9745 )
9746 .expect("catalog request frame builds");
9747 let response = handler
9748 .handle_catalog_list(request, None)
9749 .expect("catalog list succeeds");
9750 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9751 assert_eq!(
9752 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9753 "catalog.list must preserve the declared operation description verbatim"
9754 );
9755 }
9756
9757 #[test]
9758 fn catalog_list_mirrors_registered_operation_delivery() {
9759 let registry = Arc::new(Registry::default());
9760 let handler = ControlHandler::new(Arc::clone(®istry));
9761 let delivery =
9762 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9763 let mut body = serde_json::to_value(ModuleHelloBody {
9764 manifest: manifest("watch-management", PROTOCOL_VERSION),
9765 protocol_ver: PROTOCOL_VERSION,
9766 control_ops: None,
9767 launch_nonce: None,
9768 })
9769 .unwrap();
9770 body["manifest"]["provides"] = json!([{
9771 "role": "management_surface",
9772 "operations": [
9773 {"name": "status", "kind": "query"},
9774 {"name": "future", "kind": "query"},
9775 {"name": "reset", "kind": "mutate"}
9776 ],
9777 "config_schema": {}, "observability": [], "identity_scope": []
9778 }]);
9779 body["manifest"]["operation_delivery"] = delivery.clone();
9780 let hello = Frame::build(
9781 FrameType::Hello,
9782 control_flags(),
9783 0,
9784 0,
9785 99,
9786 serde_json::to_vec(&body).unwrap(),
9787 )
9788 .unwrap();
9789 let response = handler
9790 .handle_control(ConnectionId::new(99), hello)
9791 .unwrap();
9792 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9793
9794 let (_, registrations) = registry.list_modules().unwrap();
9795 let registered = ®istrations[0].manifest;
9796 assert!(registered.is_watch_operation("status"));
9797 assert!(!registered.is_watch_operation("future"));
9798 assert!(!registered.is_watch_operation("reset"));
9799 assert!(!registered.is_watch_operation("missing"));
9800
9801 let request = Frame::build(
9802 FrameType::Request,
9803 control_flags(),
9804 0,
9805 0,
9806 100,
9807 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9808 )
9809 .unwrap();
9810 let response = handler.handle_catalog_list(request, None).unwrap();
9811 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9812 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9813 let ClientControlResponse::CatalogList { modules, .. } =
9814 serde_json::from_slice(&response[0].body).unwrap()
9815 else {
9816 panic!("expected catalog.list");
9817 };
9818 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9819 }
9820
9821 #[test]
9822 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9823 let registry = Arc::new(Registry::default());
9824 let supervisor = SupervisorHandle::new();
9825 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9826 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9827 let descriptor = json!({
9828 "schema": "ckios.richui.manifest/1",
9829 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9830 "bytes": 4096,
9831 "chunks": 4
9832 });
9833 let connection = ConnectionId::new(201);
9834 let hello = handler
9835 .handle_control(
9836 connection,
9837 hello_frame_with_rich_ui(
9838 "vault",
9839 PROTOCOL_VERSION,
9840 201,
9841 Some("the-real-nonce"),
9842 descriptor.clone(),
9843 ),
9844 )
9845 .unwrap();
9846 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9847
9848 let request =
9849 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9850 let provides = registry
9851 .get_module_by_connection(connection)
9852 .unwrap()
9853 .unwrap()
9854 .manifest
9855 .provides;
9856 handler
9857 .handle_catalog_update(connection, request, provides, None, Some(false))
9858 .unwrap();
9859
9860 let request = Frame::build(
9861 FrameType::Request,
9862 control_flags(),
9863 0,
9864 0,
9865 203,
9866 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9867 )
9868 .unwrap();
9869 let response = handler.handle_catalog_list(request, None).unwrap();
9870 let ClientControlResponse::CatalogList { modules, .. } =
9871 serde_json::from_slice(&response[0].body).unwrap()
9872 else {
9873 panic!("expected catalog.list");
9874 };
9875 assert_eq!(
9876 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9877 descriptor
9878 );
9879 assert!(modules[0].nonce_verified);
9880 }
9881
9882 fn hello_with_role_declarations(declarations: Value) -> Frame {
9883 let mut body = serde_json::to_value(ModuleHelloBody {
9884 manifest: manifest("role-test", PROTOCOL_VERSION),
9885 protocol_ver: PROTOCOL_VERSION,
9886 control_ops: None,
9887 launch_nonce: None,
9888 })
9889 .unwrap();
9890 body["manifest"]["role_declarations"] = declarations;
9891 Frame::build(
9892 FrameType::Hello,
9893 control_flags(),
9894 0,
9895 0,
9896 301,
9897 serde_json::to_vec(&body).unwrap(),
9898 )
9899 .unwrap()
9900 }
9901
9902 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9903 let request = Frame::build(
9904 FrameType::Request,
9905 control_flags(),
9906 0,
9907 0,
9908 302,
9909 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9910 )
9911 .unwrap();
9912 let response = handler.handle_catalog_list(request, None).unwrap();
9913 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9914 let ClientControlResponse::CatalogList { mut modules, .. } =
9915 serde_json::from_slice(&response[0].body).unwrap()
9916 else {
9917 panic!("expected catalog.list");
9918 };
9919 (raw["modules"][0].clone(), modules.remove(0))
9920 }
9921
9922 #[test]
9923 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9924 let registry = Arc::new(Registry::default());
9925 let handler = ControlHandler::new(Arc::clone(®istry));
9926 let declarations = json!({
9927 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9928 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9929 });
9930 let connection = ConnectionId::new(301);
9931 let response = handler
9932 .handle_control(
9933 connection,
9934 hello_with_role_declarations(declarations.clone()),
9935 )
9936 .unwrap();
9937 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9938 let registered = registry
9939 .get_module_by_connection(connection)
9940 .unwrap()
9941 .unwrap()
9942 .manifest;
9943 assert_eq!(
9947 serde_json::to_value(®istered.role_declarations).unwrap(),
9948 declarations
9949 );
9950 let (raw, entry) = role_declarations_catalog(&handler);
9951 assert_eq!(raw["role_declarations"], declarations);
9952 assert_eq!(entry.role_declarations, registered.role_declarations);
9953
9954 let request =
9955 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9956 handler
9957 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9958 .unwrap();
9959 let (raw, entry) = role_declarations_catalog(&handler);
9960 assert_eq!(raw["role_declarations"], declarations);
9961 assert_eq!(entry.role_declarations, registered.role_declarations);
9962 }
9963
9964 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9965 let registry = Arc::new(Registry::default());
9966 let handler = ControlHandler::new(Arc::clone(®istry));
9967 let capture = EventCapture::default();
9968 let _subscriber =
9969 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9970 let response = handler
9971 .handle_control(
9972 ConnectionId::new(301),
9973 hello_with_role_declarations(declarations),
9974 )
9975 .unwrap();
9976 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9977 assert!(registry
9978 .get_module_by_connection(ConnectionId::new(301))
9979 .unwrap()
9980 .unwrap()
9981 .manifest
9982 .role_declarations
9983 .is_empty());
9984 let (raw, entry) = role_declarations_catalog(&handler);
9985 assert!(raw.get("role_declarations").is_none());
9986 assert!(entry.role_declarations.is_empty());
9987 let warnings: Vec<_> = capture
9988 .events()
9989 .into_iter()
9990 .filter(|event| {
9991 event.fields.get("message").map(String::as_str)
9992 == Some("dropping whole role_declarations map from HELLO")
9993 })
9994 .collect();
9995 assert_eq!(warnings.len(), 1);
9996 assert_eq!(warnings[0].level, tracing::Level::WARN);
9997 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9998 assert_eq!(warnings[0].fields["reason"], reason);
9999 }
10000
10001 #[test]
10002 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10003 for value in [
10004 json!(null),
10005 json!(false),
10006 json!(0),
10007 json!("opaque"),
10008 json!([]),
10009 ] {
10010 assert_hello_drops_role_declarations_with_warning(
10011 json!({"routing/v1": {}, "broken/v1": value}),
10012 "role_declarations values must be objects",
10013 );
10014 }
10015 assert_hello_drops_role_declarations_with_warning(
10016 json!([]),
10017 "role_declarations must be an object",
10018 );
10019 }
10020
10021 #[test]
10022 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10023 assert_hello_drops_role_declarations_with_warning(
10024 json!({"routing/v1": {}, "bad key": {}}),
10025 "role_declarations key must use capability-name grammar",
10026 );
10027 }
10028
10029 #[test]
10030 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10031 assert_hello_drops_role_declarations_with_warning(
10032 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10033 "role_declarations exceeds 16384 compact JSON bytes",
10034 );
10035 }
10036
10037 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10038 let mut body = serde_json::to_value(ModuleHelloBody {
10039 manifest: manifest("flow-ops", PROTOCOL_VERSION),
10040 protocol_ver: PROTOCOL_VERSION,
10041 control_ops: None,
10042 launch_nonce: None,
10043 })
10044 .unwrap();
10045 body["manifest"]["provides"] = json!([{
10046 "role": "management_surface",
10047 "operations": [
10048 {"name": "diagnostics.findings", "kind": "query"},
10049 {"name": "reset", "kind": "mutate"}
10050 ],
10051 "config_schema": {}, "observability": [], "identity_scope": []
10052 }]);
10053 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10054 if let Some(flow_operations) = flow_operations {
10055 body["manifest"]["flow_operations"] = flow_operations;
10056 }
10057 Frame::build(
10058 FrameType::Hello,
10059 control_flags(),
10060 0,
10061 0,
10062 401,
10063 serde_json::to_vec(&body).unwrap(),
10064 )
10065 .unwrap()
10066 }
10067
10068 #[test]
10069 fn catalog_list_mirrors_flow_operations_unchanged() {
10070 for (capability, flow_operations) in [
10073 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10074 ("call-modes/v1", json!(["diagnostics.findings"])),
10075 ("flow-scopes/v1", json!([])),
10076 ] {
10077 let registry = Arc::new(Registry::default());
10078 let handler = ControlHandler::new(Arc::clone(®istry));
10079 let connection = ConnectionId::new(401);
10080 let response = handler
10081 .handle_control(
10082 connection,
10083 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10084 )
10085 .unwrap();
10086 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10087 let registered = registry
10088 .get_module_by_connection(connection)
10089 .unwrap()
10090 .unwrap()
10091 .manifest;
10092 assert_eq!(
10093 serde_json::to_value(®istered.flow_operations).unwrap(),
10094 flow_operations
10095 );
10096 let (raw, entry) = role_declarations_catalog(&handler);
10097 assert_eq!(raw["flow_operations"], flow_operations);
10098 assert_eq!(entry.flow_operations, registered.flow_operations);
10099 }
10100
10101 let registry = Arc::new(Registry::default());
10102 let handler = ControlHandler::new(Arc::clone(®istry));
10103 let response = handler
10104 .handle_control(
10105 ConnectionId::new(401),
10106 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10107 )
10108 .unwrap();
10109 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10110 let (raw, entry) = role_declarations_catalog(&handler);
10111 assert!(raw.get("flow_operations").is_none());
10112 assert_eq!(entry.flow_operations, None);
10113 }
10114
10115 fn assert_hello_drops_flow_operations_with_warning(
10116 capabilities: Value,
10117 flow_operations: Value,
10118 reason: &str,
10119 ) {
10120 let registry = Arc::new(Registry::default());
10121 let handler = ControlHandler::new(Arc::clone(®istry));
10122 let capture = EventCapture::default();
10123 let _subscriber =
10124 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10125 let response = handler
10126 .handle_control(
10127 ConnectionId::new(401),
10128 hello_with_flow_operations(capabilities, Some(flow_operations)),
10129 )
10130 .unwrap();
10131 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10132 let registered = registry
10133 .get_module_by_connection(ConnectionId::new(401))
10134 .unwrap()
10135 .expect("the module still registers")
10136 .manifest;
10137 assert_eq!(registered.module_id, "flow-ops");
10138 assert_eq!(registered.flow_operations, None);
10139 let (raw, entry) = role_declarations_catalog(&handler);
10140 assert_eq!(raw["module_id"], "flow-ops");
10141 assert!(raw.get("flow_operations").is_none());
10142 assert_eq!(entry.flow_operations, None);
10143 let warnings: Vec<_> = capture
10144 .events()
10145 .into_iter()
10146 .filter(|event| {
10147 event.fields.get("message").map(String::as_str)
10148 == Some("dropping flow_operations from HELLO")
10149 })
10150 .collect();
10151 assert_eq!(warnings.len(), 1);
10152 assert_eq!(warnings[0].level, tracing::Level::WARN);
10153 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10154 assert_eq!(warnings[0].fields["reason"], reason);
10155 }
10156
10157 #[test]
10158 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10159 assert_hello_drops_flow_operations_with_warning(
10160 json!(["flow-scopes/v1"]),
10161 json!(["diagnostics.findings", "missing"]),
10162 "flow_operations names 'missing', which is not a declared management operation",
10163 );
10164 }
10165
10166 #[test]
10167 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10168 assert_hello_drops_flow_operations_with_warning(
10169 json!([]),
10170 json!(["diagnostics.findings"]),
10171 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10172 );
10173 assert_hello_drops_flow_operations_with_warning(
10175 json!(["agent-run-scopes/v1"]),
10176 json!([]),
10177 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10178 );
10179 }
10180
10181 #[test]
10182 fn hello_malformed_flow_operations_is_dropped_with_warning() {
10183 assert_hello_drops_flow_operations_with_warning(
10184 json!(["flow-scopes/v1"]),
10185 json!(["reset", "reset"]),
10186 "flow_operations names 'reset' more than once",
10187 );
10188 assert_hello_drops_flow_operations_with_warning(
10189 json!(["flow-scopes/v1"]),
10190 json!(vec!["reset"; 257]),
10191 "flow_operations lists 257 names, more than 256",
10192 );
10193 assert_hello_drops_flow_operations_with_warning(
10194 json!(["flow-scopes/v1"]),
10195 json!("reset"),
10196 "flow_operations must be an array of operation names",
10197 );
10198 }
10199
10200 #[test]
10201 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10202 let registry = Arc::new(Registry::default());
10203 let handler = ControlHandler::new(registry.clone());
10204 let descriptor = json!({
10205 "schema": "ckios.richui.manifest/1",
10206 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10207 "bytes": 128,
10208 "chunks": 2
10209 });
10210 let malformed = [
10211 ("schema", json!("")),
10212 ("schema", json!("a".repeat(65))),
10213 ("schema", json!("Bad schema")),
10214 (
10215 "manifest_digest",
10216 json!(format!("sha256:{}", "A".repeat(64))),
10217 ),
10218 ("manifest_digest", json!("sha256:abc")),
10219 ("bytes", json!(0)),
10220 ("bytes", json!(16_777_217)),
10221 ("bytes", json!("128")),
10222 ("chunks", json!(0)),
10223 ("chunks", json!(1025)),
10224 ("chunks", json!("2")),
10225 ];
10226 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10227 let mut invalid = descriptor.clone();
10228 invalid[field] = invalid_value;
10229 let response = handler
10230 .handle_control(
10231 ConnectionId::new(210 + index as u64),
10232 hello_frame_with_rich_ui(
10233 &format!("bad-rich-ui-{index}"),
10234 PROTOCOL_VERSION,
10235 210 + index as u64,
10236 None,
10237 invalid,
10238 ),
10239 )
10240 .unwrap();
10241 assert_eq!(response[0].header.ty, FrameType::Error);
10242 let error = parse_error(&response[0]);
10243 assert_eq!(error["code"], "invalid_manifest");
10244 assert!(error["message"]
10245 .as_str()
10246 .unwrap()
10247 .contains(&format!("rich_ui.{field}")));
10248 assert!(registry
10249 .get_module(&format!("bad-rich-ui-{index}"))
10250 .unwrap()
10251 .is_none());
10252 }
10253 }
10254
10255 #[test]
10256 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10257 let registry = Arc::new(Registry::default());
10258 let handler = ControlHandler::new(Arc::clone(®istry));
10259 let connection = ConnectionId::new(219);
10260 registry
10261 .register_with_control_ops(
10262 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10263 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10264 "bad schema",
10265 format!("sha256:{}", "a".repeat(64)),
10266 128,
10267 2,
10268 )))
10269 .build(),
10270 PROTOCOL_VERSION,
10271 connection,
10272 Vec::new(),
10273 )
10274 .unwrap();
10275 let request =
10276 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10277 let response = handler
10278 .handle_catalog_update(connection, request, Vec::new(), None, None)
10279 .unwrap();
10280 assert_eq!(response[0].header.ty, FrameType::Error);
10281 let error = parse_error(&response[0]);
10282 assert_eq!(error["code"], "invalid_manifest");
10283 assert!(error["message"]
10284 .as_str()
10285 .unwrap()
10286 .contains("rich_ui.schema"));
10287 }
10288
10289 #[test]
10290 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10291 let registry = Arc::new(Registry::default());
10292 let supervisor = SupervisorHandle::new();
10293 supervisor.open_swap("provider", "candidate-nonce".to_string());
10294 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10295 let connection = ConnectionId::new(218);
10296 let hello = handler
10297 .handle_control(
10298 connection,
10299 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10300 )
10301 .unwrap();
10302 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10303 assert!(registry.list_modules().unwrap().1.is_empty());
10304
10305 registry.promote_candidate("provider").unwrap().unwrap();
10306 let request = Frame::build(
10307 FrameType::Request,
10308 control_flags(),
10309 0,
10310 0,
10311 219,
10312 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10313 )
10314 .unwrap();
10315 let response = handler.handle_catalog_list(request, None).unwrap();
10316 let ClientControlResponse::CatalogList { modules, .. } =
10317 serde_json::from_slice(&response[0].body).unwrap()
10318 else {
10319 panic!("expected catalog.list");
10320 };
10321 assert_eq!(modules.len(), 1);
10322 assert!(modules[0].nonce_verified);
10323 }
10324
10325 #[test]
10326 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10327 let registry = Arc::new(Registry::default());
10328 let handler = ControlHandler::new(Arc::clone(®istry));
10329 let hello = Frame::build(
10330 FrameType::Hello,
10331 control_flags(),
10332 0,
10333 0,
10334 220,
10335 serde_json::to_vec(&ModuleHelloBody {
10336 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10337 protocol_ver: PROTOCOL_VERSION,
10338 control_ops: None,
10339 launch_nonce: Some("unverified".to_string()),
10340 })
10341 .unwrap(),
10342 )
10343 .unwrap();
10344 assert_eq!(
10345 handler
10346 .handle_control(ConnectionId::new(220), hello)
10347 .unwrap()[0]
10348 .header
10349 .ty,
10350 FrameType::HelloAck
10351 );
10352 let request = Frame::build(
10353 FrameType::Request,
10354 control_flags(),
10355 0,
10356 0,
10357 221,
10358 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10359 )
10360 .unwrap();
10361 let response = handler.handle_catalog_list(request, None).unwrap();
10362 let ClientControlResponse::CatalogList { modules, .. } =
10363 serde_json::from_slice(&response[0].body).unwrap()
10364 else {
10365 panic!("expected catalog.list");
10366 };
10367 assert_eq!(
10368 serde_json::to_vec(&modules[0]).unwrap(),
10369 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10370 );
10371 assert!(!modules[0].nonce_verified);
10372 }
10373
10374 #[test]
10375 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10376 let registry = Arc::new(Registry::default());
10377 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10378 [("vault".to_string(), true), ("squatter".to_string(), true)],
10379 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10380 );
10381 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10382 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10383 provides: vec!["credentials-provider/v1".to_string()],
10384 requires: Vec::new(),
10385 must_never_reach: Vec::new(),
10386 });
10387 let frame = Frame::build(
10388 FrameType::Hello,
10389 control_flags(),
10390 0,
10391 0,
10392 77,
10393 serde_json::to_vec(&ModuleHelloBody {
10394 manifest: squatter,
10395 protocol_ver: PROTOCOL_VERSION,
10396 control_ops: None,
10397 launch_nonce: None,
10398 })
10399 .expect("HELLO serializes"),
10400 )
10401 .expect("HELLO frame builds");
10402 let response = handler
10403 .handle_control(ConnectionId::new(77), frame)
10404 .expect("reserved claim receives a typed refusal");
10405 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10406 assert_eq!(
10407 registry
10408 .active_registration_count()
10409 .expect("registry reads"),
10410 0,
10411 "a reserved capability refusal must not leave a catalog entry"
10412 );
10413 }
10414
10415 #[test]
10416 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10417 for settlement in ["timeout", "inconclusive", "drop"] {
10418 let breakers = RouteBindBreakers::default();
10419 let RouteBindAdmission::Admitted {
10420 guard: mut old,
10421 probe: false,
10422 } = breakers.admit("prov")
10423 else {
10424 panic!("ordinary relay admitted")
10425 };
10426 let RouteBindAdmission::Admitted {
10427 guard: mut opener, ..
10428 } = breakers.admit("prov")
10429 else {
10430 panic!("second relay admitted")
10431 };
10432 assert!(
10433 !opener
10434 .record_timeout(1, Duration::ZERO)
10435 .unwrap()
10436 .reopened_after_probe
10437 );
10438 let RouteBindAdmission::Admitted {
10439 guard: mut probe,
10440 probe: true,
10441 } = breakers.admit("prov")
10442 else {
10443 panic!("one cooldown probe admitted")
10444 };
10445 match settlement {
10446 "timeout" => assert!(
10447 !old.record_timeout(1, Duration::ZERO)
10448 .unwrap()
10449 .reopened_after_probe
10450 ),
10451 "inconclusive" => old.record_inconclusive(),
10452 "drop" => drop(old),
10453 _ => unreachable!(),
10454 }
10455 assert!(
10456 matches!(
10457 breakers.admit("prov"),
10458 RouteBindAdmission::Refused {
10459 probe_in_flight: true,
10460 ..
10461 }
10462 ),
10463 "{settlement} of a pre-open relay cannot release the real probe"
10464 );
10465 assert!(
10466 probe
10467 .record_timeout(1, Duration::ZERO)
10468 .unwrap()
10469 .reopened_after_probe
10470 );
10471 assert!(matches!(
10472 breakers.admit("prov"),
10473 RouteBindAdmission::Admitted { probe: true, .. }
10474 ));
10475 }
10476 let breakers = RouteBindBreakers::default();
10477 let admit = || match breakers.admit("prov") {
10478 RouteBindAdmission::Admitted { guard, .. } => guard,
10479 _ => panic!("relay admitted"),
10480 };
10481 admit().record_timeout(1, Duration::ZERO);
10482 let mut old_probe = admit();
10483 breakers.reset_for_new_module_connection("prov");
10484 admit().record_timeout(1, Duration::ZERO);
10485 let _new_probe = admit();
10486 old_probe.record_inconclusive();
10487 assert!(matches!(
10488 breakers.admit("prov"),
10489 RouteBindAdmission::Refused {
10490 probe_in_flight: true,
10491 ..
10492 }
10493 ));
10494 }
10495
10496 #[tokio::test]
10497 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10498 for candidate in [false, true] {
10499 let registry = Arc::new(Registry::default());
10500 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10501 [("vault".to_string(), true), ("squatter".to_string(), true)],
10502 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10503 );
10504 let conn = ConnectionId::new(77);
10505 let (ctx, mut rx) = route_ctx(conn);
10506 let initial = capability_manifest("squatter", &[], &[]);
10507 if candidate {
10508 registry
10509 .register_candidate_with_control_ops(
10510 initial.clone(),
10511 PROTOCOL_VERSION,
10512 conn,
10513 module_baseline_control_ops(),
10514 )
10515 .unwrap();
10516 handler
10517 .forwarding
10518 .register_candidate_module_connection(
10519 conn,
10520 "squatter".to_string(),
10521 PROTOCOL_VERSION,
10522 manifest_concurrency(&initial),
10523 ctx.egress.clone(),
10524 )
10525 .unwrap();
10526 } else {
10527 hello_via_sink(
10528 &handler,
10529 &ctx,
10530 &mut rx,
10531 hello_frame_with_manifest(initial.clone(), 1),
10532 )
10533 .await;
10534 }
10535 let response = handler
10536 .handle_control_frame(
10537 &ctx,
10538 catalog_update_with_capabilities_frame(
10539 2,
10540 capability_manifest("squatter", &["credentials-provider/v1"], &[])
10541 .capabilities
10542 .unwrap(),
10543 ),
10544 )
10545 .await
10546 .unwrap();
10547 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10548 assert_eq!(
10549 registry
10550 .get_module_by_connection(conn)
10551 .unwrap()
10552 .unwrap()
10553 .manifest,
10554 initial
10555 );
10556 }
10557 }
10558
10559 #[test]
10560 fn server_describe_surfaces_required_capability_verdict_fields() {
10561 let registry = Arc::new(Registry::default());
10562 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10563 [
10564 ("consumer".to_string(), true),
10565 ("provider".to_string(), false),
10566 ],
10567 BTreeMap::new(),
10568 );
10569 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10570 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10571 provides: Vec::new(),
10572 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10573 capability: "credentials-provider/v1".to_string(),
10574 need: subc_protocol::manifest::CapabilityNeed::Required,
10575 }],
10576 must_never_reach: Vec::new(),
10577 });
10578 let hello = Frame::build(
10579 FrameType::Hello,
10580 control_flags(),
10581 0,
10582 0,
10583 78,
10584 serde_json::to_vec(&ModuleHelloBody {
10585 manifest: consumer,
10586 protocol_ver: PROTOCOL_VERSION,
10587 control_ops: None,
10588 launch_nonce: None,
10589 })
10590 .expect("HELLO serializes"),
10591 )
10592 .expect("HELLO frame builds");
10593 handler
10594 .handle_control(ConnectionId::new(78), hello)
10595 .expect("consumer registers");
10596 let describe = Frame::build(
10597 FrameType::Request,
10598 control_flags(),
10599 0,
10600 0,
10601 79,
10602 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10603 .expect("request serializes"),
10604 )
10605 .expect("describe frame builds");
10606 let response = handler
10607 .handle_server_describe(describe)
10608 .expect("server.describe succeeds");
10609 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10610 let requirement = &rendered["capability_requirements"][0];
10611 assert_eq!(requirement["consumer"], "consumer");
10612 assert_eq!(requirement["verdict"], "never_provided");
10613 assert_eq!(requirement["episode_seq"], 1);
10614 assert_eq!(requirement["config_satisfiable"], false);
10615 assert_eq!(requirement["runtime_available"], false);
10616 assert!(requirement["detail"]
10617 .as_str()
10618 .expect("detail string")
10619 .contains("credentials-provider/v1"));
10620 }
10621
10622 fn deregister_required_provider(
10628 daemon_shutdown_begun: bool,
10629 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10630 const CAPABILITY: &str = "project-identity/v1";
10631 let registry = Arc::new(Registry::default());
10632 let daemon_shutdown = DaemonShutdownFlag::default();
10633 let handler = ControlHandler::new(Arc::clone(®istry))
10634 .with_capability_config(
10635 [
10636 ("consumer".to_string(), true),
10637 ("provider".to_string(), true),
10638 ],
10639 BTreeMap::new(),
10640 )
10641 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10642 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10643 consumer.capabilities = Some(CapabilityDeclarations {
10644 provides: Vec::new(),
10645 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10646 capability: CAPABILITY.to_string(),
10647 need: subc_protocol::manifest::CapabilityNeed::Required,
10648 }],
10649 must_never_reach: Vec::new(),
10650 });
10651 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10652 let provider_conn = ConnectionId::new(182);
10653 for (conn, manifest, corr) in [
10654 (ConnectionId::new(181), consumer, 181),
10655 (provider_conn, provider, 182),
10656 ] {
10657 let hello = handler
10658 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10659 .expect("module registers");
10660 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10661 }
10662 assert_eq!(
10663 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10664 Some(CapabilityVerdict::Provided),
10665 "the registered provider must satisfy the requirement first"
10666 );
10667 if daemon_shutdown_begun {
10668 daemon_shutdown.set_for_test();
10669 }
10670
10671 let capture = EventCapture::default();
10672 let _subscriber =
10673 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10674 handler
10675 .cleanup_connection(provider_conn)
10676 .expect("provider deregisters");
10677 let events = capture
10678 .events()
10679 .into_iter()
10680 .filter(|event| {
10681 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10682 })
10683 .collect();
10684 (
10685 events,
10686 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10687 )
10688 }
10689
10690 #[test]
10691 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10692 let (events, verdict) = deregister_required_provider(true);
10693 assert_eq!(
10694 verdict,
10695 Some(CapabilityVerdict::NeverProvided),
10696 "shutdown must not change the verdict route.open refusals read"
10697 );
10698 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10699 let event = &events[0];
10700 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10701 assert_eq!(
10702 event.fields.get("cause").map(String::as_str),
10703 Some("\"daemon_shutdown\""),
10704 "{event:?}"
10705 );
10706 assert_eq!(
10707 event.fields.get("verdict").map(String::as_str),
10708 Some("\"never_provided\""),
10709 "{event:?}"
10710 );
10711 }
10712
10713 #[test]
10714 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10715 let (events, verdict) = deregister_required_provider(false);
10716 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10717 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10718 let event = &events[0];
10719 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10720 assert_eq!(event.fields.get("cause"), None, "{event:?}");
10721 assert_eq!(
10722 event.fields.get("verdict").map(String::as_str),
10723 Some("\"never_provided\""),
10724 "{event:?}"
10725 );
10726 }
10727
10728 #[test]
10729 fn catalog_list_omits_capabilities_for_legacy_manifest() {
10730 let registry = Arc::new(Registry::default());
10731 let handler = ControlHandler::new(Arc::clone(®istry));
10732 let hello = handler
10733 .handle_control(
10734 ConnectionId::new(101),
10735 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10736 )
10737 .expect("legacy HELLO registers");
10738 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10739
10740 let request = Frame::build(
10741 FrameType::Request,
10742 control_flags(),
10743 0,
10744 0,
10745 102,
10746 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10747 .expect("catalog request serializes"),
10748 )
10749 .expect("catalog request frame builds");
10750 let response = handler
10751 .handle_catalog_list(request, None)
10752 .expect("catalog list succeeds");
10753 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10754 assert!(
10755 body["modules"][0].get("capabilities").is_none(),
10756 "legacy manifest must retain an absent capabilities field on catalog.list"
10757 );
10758 }
10759
10760 #[test]
10761 fn hello_ack_omits_storage_when_no_storage_config() {
10762 let registry = Arc::new(Registry::default());
10763 let handler = ControlHandler::new(Arc::clone(®istry));
10764 let responses = handler
10765 .handle_control(
10766 ConnectionId::new(1),
10767 hello_frame("aft", PROTOCOL_VERSION, 7),
10768 )
10769 .unwrap();
10770 let ack = parse_ack(&responses[0]);
10771 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10772 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10773 }
10774
10775 #[tokio::test]
10776 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10777 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10778 let registry = Arc::new(Registry::default());
10779 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
10780 let responses = handler
10781 .handle_control(
10782 ConnectionId::new(1),
10783 hello_frame("aft", PROTOCOL_VERSION, 7),
10784 )
10785 .unwrap();
10786 let ack = parse_ack(&responses[0]);
10787 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10788
10789 let described = handler
10790 .handle_control_frame(
10791 &route_ctx(ConnectionId::new(2)).0,
10792 Frame::build(
10793 FrameType::Request,
10794 control_flags(),
10795 0,
10796 0,
10797 9,
10798 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10799 )
10800 .unwrap(),
10801 )
10802 .await
10803 .unwrap();
10804 let ClientControlResponse::ServerDescribe { machine_id, .. } =
10805 serde_json::from_slice(&described[0].body).unwrap()
10806 else {
10807 panic!("server.describe answered with another shape");
10808 };
10809 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10810 }
10811
10812 #[test]
10813 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10814 let registry = Arc::new(Registry::default());
10817 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
10818 crate::daemon_config::StorageConfig::Sqlite {
10819 data_home: std::path::PathBuf::from("/data"),
10820 },
10821 ));
10822
10823 let responses = handler
10824 .handle_control(
10825 ConnectionId::new(1),
10826 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10827 )
10828 .unwrap();
10829 let ack = parse_ack(&responses[0]);
10830 assert_eq!(
10831 ack.storage,
10832 Some(serde_json::json!({
10833 "module_id": "alfonso-routing",
10834 "storage_namespace": "default",
10835 "isolation": { "kind": "module" },
10836 "backend": {
10837 "backend": "sqlite",
10838 "path": "/data/cortexkit/alfonso-routing/store.db"
10839 }
10840 })),
10841 "the delivered descriptor is the module's own sqlite store path"
10842 );
10843 }
10844
10845 #[test]
10846 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10847 let registry = Arc::new(Registry::default());
10848 let handler = ControlHandler::new(Arc::clone(®istry));
10849 let conn = ConnectionId::new(1);
10850 let responses = handler
10851 .handle_control(
10852 conn,
10853 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10854 )
10855 .unwrap();
10856 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10857 let registration = registry.get_module("aft").unwrap().unwrap();
10858 assert_eq!(registration.control_ops, module_baseline_control_ops());
10859
10860 let frame =
10861 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10862 assert!(handler
10863 .guard_module_control_op(&frame, "aft", "route.bind")
10864 .unwrap()
10865 .is_none());
10866 let error = handler
10867 .guard_module_control_op(&frame, "aft", "test.synthetic")
10868 .unwrap()
10869 .expect("synthetic ungranted op should be rejected");
10870 assert_eq!(error.header.ty, FrameType::Error);
10871 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10872 }
10873
10874 #[test]
10875 fn hello_control_ops_some_adds_optional_grants() {
10876 let registry = Arc::new(Registry::default());
10877 let handler = ControlHandler::new(Arc::clone(®istry));
10878 handler
10879 .handle_control(
10880 ConnectionId::new(1),
10881 hello_frame_with_control_ops(
10882 "aft",
10883 PROTOCOL_VERSION,
10884 7,
10885 Some(vec![
10886 "future.synthetic".to_string(),
10887 "route.bind".to_string(),
10888 ]),
10889 ),
10890 )
10891 .unwrap();
10892 let registration = registry.get_module("aft").unwrap().unwrap();
10893 assert_eq!(
10894 registration.control_ops,
10895 vec![
10896 "route.bind".to_string(),
10897 "route.status".to_string(),
10898 "future.synthetic".to_string(),
10899 ]
10900 );
10901 let frame =
10902 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10903 assert!(handler
10904 .guard_module_control_op(&frame, "aft", "future.synthetic")
10905 .unwrap()
10906 .is_none());
10907 }
10908
10909 #[tokio::test]
10910 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10911 let registry = Arc::new(Registry::default());
10912 let forwarding = Arc::new(ForwardingTable::default());
10913 let handler =
10914 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10915 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10916 hello_via_sink(
10917 &handler,
10918 &module_ctx,
10919 &mut module_rx,
10920 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10921 )
10922 .await;
10923
10924 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10925 let responses = handler
10926 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10927 .await
10928 .unwrap();
10929 assert_eq!(responses.len(), 1);
10930 assert_eq!(responses[0].header.ty, FrameType::Error);
10931 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10932 assert!(module_rx.try_recv().is_err());
10933 }
10934
10935 #[tokio::test]
10936 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10937 let registry = Arc::new(Registry::default());
10938 let forwarding = Arc::new(ForwardingTable::default());
10939 let handler =
10940 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10941 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10942 hello_via_sink(
10943 &handler,
10944 &module_ctx,
10945 &mut module_rx,
10946 hello_frame_with_control_ops(
10947 "aft",
10948 PROTOCOL_VERSION,
10949 7,
10950 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10951 ),
10952 )
10953 .await;
10954
10955 let project_root = unique_project_root("demux");
10956 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10957 let route_handler = handler.clone();
10958 let route_task = tokio::spawn(async move {
10959 route_handler
10960 .handle_control_frame(
10961 &route_client_ctx,
10962 route_open_frame(100, "aft", project_root),
10963 )
10964 .await
10965 .unwrap()
10966 });
10967 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10968 .await
10969 .unwrap()
10970 .unwrap();
10971 assert!(matches!(
10972 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10973 ModuleControlRequest::RouteBind { .. }
10974 ));
10975
10976 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10977 let health_handler = handler.clone();
10978 let health_task = tokio::spawn(async move {
10979 health_handler
10980 .handle_control_frame(
10981 &health_client_ctx,
10982 supervisor_health_probe_frame(101, "aft"),
10983 )
10984 .await
10985 .unwrap()
10986 });
10987 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10988 .await
10989 .unwrap()
10990 .unwrap();
10991 assert_eq!(
10992 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10993 ModuleControlRequest::HealthCheck {}
10994 );
10995
10996 handler
10997 .handle_control_frame(
10998 &module_ctx,
10999 health_response(health_frame.header.corr, HealthStatus::Degraded),
11000 )
11001 .await
11002 .unwrap();
11003 let health_response = health_task.await.unwrap();
11004 assert_eq!(health_response.len(), 1);
11005 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11006 ClientControlResponse::SupervisorHealthProbe {
11007 module_id,
11008 status,
11009 detail,
11010 metrics,
11011 } => {
11012 assert_eq!(module_id, "aft");
11013 assert_eq!(status, HealthStatus::Degraded);
11014 assert_eq!(detail.as_deref(), Some("warming"));
11015 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11016 }
11017 other => panic!("unexpected health response: {other:?}"),
11018 }
11019
11020 handler
11021 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11022 .await
11023 .unwrap();
11024 let route_response = route_task.await.unwrap();
11025 assert!(route_response.is_empty());
11026 let published = route_client_rx.recv().await.unwrap();
11027 assert!(matches!(
11028 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11029 ClientControlResponse::RouteOpen { .. }
11030 ));
11031 }
11032
11033 async fn relay_route_open(
11038 handler: &ControlHandler,
11039 client_connection: ConnectionId,
11040 client_egress: &FrameSink,
11041 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11042 corr: u64,
11043 module_id: &str,
11044 project_root_label: &str,
11045 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11046 let ctx = RouteCtx {
11047 connection_id: client_connection,
11048 egress: client_egress.clone(),
11049 };
11050 let handler = handler.clone();
11051 let project_root = unique_project_root(project_root_label);
11052 let module_id = module_id.to_string();
11053 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11054 let task = tokio::spawn(async move {
11055 let _guard = tracing::dispatcher::set_default(&dispatch);
11056 handler
11057 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11058 .await
11059 .unwrap()
11060 });
11061 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11062 .await
11063 .expect("module receives the relayed route.bind")
11064 .expect("module egress is open");
11065 (task, bind.frame)
11066 }
11067
11068 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11069 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11070 ModuleControlRequest::RouteBind {
11071 route_channel,
11072 epoch,
11073 ..
11074 } => (route_channel, epoch),
11075 other => panic!("expected a route.bind request, got {other:?}"),
11076 }
11077 }
11078
11079 fn published_route(frame: &Frame) -> (u16, u32) {
11080 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11081 ClientControlResponse::RouteOpen {
11082 route_channel,
11083 route_epoch,
11084 } => (route_channel, route_epoch),
11085 other => panic!("expected a route.open response, got {other:?}"),
11086 }
11087 }
11088
11089 #[tokio::test]
11109 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11110 let registry = Arc::new(Registry::default());
11111 let forwarding = Arc::new(ForwardingTable::default());
11112 let handler =
11113 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11114
11115 let module_connection = ConnectionId::new(30);
11116 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11117 hello_via_sink(
11118 &handler,
11119 &module_ctx,
11120 &mut module_rx,
11121 hello_frame("aft", PROTOCOL_VERSION, 7),
11122 )
11123 .await;
11124
11125 let dying_client = ConnectionId::new(31);
11126 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11127
11128 let (first_task, first_bind) = relay_route_open(
11131 &handler,
11132 dying_client,
11133 &dying_ctx.egress,
11134 &mut module_rx,
11135 100,
11136 "aft",
11137 "closing-first",
11138 )
11139 .await;
11140 handler
11141 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11142 .await
11143 .unwrap();
11144 assert!(first_task.await.unwrap().is_empty());
11145 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11146
11147 let (second_task, second_bind) = relay_route_open(
11149 &handler,
11150 dying_client,
11151 &dying_ctx.egress,
11152 &mut module_rx,
11153 101,
11154 "aft",
11155 "closing-second",
11156 )
11157 .await;
11158 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11159
11160 assert!(forwarding
11163 .escalate_client_delivery_failure(
11164 dying_client,
11165 first_channel,
11166 first_epoch,
11167 CloseReason::new(
11168 "module_to_client_delivery_failed",
11169 "client egress refused a module frame",
11170 ),
11171 crate::forwarding::UndeliveredFrame {
11172 module_id: None,
11173 sink: &dying_ctx.egress,
11174 },
11175 )
11176 .unwrap());
11177 assert!(!dying_ctx.egress.is_closed());
11178
11179 let ack = handler
11181 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11182 .await;
11183 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11184 if module_loop_error.is_some() {
11185 handler.cleanup_connection(module_connection).unwrap();
11189 }
11190 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11194 .await
11195 .ok()
11196 .flatten();
11197
11198 assert!(
11200 registry
11201 .get_module_by_connection(module_connection)
11202 .unwrap()
11203 .is_some(),
11204 "one client's closing connection ended the shared module connection: \
11205 {module_loop_error:?}"
11206 );
11207 let cotenant = ConnectionId::new(32);
11209 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11210 let (cotenant_task, cotenant_bind) = relay_route_open(
11211 &handler,
11212 cotenant,
11213 &cotenant_ctx.egress,
11214 &mut module_rx,
11215 102,
11216 "aft",
11217 "closing-cotenant",
11218 )
11219 .await;
11220 handler
11221 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11222 .await
11223 .unwrap();
11224 assert!(cotenant_task.await.unwrap().is_empty());
11225 let (cotenant_channel, cotenant_epoch) =
11226 published_route(&cotenant_rx.recv().await.unwrap());
11227 assert!(matches!(
11228 forwarding
11229 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11230 .unwrap(),
11231 DataRoute::Client(DataRouteState::Bound(_))
11232 ));
11233
11234 let goodbye = post_ack_module_frame
11237 .expect("module receives a GOODBYE for the abandoned route channel");
11238 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11239 assert_eq!(goodbye.header.channel, abandoned_channel);
11240 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11241
11242 assert!(dying_rx.try_recv().is_err());
11246 let second_response = second_task.await.unwrap();
11247 assert_eq!(second_response.len(), 1);
11248 assert_eq!(
11249 parse_error(&second_response[0])["code"],
11250 "target_unavailable"
11251 );
11252 }
11253
11254 async fn catalog_snapshot(
11257 handler: &ControlHandler,
11258 client_ctx: &RouteCtx,
11259 corr: u64,
11260 ) -> (u64, Vec<(String, bool)>) {
11261 let body =
11262 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11263 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11264 let reply = handler
11265 .handle_control_frame(client_ctx, request)
11266 .await
11267 .unwrap()
11268 .pop()
11269 .unwrap();
11270 let ClientControlResponse::CatalogList {
11271 generation,
11272 modules,
11273 ..
11274 } = serde_json::from_slice(&reply.body).unwrap()
11275 else {
11276 panic!("not a catalog.list reply");
11277 };
11278 (
11279 generation,
11280 modules
11281 .into_iter()
11282 .map(|entry| (entry.module_id, entry.ready))
11283 .collect(),
11284 )
11285 }
11286
11287 fn hello_held_before_forwarding() -> (
11293 ControlHandler,
11294 Arc<std::sync::Barrier>,
11295 std::thread::JoinHandle<Vec<Frame>>,
11296 RouteCtx,
11297 mpsc::Receiver<crate::router::OutboundFrame>,
11298 ) {
11299 let hold = Arc::new(std::sync::Barrier::new(2));
11300 let handler = ControlHandler::with_forwarding(
11301 Arc::new(Registry::default()),
11302 Arc::new(ForwardingTable::default()),
11303 )
11304 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11305 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11306 let hello = {
11307 let handler = handler.clone();
11308 let module_ctx = module_ctx.clone();
11309 std::thread::spawn(move || {
11310 handler
11311 .handle_hello(
11312 module_ctx.connection_id,
11313 Some(module_ctx.egress.clone()),
11314 hello_frame("aft", PROTOCOL_VERSION, 7),
11315 )
11316 .unwrap()
11317 })
11318 };
11319 (handler, hold, hello, module_ctx, module_rx)
11320 }
11321
11322 #[tokio::test]
11331 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11332 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11333 let client = ConnectionId::new(41);
11334 let (client_ctx, mut client_rx) = route_ctx(client);
11335
11336 hold.wait();
11337 assert!(handler.registry.get_module("aft").unwrap().is_some());
11339 assert!(!handler
11340 .forwarding
11341 .has_live_module_connection("aft")
11342 .unwrap());
11343 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11344 assert!(
11345 listed.is_empty(),
11346 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11347 );
11348 let refused = handler
11349 .handle_control_frame(
11350 &client_ctx,
11351 route_open_frame(301, "aft", unique_project_root("listed-window")),
11352 )
11353 .await
11354 .unwrap();
11355 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11356 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11357 let duplicate = handler
11358 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11359 .await
11360 .unwrap();
11361 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11362
11363 hold.wait();
11364 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11365 let ack = module_rx.try_recv().unwrap().frame;
11366 assert_eq!(ack.header.ty, FrameType::HelloAck);
11367 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11368 assert_eq!(listed, vec![("aft".to_string(), true)]);
11369
11370 let (open_task, bind) = relay_route_open(
11371 &handler,
11372 client,
11373 &client_ctx.egress,
11374 &mut module_rx,
11375 303,
11376 "aft",
11377 "listed-after",
11378 )
11379 .await;
11380 assert!(handler
11381 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11382 .await
11383 .unwrap()
11384 .is_empty());
11385 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11386 published_route(&client_rx.try_recv().unwrap().frame);
11387 }
11388
11389 #[tokio::test]
11396 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11397 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11398 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11399 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11400
11401 hold.wait();
11402 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11403 assert!(listed.is_empty());
11404 assert_eq!(
11405 during, before,
11406 "the generation moved while the module was not listed"
11407 );
11408
11409 hold.wait();
11410 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11411 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11412 assert_eq!(listed, vec![("aft".to_string(), true)]);
11413 assert_eq!(
11414 after,
11415 before + 1,
11416 "the module appeared without exactly one generation bump"
11417 );
11418 }
11419
11420 #[tokio::test]
11425 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11426 let registry = Arc::new(Registry::default());
11427 let handler = ControlHandler::with_forwarding(
11428 Arc::clone(®istry),
11429 Arc::new(ForwardingTable::default()),
11430 );
11431 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11432 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11433 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11434 drop(module_rx);
11435
11436 let replies = handler
11437 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11438 .await
11439 .unwrap();
11440 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11441 assert!(registry.get_module("aft").unwrap().is_none());
11442 assert!(registry
11443 .get_module_by_connection(module_ctx.connection_id)
11444 .unwrap()
11445 .is_none());
11446 assert_eq!(
11447 catalog_snapshot(&handler, &client_ctx, 301).await,
11448 (before, Vec::new())
11449 );
11450 }
11451
11452 async fn route_open_held_after_bind_send() -> (
11456 ControlHandler,
11457 Arc<tokio::sync::Barrier>,
11458 RouteCtx,
11459 mpsc::Receiver<crate::router::OutboundFrame>,
11460 ) {
11461 let hold = Arc::new(tokio::sync::Barrier::new(2));
11462 let handler = ControlHandler::with_forwarding(
11463 Arc::new(Registry::default()),
11464 Arc::new(ForwardingTable::default()),
11465 )
11466 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11467 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11468 hello_via_sink(
11469 &handler,
11470 &module_ctx,
11471 &mut module_rx,
11472 hello_frame("aft", PROTOCOL_VERSION, 7),
11473 )
11474 .await;
11475 (handler, hold, module_ctx, module_rx)
11476 }
11477
11478 #[tokio::test]
11485 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11486 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11487 let client = ConnectionId::new(41);
11488 let (client_ctx, mut client_rx) = route_ctx(client);
11489
11490 let (open_task, bind) = relay_route_open(
11491 &handler,
11492 client,
11493 &client_ctx.egress,
11494 &mut module_rx,
11495 200,
11496 "aft",
11497 "acked-before-mark",
11498 )
11499 .await;
11500 let (module_channel, module_epoch) = route_bind_channel(&bind);
11501 hold.wait().await;
11503
11504 assert!(handler
11506 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11507 .await
11508 .unwrap()
11509 .is_empty());
11510 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11511
11512 hold.wait().await;
11513 assert!(
11514 open_task.await.unwrap().is_empty(),
11515 "route.open answers accepted"
11516 );
11517
11518 match module_rx.try_recv() {
11521 Err(mpsc::error::TryRecvError::Empty) => {}
11522 Ok(outbound) => panic!(
11523 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11524 outbound.frame.header.ty,
11525 outbound.frame.header.channel,
11526 outbound.frame.header.epoch
11527 ),
11528 Err(err) => panic!("module egress closed: {err}"),
11529 }
11530
11531 let router = Router::with_control_handler(Arc::new(handler.clone()));
11534 let request = Frame::build(
11535 FrameType::Request,
11536 Flags::new(true, Priority::Interactive, false),
11537 client_channel,
11538 client_epoch,
11539 201,
11540 b"first request".to_vec(),
11541 )
11542 .unwrap();
11543 router
11544 .route_for_connection(&client_ctx, request)
11545 .await
11546 .unwrap();
11547 let delivered = module_rx.try_recv().unwrap().frame;
11548 assert_eq!(delivered.header.ty, FrameType::Request);
11549 assert_eq!(
11550 (delivered.header.channel, delivered.header.epoch),
11551 (module_channel, module_epoch)
11552 );
11553 assert_eq!(delivered.body, b"first request");
11554 }
11555
11556 #[tokio::test]
11562 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11563 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11564 let client = ConnectionId::new(42);
11565 let (client_ctx, _client_rx) = route_ctx(client);
11566
11567 let (open_task, bind) = relay_route_open(
11568 &handler,
11569 client,
11570 &client_ctx.egress,
11571 &mut module_rx,
11572 300,
11573 "aft",
11574 "released-before-mark",
11575 )
11576 .await;
11577 let (module_channel, module_epoch) = route_bind_channel(&bind);
11578 hold.wait().await;
11579
11580 handler.cleanup_connection(client).unwrap();
11581 assert!(
11582 module_rx.try_recv().is_err(),
11583 "client teardown must not GOODBYE a bind route.open is still queueing"
11584 );
11585
11586 hold.wait().await;
11587 let response = open_task.await.unwrap();
11588 assert_eq!(response.len(), 1);
11589 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11590
11591 let goodbye = module_rx
11592 .try_recv()
11593 .expect("module is told to drop the binding the daemon abandoned")
11594 .frame;
11595 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11596 assert_eq!(
11597 (goodbye.header.channel, goodbye.header.epoch),
11598 (module_channel, module_epoch)
11599 );
11600 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11601 }
11602
11603 #[tokio::test]
11608 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11609 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11610 let client = ConnectionId::new(43);
11611 let (client_ctx, client_rx) = route_ctx(client);
11612
11613 let (open_task, bind) = relay_route_open(
11614 &handler,
11615 client,
11616 &client_ctx.egress,
11617 &mut module_rx,
11618 400,
11619 "aft",
11620 "gone-client-before-mark",
11621 )
11622 .await;
11623 let (module_channel, module_epoch) = route_bind_channel(&bind);
11624 hold.wait().await;
11625
11626 drop(client_rx);
11627 assert!(client_ctx.egress.is_closed());
11628 handler
11629 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11630 .await
11631 .unwrap();
11632
11633 hold.wait().await;
11634 let response = open_task.await.unwrap();
11635 assert_eq!(response.len(), 1);
11636 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11637
11638 let goodbye = module_rx
11639 .try_recv()
11640 .expect("module is told to drop the binding it acked")
11641 .frame;
11642 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11643 assert_eq!(
11644 (goodbye.header.channel, goodbye.header.epoch),
11645 (module_channel, module_epoch)
11646 );
11647 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11648 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11649 }
11650
11651 #[test]
11658 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11659 let handler = ControlHandler::default();
11660 let module_connection = ConnectionId::new(30);
11661 let client_connection = ConnectionId::new(31);
11662
11663 handler
11664 .refuse_to_end_module_connection_for_a_client(
11665 module_connection,
11666 77,
11667 ForwardingError::ConnectionClosing {
11668 connection_id: client_connection,
11669 },
11670 )
11671 .expect("a closing client must never end the module connection");
11672
11673 assert!(matches!(
11674 handler.refuse_to_end_module_connection_for_a_client(
11675 module_connection,
11676 78,
11677 ForwardingError::ConnectionClosing {
11678 connection_id: module_connection,
11679 },
11680 ),
11681 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11682 connection_id
11683 })) if connection_id == module_connection
11684 ));
11685 assert!(matches!(
11686 handler.refuse_to_end_module_connection_for_a_client(
11687 module_connection,
11688 79,
11689 ForwardingError::Poisoned,
11690 ),
11691 Err(RouterError::Forwarding(ForwardingError::Poisoned))
11692 ));
11693 assert!(matches!(
11694 handler.refuse_to_end_module_connection_for_a_client(
11695 module_connection,
11696 80,
11697 ForwardingError::StaleModuleEndpoint,
11698 ),
11699 Err(RouterError::Forwarding(
11700 ForwardingError::StaleModuleEndpoint
11701 ))
11702 ));
11703 }
11704
11705 #[tokio::test]
11718 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11719 let registry = Arc::new(Registry::default());
11720 let forwarding = Arc::new(ForwardingTable::default());
11721 let supervisor = SupervisorHandle::new();
11722 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11723 let handler =
11724 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11725 .with_supervisor(supervisor);
11726
11727 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11728 hello_via_sink(
11729 &handler,
11730 &target_ctx,
11731 &mut target_rx,
11732 hello_frame("target", PROTOCOL_VERSION, 1),
11733 )
11734 .await;
11735
11736 let wrong_nonce = handler
11740 .handle_control_frame(
11741 &route_ctx(ConnectionId::new(91)).0,
11742 route_open_frame_with_admission_facts(
11743 20,
11744 "target",
11745 unique_project_root("admission-facts"),
11746 Some(subc_control::ConsumerIdentity {
11747 module_id: "fed".to_string(),
11748 launch_nonce: "not-the-real-nonce".to_string(),
11749 }),
11750 None,
11751 ),
11752 )
11753 .await
11754 .unwrap();
11755 assert_eq!(
11756 parse_error(&wrong_nonce[0])["code"],
11757 "bad_consumer_identity",
11758 "a mismatched launch nonce must be refused, not stamped Reserved"
11759 );
11760
11761 let never_spawned = handler
11765 .handle_control_frame(
11766 &route_ctx(ConnectionId::new(92)).0,
11767 route_open_frame_with_admission_facts(
11768 21,
11769 "target",
11770 unique_project_root("admission-facts"),
11771 Some(subc_control::ConsumerIdentity {
11772 module_id: "never-spawned".to_string(),
11773 launch_nonce: "any-nonce".to_string(),
11774 }),
11775 None,
11776 ),
11777 )
11778 .await
11779 .unwrap();
11780 assert_eq!(
11781 parse_error(&never_spawned[0])["code"],
11782 "bad_consumer_identity",
11783 "an unspawned module_id must be refused rather than accepted for lack of a record"
11784 );
11785 }
11786
11787 #[tokio::test]
11810 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11811 let registry = Arc::new(Registry::default());
11812 let forwarding = Arc::new(ForwardingTable::default());
11813 let supervisor = SupervisorHandle::new();
11814 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11815 let handler =
11816 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11817 .with_supervisor(supervisor);
11818
11819 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11820 hello_via_sink(
11821 &handler,
11822 &target_ctx,
11823 &mut target_rx,
11824 hello_frame("target", PROTOCOL_VERSION, 1),
11825 )
11826 .await;
11827
11828 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11829 let route_handler = handler.clone();
11830 let route_task = tokio::spawn(async move {
11831 route_handler
11832 .handle_control_frame(
11833 &client_ctx,
11834 route_open_frame_with_admission_facts(
11835 30,
11836 "target",
11837 unique_project_root("admission-facts"),
11838 Some(subc_control::ConsumerIdentity {
11839 module_id: "fed".to_string(),
11840 launch_nonce: "fed-nonce".to_string(),
11841 }),
11842 None,
11843 ),
11844 )
11845 .await
11846 .unwrap()
11847 });
11848
11849 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11856 .await
11857 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11858 .expect("module control channel closed before route.bind");
11859 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11860 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11861 panic!("expected route.bind")
11862 };
11863 assert_eq!(
11864 principal,
11865 Some(Principal::Reserved {
11866 module_id: "fed".to_string()
11867 }),
11868 "a correctly attested consumer must be stamped Reserved for its own id"
11869 );
11870
11871 handler
11872 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11873 .await
11874 .unwrap();
11875 assert!(route_task.await.unwrap().is_empty());
11876 assert!(
11877 matches!(
11878 serde_json::from_slice::<ClientControlResponse>(
11879 &client_rx.recv().await.unwrap().body
11880 )
11881 .unwrap(),
11882 ClientControlResponse::RouteOpen { .. }
11883 ),
11884 "the route must actually open, not merely avoid an error"
11885 );
11886 }
11887
11888 #[tokio::test(start_paused = true)]
11889 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11890 let registry = Arc::new(Registry::default());
11891 let forwarding = Arc::new(ForwardingTable::default());
11892 let supervisor = SupervisorHandle::new();
11893 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11894 let handler =
11895 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11896 .with_supervisor(supervisor);
11897
11898 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11899 hello_via_sink(
11900 &handler,
11901 &target_ctx,
11902 &mut target_rx,
11903 hello_frame("target", PROTOCOL_VERSION, 1),
11904 )
11905 .await;
11906
11907 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11908 let direct_handler = handler.clone();
11909 let direct_open = tokio::spawn(async move {
11910 direct_handler
11911 .handle_control_frame(
11912 &direct_ctx,
11913 route_open_frame(2, "target", unique_project_root("route-census-direct")),
11914 )
11915 .await
11916 .unwrap()
11917 });
11918 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11919 .await
11920 .expect("no direct route.bind within 5s")
11921 .expect("target control channel closed before direct route.bind");
11922 handler
11923 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11924 .await
11925 .unwrap();
11926 assert!(direct_open.await.unwrap().is_empty());
11927 let _ = direct_rx.recv().await.unwrap();
11928
11929 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11930 let reserved_handler = handler.clone();
11931 let reserved_open = tokio::spawn(async move {
11932 reserved_handler
11933 .handle_control_frame(
11934 &reserved_ctx,
11935 route_open_frame_with_admission_facts(
11936 3,
11937 "target",
11938 unique_project_root("admission-facts"),
11939 Some(ConsumerIdentity {
11940 module_id: "fed".to_string(),
11941 launch_nonce: "fed-nonce".to_string(),
11942 }),
11943 None,
11944 ),
11945 )
11946 .await
11947 .unwrap()
11948 });
11949 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11950 .await
11951 .expect("no reserved route.bind within 5s")
11952 .expect("target control channel closed before reserved route.bind");
11953 handler
11954 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11955 .await
11956 .unwrap();
11957 assert!(reserved_open.await.unwrap().is_empty());
11958 let _ = reserved_rx.recv().await.unwrap();
11959
11960 forwarding
11961 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11962 .unwrap();
11963 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11964 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11965 module_id: Some("target".to_string()),
11966 })
11967 .unwrap();
11968 let census_frame =
11969 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11970 let response = handler
11971 .handle_control_frame(&census_ctx, census_frame)
11972 .await
11973 .unwrap()
11974 .pop()
11975 .unwrap();
11976 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11977 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11978 assert!(matches!(
11979 decoded,
11980 ClientControlResponse::SupervisorRoutes { .. }
11981 ));
11982 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11983 assert_eq!(routes.len(), 2);
11984 assert!(routes.iter().all(|route| route["draining"] == true));
11985 assert!(
11988 routes.iter().all(|route| route["drain_reason"] == "reload"),
11989 "draining routes must name the drain's reason: {routes:?}"
11990 );
11991 assert!(routes.iter().any(|route| {
11992 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11993 }));
11994 assert!(routes.iter().any(|route| {
11995 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11996 }));
11997
11998 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11999 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12000 if std::env::var_os("UPDATE_GOLDEN").is_some() {
12001 std::fs::write(
12002 &golden_path,
12003 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12004 )
12005 .unwrap();
12006 }
12007 let expected: Value =
12008 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12009 assert_eq!(actual, expected);
12010 }
12011
12012 async fn query_live_roots(
12013 handler: &ControlHandler,
12014 module_ctx: &RouteCtx,
12015 ) -> ModuleControlResponseToModule {
12016 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12017 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12018 let response = handler
12019 .handle_control_frame(module_ctx, frame)
12020 .await
12021 .unwrap()
12022 .pop()
12023 .unwrap();
12024 serde_json::from_slice(&response.body).unwrap()
12025 }
12026
12027 #[tokio::test(start_paused = true)]
12028 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12029 let registry = Arc::new(Registry::default());
12030 let forwarding = Arc::new(ForwardingTable::default());
12031 let handler = ControlHandler::with_forwarding(registry, forwarding);
12032 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12033 hello_via_sink(
12034 &handler,
12035 &target_ctx,
12036 &mut target_rx,
12037 hello_frame("target", PROTOCOL_VERSION, 1),
12038 )
12039 .await;
12040 let root = unique_project_root("live-roots-known");
12041 let path = ProjectRootId::from_path_allowing_missing(root.path())
12042 .unwrap()
12043 .as_path()
12044 .to_path_buf();
12045 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12046 let open_handler = handler.clone();
12047 let opened = tokio::spawn(async move {
12048 open_handler
12049 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
12050 .await
12051 .unwrap()
12052 });
12053 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12054 .await
12055 .unwrap()
12056 .unwrap();
12057 handler
12058 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12059 .await
12060 .unwrap();
12061 assert!(opened.await.unwrap().is_empty());
12062 let _ = client_rx.recv().await.unwrap();
12063
12064 let root = unique_project_root("live-roots-pending");
12065 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12066 .unwrap()
12067 .as_path()
12068 .to_path_buf();
12069 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12070 let open_handler = handler.clone();
12071 let pending = tokio::spawn(async move {
12072 open_handler
12073 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
12074 .await
12075 .unwrap()
12076 });
12077 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12078 .await
12079 .unwrap()
12080 .unwrap();
12081 let actual = query_live_roots(&handler, &target_ctx).await;
12082 let ModuleControlResponseToModule::LiveRoots {
12083 roots,
12084 unknown_root_bindings,
12085 total_bindings,
12086 } = actual
12087 else {
12088 panic!("expected live roots")
12089 };
12090 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12091 assert_eq!(unknown_root_bindings, 0);
12092 assert_eq!(
12093 roots.len(),
12094 2,
12095 "root-known arm must retain each canonical root"
12096 );
12097 assert_eq!(
12098 total_bindings,
12099 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12100 );
12101 let counts = roots
12102 .iter()
12103 .map(|root| (root.project_root.clone(), root.bound, root.pending))
12104 .collect::<Vec<_>>();
12105 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12106 expected.sort_by(|a, b| a.0.cmp(&b.0));
12107 assert_eq!(
12108 counts, expected,
12109 "roots must sort by path and count pending separately"
12110 );
12111 handler
12112 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12113 .await
12114 .unwrap();
12115 assert!(pending.await.unwrap().is_empty());
12116 }
12117
12118 #[tokio::test(start_paused = true)]
12119 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12120 let forwarding = Arc::new(ForwardingTable::default());
12121 let handler =
12122 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12123 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12124 hello_via_sink(
12125 &handler,
12126 &target_ctx,
12127 &mut target_rx,
12128 hello_frame("target", PROTOCOL_VERSION, 1),
12129 )
12130 .await;
12131 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12132 let pending = forwarding
12133 .begin_route_bind_relay_for_test(
12134 client_ctx.connection_id,
12135 client_ctx.egress.clone(),
12136 2,
12137 "target",
12138 )
12139 .unwrap();
12140 forwarding
12141 .complete_pending_relay(
12142 target_ctx.connection_id,
12143 pending.corr,
12144 RouteBindRelayOutcome::Accepted,
12145 )
12146 .unwrap();
12147 let actual = query_live_roots(&handler, &target_ctx).await;
12148 let ModuleControlResponseToModule::LiveRoots {
12149 roots,
12150 unknown_root_bindings,
12151 total_bindings,
12152 } = actual
12153 else {
12154 panic!("expected live roots")
12155 };
12156 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12157 assert_eq!(
12158 unknown_root_bindings, 1,
12159 "unknown-root arm must not read as no bindings"
12160 );
12161 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12162 assert_eq!(
12163 total_bindings,
12164 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12165 );
12166 }
12167
12168 #[tokio::test(start_paused = true)]
12176 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12177 let forwarding = Arc::new(ForwardingTable::default());
12178 let handler =
12179 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12180 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12181 let replies = handler
12182 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12183 .await
12184 .unwrap();
12185 let queued_by_hello = module_rx.len();
12186
12187 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12188 let open_handler = handler.clone();
12189 let open = tokio::spawn(async move {
12190 open_handler
12191 .handle_control_frame(
12192 &client_ctx,
12193 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12194 )
12195 .await
12196 .unwrap()
12197 });
12198 let mut spins = 0;
12200 while module_rx.len() == queued_by_hello {
12201 spins += 1;
12202 assert!(spins < 10_000, "route.open never queued a route.bind");
12203 tokio::task::yield_now().await;
12204 }
12205
12206 for reply in replies {
12208 module_ctx.egress.send(reply).await.unwrap();
12209 }
12210
12211 let first = module_rx.recv().await.unwrap().frame;
12212 assert_eq!(
12213 first.header.ty,
12214 FrameType::HelloAck,
12215 "the first frame a registering module reads must be its HELLO_ACK"
12216 );
12217 assert_eq!(first.header.corr, 7);
12218 let second = module_rx.recv().await.unwrap().frame;
12219 assert_eq!(second.header.ty, FrameType::Request);
12220 assert!(
12221 matches!(
12222 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12223 ModuleControlRequest::RouteBind { .. }
12224 ),
12225 "the route.bind follows the ack"
12226 );
12227 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12228
12229 handler
12230 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12231 .await
12232 .unwrap();
12233 assert!(open.await.unwrap().is_empty());
12234 let _ = client_rx.recv().await.unwrap();
12235 }
12236
12237 #[tokio::test(start_paused = true)]
12238 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12239 let handler = ControlHandler::with_forwarding(
12240 Arc::new(Registry::default()),
12241 Arc::new(ForwardingTable::default()),
12242 );
12243 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12244 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12245 hello_via_sink(
12246 &handler,
12247 &first_ctx,
12248 &mut first_rx,
12249 hello_frame("first", PROTOCOL_VERSION, 1),
12250 )
12251 .await;
12252 hello_via_sink(
12253 &handler,
12254 &second_ctx,
12255 &mut second_rx,
12256 hello_frame("second", PROTOCOL_VERSION, 2),
12257 )
12258 .await;
12259 let root = unique_project_root("second-only");
12260 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12261 let cloned = handler.clone();
12262 let open = tokio::spawn(async move {
12263 cloned
12264 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12265 .await
12266 .unwrap()
12267 });
12268 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12269 .await
12270 .unwrap()
12271 .unwrap();
12272 let first = query_live_roots(&handler, &first_ctx).await;
12273 let second = query_live_roots(&handler, &second_ctx).await;
12274 assert!(
12275 matches!(
12276 first,
12277 ModuleControlResponseToModule::LiveRoots {
12278 total_bindings: 0,
12279 ..
12280 }
12281 ),
12282 "cross-module scope must not expose another module's roots"
12283 );
12284 assert!(
12285 matches!(
12286 second,
12287 ModuleControlResponseToModule::LiveRoots {
12288 total_bindings: 1,
12289 ..
12290 }
12291 ),
12292 "second module must see its pending route"
12293 );
12294 handler
12295 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12296 .await
12297 .unwrap();
12298 assert!(open.await.unwrap().is_empty());
12299 }
12300
12301 #[tokio::test(start_paused = true)]
12302 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12303 let handler = ControlHandler::with_forwarding(
12304 Arc::new(Registry::default()),
12305 Arc::new(ForwardingTable::default()),
12306 );
12307 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12308 hello_via_sink(
12309 &handler,
12310 &target_ctx,
12311 &mut target_rx,
12312 hello_frame("target", PROTOCOL_VERSION, 1),
12313 )
12314 .await;
12315 let actual = query_live_roots(&handler, &target_ctx).await;
12316 let ModuleControlResponseToModule::LiveRoots {
12317 roots,
12318 unknown_root_bindings,
12319 total_bindings,
12320 } = actual
12321 else {
12322 panic!("expected live roots")
12323 };
12324 assert!(roots.is_empty());
12325 assert_eq!(unknown_root_bindings, 0);
12326 assert_eq!(total_bindings, 0);
12327 assert_eq!(
12328 total_bindings,
12329 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12330 );
12331 }
12332
12333 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12340 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12341 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12342 let text = std::fs::read_to_string(&path)
12343 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12344 let vectors: Vec<(String, Value)> = text
12345 .lines()
12346 .filter(|line| !line.trim().is_empty())
12347 .map(|line| {
12348 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12349 let id = entry["corpus_id"]
12350 .as_str()
12351 .expect("every vector carries a corpus_id")
12352 .to_string();
12353 (id, entry["package"].clone())
12354 })
12355 .collect();
12356 assert_eq!(
12360 vectors.len(),
12361 3,
12362 "vendored fed corpus changed size; re-sync from subc-federation"
12363 );
12364
12365 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12380 let richest = vectors
12381 .iter()
12382 .filter_map(|(_, package)| package.as_object())
12383 .map(|object| {
12384 object
12385 .keys()
12386 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12387 .count()
12388 })
12389 .max()
12390 .unwrap_or(0);
12391 assert!(
12392 richest >= 2,
12393 "vendored corpus no longer carries a package with unmodelled fields, \
12394 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12395 );
12396
12397 vectors
12398 }
12399
12400 #[tokio::test]
12408 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12409 for (corpus_id, package) in fed_admission_facts_vectors() {
12410 let registry = Arc::new(Registry::default());
12411 let forwarding = Arc::new(ForwardingTable::default());
12412 let supervisor = SupervisorHandle::new();
12413 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12414 let handler =
12415 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12416 .with_supervisor(supervisor)
12417 .with_admission_facts_config(
12418 Some("fed".to_string()),
12419 Some(vec!["target".to_string()]),
12420 );
12421
12422 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12423 hello_via_sink(
12424 &handler,
12425 &target_ctx,
12426 &mut target_rx,
12427 hello_frame("target", PROTOCOL_VERSION, 1),
12428 )
12429 .await;
12430
12431 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12432 let route_handler = handler.clone();
12433 let expected = package.clone();
12434 let route_task = tokio::spawn(async move {
12435 route_handler
12436 .handle_control_frame(
12437 &client_ctx,
12438 route_open_frame_with_admission_facts(
12439 20,
12440 "target",
12441 unique_project_root("admission-facts"),
12442 Some(subc_control::ConsumerIdentity {
12443 module_id: "fed".to_string(),
12444 launch_nonce: "fed-nonce".to_string(),
12445 }),
12446 Some(package),
12447 ),
12448 )
12449 .await
12450 .unwrap()
12451 });
12452
12453 let bind_frame = target_rx.recv().await.unwrap();
12454 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12455 let ModuleControlRequest::RouteBind {
12456 admission_facts, ..
12457 } = bind
12458 else {
12459 panic!("{corpus_id}: expected route.bind")
12460 };
12461 assert_eq!(
12462 admission_facts,
12463 Some(expected),
12464 "{corpus_id}: relay must not add, drop or reshape any field"
12465 );
12466
12467 handler
12468 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12469 .await
12470 .unwrap();
12471 route_task.await.unwrap();
12472 }
12473 }
12474
12475 #[tokio::test]
12476 async fn route_bind_never_relays_the_openers_build_class() {
12477 let registry = Arc::new(Registry::default());
12478 let forwarding = Arc::new(ForwardingTable::default());
12479 let handler =
12480 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12481 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12482 hello_via_sink(
12483 &handler,
12484 &target_ctx,
12485 &mut target_rx,
12486 hello_frame("target", PROTOCOL_VERSION, 1),
12487 )
12488 .await;
12489
12490 let root = unique_project_root("build-class");
12491 let mut identity = BindIdentity::new(
12492 root.path().to_path_buf(),
12493 "unit".to_string(),
12494 "s".to_string(),
12495 );
12496 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12497 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12498 target: RouteTarget::ToolProvider {
12499 module_id: "target".to_string(),
12500 },
12501 identity,
12502 consumer_identity: None,
12503 consumer_capabilities: None,
12504 role_versions: None,
12505 admission_facts: None,
12506 scope: None,
12507 })
12508 .unwrap();
12509 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12510
12511 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12512 let route_handler = handler.clone();
12513 let route_task = tokio::spawn(async move {
12514 route_handler
12515 .handle_control_frame(&client_ctx, open)
12516 .await
12517 .unwrap()
12518 });
12519 let bind_frame = target_rx.recv().await.unwrap();
12520 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12521 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12522 panic!("expected route.bind")
12523 };
12524 assert_eq!(
12525 identity.build_class, None,
12526 "route.bind carried the build class"
12527 );
12528 assert!(
12529 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12530 "the route.bind bytes still name build_class"
12531 );
12532 handler
12533 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12534 .await
12535 .unwrap();
12536 route_task.await.unwrap();
12537 }
12538
12539 #[tokio::test]
12540 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12541 let registry = Arc::new(Registry::default());
12542 let forwarding = Arc::new(ForwardingTable::default());
12543 let supervisor = SupervisorHandle::new();
12544 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12545 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12546 let handler =
12547 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12548 .with_supervisor(supervisor)
12549 .with_admission_facts_config(
12550 Some("fed".to_string()),
12551 Some(vec!["target".to_string()]),
12552 );
12553
12554 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12555 hello_via_sink(
12556 &handler,
12557 &target_ctx,
12558 &mut target_rx,
12559 hello_frame("target", PROTOCOL_VERSION, 1),
12560 )
12561 .await;
12562 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12563 hello_via_sink(
12564 &handler,
12565 &other_ctx,
12566 &mut other_rx,
12567 hello_frame("other", PROTOCOL_VERSION, 2),
12568 )
12569 .await;
12570
12571 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12572 let expected_facts = facts.clone();
12573 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12574 let route_handler = handler.clone();
12575 let route_task = tokio::spawn(async move {
12576 route_handler
12577 .handle_control_frame(
12578 &client_ctx,
12579 route_open_frame_with_admission_facts(
12580 10,
12581 "target",
12582 unique_project_root("admission-facts"),
12583 Some(subc_control::ConsumerIdentity {
12584 module_id: "fed".to_string(),
12585 launch_nonce: "fed-nonce".to_string(),
12586 }),
12587 Some(facts.clone()),
12588 ),
12589 )
12590 .await
12591 .unwrap()
12592 });
12593 let bind_frame = target_rx.recv().await.unwrap();
12594 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12595 let ModuleControlRequest::RouteBind {
12596 admission_facts, ..
12597 } = bind
12598 else {
12599 panic!("expected route.bind")
12600 };
12601 assert_eq!(admission_facts, Some(expected_facts));
12602 handler
12603 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12604 .await
12605 .unwrap();
12606 assert!(route_task.await.unwrap().is_empty());
12607 assert!(matches!(
12608 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12609 .unwrap(),
12610 ClientControlResponse::RouteOpen { .. }
12611 ));
12612
12613 let direct = handler
12614 .handle_control_frame(
12615 &route_ctx(ConnectionId::new(73)).0,
12616 route_open_frame_with_admission_facts(
12617 11,
12618 "target",
12619 unique_project_root("admission-facts"),
12620 None,
12621 Some(json!({"x": 1})),
12622 ),
12623 )
12624 .await
12625 .unwrap();
12626 assert_eq!(
12627 parse_error(&direct[0])["code"],
12628 "admission_facts_not_permitted"
12629 );
12630
12631 let different_reserved = handler
12632 .handle_control_frame(
12633 &route_ctx(ConnectionId::new(77)).0,
12634 route_open_frame_with_admission_facts(
12635 15,
12636 "target",
12637 unique_project_root("admission-facts"),
12638 Some(subc_control::ConsumerIdentity {
12639 module_id: "other".to_string(),
12640 launch_nonce: "other-nonce".to_string(),
12641 }),
12642 Some(json!({"x": 1})),
12643 ),
12644 )
12645 .await
12646 .unwrap();
12647 assert_eq!(
12648 parse_error(&different_reserved[0])["code"],
12649 "admission_facts_not_permitted"
12650 );
12651
12652 let other_target = handler
12653 .handle_control_frame(
12654 &route_ctx(ConnectionId::new(74)).0,
12655 route_open_frame_with_admission_facts(
12656 12,
12657 "other",
12658 unique_project_root("admission-facts"),
12659 Some(subc_control::ConsumerIdentity {
12660 module_id: "fed".to_string(),
12661 launch_nonce: "fed-nonce".to_string(),
12662 }),
12663 Some(json!({"x": 1})),
12664 ),
12665 )
12666 .await
12667 .unwrap();
12668 assert_eq!(
12669 parse_error(&other_target[0])["code"],
12670 "admission_facts_target_not_allowed"
12671 );
12672
12673 let nonexistent = handler
12674 .handle_control_frame(
12675 &route_ctx(ConnectionId::new(75)).0,
12676 route_open_frame_with_admission_facts(
12677 13,
12678 "missing",
12679 unique_project_root("admission-facts"),
12680 None,
12681 Some(json!({"x": 1})),
12682 ),
12683 )
12684 .await
12685 .unwrap();
12686 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12687
12688 let described = handler
12689 .handle_control_frame(
12690 &route_ctx(ConnectionId::new(76)).0,
12691 Frame::build(
12692 FrameType::Request,
12693 control_flags(),
12694 0,
12695 0,
12696 14,
12697 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12698 )
12699 .unwrap(),
12700 )
12701 .await
12702 .unwrap();
12703 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12704 serde_json::from_slice(&described[0].body).unwrap()
12705 else {
12706 panic!("expected server.describe response")
12707 };
12708 assert!(capabilities
12709 .iter()
12710 .any(|cap| cap == "admission_facts_relay_v1"));
12711 }
12712
12713 #[tokio::test]
12714 async fn admission_facts_without_configured_carrier_are_rejected() {
12715 let registry = Arc::new(Registry::default());
12716 let forwarding = Arc::new(ForwardingTable::default());
12717 let handler = ControlHandler::with_forwarding(registry, forwarding);
12718 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12719 hello_via_sink(
12720 &handler,
12721 &target_ctx,
12722 &mut target_rx,
12723 hello_frame("target", PROTOCOL_VERSION, 1),
12724 )
12725 .await;
12726
12727 let responses = handler
12728 .handle_control_frame(
12729 &route_ctx(ConnectionId::new(79)).0,
12730 route_open_frame_with_admission_facts(
12731 16,
12732 "target",
12733 unique_project_root("admission-facts"),
12734 None,
12735 Some(json!({"x": 1})),
12736 ),
12737 )
12738 .await
12739 .unwrap();
12740 assert_eq!(
12741 parse_error(&responses[0])["code"],
12742 "admission_facts_not_permitted"
12743 );
12744 }
12745
12746 #[tokio::test]
12747 async fn route_open_relays_consumer_capabilities_verbatim() {
12748 let registry = Arc::new(Registry::default());
12749 let forwarding = Arc::new(ForwardingTable::default());
12750 let handler =
12751 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12752 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12753 hello_via_sink(
12754 &handler,
12755 &module_ctx,
12756 &mut module_rx,
12757 hello_frame("aft", PROTOCOL_VERSION, 7),
12758 )
12759 .await;
12760
12761 let expected = vec!["elicitation".to_string(), "roots".to_string()];
12762 let expected_for_request = expected.clone();
12763 let project_root = unique_project_root("consumer-capabilities-present");
12764 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12765 let route_handler = handler.clone();
12766 let route_task = tokio::spawn(async move {
12767 route_handler
12768 .handle_control_frame(
12769 &client_ctx,
12770 route_open_frame_with_consumer_capabilities(
12771 401,
12772 "aft",
12773 project_root,
12774 Some(expected_for_request),
12775 ),
12776 )
12777 .await
12778 .unwrap()
12779 });
12780 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12781 .await
12782 .unwrap()
12783 .unwrap();
12784 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12785 let ModuleControlRequest::RouteBind {
12786 consumer_capabilities,
12787 ..
12788 } = bind
12789 else {
12790 panic!("expected route.bind request, got {bind:?}");
12791 };
12792 assert_eq!(consumer_capabilities, Some(expected.clone()));
12793
12794 handler
12795 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12796 .await
12797 .unwrap();
12798 let route_response = route_task.await.unwrap();
12799 assert!(route_response.is_empty());
12800 let published = client_rx.recv().await.unwrap();
12801 assert!(matches!(
12802 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12803 ClientControlResponse::RouteOpen { .. }
12804 ));
12805 }
12806
12807 #[tokio::test]
12808 async fn route_open_without_consumer_capabilities_relays_none() {
12809 let registry = Arc::new(Registry::default());
12810 let forwarding = Arc::new(ForwardingTable::default());
12811 let handler =
12812 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12813 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12814 hello_via_sink(
12815 &handler,
12816 &module_ctx,
12817 &mut module_rx,
12818 hello_frame("aft", PROTOCOL_VERSION, 7),
12819 )
12820 .await;
12821
12822 let project_root = unique_project_root("consumer-capabilities-absent");
12823 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12824 let route_handler = handler.clone();
12825 let route_task = tokio::spawn(async move {
12826 route_handler
12827 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12828 .await
12829 .unwrap()
12830 });
12831 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12832 .await
12833 .unwrap()
12834 .unwrap();
12835 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12836 let ModuleControlRequest::RouteBind {
12837 consumer_capabilities,
12838 ..
12839 } = bind
12840 else {
12841 panic!("expected route.bind request, got {bind:?}");
12842 };
12843 assert_eq!(consumer_capabilities, None);
12844
12845 handler
12846 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12847 .await
12848 .unwrap();
12849 let route_response = route_task.await.unwrap();
12850 assert!(route_response.is_empty());
12851 let published = client_rx.recv().await.unwrap();
12852 assert!(matches!(
12853 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12854 ClientControlResponse::RouteOpen { .. }
12855 ));
12856 }
12857
12858 async fn bind_role_versions_for(
12862 sent: Option<BTreeMap<String, String>>,
12863 connection: u64,
12864 ) -> Option<BTreeMap<String, String>> {
12865 let registry = Arc::new(Registry::default());
12866 let forwarding = Arc::new(ForwardingTable::default());
12867 let handler =
12868 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12869 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12870 hello_via_sink(
12871 &handler,
12872 &module_ctx,
12873 &mut module_rx,
12874 hello_frame("aft", PROTOCOL_VERSION, 7),
12875 )
12876 .await;
12877 let project_root = unique_project_root("role-versions");
12878 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12879 let route_handler = handler.clone();
12880 let route_task = tokio::spawn(async move {
12881 route_handler
12882 .handle_control_frame(
12883 &client_ctx,
12884 route_open_frame_with_role_versions(403, "aft", project_root, sent),
12885 )
12886 .await
12887 .unwrap()
12888 });
12889 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12890 .await
12891 .expect("a well-formed route.open reaches the module as a bind")
12892 .unwrap();
12893 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12894 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12895 panic!("expected route.bind request, got {bind:?}");
12896 };
12897 handler
12898 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12899 .await
12900 .unwrap();
12901 assert!(route_task.await.unwrap().is_empty());
12902 let published = client_rx.recv().await.unwrap();
12903 assert!(matches!(
12904 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12905 ClientControlResponse::RouteOpen { .. }
12906 ));
12907 role_versions
12908 }
12909
12910 #[tokio::test]
12911 async fn route_open_relays_role_versions_verbatim() {
12912 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12913 assert_eq!(
12914 bind_role_versions_for(Some(sent.clone()), 141).await,
12915 Some(sent)
12916 );
12917 }
12918
12919 #[tokio::test]
12922 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12923 assert_eq!(bind_role_versions_for(None, 143).await, None);
12924 assert_eq!(
12925 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12926 None
12927 );
12928 }
12929
12930 #[tokio::test]
12931 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12932 let registry = Arc::new(Registry::default());
12933 let forwarding = Arc::new(ForwardingTable::default());
12934 let handler =
12935 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12936 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12937 hello_via_sink(
12938 &handler,
12939 &module_ctx,
12940 &mut module_rx,
12941 hello_frame("aft", PROTOCOL_VERSION, 7),
12942 )
12943 .await;
12944
12945 let nine: BTreeMap<String, String> = (0..9)
12946 .map(|index| (format!("role-{index}"), "v1".to_string()))
12947 .collect();
12948 for (label, malformed) in [
12949 (
12950 "invalid role name",
12951 role_versions(&[("Tool_Provider", "v1")]),
12952 ),
12953 ("invalid version", role_versions(&[("tool-provider", "v0")])),
12954 ("nine entries", nine),
12955 ] {
12956 let responses = tokio::time::timeout(
12959 Duration::from_secs(1),
12960 handler.handle_control_frame(
12961 &route_ctx(ConnectionId::new(148)).0,
12962 route_open_frame_with_role_versions(
12963 404,
12964 "aft",
12965 unique_project_root("role-versions-malformed"),
12966 Some(malformed),
12967 ),
12968 ),
12969 )
12970 .await
12971 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12972 .unwrap();
12973 assert_eq!(responses.len(), 1, "{label}");
12974 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12975 let error = parse_error(&responses[0]);
12976 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12977 assert_eq!(
12978 error["detail"]["field"], "role_versions",
12979 "{label}: {error}"
12980 );
12981 assert!(
12982 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12983 "{label}: a malformed declaration is terminal"
12984 );
12985 assert!(
12986 module_rx.try_recv().is_err(),
12987 "{label}: the module must never see a bind"
12988 );
12989 }
12990 }
12991
12992 #[tokio::test]
12996 async fn route_role_versions_capability_is_advertised() {
12997 let handler = ControlHandler::new(Arc::new(Registry::default()));
12998 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12999 let ack = hello_via_sink(
13000 &handler,
13001 &ctx,
13002 &mut rx,
13003 hello_frame("m", PROTOCOL_VERSION, 1),
13004 )
13005 .await;
13006 let ack = parse_ack(&ack);
13007 assert!(
13008 ack.subc_capabilities
13009 .iter()
13010 .any(|c| c == "route-role-versions/v1"),
13011 "{:?}",
13012 ack.subc_capabilities
13013 );
13014
13015 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13016 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13017 let reply = handler
13018 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13019 .await
13020 .unwrap()
13021 .pop()
13022 .unwrap();
13023 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13024 serde_json::from_slice(&reply.body).unwrap()
13025 else {
13026 panic!("not a server.describe reply");
13027 };
13028 assert!(
13029 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13030 "{capabilities:?}"
13031 );
13032 }
13033
13034 #[tokio::test]
13038 async fn catalog_events_capability_is_advertised() {
13039 let handler = ControlHandler::new(Arc::new(Registry::default()));
13040 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13041 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13042 let reply = handler
13043 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13044 .await
13045 .unwrap()
13046 .pop()
13047 .unwrap();
13048 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13049 serde_json::from_slice(&reply.body).unwrap()
13050 else {
13051 panic!("not a server.describe reply");
13052 };
13053 assert!(
13054 capabilities
13055 .iter()
13056 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13057 "{capabilities:?}"
13058 );
13059 }
13060
13061 #[tokio::test]
13062 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13063 let handler = ControlHandler::new(Arc::new(Registry::default()));
13064 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13065 let ack = hello_via_sink(
13066 &handler,
13067 &module,
13068 &mut module_rx,
13069 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13070 )
13071 .await;
13072 let ack = parse_ack(&ack);
13073 assert!(
13074 ack.subc_capabilities
13075 .iter()
13076 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13077 "{:?}",
13078 ack.subc_capabilities
13079 );
13080 assert!(ack
13081 .subc_capabilities
13082 .iter()
13083 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13084
13085 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13086 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13087 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13088 let reply = handler
13089 .handle_control_frame(&client, frame)
13090 .await
13091 .unwrap()
13092 .pop()
13093 .unwrap();
13094 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13095 serde_json::from_slice(&reply.body).unwrap()
13096 else {
13097 panic!("not a server.describe reply");
13098 };
13099 assert!(
13100 capabilities
13101 .iter()
13102 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13103 "{capabilities:?}"
13104 );
13105 assert!(capabilities
13106 .iter()
13107 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13108 }
13109
13110 #[tokio::test]
13111 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13112 let handler = ControlHandler::new(Arc::new(Registry::default()));
13113 let (module, mut rx) = route_ctx(ConnectionId::new(156));
13114 let ack = hello_via_sink(
13115 &handler,
13116 &module,
13117 &mut rx,
13118 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13119 )
13120 .await;
13121 assert!(parse_ack(&ack)
13122 .subc_capabilities
13123 .iter()
13124 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13125 assert!(parse_ack(&ack)
13126 .subc_capabilities
13127 .iter()
13128 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13129 let (client, _rx) = route_ctx(ConnectionId::new(157));
13130 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13131 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13132 let reply = handler
13133 .handle_control_frame(&client, frame)
13134 .await
13135 .unwrap()
13136 .pop()
13137 .unwrap();
13138 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13139 serde_json::from_slice(&reply.body).unwrap()
13140 else {
13141 panic!("server.describe reply")
13142 };
13143 assert!(capabilities
13144 .iter()
13145 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13146 assert!(capabilities
13147 .iter()
13148 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13149 }
13150
13151 #[tokio::test]
13152 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13153 let handler = ControlHandler::new(Arc::new(Registry::default()));
13154 let (module, mut rx) = route_ctx(ConnectionId::new(158));
13155 let ack = hello_via_sink(
13156 &handler,
13157 &module,
13158 &mut rx,
13159 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13160 )
13161 .await;
13162 assert!(parse_ack(&ack)
13163 .subc_capabilities
13164 .iter()
13165 .any(|cap| cap == "scope-workdir/v1"));
13166 let (client, _rx) = route_ctx(ConnectionId::new(159));
13167 let frame = Frame::build(
13168 FrameType::Request,
13169 control_flags(),
13170 0,
13171 0,
13172 5,
13173 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13174 )
13175 .unwrap();
13176 let reply = handler
13177 .handle_control_frame(&client, frame)
13178 .await
13179 .unwrap()
13180 .remove(0);
13181 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13182 serde_json::from_slice(&reply.body).unwrap()
13183 else {
13184 panic!("server.describe reply")
13185 };
13186 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13187 }
13188
13189 #[tokio::test]
13190 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13191 let handler = ControlHandler::new(Arc::new(Registry::default()));
13192 let (module, mut rx) = route_ctx(ConnectionId::new(160));
13193 let ack = hello_via_sink(
13194 &handler,
13195 &module,
13196 &mut rx,
13197 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13198 )
13199 .await;
13200 assert!(parse_ack(&ack)
13201 .subc_capabilities
13202 .iter()
13203 .any(|cap| cap == "worker-runs/v1"));
13204 let (client, _rx) = route_ctx(ConnectionId::new(161));
13205 let frame = Frame::build(
13206 FrameType::Request,
13207 control_flags(),
13208 0,
13209 0,
13210 5,
13211 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13212 )
13213 .unwrap();
13214 let reply = handler
13215 .handle_control_frame(&client, frame)
13216 .await
13217 .unwrap()
13218 .remove(0);
13219 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13220 serde_json::from_slice(&reply.body).unwrap()
13221 else {
13222 panic!("server.describe reply")
13223 };
13224 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13225 }
13226
13227 #[tokio::test]
13228 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13229 let handler = ControlHandler::new(Arc::new(Registry::default()));
13230 let (module, mut rx) = route_ctx(ConnectionId::new(154));
13231 let ack = hello_via_sink(
13232 &handler,
13233 &module,
13234 &mut rx,
13235 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13236 )
13237 .await;
13238 assert!(parse_ack(&ack)
13239 .subc_capabilities
13240 .iter()
13241 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13242 let (client, _rx) = route_ctx(ConnectionId::new(155));
13243 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13244 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13245 let reply = handler
13246 .handle_control_frame(&client, frame)
13247 .await
13248 .unwrap()
13249 .pop()
13250 .unwrap();
13251 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13252 serde_json::from_slice(&reply.body).unwrap()
13253 else {
13254 panic!("not a server.describe reply")
13255 };
13256 assert!(capabilities
13257 .iter()
13258 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13259 }
13260
13261 #[tokio::test]
13262 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13263 let registry = Arc::new(Registry::default());
13264 let forwarding = Arc::new(ForwardingTable::default());
13265 let handler =
13266 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13267 .with_health_probe_timeout(Duration::from_secs(5));
13268 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13269 hello_via_sink(
13270 &handler,
13271 &module_ctx,
13272 &mut module_rx,
13273 non_routable_hello_frame_with_control_ops(
13274 "mcp",
13275 300,
13276 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13277 ),
13278 )
13279 .await;
13280 assert!(registry
13281 .get_module("mcp")
13282 .unwrap()
13283 .unwrap()
13284 .manifest
13285 .provides
13286 .is_empty());
13287
13288 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13289 let route_response = handler
13290 .handle_control_frame(
13291 &route_client_ctx,
13292 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13293 )
13294 .await
13295 .unwrap();
13296 assert_eq!(route_response[0].header.ty, FrameType::Error);
13297 assert_eq!(
13298 parse_error(&route_response[0])["code"],
13299 "target_unavailable"
13300 );
13301 assert!(parse_error(&route_response[0])["message"]
13302 .as_str()
13303 .unwrap()
13304 .contains("does not provide the requested target"));
13305 assert!(module_rx.try_recv().is_err());
13306
13307 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13308 let health_handler = handler.clone();
13309 let health_task = tokio::spawn(async move {
13310 health_handler
13311 .handle_control_frame(
13312 &health_client_ctx,
13313 supervisor_health_probe_frame(302, "mcp"),
13314 )
13315 .await
13316 .unwrap()
13317 });
13318 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13319 .await
13320 .unwrap()
13321 .unwrap();
13322 assert_eq!(
13323 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13324 ModuleControlRequest::HealthCheck {}
13325 );
13326 handler
13327 .handle_control_frame(
13328 &module_ctx,
13329 health_response(health_frame.header.corr, HealthStatus::Ok),
13330 )
13331 .await
13332 .unwrap();
13333 let health_response = health_task.await.unwrap();
13334 assert_eq!(health_response[0].header.ty, FrameType::Response);
13335 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13336 ClientControlResponse::SupervisorHealthProbe {
13337 module_id, status, ..
13338 } => {
13339 assert_eq!(module_id, "mcp");
13340 assert_eq!(status, HealthStatus::Ok);
13341 }
13342 other => panic!("unexpected health response: {other:?}"),
13343 }
13344
13345 forwarding
13350 .cleanup_connection(module_ctx.connection_id)
13351 .unwrap();
13352 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13353 let cleanup_response = tokio::time::timeout(
13354 Duration::from_millis(200),
13355 handler.handle_control_frame(
13356 &cleanup_probe_ctx,
13357 supervisor_health_probe_frame(303, "mcp"),
13358 ),
13359 )
13360 .await
13361 .expect("probe should fail immediately when the control lane is gone")
13362 .unwrap();
13363 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13364 assert_eq!(
13365 parse_error(&cleanup_response[0])["code"],
13366 "target_unavailable"
13367 );
13368 assert!(parse_error(&cleanup_response[0])["message"]
13369 .as_str()
13370 .unwrap()
13371 .contains("no module connection"));
13372
13373 handler
13374 .cleanup_connection(module_ctx.connection_id)
13375 .unwrap();
13376 }
13377
13378 #[tokio::test]
13379 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13380 let registry = Arc::new(Registry::default());
13381 let supervisor_handle = SupervisorHandle::new();
13382 let supervisor =
13383 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13384 .with_handle(supervisor_handle.clone())
13385 .with_connection_file_path(
13386 std::env::temp_dir()
13387 .join(format!("subc-route-open-warming-{}", std::process::id())),
13388 );
13389 let module = supervisor
13390 .supervise_configured(
13391 ModuleSpec {
13392 module_id: "warming".to_string(),
13393 program: fake_aft_stub_path(),
13394 args: Vec::new(),
13395 env: Vec::new(),
13396 reserved: false,
13397 reserved_prefixes: Vec::new(),
13398 protocol: ModuleProtocol::Subc,
13399 overlap: Default::default(),
13400 },
13401 true,
13402 )
13403 .unwrap();
13404 assert_eq!(module.state().unwrap(), ModuleState::Running);
13405
13406 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13407 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13408 let response = handler
13409 .handle_control_frame(
13410 &ctx,
13411 route_open_frame(304, "warming", unique_project_root("warming")),
13412 )
13413 .await
13414 .unwrap();
13415 module.stop().await.unwrap();
13416
13417 assert_eq!(response[0].header.ty, FrameType::Error);
13418 let error = parse_error(&response[0]);
13419 assert_eq!(error["code"], "module_warming");
13420 assert!(error["message"]
13421 .as_str()
13422 .unwrap()
13423 .contains("state=running, enabled=true, live=false"));
13424 }
13425
13426 #[test]
13427 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13428 let handler = ControlHandler::new(Arc::new(Registry::default()));
13429 let capture = EventCapture::default();
13430 let _subscriber =
13431 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13432 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13433 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13434 let pending = (0..limit).collect::<Vec<_>>();
13435 let response = handler
13436 .route_open_capacity_refusal(
13437 &ctx,
13438 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13439 "busy",
13440 pending.len(),
13441 limit,
13442 )
13443 .unwrap();
13444 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13445 let event = capture
13446 .events()
13447 .into_iter()
13448 .find(|event| {
13449 event.target == "control"
13450 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13451 })
13452 .expect("connection admission refusal event");
13453 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13454 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13455 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13458 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13459 }
13460
13461 #[test]
13462 fn route_open_target_cap_logs_admission_reason_and_capacity() {
13463 let handler = ControlHandler::new(Arc::new(Registry::default()));
13464 let capture = EventCapture::default();
13465 let _subscriber =
13466 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13467 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13468 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13469 let guards = (0..limit)
13470 .map(|_| {
13471 handler
13472 .route_bind_concurrency
13473 .try_admit("busy", limit)
13474 .unwrap()
13475 })
13476 .collect::<Vec<_>>();
13477 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13478 Err(in_flight) => in_flight,
13479 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13480 };
13481 let response = handler
13482 .route_open_target_capacity_refusal(
13483 &ctx,
13484 &RouteOpenCaller::unknown(),
13485 &route_open_frame(397, "busy", unique_project_root("target-cap")),
13486 "busy",
13487 in_flight,
13488 )
13489 .unwrap();
13490 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13491 let event = capture
13492 .events()
13493 .into_iter()
13494 .find(|event| {
13495 event.target == "control"
13496 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13497 })
13498 .expect("target admission refusal event");
13499 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13500 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13501 drop(guards);
13502 }
13503
13504 #[cfg(unix)]
13505 #[tokio::test(start_paused = true)]
13506 async fn supervisor_list_shows_a_failed_modules_next_retry() {
13507 let registry = Arc::new(Registry::default());
13508 let handle = SupervisorHandle::new();
13509 let supervisor =
13510 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13511 .with_handle(handle.clone());
13512 let module = supervisor
13513 .spawn(crate::ModuleSpec {
13514 module_id: "retry-list".into(),
13515 program: PathBuf::from("/bin/sh"),
13516 args: vec!["-c".into(), "exit 17".into()],
13517 env: vec![],
13518 reserved: false,
13519 reserved_prefixes: vec![],
13520 protocol: ModuleProtocol::None,
13521 overlap: Default::default(),
13522 })
13523 .unwrap();
13524 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13525 while module.state().unwrap() != crate::ModuleState::Failed {
13526 assert!(
13527 std::time::Instant::now() < deadline,
13528 "waiting for budget failure"
13529 );
13530 tokio::task::yield_now().await;
13531 }
13532 let handler = ControlHandler::new(registry).with_supervisor(handle);
13533 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13534 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13535 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13536 assert_eq!(body["modules"][0]["state"], "failed");
13537 let retry = body["modules"][0]["next_retry_at_ms"]
13538 .as_u64()
13539 .expect("scheduled retry on the wire");
13540 let now = std::time::SystemTime::now()
13541 .duration_since(std::time::UNIX_EPOCH)
13542 .unwrap()
13543 .as_millis() as u64;
13544 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13545 module.set_enabled(false).await.unwrap();
13546 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13547 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13548 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13549 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13550 }
13551
13552 #[tokio::test]
13553 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
13554 let registry = Arc::new(Registry::default());
13555 let handle = SupervisorHandle::new();
13556 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13557 .with_handle(handle.clone());
13558 supervisor
13559 .supervise_configured(
13560 crate::ModuleSpec {
13561 module_id: "drift".into(),
13562 program: PathBuf::from("disabled-module"),
13563 args: Vec::new(),
13564 env: Vec::new(),
13565 reserved: false,
13566 reserved_prefixes: Vec::new(),
13567 protocol: ModuleProtocol::Subc,
13568 overlap: Default::default(),
13569 },
13570 false,
13571 )
13572 .unwrap();
13573 let handler = ControlHandler::new(registry).with_supervisor(handle);
13574 for configured in [None, Some(false), Some(true)] {
13575 handler.capability_evaluator.configure(
13576 configured.map(|enabled| ("drift".to_string(), enabled)),
13577 BTreeMap::new(),
13578 );
13579 let frame =
13580 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13581 let replies = handler
13582 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
13583 .await
13584 .unwrap();
13585 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13586 assert!(body.get("configured_enabled").is_none());
13587 assert_eq!(body["op"], "supervisor.ack");
13588 let frame =
13589 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13590 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13591 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13592 assert_eq!(
13593 body["modules"][0].get("configured_enabled"),
13594 configured.map(Value::Bool).as_ref()
13595 );
13596 }
13597 }
13598
13599 #[tokio::test(flavor = "current_thread")]
13600 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
13601 let handler = ControlHandler::new(Arc::new(Registry::default()));
13602 let capture = EventCapture::default();
13603 let _subscriber =
13604 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13605 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
13606 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13607
13608 let _ = handler
13609 .handle_client_control_request(
13610 &ctx,
13611 frame,
13612 ClientControlRequest::SupervisorSetEnabled {
13613 module_id: "broca".to_string(),
13614 enabled: false,
13615 },
13616 &mut None,
13617 )
13618 .await
13619 .unwrap();
13620
13621 let events = capture
13622 .events()
13623 .into_iter()
13624 .filter(|event| {
13625 event.target == "control"
13626 && event.fields.get("message").map(String::as_str)
13627 == Some("supervisor request received")
13628 })
13629 .collect::<Vec<_>>();
13630 assert_eq!(events.len(), 1, "one supervisor request log line");
13631 let event = &events[0];
13632 assert_eq!(event.level, tracing::Level::INFO);
13633 assert_eq!(
13634 event.fields.get("op"),
13635 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
13636 );
13637 assert_eq!(
13638 event.fields.get("module_id"),
13639 Some(&"\"broca\"".to_string())
13640 );
13641 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
13642 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
13643 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
13644 }
13645
13646 #[tokio::test(flavor = "current_thread")]
13647 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
13648 let registry = Arc::new(Registry::default());
13649 let module_connection = ConnectionId::new(303);
13650 registry
13651 .register_with_control_ops(
13652 manifest("aft", PROTOCOL_VERSION),
13653 PROTOCOL_VERSION,
13654 module_connection,
13655 module_baseline_control_ops(),
13656 )
13657 .unwrap();
13658 let handler = ControlHandler::new(registry);
13659 let capture = EventCapture::default();
13660 let _subscriber =
13661 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13662 let (ctx, _rx) = route_ctx(module_connection);
13663 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13664
13665 let _ = handler
13666 .handle_client_control_request(
13667 &ctx,
13668 frame,
13669 ClientControlRequest::SupervisorSetEnabled {
13670 module_id: "broca".to_string(),
13671 enabled: false,
13672 },
13673 &mut None,
13674 )
13675 .await
13676 .unwrap();
13677
13678 let callers = capture
13679 .events()
13680 .into_iter()
13681 .filter(|event| {
13682 event.fields.get("message").map(String::as_str)
13683 == Some("supervisor request received")
13684 })
13685 .map(|event| event.fields.get("caller").cloned())
13686 .collect::<Vec<_>>();
13687 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13688 }
13689
13690 #[tokio::test(flavor = "current_thread")]
13691 async fn supervisor_list_does_not_log_request_received() {
13692 let handler = ControlHandler::new(Arc::new(Registry::default()));
13693 let capture = EventCapture::default();
13694 let _subscriber =
13695 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13696 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13697 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13698
13699 handler
13700 .handle_client_control_request(
13701 &ctx,
13702 frame,
13703 ClientControlRequest::SupervisorList {},
13704 &mut None,
13705 )
13706 .await
13707 .unwrap();
13708
13709 assert!(capture.events().into_iter().all(|event| {
13710 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13711 }));
13712 }
13713
13714 #[tokio::test(flavor = "current_thread")]
13715 async fn supervisor_rescan_preview_does_not_log_request_received() {
13716 let handler = ControlHandler::new(Arc::new(Registry::default()));
13717 let capture = EventCapture::default();
13718 let _subscriber =
13719 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13720 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13721 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13722
13723 handler
13724 .handle_client_control_request(
13725 &ctx,
13726 frame,
13727 ClientControlRequest::SupervisorRescan { preview: true },
13728 &mut None,
13729 )
13730 .await
13731 .unwrap();
13732
13733 assert!(capture.events().into_iter().all(|event| {
13734 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13735 }));
13736 }
13737
13738 struct ApproveOperator;
13739 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13740 fn prompt(
13741 &self,
13742 _: &str,
13743 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13744 ) -> crate::operator_confirm::ProviderResult {
13745 crate::operator_confirm::ProviderResult::Approved
13746 }
13747 }
13748
13749 #[tokio::test]
13750 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13751 assert_eq!(
13752 ControlHandler::default().privileged_operator_prompt,
13753 PrivilegedOperatorPrompt::Log
13754 );
13755 for (initial, changed, running) in [
13756 ("log", "enforce", PrivilegedOperatorPrompt::Log),
13757 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13758 ] {
13759 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13760 let path = dir.join("subc.jsonc");
13761 std::fs::write(
13762 &path,
13763 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13764 )
13765 .unwrap();
13766 let config = crate::daemon_config::load(&path).unwrap().unwrap();
13767 let registry = Arc::new(Registry::default());
13768 let handle = SupervisorHandle::new();
13769 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13770 .with_handle(handle.clone());
13771 let handler = ControlHandler::new(registry)
13772 .with_supervisor(handle)
13773 .with_privileged_operator_prompt(config.privileged_operator_prompt)
13774 .with_supervisor_rescan(supervisor, &path, config.port);
13775 handler.forwarding.operator_confirms().configure(
13776 Arc::new(ApproveOperator),
13777 crate::operator_confirm::OperatorLimits::default(),
13778 );
13779 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13780
13781 for (disk_mode, needs_restart) in [
13782 (Some(initial), false),
13783 (Some(changed), true),
13784 (Some(changed), true),
13785 (None, running == PrivilegedOperatorPrompt::Enforce),
13786 (Some(initial), false),
13787 ] {
13788 let mut doc = json!({"version":1});
13789 if let Some(mode) = disk_mode {
13790 doc["privileged_operator_prompt"] = json!(mode);
13791 }
13792 std::fs::write(&path, doc.to_string()).unwrap();
13793 for preview in [true, false] {
13794 let request = ClientControlRequest::SupervisorRescan { preview };
13795 let frame = Frame::build(
13796 FrameType::Request,
13797 control_flags(),
13798 0,
13799 0,
13800 1,
13801 serde_json::to_vec(&request).unwrap(),
13802 )
13803 .unwrap();
13804 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13805 assert_eq!(frames.len(), 1);
13806 let ClientControlResponse::SupervisorRescan { result } =
13807 serde_json::from_slice(&frames[0].body).unwrap()
13808 else {
13809 panic!("expected a rescan response");
13810 };
13811 let expected = if needs_restart {
13812 vec!["privileged_operator_prompt"]
13813 } else {
13814 Vec::new()
13815 };
13816 assert_eq!(result.restart_required, expected);
13817 assert_eq!(handler.privileged_operator_prompt, running);
13818 assert_eq!(
13819 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13820 running
13821 );
13822 }
13823 }
13824 }
13825 }
13826
13827 #[tokio::test]
13828 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13829 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13830 let path = dir.join("subc.jsonc");
13831 let registry = Arc::new(Registry::default());
13832 let handle = SupervisorHandle::new();
13833 let supervisor =
13834 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13835 let handler = ControlHandler::new(registry)
13836 .with_supervisor(handle)
13837 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13838 .with_supervisor_rescan(supervisor, &path, None);
13839 handler.forwarding.operator_confirms().configure(
13840 Arc::new(ApproveOperator),
13841 crate::operator_confirm::OperatorLimits::default(),
13842 );
13843 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13844 for preview in [true, false] {
13845 std::fs::write(
13846 &path,
13847 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13848 )
13849 .unwrap();
13850 let request = ClientControlRequest::SupervisorRescan { preview };
13851 let frame = Frame::build(
13852 FrameType::Request,
13853 control_flags(),
13854 0,
13855 0,
13856 1,
13857 serde_json::to_vec(&request).unwrap(),
13858 )
13859 .unwrap();
13860 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13861 assert_eq!(frames.len(), 1);
13862 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13863 assert_eq!(
13864 handler.privileged_operator_prompt,
13865 PrivilegedOperatorPrompt::Enforce
13866 );
13867 }
13868 }
13869
13870 #[tokio::test]
13875 async fn route_open_refusal_names_the_check_that_refused() {
13876 let handler = ControlHandler::new(Arc::new(Registry::default()));
13877 let capture = EventCapture::default();
13878 let _subscriber =
13879 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13880 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13881 let response = handler
13882 .handle_control_frame(
13883 &ctx,
13884 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13885 )
13886 .await
13887 .unwrap();
13888
13889 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13890 let event = capture
13891 .events()
13892 .into_iter()
13893 .find(|event| {
13894 event.target == "control"
13895 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13896 })
13897 .expect("route.open refusal event");
13898 assert_eq!(
13899 event.fields.get("reason"),
13900 Some(&"\"not_registered\"".to_string())
13901 );
13902 }
13903
13904 #[tokio::test]
13905 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13906 let registry = Arc::new(Registry::default());
13907 let supervisor_handle = SupervisorHandle::new();
13908 let supervisor =
13909 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13910 .with_handle(supervisor_handle.clone())
13911 .with_connection_file_path(std::env::temp_dir().join(format!(
13912 "subc-route-open-refusal-info-{}",
13913 std::process::id()
13914 )));
13915 let module = supervisor
13916 .supervise_configured(
13917 ModuleSpec {
13918 module_id: "warming".to_string(),
13919 program: fake_aft_stub_path(),
13920 args: Vec::new(),
13921 env: Vec::new(),
13922 reserved: false,
13923 reserved_prefixes: Vec::new(),
13924 protocol: ModuleProtocol::Subc,
13925 overlap: Default::default(),
13926 },
13927 true,
13928 )
13929 .unwrap();
13930 assert_eq!(module.state().unwrap(), ModuleState::Running);
13931
13932 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13933 assert!(handler
13934 .counters()
13935 .snapshot()
13936 .get("route_open_refused_by_code")
13937 .is_none());
13938 let capture = EventCapture::default();
13939 let _subscriber =
13940 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13941 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13942 let response = handler
13943 .handle_control_frame(
13944 &ctx,
13945 route_open_frame(394, "warming", unique_project_root("refusal-info")),
13946 )
13947 .await
13948 .unwrap();
13949 module.stop().await.unwrap();
13950
13951 assert_eq!(parse_error(&response[0])["code"], "module_warming");
13952 let event = capture
13953 .events()
13954 .into_iter()
13955 .find(|event| {
13956 event.target == "control"
13957 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13958 })
13959 .expect("route.open refusal event");
13960 assert_eq!(
13961 event.fields.get("module_id"),
13962 Some(&"\"warming\"".to_string())
13963 );
13964 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13965 assert_eq!(
13966 event.fields.get("reason"),
13967 Some(&"\"supervised_not_registered\"".to_string())
13968 );
13969 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13970 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13971 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13972 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13973 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13974 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13975 assert_eq!(
13976 handler.counters().snapshot()["route_open_refused_by_code"],
13977 json!({ "module_warming": 1 })
13978 );
13979 }
13980
13981 const OUTAGE_START: &str = "route.open refusing module: not serving";
13982 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13983
13984 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13985 capture
13986 .events()
13987 .into_iter()
13988 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13989 .collect()
13990 }
13991
13992 fn supervise_stub(
13993 registry: &Arc<Registry>,
13994 module_id: &str,
13995 enabled: bool,
13996 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13997 let supervisor_handle = SupervisorHandle::new();
13998 let supervisor =
13999 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
14000 .with_handle(supervisor_handle.clone())
14001 .with_connection_file_path(std::env::temp_dir().join(format!(
14002 "subc-route-outage-{module_id}-{}",
14003 std::process::id()
14004 )));
14005 let module = supervisor
14006 .supervise_configured(
14007 ModuleSpec {
14008 module_id: module_id.to_string(),
14009 program: fake_aft_stub_path(),
14010 args: Vec::new(),
14011 env: Vec::new(),
14012 reserved: false,
14013 reserved_prefixes: Vec::new(),
14014 protocol: ModuleProtocol::Subc,
14015 overlap: Default::default(),
14016 },
14017 enabled,
14018 )
14019 .unwrap();
14020 (supervisor_handle, module)
14021 }
14022
14023 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
14024 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
14025 module_id: module_id.to_string(),
14026 drain_timeout_ms: Some(50),
14027 })
14028 .unwrap();
14029 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
14030 }
14031
14032 #[test]
14036 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
14037 let registry = Arc::new(Registry::default());
14038 let forwarding = Arc::new(ForwardingTable::default());
14039 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14040 let second = ControlHandler::with_forwarding(registry, forwarding);
14041 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
14042 }
14043
14044 #[tokio::test(flavor = "current_thread")]
14048 async fn route_open_unknown_module_refusals_add_no_outage_state() {
14049 let handler = ControlHandler::new(Arc::new(Registry::default()));
14050 let capture = EventCapture::default();
14051 let _subscriber =
14052 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14053 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
14054 for corr in 0..8 {
14055 let response = handler
14056 .handle_control_frame(
14057 &ctx,
14058 route_open_frame(
14059 380 + corr,
14060 &format!("nobody-{corr}"),
14061 unique_project_root("outage-unknown"),
14062 ),
14063 )
14064 .await
14065 .unwrap();
14066 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14067 }
14068
14069 assert_eq!(handler.route_outages.tracked_module_count(), 0);
14070 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
14071 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
14072 }
14073
14074 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14079 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
14080 let registry = Arc::new(Registry::default());
14081 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
14082 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14083 let capture = EventCapture::default();
14084 let _subscriber =
14085 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14086 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
14087 handler.route_outages.record_accepted("outage-restart");
14090
14091 let response = handler
14092 .handle_control_frame(
14093 &ctx,
14094 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
14095 )
14096 .await
14097 .unwrap();
14098 assert_eq!(response[0].header.ty, FrameType::Error);
14099 let starts = outage_lines(&capture, OUTAGE_START);
14100 assert_eq!(starts.len(), 1, "{starts:?}");
14101 assert_eq!(starts[0].level, tracing::Level::WARN);
14102 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
14103 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
14104 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
14105 handler.route_outages.record_accepted("outage-restart");
14106 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
14107
14108 let restart = handler
14109 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
14110 .await
14111 .unwrap();
14112 assert_eq!(
14113 restart[0].header.ty,
14114 FrameType::Response,
14115 "{:?}",
14116 parse_error(&restart[0])
14117 );
14118 handler
14119 .handle_control_frame(
14120 &ctx,
14121 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
14122 )
14123 .await
14124 .unwrap();
14125 module.stop().await.unwrap();
14126
14127 let starts = outage_lines(&capture, OUTAGE_START);
14128 assert_eq!(starts.len(), 2, "{starts:?}");
14129 assert_eq!(starts[1].level, tracing::Level::INFO);
14130 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
14131 }
14132
14133 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14137 async fn failed_operator_restart_leaves_no_operator_mark() {
14138 let registry = Arc::new(Registry::default());
14139 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
14140 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14141 let capture = EventCapture::default();
14142 let _subscriber =
14143 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14144 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
14145 handler.route_outages.record_accepted("outage-disabled");
14146
14147 let restart = handler
14148 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
14149 .await
14150 .unwrap();
14151 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
14152 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
14153
14154 handler
14155 .handle_control_frame(
14156 &ctx,
14157 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
14158 )
14159 .await
14160 .unwrap();
14161 let starts = outage_lines(&capture, OUTAGE_START);
14162 assert_eq!(starts.len(), 1, "{starts:?}");
14163 assert_eq!(starts[0].level, tracing::Level::WARN);
14164 }
14165
14166 #[tokio::test(flavor = "current_thread")]
14167 async fn route_open_unknown_module_escapes_target_module_id() {
14168 let handler = ControlHandler::new(Arc::new(Registry::default()));
14169 let capture = EventCapture::default();
14170 let _subscriber =
14171 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14172 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
14173 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14174 let response = handler
14175 .handle_control_frame(
14176 &ctx,
14177 route_open_frame(
14178 395,
14179 hostile_module_id,
14180 unique_project_root("hostile-target-module-id"),
14181 ),
14182 )
14183 .await
14184 .unwrap();
14185
14186 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14187 let event = capture
14188 .events()
14189 .into_iter()
14190 .find(|event| {
14191 event.target == "control"
14192 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14193 })
14194 .expect("route.open unknown-module refusal event");
14195 let logged = event.fields.get("module_id").expect("module_id field");
14196 assert!(!logged.bytes().any(|byte| byte < 0x20));
14197 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14198 }
14199
14200 #[tokio::test(flavor = "current_thread")]
14203 async fn route_open_refusal_carries_daemon_origin() {
14204 let handler = ControlHandler::new(Arc::new(Registry::default()));
14205 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14206 let response = handler
14207 .handle_control_frame(
14208 &ctx,
14209 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
14210 )
14211 .await
14212 .unwrap();
14213
14214 assert_eq!(response.len(), 1);
14215 assert_eq!(response[0].header.ty, FrameType::Error);
14216 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14217 assert!(
14218 response[0].header.flags.is_daemon_origin(),
14219 "route.open refusal must carry daemon origin: {:?}",
14220 response[0].header.flags
14221 );
14222 }
14223
14224 #[tokio::test(flavor = "current_thread")]
14228 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
14229 let registry = Arc::new(Registry::default());
14230 let forwarding = Arc::new(ForwardingTable::default());
14231 let handler =
14232 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14233 let module_connection = ConnectionId::new(95);
14234 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14235 hello_via_sink(
14236 &handler,
14237 &module_ctx,
14238 &mut module_rx,
14239 hello_frame("aft", PROTOCOL_VERSION, 395),
14240 )
14241 .await;
14242 forwarding
14243 .begin_module_drain("aft", RouteCloseReason::Restart)
14244 .unwrap()
14245 .expect("the module endpoint drains");
14246
14247 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
14248 let response = handler
14249 .handle_control_frame(
14250 &client_ctx,
14251 route_open_frame(396, "aft", unique_project_root("drain-origin")),
14252 )
14253 .await
14254 .unwrap();
14255
14256 assert_eq!(response.len(), 1);
14257 assert_eq!(response[0].header.ty, FrameType::Error);
14258 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14259 assert!(
14260 response[0].header.flags.is_daemon_origin(),
14261 "drain module_reloading must carry daemon origin: {:?}",
14262 response[0].header.flags
14263 );
14264 }
14265
14266 #[tokio::test(flavor = "current_thread")]
14270 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
14271 let registry = Arc::new(Registry::default());
14272 let forwarding = Arc::new(ForwardingTable::default());
14273 let handler =
14274 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14275 let module_connection = ConnectionId::new(95);
14276 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14277 hello_via_sink(
14278 &handler,
14279 &module_ctx,
14280 &mut module_rx,
14281 hello_frame("aft", PROTOCOL_VERSION, 395),
14282 )
14283 .await;
14284
14285 let client_connection = ConnectionId::new(96);
14286 let (client_ctx, _client_rx) = route_ctx(client_connection);
14287 let (route_task, bind) = relay_route_open(
14288 &handler,
14289 client_connection,
14290 &client_ctx.egress,
14291 &mut module_rx,
14292 396,
14293 "aft",
14294 "spoofed-reloading",
14295 )
14296 .await;
14297 let rejection = Frame::build(
14300 FrameType::Error,
14301 crate::router::daemon_error_flags(),
14302 0,
14303 0,
14304 bind.header.corr,
14305 serde_json::to_vec(&ErrorBody::new(
14306 "module_reloading",
14307 "module refused route.bind",
14308 ))
14309 .unwrap(),
14310 )
14311 .unwrap();
14312 handler
14313 .handle_control_frame(&module_ctx, rejection)
14314 .await
14315 .unwrap();
14316
14317 let response = route_task.await.unwrap();
14318 assert_eq!(response.len(), 1);
14319 assert_eq!(response[0].header.ty, FrameType::Error);
14320 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14321 assert!(
14322 !response[0].header.flags.is_daemon_origin(),
14323 "a module's own rejection must not carry daemon origin: {:?}",
14324 response[0].header.flags
14325 );
14326 }
14327
14328 #[tokio::test(flavor = "current_thread")]
14329 async fn route_open_module_rejection_uses_daemon_counter_key() {
14330 let registry = Arc::new(Registry::default());
14331 let forwarding = Arc::new(ForwardingTable::default());
14332 let handler =
14333 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14334 let module_connection = ConnectionId::new(95);
14335 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14336 hello_via_sink(
14337 &handler,
14338 &module_ctx,
14339 &mut module_rx,
14340 hello_frame("aft", PROTOCOL_VERSION, 395),
14341 )
14342 .await;
14343
14344 let client_connection = ConnectionId::new(96);
14345 let (client_ctx, _client_rx) = route_ctx(client_connection);
14346 let capture = EventCapture::default();
14347 let _subscriber =
14348 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14349 let (route_task, bind) = relay_route_open(
14350 &handler,
14351 client_connection,
14352 &client_ctx.egress,
14353 &mut module_rx,
14354 396,
14355 "aft",
14356 "hostile-module-code",
14357 )
14358 .await;
14359 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
14360 let rejection = Frame::build(
14361 FrameType::Error,
14362 control_flags(),
14363 0,
14364 0,
14365 bind.header.corr,
14366 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
14367 )
14368 .unwrap();
14369 handler
14370 .handle_control_frame(&module_ctx, rejection)
14371 .await
14372 .unwrap();
14373
14374 let response = route_task.await.unwrap();
14375 assert_eq!(parse_error(&response[0])["code"], hostile_code);
14376 let counters = handler.counters().snapshot();
14377 assert_eq!(
14378 counters["route_open_refused_by_code"],
14379 json!({ "module_rejected": 1 })
14380 );
14381 assert!(counters["route_open_refused_by_code"]
14382 .get(hostile_code)
14383 .is_none());
14384
14385 let event = capture
14386 .events()
14387 .into_iter()
14388 .find(|event| {
14389 event.target == "control"
14390 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
14391 })
14392 .expect("route.open module-rejection refusal event");
14393 let logged = event.fields.get("module_code").expect("module_code field");
14394 assert!(!logged.bytes().any(|byte| byte < 0x20));
14395 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14396 }
14397
14398 #[tokio::test]
14399 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
14400 let registry = Arc::new(Registry::default());
14401 let supervisor_handle = SupervisorHandle::new();
14402 let missing_program = std::env::temp_dir().join(format!(
14403 "subc-route-open-missing-program-{}",
14404 std::process::id()
14405 ));
14406 let supervisor =
14407 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14408 .with_handle(supervisor_handle.clone());
14409 let module = supervisor
14410 .supervise_configured(
14411 ModuleSpec {
14412 module_id: "failed".to_string(),
14413 program: missing_program,
14414 args: Vec::new(),
14415 env: Vec::new(),
14416 reserved: false,
14417 reserved_prefixes: Vec::new(),
14418 protocol: ModuleProtocol::Subc,
14419 overlap: Default::default(),
14420 },
14421 true,
14422 )
14423 .unwrap();
14424 assert_eq!(module.state().unwrap(), ModuleState::Failed);
14425
14426 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14427 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
14428 let response = handler
14429 .handle_control_frame(
14430 &ctx,
14431 route_open_frame(305, "failed", unique_project_root("failed")),
14432 )
14433 .await
14434 .unwrap();
14435
14436 assert_eq!(response[0].header.ty, FrameType::Error);
14437 let error = parse_error(&response[0]);
14438 assert_eq!(error["code"], "target_unavailable");
14439 assert!(error["message"]
14440 .as_str()
14441 .unwrap()
14442 .contains("state=failed, enabled=true, live=false"));
14443 }
14444
14445 #[tokio::test]
14446 async fn route_open_role_mismatch_remains_target_unavailable() {
14447 let registry = Arc::new(Registry::default());
14448 let handler = ControlHandler::new(Arc::clone(®istry));
14449 handler
14450 .handle_control(
14451 ConnectionId::new(41),
14452 non_routable_hello_frame_with_control_ops("health-only", 306, None),
14453 )
14454 .unwrap();
14455
14456 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
14457 let response = handler
14458 .handle_control_frame(
14459 &ctx,
14460 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
14461 )
14462 .await
14463 .unwrap();
14464
14465 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14466 assert!(parse_error(&response[0])["message"]
14467 .as_str()
14468 .unwrap()
14469 .contains("does not provide the requested target"));
14470 }
14471
14472 #[tokio::test]
14473 async fn route_open_inactive_registration_remains_target_unavailable() {
14474 let registry = Arc::new(Registry::default());
14475 let handler = ControlHandler::new(Arc::clone(®istry));
14476 handler
14477 .handle_control(
14478 ConnectionId::new(43),
14479 hello_frame("inactive", PROTOCOL_VERSION, 308),
14480 )
14481 .unwrap();
14482 assert!(registry
14483 .set_module_state_for_test("inactive", ChannelState::Closed)
14484 .unwrap());
14485
14486 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
14487 let response = handler
14488 .handle_control_frame(
14489 &ctx,
14490 route_open_frame(309, "inactive", unique_project_root("inactive")),
14491 )
14492 .await
14493 .unwrap();
14494
14495 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14496 assert!(parse_error(&response[0])["message"]
14497 .as_str()
14498 .unwrap()
14499 .contains("is not active"));
14500 }
14501
14502 #[tokio::test]
14503 async fn late_health_reply_is_recorded_through_the_module_response_path() {
14504 let registry = Arc::new(Registry::default());
14505 let forwarding = Arc::new(ForwardingTable::default());
14506 let supervisor_handle = SupervisorHandle::new();
14507 let supervisor =
14508 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
14509 .with_forwarding(Arc::clone(&forwarding))
14510 .with_handle(supervisor_handle.clone());
14511 let module = supervisor
14512 .supervise_configured(
14513 crate::ModuleSpec {
14514 module_id: "late-health-response".to_string(),
14515 program: PathBuf::from("disabled-module"),
14516 args: Vec::new(),
14517 env: Vec::new(),
14518 reserved: false,
14519 reserved_prefixes: Vec::new(),
14520 protocol: ModuleProtocol::Subc,
14521 overlap: Default::default(),
14522 },
14523 false,
14524 )
14525 .unwrap();
14526 let handler =
14527 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14528 .with_supervisor(supervisor_handle);
14529 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
14530 handler
14531 .handle_control_frame(
14532 &module_ctx,
14533 hello_frame_with_control_ops(
14534 "late-health-response",
14535 PROTOCOL_VERSION,
14536 7,
14537 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14538 ),
14539 )
14540 .await
14541 .unwrap();
14542 let probe_started_at = Instant::now() - Duration::from_millis(80);
14543 let pending = forwarding
14544 .begin_health_probe_rpc_for(
14545 "late-health-response",
14546 MODULE_CONTROL_OP_HEALTH_CHECK,
14547 probe_started_at,
14548 Instant::now() - Duration::from_millis(1),
14549 )
14550 .unwrap();
14551 assert!(forwarding
14552 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
14553 .unwrap());
14554
14555 let responses = handler
14556 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
14557 .await
14558 .unwrap();
14559
14560 assert!(responses.is_empty());
14561 let health = module.status().unwrap().health;
14562 assert_eq!(health.late_answer_count, 1);
14563 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
14564 }
14565
14566 #[tokio::test]
14567 async fn health_probe_timeout_and_module_death_are_typed() {
14568 let registry = Arc::new(Registry::default());
14569 let forwarding = Arc::new(ForwardingTable::default());
14570 let handler =
14571 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14572 .with_health_probe_timeout(Duration::from_millis(50));
14573 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14574 hello_via_sink(
14575 &handler,
14576 &module_ctx,
14577 &mut module_rx,
14578 hello_frame_with_control_ops(
14579 "aft",
14580 PROTOCOL_VERSION,
14581 7,
14582 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14583 ),
14584 )
14585 .await;
14586
14587 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14588 let responses = handler
14589 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
14590 .await
14591 .unwrap();
14592 assert_eq!(responses[0].header.ty, FrameType::Error);
14593 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
14594 let _ = module_rx.try_recv();
14595
14596 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
14597 let health_handler = handler.clone();
14598 let death_task = tokio::spawn(async move {
14599 health_handler
14600 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
14601 .await
14602 .unwrap()
14603 });
14604 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14605 .await
14606 .unwrap()
14607 .unwrap();
14608 handler
14609 .cleanup_connection(module_ctx.connection_id)
14610 .unwrap();
14611 let responses = death_task.await.unwrap();
14612 assert_eq!(responses[0].header.ty, FrameType::Error);
14613 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
14614 }
14615
14616 #[test]
14617 fn hello_requires_exact_protocol_version() {
14618 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
14619 let registry = Arc::new(Registry::default());
14620 let handler = ControlHandler::new(Arc::clone(®istry));
14621 let responses = handler
14622 .handle_control(
14623 ConnectionId::new(connection),
14624 hello_frame("aft", offered, 9),
14625 )
14626 .unwrap();
14627
14628 assert_eq!(responses.len(), 1);
14629 assert_eq!(responses[0].header.ty, FrameType::Error);
14630 let error = parse_error(&responses[0]);
14631 assert_eq!(error["code"], "version_unsupported");
14632 assert!(registry.get_module("aft").unwrap().is_none());
14633 assert_eq!(registry.active_registration_count().unwrap(), 0);
14634 }
14635 }
14636
14637 #[test]
14638 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
14639 let registry = Arc::new(Registry::default());
14640 let forwarding = Arc::new(ForwardingTable::default());
14641 let handler =
14642 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14643 let module_connection = ConnectionId::new(301);
14644 let registration = registry
14645 .register_with_control_ops(
14646 manifest("aft-push", PROTOCOL_VERSION),
14647 PROTOCOL_VERSION,
14648 module_connection,
14649 module_baseline_control_ops(),
14650 )
14651 .unwrap();
14652 let (module_tx, _module_rx) = mpsc::channel(8);
14653 let endpoint = forwarding
14654 .register_module_connection(
14655 module_connection,
14656 "aft-push".to_string(),
14657 PROTOCOL_VERSION,
14658 manifest_concurrency(®istration.manifest),
14659 FrameSink::new(module_tx),
14660 )
14661 .unwrap();
14662
14663 let unknown = Frame::build(
14665 FrameType::Push,
14666 control_flags(),
14667 0,
14668 0,
14669 5,
14670 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
14671 )
14672 .unwrap();
14673 let out = handler.handle_status_update(endpoint, unknown).unwrap();
14674 assert!(
14675 out.is_empty(),
14676 "unknown push op must be ignored, got {out:?}"
14677 );
14678
14679 let malformed = Frame::build(
14681 FrameType::Push,
14682 control_flags(),
14683 0,
14684 0,
14685 6,
14686 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
14687 )
14688 .unwrap();
14689 let out = handler.handle_status_update(endpoint, malformed).unwrap();
14690 assert_eq!(out.len(), 1);
14691 assert_eq!(out[0].header.ty, FrameType::Error);
14692 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
14693 }
14694
14695 #[test]
14696 fn hello_rejected_when_connection_already_owns_client_routes() {
14697 let registry = Arc::new(Registry::default());
14698 let forwarding = Arc::new(ForwardingTable::default());
14699 let handler =
14700 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14701 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
14703 let client_connection = ConnectionId::new(202);
14704
14705 let responses = handler
14708 .handle_control(
14709 client_connection,
14710 hello_frame("aft-second", PROTOCOL_VERSION, 9),
14711 )
14712 .unwrap();
14713 assert_eq!(responses[0].header.ty, FrameType::Error);
14714 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
14715 assert!(registry.get_module("aft-second").unwrap().is_none());
14716 }
14717
14718 #[tokio::test]
14719 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
14720 let registry = Arc::new(Registry::default());
14721 let forwarding = Arc::new(ForwardingTable::default());
14722 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
14723 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
14724 hello_via_sink(
14725 &handler,
14726 &module_ctx,
14727 &mut module_rx,
14728 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
14729 )
14730 .await;
14731 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
14732 let pending = forwarding
14733 .begin_route_bind_relay_for_test(
14734 client_ctx.connection_id,
14735 client_ctx.egress.clone(),
14736 2,
14737 "alpha",
14738 )
14739 .unwrap();
14740 forwarding
14741 .complete_pending_relay(
14742 module_ctx.connection_id,
14743 pending.corr,
14744 RouteBindRelayOutcome::Accepted,
14745 )
14746 .unwrap();
14747 client_rx.try_recv().unwrap();
14748 for module_id in ["beta", "alpha"] {
14749 let replies = handler
14750 .handle_control_frame(
14751 &module_ctx,
14752 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
14753 )
14754 .await
14755 .unwrap();
14756 assert_eq!(replies.len(), 1, "second HELLO must be refused");
14757 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
14758 }
14759 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
14760 assert!(registry.get_module("beta").unwrap().is_none());
14761 assert!(matches!(
14762 forwarding
14763 .lookup_data_route(
14764 client_ctx.connection_id,
14765 pending.client_channel,
14766 pending.client_epoch,
14767 )
14768 .unwrap(),
14769 DataRoute::Client(DataRouteState::Bound(_))
14770 ));
14771 assert!(handler
14772 .hello_launch_nonces
14773 .lock()
14774 .unwrap()
14775 .presented(module_ctx.connection_id, Some("alpha-nonce")));
14776 assert!(module_rx.try_recv().is_err());
14777 }
14778
14779 #[test]
14780 fn reserved_module_hello_requires_matching_launch_nonce() {
14781 let registry = Arc::new(Registry::default());
14782 let supervisor = SupervisorHandle::new();
14783 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
14786 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14787
14788 let no_nonce = handler
14790 .handle_control(
14791 ConnectionId::new(1),
14792 hello_frame("vault", PROTOCOL_VERSION, 1),
14793 )
14794 .unwrap();
14795 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
14796 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
14797 assert!(registry.get_module("vault").unwrap().is_none());
14798
14799 let wrong = handler
14801 .handle_control(
14802 ConnectionId::new(2),
14803 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
14804 )
14805 .unwrap();
14806 assert_eq!(wrong[0].header.ty, FrameType::Error);
14807 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14808 assert!(registry.get_module("vault").unwrap().is_none());
14809
14810 let ok = handler
14812 .handle_control(
14813 ConnectionId::new(3),
14814 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14815 )
14816 .unwrap();
14817 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14818 assert!(registry.get_module("vault").unwrap().is_some());
14819 }
14820
14821 #[test]
14822 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14823 let registry = Arc::new(Registry::default());
14824 let supervisor = SupervisorHandle::new();
14825 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14826 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14827 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14828
14829 let squat = handler
14830 .handle_control(
14831 ConnectionId::new(1),
14832 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14833 )
14834 .unwrap();
14835 assert_eq!(squat[0].header.ty, FrameType::Error);
14836 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14837 assert!(parse_error(&squat[0])["message"]
14838 .as_str()
14839 .unwrap()
14840 .contains("fed:"));
14841
14842 let accepted_peer = handler
14843 .handle_control(
14844 ConnectionId::new(2),
14845 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14846 )
14847 .unwrap();
14848 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14849
14850 let accepted_short = handler
14851 .handle_control(
14852 ConnectionId::new(3),
14853 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14854 )
14855 .unwrap();
14856 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14857
14858 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14859 let response = handler
14860 .handle_control(
14861 ConnectionId::new(conn),
14862 hello_frame(module_id, PROTOCOL_VERSION, conn),
14863 )
14864 .unwrap();
14865 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14866 }
14867 }
14868
14869 #[test]
14870 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14871 let registry = Arc::new(Registry::default());
14872 let supervisor = SupervisorHandle::new();
14873 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14874 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14875 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14876 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14877
14878 let owner_nonce = handler
14879 .handle_control(
14880 ConnectionId::new(1),
14881 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14882 )
14883 .unwrap();
14884 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14885 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14886 assert!(registry.get_module("fed:special").unwrap().is_none());
14887
14888 let exact_nonce = handler
14889 .handle_control(
14890 ConnectionId::new(2),
14891 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14892 )
14893 .unwrap();
14894 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14895 assert!(registry.get_module("fed:special").unwrap().is_some());
14896 }
14897
14898 #[test]
14899 fn non_reserved_module_ignores_launch_nonce() {
14900 let registry = Arc::new(Registry::default());
14901 let handler = ControlHandler::new(Arc::clone(®istry));
14904 let no_nonce = handler
14905 .handle_control(
14906 ConnectionId::new(1),
14907 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14908 )
14909 .unwrap();
14910 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14911 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14912
14913 let echoed_nonce = handler
14914 .handle_control(
14915 ConnectionId::new(2),
14916 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14917 )
14918 .unwrap();
14919 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14920 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14921 }
14922
14923 #[test]
14924 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14925 let handler = ControlHandler::default();
14926 let conn = ConnectionId::new(1);
14927 let malformed = Frame::build(
14928 FrameType::Hello,
14929 control_flags(),
14930 0,
14931 0,
14932 3,
14933 b"{not json".to_vec(),
14934 )
14935 .unwrap();
14936
14937 let error = handler.handle_control(conn, malformed).unwrap();
14938 assert_eq!(error[0].header.ty, FrameType::Error);
14939 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14940
14941 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14942 let pong = handler.handle_control(conn, ping).unwrap();
14943 assert_eq!(pong[0].header.ty, FrameType::Pong);
14944 assert_eq!(pong[0].header.corr, 4);
14945 }
14946
14947 #[test]
14948 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14949 let registry = Arc::new(Registry::default());
14950 let handler = ControlHandler::new(Arc::clone(®istry));
14951
14952 handler
14953 .handle_control(
14954 ConnectionId::new(1),
14955 hello_frame("aft", PROTOCOL_VERSION, 1),
14956 )
14957 .unwrap();
14958 let duplicate = handler
14959 .handle_control(
14960 ConnectionId::new(2),
14961 hello_frame("aft", PROTOCOL_VERSION, 2),
14962 )
14963 .unwrap();
14964
14965 assert_eq!(duplicate[0].header.ty, FrameType::Error);
14966 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14967 let registration = registry.get_module("aft").unwrap().unwrap();
14968 assert_eq!(registration.connection_id, ConnectionId::new(1));
14969 }
14970
14971 #[test]
14972 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14973 let registry = Arc::new(Registry::default());
14974 let forwarding = Arc::new(ForwardingTable::default());
14975 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14976 let handler =
14977 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14978 .with_process_liveness(process_liveness);
14979 let (ctx, route_channel, route_epoch) =
14980 bind_liveness_route(®istry, &forwarding, "aft-dead");
14981 let responses = handler
14982 .handle_route_poll(
14983 &ctx,
14984 route_poll_frame(41, PollKind::Liveness, route_channel),
14985 route_channel,
14986 route_epoch,
14987 PollKind::Liveness,
14988 )
14989 .unwrap();
14990
14991 assert_eq!(responses.len(), 1);
14992 assert_eq!(responses[0].header.ty, FrameType::Response);
14993 assert_route_poll_liveness(&responses[0], false);
14994 }
14995
14996 #[test]
14997 fn liveness_poll_without_process_source_uses_bound_route() {
14998 let registry = Arc::new(Registry::default());
14999 let forwarding = Arc::new(ForwardingTable::default());
15000 let handler =
15001 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15002 let (ctx, route_channel, route_epoch) =
15003 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
15004 let responses = handler
15005 .handle_route_poll(
15006 &ctx,
15007 route_poll_frame(42, PollKind::Liveness, route_channel),
15008 route_channel,
15009 route_epoch,
15010 PollKind::Liveness,
15011 )
15012 .unwrap();
15013
15014 assert_route_poll_liveness(&responses[0], true);
15015 }
15016
15017 #[test]
15018 fn liveness_poll_untracked_process_source_uses_bound_route() {
15019 let registry = Arc::new(Registry::default());
15020 let forwarding = Arc::new(ForwardingTable::default());
15021 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
15022 let handler =
15023 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15024 .with_process_liveness(process_liveness);
15025 let (ctx, route_channel, route_epoch) =
15026 bind_liveness_route(®istry, &forwarding, "aft-untracked");
15027 let responses = handler
15028 .handle_route_poll(
15029 &ctx,
15030 route_poll_frame(43, PollKind::Liveness, route_channel),
15031 route_channel,
15032 route_epoch,
15033 PollKind::Liveness,
15034 )
15035 .unwrap();
15036
15037 assert_route_poll_liveness(&responses[0], true);
15038 }
15039
15040 #[tokio::test]
15041 async fn unknown_op_returns_unknown_control_op() {
15042 let handler = ControlHandler::default();
15043 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
15044 let request = Frame::build(
15045 FrameType::Request,
15046 control_flags(),
15047 0,
15048 0,
15049 55,
15050 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
15051 )
15052 .unwrap();
15053
15054 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15055
15056 assert_eq!(response.len(), 1);
15057 assert_eq!(response[0].header.ty, FrameType::Error);
15058 assert_eq!(response[0].header.corr, 55);
15059 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
15060 }
15061
15062 #[tokio::test]
15063 async fn supervisor_provenance_rejects_unknown_exact_module() {
15064 let handler = ControlHandler::default();
15065 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
15066 let request = Frame::build(
15067 FrameType::Request,
15068 control_flags(),
15069 0,
15070 0,
15071 57,
15072 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
15073 )
15074 .unwrap();
15075
15076 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15077
15078 assert_eq!(response.len(), 1);
15079 assert_eq!(response[0].header.ty, FrameType::Error);
15080 assert_eq!(response[0].header.corr, 57);
15081 let error = parse_error(&response[0]);
15082 assert_eq!(error["code"], "unknown_module");
15083 assert_eq!(error["message"], "module_id 'missing' is not supervised");
15084 }
15085
15086 #[test]
15087 fn provenance_probe_override_keeps_handler_tests_deterministic() {
15088 let expected = subc_control::RunningImageAgreement::Unavailable {
15089 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15090 };
15091 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
15092 assert_eq!(handler.provenance_probe_override, Some(expected));
15093 }
15094
15095 #[test]
15096 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
15097 let verdict = reload_verdict(
15098 std::path::Path::new("/bin/new"),
15099 Some(std::path::Path::new("/bin/old")),
15100 subc_control::RunningImageAgreement::Unavailable {
15101 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15102 },
15103 );
15104 assert!(matches!(
15105 verdict.path,
15106 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
15107 if configured == std::path::Path::new("/bin/new")
15108 && spawned_from == std::path::Path::new("/bin/old")
15109 ));
15110 }
15111
15112 #[test]
15113 fn launch_spec_without_running_digest_is_unavailable_not_match() {
15114 use subc_control::{
15115 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
15116 };
15117 let configured = crate::supervise::ModuleSpec {
15118 module_id: "legacy".into(),
15119 program: "/module".into(),
15120 args: Vec::new(),
15121 env: Vec::new(),
15122 reserved: false,
15123 reserved_prefixes: Vec::new(),
15124 protocol: subc_control::ModuleProtocol::Subc,
15125 overlap: Default::default(),
15126 };
15127 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
15128 assert_eq!(
15129 launch_spec_agreement(&configured, false, Some(&digest)),
15130 Agreement::Unavailable {
15131 reason: Reason::NotRunning
15132 }
15133 );
15134 assert_eq!(
15135 launch_spec_agreement(&configured, true, None),
15136 Agreement::Unavailable {
15137 reason: Reason::SpawnedSpecUnavailable
15138 }
15139 );
15140 assert_eq!(
15141 launch_spec_agreement(&configured, true, Some(&digest)),
15142 Agreement::Match
15143 );
15144 }
15145
15146 #[cfg(windows)]
15147 #[test]
15148 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
15149 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
15150 let configured = dir.join("module.exe");
15151 std::fs::write(&configured, b"executable path fixture").unwrap();
15152 let spawned = std::fs::canonicalize(&configured).unwrap();
15153 assert_ne!(
15154 configured, spawned,
15155 "fixture must exercise different path spellings"
15156 );
15157 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
15158 let lower_case = configured.to_string_lossy().to_lowercase();
15159 for spelling in [
15160 &configured,
15161 Path::new(&forward_slashes),
15162 Path::new(&lower_case),
15163 ] {
15164 let verdict = reload_verdict(
15165 spelling,
15166 Some(&spawned),
15167 subc_control::RunningImageAgreement::Unavailable {
15168 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15169 },
15170 );
15171 assert_eq!(
15172 verdict.path,
15173 ReloadPathAgreement::Match,
15174 "configured {spelling:?}, spawned {spawned:?}"
15175 );
15176 }
15177 }
15178
15179 #[cfg(windows)]
15180 #[test]
15181 fn windows_reload_path_keeps_different_existing_program_pending() {
15182 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
15183 let old = dir.join("old.exe");
15184 let new = dir.join("new.exe");
15185 std::fs::write(&old, b"same executable content").unwrap();
15186 std::fs::copy(&old, &new).unwrap();
15187 let spawned = std::fs::canonicalize(&old).unwrap();
15188 let image = subc_control::RunningImageAgreement::Unavailable {
15189 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15190 };
15191 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
15192 assert!(
15193 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
15194 if configured == new && spawned_from == spawned)
15195 );
15196 let missing = dir.join("missing.exe");
15197 assert!(matches!(
15198 reload_verdict(&missing, Some(&spawned), image).path,
15199 ReloadPathAgreement::Mismatch { .. }
15200 ));
15201 }
15202
15203 #[test]
15204 fn reload_verdict_detects_replaced_image_at_same_path() {
15205 let image = subc_control::RunningImageAgreement::Mismatch {
15206 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
15207 digest: "old".into(),
15208 },
15209 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
15210 digest: "new".into(),
15211 },
15212 };
15213 let verdict = reload_verdict(
15214 std::path::Path::new("/bin/same"),
15215 Some(std::path::Path::new("/bin/same")),
15216 image.clone(),
15217 );
15218 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15219 assert_eq!(verdict.image, image);
15220 }
15221
15222 #[test]
15223 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
15224 let image = subc_control::RunningImageAgreement::Unavailable {
15225 reason: subc_control::RunningImageUnavailableReason::NotRunning,
15226 };
15227 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
15228 assert_eq!(
15229 verdict.path,
15230 subc_control::ReloadPathAgreement::Unavailable {
15231 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
15232 }
15233 );
15234 assert_eq!(verdict.image, image);
15235
15236 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
15237 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15238 };
15239 let verdict = reload_verdict(
15240 std::path::Path::new("/bin/same"),
15241 Some(std::path::Path::new("/bin/same")),
15242 unconfirmed.clone(),
15243 );
15244 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15245 assert_eq!(verdict.image, unconfirmed);
15246 }
15247
15248 #[test]
15249 fn reload_verdict_preserves_each_image_unavailability_reason() {
15250 use subc_control::RunningImageUnavailableReason as Reason;
15251
15252 for reason in [
15253 Reason::NotRunning,
15254 Reason::UnsupportedPlatform,
15255 Reason::RunningExecutableUnreadable,
15256 Reason::SpawnedPathUnreadable,
15257 Reason::HashFailed,
15258 Reason::ProcessIdentityUnconfirmed,
15259 Reason::Unknown("future_probe_reason".to_string()),
15260 ] {
15261 let image = subc_control::RunningImageAgreement::Unavailable {
15262 reason: reason.clone(),
15263 };
15264 let verdict = reload_verdict(
15265 std::path::Path::new("/bin/same"),
15266 Some(std::path::Path::new("/bin/same")),
15267 image.clone(),
15268 );
15269 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15270 assert_eq!(verdict.image, image, "{reason:?}");
15271 }
15272 }
15273
15274 #[tokio::test]
15275 async fn malformed_control_bodies_return_invalid_control_body() {
15276 let handler = ControlHandler::default();
15277 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
15278
15279 for (corr, body) in [
15280 (56, br#"{"route_channel":1}"#.as_slice()),
15281 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
15282 (
15283 58,
15284 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
15285 ),
15286 ] {
15287 let request = Frame::build(
15288 FrameType::Request,
15289 control_flags(),
15290 0,
15291 0,
15292 corr,
15293 body.to_vec(),
15294 )
15295 .unwrap();
15296 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15297
15298 assert_eq!(response.len(), 1);
15299 assert_eq!(response[0].header.ty, FrameType::Error);
15300 assert_eq!(response[0].header.corr, corr);
15301 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
15302 }
15303 }
15304
15305 #[tokio::test]
15306 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
15307 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15308 let registry = Arc::new(Registry::default());
15309 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15310 let router = Router::with_control_handler(Arc::clone(&control));
15311 let connection = router.begin_connection();
15312 let (ctx, mut rx) = route_ctx(connection.id());
15313
15314 router
15315 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
15316 .await
15317 .unwrap();
15318 let response = rx.recv().await.unwrap();
15319 let ack = parse_ack(&response);
15320 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15321 let channel = 1;
15322
15323 let goodbye =
15324 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
15325 router.route_for_connection(&ctx, goodbye).await.unwrap();
15326 assert!(rx.try_recv().is_err());
15327 assert!(registry.get_module("aft").unwrap().is_none());
15328
15329 router
15330 .route_for_connection(&ctx, channel_request(channel, 13))
15331 .await
15332 .unwrap();
15333 let error_frame = rx.recv().await.unwrap();
15334 assert_eq!(error_frame.header.ty, FrameType::Error);
15335 assert_eq!(error_frame.header.channel, channel);
15336 let captured = crate::router::test_log::captured_logs(&logs);
15337 assert_eq!(
15338 captured
15339 .lines()
15340 .filter(|line| {
15341 line.contains("module_id=aft")
15342 && line.contains("reason=explicit_goodbye")
15343 && line.contains("module registration ended")
15344 })
15345 .count(),
15346 1,
15347 "unexpected GOODBYE registry log: {captured}"
15348 );
15349 }
15350
15351 #[tokio::test]
15352 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
15353 let registry = Arc::new(Registry::default());
15354 let handler = ControlHandler::new(registry).with_capability_config(
15355 [("prov".to_string(), true), ("cons".to_string(), true)],
15356 BTreeMap::new(),
15357 );
15358 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
15359 register_capability_manifest(
15360 &handler,
15361 &provider_ctx,
15362 &mut provider_rx,
15363 capability_manifest("prov", &["thing/v1"], &[]),
15364 1,
15365 )
15366 .await;
15367 let mut consumer = capability_manifest("cons", &[], &[]);
15368 consumer.capabilities.as_mut().unwrap().requires.push(
15369 subc_protocol::manifest::CapabilityRequirement {
15370 capability: "thing/v1".to_string(),
15371 need: subc_protocol::manifest::CapabilityNeed::Required,
15372 },
15373 );
15374 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
15375 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
15376 assert_eq!(
15377 handler.capability_evaluator.verdict("cons", "thing/v1"),
15378 Some(CapabilityVerdict::Provided)
15379 );
15380 let (mut client_rx, _) = open_route_for_capability_test(
15381 &handler,
15382 &provider_ctx,
15383 &mut provider_rx,
15384 703,
15385 3,
15386 "prov",
15387 None,
15388 )
15389 .await;
15390 let goodbye =
15391 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15392 handler
15393 .handle_control_frame(&provider_ctx, goodbye)
15394 .await
15395 .unwrap();
15396 assert_eq!(
15397 handler.capability_evaluator.verdict("cons", "thing/v1"),
15398 Some(CapabilityVerdict::NeverProvided)
15399 );
15400 let closed = client_rx
15401 .try_recv()
15402 .expect("GOODBYE pushes route.closed before route GOODBYE");
15403 assert!(
15404 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
15405 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
15406 );
15407 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
15408 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
15409 }
15410
15411 struct RouteClosedFixture {
15412 handler: ControlHandler,
15413 #[cfg(unix)]
15414 supervisor: Supervisor,
15415 module_ctx: RouteCtx,
15416 #[cfg(unix)]
15417 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15418 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15419 route: Arc<crate::forwarding::RouteBinding>,
15420 }
15421
15422 async fn route_closed_fixture() -> RouteClosedFixture {
15423 let registry = Arc::new(Registry::default());
15424 let handler = ControlHandler::new(Arc::clone(®istry));
15425 #[cfg(unix)]
15426 let supervisor = Supervisor::new(registry, RestartPolicy::default())
15427 .with_forwarding(Arc::clone(&handler.forwarding));
15428 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
15429 register_capability_manifest(
15430 &handler,
15431 &module_ctx,
15432 &mut module_rx,
15433 capability_manifest("prov", &[], &[]),
15434 1,
15435 )
15436 .await;
15437 let (client_rx, bind) = open_route_for_capability_test(
15438 &handler,
15439 &module_ctx,
15440 &mut module_rx,
15441 702,
15442 2,
15443 "prov",
15444 None,
15445 )
15446 .await;
15447 let ModuleControlRequest::RouteBind {
15448 route_channel: channel,
15449 epoch: route_epoch,
15450 ..
15451 } = bind
15452 else {
15453 panic!("expected route.bind");
15454 };
15455 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
15456 handler
15457 .forwarding
15458 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
15459 .unwrap()
15460 else {
15461 panic!("the acknowledged route must be bound");
15462 };
15463 RouteClosedFixture {
15464 handler,
15465 #[cfg(unix)]
15466 supervisor,
15467 module_ctx,
15468 #[cfg(unix)]
15469 module_rx,
15470 client_rx,
15471 route,
15472 }
15473 }
15474
15475 impl RouteClosedFixture {
15476 #[cfg(unix)]
15477 async fn cut(&mut self) {
15478 let notices = async {
15479 let mut draining = self.module_rx.recv().await.unwrap();
15480 assert!(matches!(
15481 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
15482 &draining.body
15483 )
15484 .unwrap(),
15485 subc_protocol::session::ModuleControlCommand::Draining {
15486 reason: RouteCloseReason::Restart,
15487 ..
15488 }
15489 ));
15490 draining.flushed.take().unwrap().send(()).unwrap();
15491 let mut closing = self.client_rx.recv().await.unwrap();
15492 assert!(matches!(
15493 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
15494 ClientControlPush::RouteClosing {
15495 reason: RouteCloseReason::Restart,
15496 ..
15497 }
15498 ));
15499 closing.flushed.take().unwrap().send(()).unwrap();
15500 };
15501 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
15502 result.unwrap();
15503 }
15504
15505 fn disconnect(&mut self) -> ClientControlPush {
15506 self.handler
15507 .cleanup_connection(self.module_ctx.connection_id)
15508 .unwrap();
15509 let closed = self.client_rx.try_recv().unwrap();
15510 assert_eq!(closed.header.ty, FrameType::Push);
15511 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
15512 assert!(matches!(
15513 &push,
15514 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
15515 if channels == &[self.route.client_channel]
15516 ));
15517 assert_eq!(
15518 self.client_rx.try_recv().unwrap().header.ty,
15519 FrameType::Goodbye
15520 );
15521 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
15522 push
15523 }
15524 }
15525
15526 #[cfg(unix)]
15527 #[tokio::test(start_paused = true)]
15528 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
15529 let mut fixture = route_closed_fixture().await;
15530 fixture.cut().await;
15531 assert!(matches!(
15532 fixture.disconnect(),
15533 ClientControlPush::RouteClosed {
15534 reason: RouteCloseReason::Restart,
15535 drained: true,
15536 ..
15537 }
15538 ));
15539 }
15540
15541 #[cfg(unix)]
15542 #[tokio::test(start_paused = true)]
15543 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
15544 let mut fixture = route_closed_fixture().await;
15545 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
15546 let started = tokio::time::Instant::now();
15547 fixture.cut().await;
15548 assert!(started.elapsed() >= Duration::from_secs(2));
15549 assert_eq!(fixture.route.flow.in_flight(), 1);
15550 assert!(matches!(
15551 fixture.disconnect(),
15552 ClientControlPush::RouteClosed {
15553 reason: RouteCloseReason::Restart,
15554 drained: false,
15555 ..
15556 }
15557 ));
15558 }
15559
15560 #[tokio::test]
15561 async fn module_connection_crash_reports_crash_and_not_drained() {
15562 let mut fixture = route_closed_fixture().await;
15563 assert!(matches!(
15567 fixture.disconnect(),
15568 ClientControlPush::RouteClosed {
15569 reason: RouteCloseReason::Crash,
15570 drained: false,
15571 ..
15572 }
15573 ));
15574 }
15575
15576 #[tokio::test]
15577 async fn dropping_router_connection_releases_registration() {
15578 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15579 let registry = Arc::new(Registry::default());
15580 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15581 let router = Router::with_control_handler(Arc::clone(&control));
15582 let connection = router.begin_connection();
15583 let connection_id = connection.id();
15584 let (ctx, mut rx) = route_ctx(connection_id);
15585
15586 router
15587 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
15588 .await
15589 .unwrap();
15590 let response = rx.recv().await.unwrap();
15591 let ack = parse_ack(&response);
15592 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15593 assert!(registry.get_module("aft").unwrap().is_some());
15594
15595 drop(connection);
15596
15597 assert!(registry.get_module("aft").unwrap().is_none());
15598 assert_eq!(registry.active_registration_count().unwrap(), 0);
15599
15600 control
15601 .cleanup_connection(ConnectionId::new(u64::MAX))
15602 .unwrap();
15603 let captured = crate::router::test_log::captured_logs(&logs);
15604 println!("captured registry lifecycle logs:\n{captured}");
15605 let events: Vec<_> = captured
15606 .lines()
15607 .filter(|line| {
15608 line.contains("module registered module_id=aft ")
15609 || line.contains("module registration ended")
15610 })
15611 .collect();
15612 assert_eq!(
15613 events.len(),
15614 2,
15615 "unexpected registry lifecycle logs: {captured}"
15616 );
15617 assert!(events[0].contains("module registered module_id=aft "));
15618 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
15619 assert!(events[1].contains(&format!(
15620 "module_id=aft connection_id={}",
15621 connection_id.get()
15622 )));
15623 assert!(events[1].contains("reason=connection_closed"));
15624 assert!(events[1].contains("module registration ended"));
15625 assert_eq!(
15626 captured
15627 .lines()
15628 .filter(|line| line.contains("module registered module_id=aft "))
15629 .count(),
15630 1,
15631 "legacy registration admission line must appear once: {captured}"
15632 );
15633 }
15634
15635 #[tokio::test]
15636 async fn hello_registration_keeps_legacy_module_registered_line_once() {
15637 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15638 let registry = Arc::new(Registry::default());
15639 let control = ControlHandler::new(Arc::clone(®istry));
15640 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
15641 hello_via_sink(
15642 &control,
15643 &ctx,
15644 &mut rx,
15645 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
15646 )
15647 .await;
15648
15649 let captured = crate::router::test_log::captured_logs(&logs);
15650 let admissions: Vec<_> = captured
15651 .lines()
15652 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
15653 .collect();
15654 assert_eq!(
15655 admissions.len(),
15656 1,
15657 "legacy admission line must remain exactly once: {captured}"
15658 );
15659 assert!(admissions[0].contains("routable_provider=true"));
15660 assert!(admissions[0].contains("connection_id=777"));
15661 }
15662
15663 fn capability_manifest(
15664 module_id: &str,
15665 provides: &[&str],
15666 must_never_reach: &[&str],
15667 ) -> ModuleManifest {
15668 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
15669 manifest.capabilities = Some(CapabilityDeclarations {
15670 provides: provides
15671 .iter()
15672 .map(|capability| (*capability).to_string())
15673 .collect(),
15674 requires: Vec::new(),
15675 must_never_reach: must_never_reach
15676 .iter()
15677 .map(|capability| (*capability).to_string())
15678 .collect(),
15679 });
15680 manifest
15681 }
15682
15683 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
15684 Frame::build(
15685 FrameType::Hello,
15686 control_flags(),
15687 0,
15688 0,
15689 corr,
15690 serde_json::to_vec(&ModuleHelloBody {
15691 protocol_ver: manifest.protocol_ver,
15692 manifest,
15693 control_ops: None,
15694 launch_nonce: None,
15695 })
15696 .expect("capability test HELLO serializes"),
15697 )
15698 .expect("capability test HELLO frame builds")
15699 }
15700
15701 fn catalog_update_with_capabilities_frame(
15702 corr: u64,
15703 capabilities: CapabilityDeclarations,
15704 ) -> Frame {
15705 Frame::build(
15706 FrameType::Request,
15707 control_flags(),
15708 0,
15709 0,
15710 corr,
15711 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15712 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
15713 capabilities: Some(capabilities),
15714 ready: None,
15715 })
15716 .expect("capability catalog.update serializes"),
15717 )
15718 .expect("capability catalog.update frame builds")
15719 }
15720
15721 async fn register_capability_manifest(
15722 handler: &ControlHandler,
15723 ctx: &RouteCtx,
15724 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15725 manifest: ModuleManifest,
15726 corr: u64,
15727 ) {
15728 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
15729 }
15730
15731 async fn open_route_for_capability_test(
15732 handler: &ControlHandler,
15733 target_ctx: &RouteCtx,
15734 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15735 client_connection_id: u64,
15736 corr: u64,
15737 target_module_id: &str,
15738 consumer_identity: Option<ConsumerIdentity>,
15739 ) -> (
15740 mpsc::Receiver<crate::router::OutboundFrame>,
15741 ModuleControlRequest,
15742 ) {
15743 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
15744 let route_handler = handler.clone();
15745 let target_module_id = target_module_id.to_string();
15746 let route_task = tokio::spawn(async move {
15747 route_handler
15748 .handle_control_frame(
15749 &client_ctx,
15750 route_open_frame_with_admission_facts(
15751 corr,
15752 &target_module_id,
15753 unique_project_root("admission-facts"),
15754 consumer_identity,
15755 None,
15756 ),
15757 )
15758 .await
15759 .expect("capability test route.open succeeds")
15760 });
15761 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
15762 .await
15763 .expect("capability test route.open must reach route.bind")
15764 .expect("target control receiver stays open");
15765 let bind_request: ModuleControlRequest =
15766 serde_json::from_slice(&bind.body).expect("route.bind decodes");
15767 handler
15768 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
15769 .await
15770 .expect("capability test route.bind ACK succeeds");
15771 assert!(route_task.await.expect("route.open task joins").is_empty());
15772 let opened = client_rx
15773 .recv()
15774 .await
15775 .expect("successful route.open publishes a response");
15776 assert!(matches!(
15777 serde_json::from_slice::<ClientControlResponse>(&opened.body),
15778 Ok(ClientControlResponse::RouteOpen { .. })
15779 ));
15780 (client_rx, bind_request)
15781 }
15782
15783 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
15784 assert_eq!(frame.header.ty, FrameType::Push);
15785 assert_eq!(frame.header.channel, 0);
15786 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
15787 .expect("route.closed control push decodes");
15788 let ClientControlPush::RouteClosed { channels, .. } = &push else {
15789 panic!("expected route.closed");
15790 };
15791 assert_eq!(channels.len(), 1, "exactly one violating route closed");
15792 let closed_channel = channels[0];
15793 let channels = channels.clone();
15794 assert_eq!(
15795 push,
15796 ClientControlPush::RouteClosed {
15797 module_id: target_module_id.to_string(),
15798 channels,
15799 reason: RouteCloseReason::CapabilityDenied,
15800 drained: false,
15801 abandoned: 0,
15802 excluded_subscriptions: 0,
15803 terminal: Some(false),
15804 }
15805 );
15806 closed_channel
15807 }
15808
15809 #[tokio::test]
15810 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15811 let registry = Arc::new(Registry::default());
15812 let forwarding = Arc::new(ForwardingTable::default());
15813 let supervisor = SupervisorHandle::new();
15814 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15815 let handler =
15816 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15817 .with_supervisor(supervisor);
15818 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15819 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15820 register_capability_manifest(
15821 &handler,
15822 &target_ctx,
15823 &mut target_rx,
15824 capability_manifest("target", &["credentials-provider/v1"], &[]),
15825 1,
15826 )
15827 .await;
15828 register_capability_manifest(
15829 &handler,
15830 &opener_ctx,
15831 &mut opener_rx,
15832 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15833 2,
15834 )
15835 .await;
15836
15837 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15838 let replies = handler
15839 .handle_control_frame(
15840 &client_ctx,
15841 route_open_frame_with_admission_facts(
15842 3,
15843 "target",
15844 unique_project_root("admission-facts"),
15845 Some(ConsumerIdentity {
15846 module_id: "opener".to_string(),
15847 launch_nonce: "opener-nonce".to_string(),
15848 }),
15849 None,
15850 ),
15851 )
15852 .await
15853 .expect("denied route.open returns a typed frame");
15854 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15855 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15856 assert!(
15857 target_rx.try_recv().is_err(),
15858 "forbidden route.open must not relay route.bind"
15859 );
15860 }
15861
15862 #[tokio::test]
15863 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15864 let registry = Arc::new(Registry::default());
15865 let forwarding = Arc::new(ForwardingTable::default());
15866 let supervisor = SupervisorHandle::new();
15867 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15868 let handler =
15869 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15870 .with_supervisor(supervisor);
15871 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15872 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15873 register_capability_manifest(
15874 &handler,
15875 &target_ctx,
15876 &mut target_rx,
15877 capability_manifest("target", &["credentials-provider/v1"], &[]),
15878 1,
15879 )
15880 .await;
15881 register_capability_manifest(
15882 &handler,
15883 &old_opener_ctx,
15884 &mut old_opener_rx,
15885 capability_manifest("opener", &[], &[]),
15886 2,
15887 )
15888 .await;
15889 let (mut client_rx, _) = open_route_for_capability_test(
15890 &handler,
15891 &target_ctx,
15892 &mut target_rx,
15893 712,
15894 3,
15895 "target",
15896 Some(ConsumerIdentity {
15897 module_id: "opener".to_string(),
15898 launch_nonce: "opener-nonce".to_string(),
15899 }),
15900 )
15901 .await;
15902 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15903
15904 handler
15905 .cleanup_connection(old_opener_ctx.connection_id)
15906 .expect("old opener registration cleans up");
15907 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15908 register_capability_manifest(
15909 &handler,
15910 &new_opener_ctx,
15911 &mut new_opener_rx,
15912 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15913 4,
15914 )
15915 .await;
15916
15917 let closed_channel = assert_capability_denied_push(
15918 client_rx
15919 .try_recv()
15920 .expect("HELLO deny addition must emit route.closed")
15921 .frame,
15922 "target",
15923 );
15924 let goodbye = client_rx
15925 .try_recv()
15926 .expect("HELLO deny addition must send the client a route GOODBYE")
15927 .frame;
15928 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15929 assert_eq!(goodbye.header.channel, closed_channel);
15930 assert_ne!(goodbye.header.channel, 0);
15931 assert_ne!(goodbye.header.epoch, 0);
15932 assert!(client_rx.try_recv().is_err());
15933 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15934 assert!(matches!(
15935 target_rx.try_recv(),
15936 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15937 ));
15938 }
15939
15940 #[tokio::test]
15941 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15942 let registry = Arc::new(Registry::default());
15943 let forwarding = Arc::new(ForwardingTable::default());
15944 let supervisor = SupervisorHandle::new();
15945 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15946 let handler =
15947 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15948 .with_supervisor(supervisor);
15949 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15950 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15951 register_capability_manifest(
15952 &handler,
15953 &target_ctx,
15954 &mut target_rx,
15955 capability_manifest("target", &[], &[]),
15956 1,
15957 )
15958 .await;
15959 register_capability_manifest(
15960 &handler,
15961 &opener_ctx,
15962 &mut opener_rx,
15963 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15964 2,
15965 )
15966 .await;
15967 let (mut client_rx, _) = open_route_for_capability_test(
15968 &handler,
15969 &target_ctx,
15970 &mut target_rx,
15971 722,
15972 3,
15973 "target",
15974 Some(ConsumerIdentity {
15975 module_id: "opener".to_string(),
15976 launch_nonce: "opener-nonce".to_string(),
15977 }),
15978 )
15979 .await;
15980 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15981
15982 let replies = handler
15983 .handle_control_frame(
15984 &target_ctx,
15985 catalog_update_with_capabilities_frame(
15986 4,
15987 CapabilityDeclarations {
15988 provides: vec!["credentials-provider/v1".to_string()],
15989 requires: Vec::new(),
15990 must_never_reach: Vec::new(),
15991 },
15992 ),
15993 )
15994 .await
15995 .expect("claim catalog.update succeeds");
15996 assert!(matches!(
15997 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15998 Ok(ModuleControlResponseToModule::CatalogUpdate {})
15999 ));
16000 let closed_channel = assert_capability_denied_push(
16001 client_rx
16002 .try_recv()
16003 .expect("claim addition must emit route.closed")
16004 .frame,
16005 "target",
16006 );
16007 let goodbye = client_rx
16008 .try_recv()
16009 .expect("claim addition must send the client a route GOODBYE")
16010 .frame;
16011 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16012 assert_eq!(goodbye.header.channel, closed_channel);
16013 assert_ne!(goodbye.header.channel, 0);
16014 assert_ne!(goodbye.header.epoch, 0);
16015 assert!(client_rx.try_recv().is_err());
16016 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16017 assert!(matches!(
16018 target_rx.try_recv(),
16019 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16020 ));
16021 }
16022
16023 #[tokio::test]
16024 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
16025 let registry = Arc::new(Registry::default());
16026 let forwarding = Arc::new(ForwardingTable::default());
16027 let supervisor = SupervisorHandle::new();
16028 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16029 let handler =
16030 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16031 .with_supervisor(supervisor);
16032 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
16033 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
16034 register_capability_manifest(
16035 &handler,
16036 &target_ctx,
16037 &mut target_rx,
16038 capability_manifest("target", &["credentials-provider/v1"], &[]),
16039 1,
16040 )
16041 .await;
16042 register_capability_manifest(
16043 &handler,
16044 &opener_ctx,
16045 &mut opener_rx,
16046 capability_manifest("opener", &[], &[]),
16047 2,
16048 )
16049 .await;
16050 let (mut client_rx, _) = open_route_for_capability_test(
16051 &handler,
16052 &target_ctx,
16053 &mut target_rx,
16054 732,
16055 3,
16056 "target",
16057 Some(ConsumerIdentity {
16058 module_id: "opener".to_string(),
16059 launch_nonce: "opener-nonce".to_string(),
16060 }),
16061 )
16062 .await;
16063 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16064
16065 handler
16066 .handle_control_frame(
16067 &target_ctx,
16068 catalog_update_with_capabilities_frame(
16069 4,
16070 CapabilityDeclarations {
16071 provides: Vec::new(),
16072 requires: Vec::new(),
16073 must_never_reach: Vec::new(),
16074 },
16075 ),
16076 )
16077 .await
16078 .expect("claim removal catalog.update succeeds");
16079 assert_eq!(
16080 forwarding.active_binding_count().unwrap(),
16081 1,
16082 "removing an attested target claim must leave the route census unchanged"
16083 );
16084 assert!(
16085 client_rx.try_recv().is_err(),
16086 "claim removal must not emit route.closed or a client route GOODBYE"
16087 );
16088 assert!(
16089 target_rx.try_recv().is_err(),
16090 "claim removal must not send the target a route GOODBYE"
16091 );
16092 }
16093
16094 #[tokio::test]
16097 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
16098 let registry = Arc::new(Registry::default());
16099 let forwarding = Arc::new(ForwardingTable::default());
16100 let supervisor = SupervisorHandle::new();
16101 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16102 let handler =
16103 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16104 .with_supervisor(supervisor);
16105 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
16106 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
16107 register_capability_manifest(
16108 &handler,
16109 &target_ctx,
16110 &mut target_rx,
16111 capability_manifest("target", &["credentials-provider/v1"], &[]),
16112 1,
16113 )
16114 .await;
16115 register_capability_manifest(
16116 &handler,
16117 &opener_ctx,
16118 &mut opener_rx,
16119 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16120 2,
16121 )
16122 .await;
16123
16124 let (_client_rx, bind) = open_route_for_capability_test(
16125 &handler,
16126 &target_ctx,
16127 &mut target_rx,
16128 742,
16129 3,
16130 "target",
16131 None,
16132 )
16133 .await;
16134 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
16135 panic!("direct scope-honesty route must bind");
16136 };
16137 assert_eq!(principal, Some(Principal::Direct));
16138 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16139 }
16140
16141 #[tokio::test]
16144 async fn must_never_reach_self_route_is_capability_forbidden() {
16145 let registry = Arc::new(Registry::default());
16146 let forwarding = Arc::new(ForwardingTable::default());
16147 let supervisor = SupervisorHandle::new();
16148 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
16149 let handler =
16150 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16151 .with_supervisor(supervisor);
16152 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
16153 register_capability_manifest(
16154 &handler,
16155 &self_ctx,
16156 &mut self_rx,
16157 capability_manifest(
16158 "self-provider",
16159 &["credentials-provider/v1"],
16160 &["credentials-provider/v1"],
16161 ),
16162 1,
16163 )
16164 .await;
16165
16166 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
16167 let replies = handler
16168 .handle_control_frame(
16169 &client_ctx,
16170 route_open_frame_with_admission_facts(
16171 2,
16172 "self-provider",
16173 unique_project_root("admission-facts"),
16174 Some(ConsumerIdentity {
16175 module_id: "self-provider".to_string(),
16176 launch_nonce: "self-nonce".to_string(),
16177 }),
16178 None,
16179 ),
16180 )
16181 .await
16182 .expect("self-route refusal returns a typed frame");
16183 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16184 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16185 assert!(
16186 self_rx.try_recv().is_err(),
16187 "self denial must not relay route.bind"
16188 );
16189 }
16190
16191 #[test]
16192 fn unsupported_channel_zero_frame_returns_error() {
16193 let handler = ControlHandler::default();
16194 let request = Frame::build(
16195 FrameType::Request,
16196 control_flags(),
16197 0,
16198 0,
16199 21,
16200 b"opaque".to_vec(),
16201 )
16202 .unwrap();
16203
16204 let response = handler
16205 .handle_control(ConnectionId::new(1), request)
16206 .unwrap();
16207
16208 assert_eq!(response[0].header.ty, FrameType::Error);
16209 assert_eq!(
16210 parse_error(&response[0])["code"],
16211 "unsupported_control_frame"
16212 );
16213 }
16214
16215 mod swap {
16220 use super::*;
16221
16222 const INCUMBENT: ConnectionId = ConnectionId::new(30);
16223 const CANDIDATE: ConnectionId = ConnectionId::new(40);
16224
16225 struct Swap {
16226 registry: Arc<Registry>,
16227 forwarding: Arc<ForwardingTable>,
16228 handler: ControlHandler,
16229 incumbent_ctx: RouteCtx,
16230 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16231 candidate_ctx: RouteCtx,
16232 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16233 }
16234
16235 async fn swap_with_incumbent() -> Swap {
16236 let registry = Arc::new(Registry::default());
16237 let forwarding = Arc::new(ForwardingTable::default());
16238 let handler =
16239 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16240 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
16241 hello_via_sink(
16242 &handler,
16243 &incumbent_ctx,
16244 &mut incumbent_rx,
16245 hello_frame("aft", PROTOCOL_VERSION, 7),
16246 )
16247 .await;
16248 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
16249 Swap {
16250 registry,
16251 forwarding,
16252 handler,
16253 incumbent_ctx,
16254 incumbent_rx,
16255 candidate_ctx,
16256 candidate_rx,
16257 }
16258 }
16259
16260 fn register_candidate(swap: &Swap, ready: Option<bool>) {
16261 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
16262 candidate_manifest.ready = ready;
16263 let registration = swap
16264 .registry
16265 .register_candidate_with_control_ops(
16266 candidate_manifest,
16267 PROTOCOL_VERSION,
16268 CANDIDATE,
16269 module_baseline_control_ops(),
16270 )
16271 .unwrap();
16272 swap.forwarding
16273 .register_candidate_module_connection(
16274 CANDIDATE,
16275 "aft".to_string(),
16276 PROTOCOL_VERSION,
16277 manifest_concurrency(®istration.manifest),
16278 swap.candidate_ctx.egress.clone(),
16279 )
16280 .unwrap();
16281 }
16282
16283 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
16284 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
16285 swap.registry.promote_candidate("aft").unwrap().unwrap();
16286 cutover.incumbent.unwrap()
16287 }
16288
16289 #[tokio::test]
16298 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
16299 let mut swap = swap_with_incumbent().await;
16300 let hold = Arc::new(std::sync::Barrier::new(2));
16301 let handler = swap
16302 .handler
16303 .clone()
16304 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
16305 handler
16306 .supervisor
16307 .open_swap("aft", "candidate-nonce".to_string());
16308 let hello = {
16309 let handler = handler.clone();
16310 let candidate_ctx = swap.candidate_ctx.clone();
16311 std::thread::spawn(move || {
16312 handler
16313 .handle_hello(
16314 candidate_ctx.connection_id,
16315 Some(candidate_ctx.egress.clone()),
16316 hello_frame_with_nonce(
16317 "aft",
16318 PROTOCOL_VERSION,
16319 8,
16320 Some("candidate-nonce"),
16321 ),
16322 )
16323 .unwrap()
16324 })
16325 };
16326
16327 hold.wait();
16328 assert!(swap
16331 .registry
16332 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
16333 .unwrap()
16334 .is_some());
16335 assert!(swap
16336 .registry
16337 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16338 .unwrap()
16339 .is_none());
16340 assert!(swap
16341 .forwarding
16342 .module_endpoint_for_connection(CANDIDATE)
16343 .unwrap()
16344 .is_none());
16345
16346 hold.wait();
16347 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
16348 assert_eq!(
16349 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
16350 FrameType::HelloAck
16351 );
16352 let published = swap
16353 .registry
16354 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16355 .unwrap()
16356 .expect("candidate is published once its HELLO completes");
16357 assert_eq!(published.connection_id, CANDIDATE);
16358 assert!(swap
16359 .forwarding
16360 .module_endpoint_for_connection(CANDIDATE)
16361 .unwrap()
16362 .is_some());
16363 }
16364
16365 fn keyed_total(counters: &Value, key: &str) -> u64 {
16366 counters[key]
16367 .as_object()
16368 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
16369 .unwrap_or(0)
16370 }
16371
16372 #[tokio::test]
16373 async fn replacement_logs_old_end_and_new_admission_once() {
16374 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16375 let mut swap = swap_with_incumbent().await;
16376 swap.handler
16377 .supervisor
16378 .open_swap("aft", "candidate-nonce".to_string());
16379 hello_via_sink(
16380 &swap.handler,
16381 &swap.candidate_ctx,
16382 &mut swap.candidate_rx,
16383 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
16384 )
16385 .await;
16386 cutover(&swap);
16387 swap.handler.cleanup_connection(INCUMBENT).unwrap();
16388
16389 let captured = crate::router::test_log::captured_logs(&logs);
16390 println!("captured replacement registry logs:\n{captured}");
16391 let new_admissions: Vec<_> = captured
16392 .lines()
16393 .filter(|line| {
16394 line.contains("connection_id=40")
16395 && line.contains("swap candidate registered; not routable until cutover")
16396 })
16397 .collect();
16398 assert_eq!(
16399 new_admissions.len(),
16400 1,
16401 "unexpected admission logs: {captured}"
16402 );
16403 assert!(new_admissions[0].contains("module_id=aft"));
16404 assert!(new_admissions[0].contains("connection_id=40"));
16405 assert!(
16406 new_admissions[0].contains("swap candidate registered; not routable until cutover")
16407 );
16408 assert!(new_admissions[0].contains("ready=true"));
16409
16410 let old_admissions: Vec<_> = captured
16411 .lines()
16412 .filter(|line| {
16413 line.contains("module registered module_id=aft ")
16414 && line.contains("connection_id=30")
16415 })
16416 .collect();
16417 assert_eq!(
16418 old_admissions.len(),
16419 1,
16420 "unexpected incumbent admission: {captured}"
16421 );
16422
16423 let old_ends: Vec<_> = captured
16424 .lines()
16425 .filter(|line| {
16426 line.contains("connection_id=30") && line.contains("module registration ended")
16427 })
16428 .collect();
16429 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
16430 assert!(old_ends[0].contains("module_id=aft"));
16431 assert!(old_ends[0].contains("connection_id=30"));
16432 assert!(old_ends[0].contains("reason=replaced"));
16433 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
16434 assert_eq!(
16435 captured
16436 .lines()
16437 .filter(|line| line.contains("module registration promoted"))
16438 .count(),
16439 1,
16440 "unexpected promotion logs: {captured}"
16441 );
16442 }
16443
16444 #[tokio::test]
16450 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
16451 let mut swap = swap_with_incumbent().await;
16452 let handler = swap.handler.clone();
16453
16454 let cotenant = ConnectionId::new(31);
16456 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
16457 let (cotenant_task, cotenant_bind) = relay_route_open(
16458 &handler,
16459 cotenant,
16460 &cotenant_ctx.egress,
16461 &mut swap.incumbent_rx,
16462 100,
16463 "aft",
16464 "swap-cotenant",
16465 )
16466 .await;
16467 handler
16468 .handle_control_frame(
16469 &swap.incumbent_ctx,
16470 route_bind_ack(cotenant_bind.header.corr),
16471 )
16472 .await
16473 .unwrap();
16474 assert!(cotenant_task.await.unwrap().is_empty());
16475 let (cotenant_channel, cotenant_epoch) =
16476 published_route(&cotenant_rx.recv().await.unwrap());
16477
16478 let caller = ConnectionId::new(32);
16480 let (caller_ctx, mut caller_rx) = route_ctx(caller);
16481 let (caller_task, caller_bind) = relay_route_open(
16482 &handler,
16483 caller,
16484 &caller_ctx.egress,
16485 &mut swap.incumbent_rx,
16486 101,
16487 "aft",
16488 "swap-caller",
16489 )
16490 .await;
16491 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
16492
16493 register_candidate(&swap, None);
16494 cutover(&swap);
16495
16496 let ack = handler
16498 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
16499 .await;
16500 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
16501 if module_loop_error.is_some() {
16502 handler.cleanup_connection(INCUMBENT).unwrap();
16505 }
16506
16507 assert!(
16509 cotenant_rx.try_recv().is_err(),
16510 "the co-tenant route on the incumbent was torn down by one late ack: \
16511 {module_loop_error:?}"
16512 );
16513 assert!(matches!(
16514 swap.forwarding
16515 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16516 .unwrap(),
16517 DataRoute::Client(DataRouteState::Bound(_))
16518 ));
16519 assert_eq!(module_loop_error, None);
16520 assert!(swap
16521 .registry
16522 .get_module_by_connection(INCUMBENT)
16523 .unwrap()
16524 .is_some());
16525
16526 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
16528 .await
16529 .expect("the incumbent is told to drop the abandoned binding")
16530 .unwrap()
16531 .frame;
16532 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16533 assert_eq!(goodbye.header.channel, abandoned_channel);
16534 assert_eq!(goodbye.header.epoch, abandoned_epoch);
16535 assert!(swap.incumbent_rx.try_recv().is_err());
16536
16537 let response = caller_task.await.unwrap();
16539 assert_eq!(response.len(), 1);
16540 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
16541 assert!(caller_rx.try_recv().is_err());
16542
16543 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
16547 let counters = handler.counters().snapshot();
16548 assert_eq!(
16549 keyed_total(&counters, "route_open_accepted_by_principal"),
16550 1
16551 );
16552 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
16553 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
16554 }
16555
16556 #[tokio::test]
16560 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
16561 let mut swap = swap_with_incumbent().await;
16562 register_candidate(&swap, None);
16563 let incumbent = cutover(&swap);
16564 swap.forwarding
16565 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
16566 .unwrap()
16567 .expect("the incumbent is still registered");
16568
16569 let client = ConnectionId::new(33);
16570 let (client_ctx, mut client_rx) = route_ctx(client);
16571 let route_handler = swap.handler.clone();
16572 let open_ctx = RouteCtx {
16573 connection_id: client,
16574 egress: client_ctx.egress.clone(),
16575 };
16576 let mut route_task = tokio::spawn(async move {
16577 route_handler
16578 .handle_control_frame(
16579 &open_ctx,
16580 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
16581 )
16582 .await
16583 .unwrap()
16584 });
16585 let bind = tokio::select! {
16586 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
16587 response = &mut route_task => {
16588 let response = response.unwrap();
16589 panic!(
16590 "post-cutover route.open was refused instead of relayed to the candidate: {}",
16591 parse_error(&response[0])["code"]
16592 );
16593 }
16594 };
16595 swap.handler
16596 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
16597 .await
16598 .unwrap();
16599 assert!(route_task.await.unwrap().is_empty());
16600 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
16601 match swap
16602 .forwarding
16603 .lookup_data_route(client, channel, epoch)
16604 .unwrap()
16605 {
16606 DataRoute::Client(DataRouteState::Bound(route)) => {
16607 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
16608 }
16609 other => panic!("expected a bound route on the candidate, got {other:?}"),
16610 }
16611 assert!(swap.incumbent_rx.try_recv().is_err());
16612 }
16613
16614 #[tokio::test]
16619 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
16620 let swap = swap_with_incumbent().await;
16621 register_candidate(&swap, Some(false));
16622 let update = Frame::build(
16623 FrameType::Request,
16624 control_flags(),
16625 0,
16626 0,
16627 55,
16628 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16629 provides: manifest("aft", PROTOCOL_VERSION).provides,
16630 capabilities: None,
16631 ready: Some(true),
16632 })
16633 .unwrap(),
16634 )
16635 .unwrap();
16636
16637 let replies = swap
16638 .handler
16639 .handle_control_frame(&swap.candidate_ctx, update)
16640 .await
16641 .unwrap();
16642
16643 assert_eq!(replies.len(), 1);
16644 assert_eq!(
16645 replies[0].header.ty,
16646 FrameType::Response,
16647 "candidate catalog.update was refused: {:?}",
16648 serde_json::from_slice::<Value>(&replies[0].body).ok()
16649 );
16650 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
16651 assert_eq!(
16652 swap.registry
16653 .get_module("aft")
16654 .unwrap()
16655 .unwrap()
16656 .connection_id,
16657 INCUMBENT
16658 );
16659 }
16660 }
16661
16662 mod swap_admission {
16666 use super::*;
16667
16668 const INCUMBENT_NONCE: &str = "incumbent-nonce";
16669 const CANDIDATE_NONCE: &str = "candidate-nonce";
16670
16671 fn handler_with_incumbent(
16672 module_id: &str,
16673 reserved: bool,
16674 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
16675 let registry = Arc::new(Registry::default());
16676 let supervisor = SupervisorHandle::new();
16677 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
16678 if reserved {
16679 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
16680 }
16681 let handler =
16682 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
16683 let incumbent = handler
16684 .handle_control(
16685 ConnectionId::new(1),
16686 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
16687 )
16688 .unwrap();
16689 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
16690 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
16691 (registry, supervisor, handler)
16692 }
16693
16694 #[test]
16700 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
16701 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
16702
16703 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
16704 let replies = handler
16705 .handle_control(
16706 ConnectionId::new(connection),
16707 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
16708 )
16709 .unwrap();
16710 assert_eq!(replies[0].header.ty, FrameType::Error);
16711 assert_eq!(
16712 parse_error(&replies[0])["code"],
16713 "swap_token_invalid",
16714 "nonce {nonce:?}"
16715 );
16716 }
16717 assert!(registry.get_candidate("aft").unwrap().is_none());
16718 assert_eq!(
16719 registry.get_module("aft").unwrap().unwrap().connection_id,
16720 ConnectionId::new(1)
16721 );
16722
16723 let admitted = handler
16726 .handle_control(
16727 ConnectionId::new(4),
16728 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
16729 )
16730 .unwrap();
16731 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
16732 assert_eq!(
16733 registry
16734 .get_candidate("aft")
16735 .unwrap()
16736 .unwrap()
16737 .connection_id,
16738 ConnectionId::new(4)
16739 );
16740 assert_eq!(
16741 registry.get_module("aft").unwrap().unwrap().connection_id,
16742 ConnectionId::new(1),
16743 "the candidate must not take the active slot"
16744 );
16745 let replayed = handler
16746 .handle_control(
16747 ConnectionId::new(5),
16748 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
16749 )
16750 .unwrap();
16751 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
16752
16753 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
16757 let squatter = handler
16758 .handle_control(
16759 ConnectionId::new(6),
16760 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
16761 )
16762 .unwrap();
16763 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
16764 assert!(
16765 registry.get_module("aft").unwrap().is_none(),
16766 "a squatter took the active slot of an id being swapped"
16767 );
16768 }
16769
16770 #[test]
16775 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
16776 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
16777
16778 let replies = handler
16779 .handle_control(
16780 ConnectionId::new(2),
16781 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16782 )
16783 .unwrap();
16784
16785 assert_eq!(
16786 replies[0].header.ty,
16787 FrameType::HelloAck,
16788 "reserved candidate refused: {:?}",
16789 serde_json::from_slice::<Value>(&replies[0].body).ok()
16790 );
16791 assert_eq!(
16792 registry
16793 .get_candidate("vault")
16794 .unwrap()
16795 .unwrap()
16796 .connection_id,
16797 ConnectionId::new(2)
16798 );
16799 }
16800
16801 #[test]
16804 fn without_an_open_swap_the_ordinary_gates_decide() {
16805 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16806 supervisor.close_swap("vault");
16807
16808 let candidate = handler
16809 .handle_control(
16810 ConnectionId::new(2),
16811 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16812 )
16813 .unwrap();
16814 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16815 let duplicate = handler
16816 .handle_control(
16817 ConnectionId::new(3),
16818 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16819 )
16820 .unwrap();
16821 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16822 assert!(registry.get_candidate("vault").unwrap().is_none());
16823 }
16824 }
16825
16826 mod scopes {
16830 mod idle_tests {
16831 use super::*;
16832 include!("scope_idle_tests.rs");
16833 }
16834 mod route_open_timings {
16835 use super::*;
16836 include!("route_open_timing_tests.rs");
16837 }
16838 mod route_open_refusal_log {
16839 use super::*;
16840 include!("route_open_refusal_log_tests.rs");
16841 }
16842 mod activity {
16843 include!("activity_scope_tests.rs");
16844 }
16845 mod delta {
16846 include!("scope_handler_delta_tests.rs");
16847 }
16848 mod call_modes {
16849 include!("scope_call_mode_tests.rs");
16850 }
16851 #[tokio::test]
16852 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16853 let rig = rig().await;
16854 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16855 "s".into(),
16856 1,
16857 "tool".into(),
16858 ))
16859 .unwrap();
16860 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16861 let (direct, _rx) = wide_ctx(999);
16862 for (ctx, expected) in [
16863 (&rig.owner, FrameType::Response),
16864 (&direct, FrameType::Error),
16865 ] {
16866 let frame =
16867 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16868 .unwrap();
16869 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16870 assert_eq!(replies[0].header.ty, expected);
16871 if expected == FrameType::Response {
16872 let reply: subc_control::ScopeServedByReply =
16873 serde_json::from_slice(&replies[0].body).unwrap();
16874 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16875 } else {
16876 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16877 }
16878 }
16879 let reply = call(
16880 &rig.handler,
16881 &rig.owner,
16882 &ModuleControlRequestFromModule::ScopeDescribe {
16883 owner: Principal::Reserved {
16884 module_id: OWNER.into(),
16885 },
16886 scope_ref: "s".into(),
16887 },
16888 )
16889 .await;
16890 assert!(matches!(
16891 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16892 ModuleControlResponseToModule::ScopeDescribe { .. }
16893 ));
16894 }
16895
16896 use subc_protocol::scope::{
16897 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16898 ScopeStatus,
16899 };
16900
16901 use super::*;
16902
16903 const OWNER: &str = "prefrontal-core";
16904
16905 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16906 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16907 }
16908
16909 async fn call(
16910 handler: &ControlHandler,
16911 ctx: &RouteCtx,
16912 request: &ModuleControlRequestFromModule,
16913 ) -> Frame {
16914 let body = serde_json::to_vec(request).unwrap();
16915 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16916 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16917 assert_eq!(replies.len(), 1, "{replies:?}");
16918 replies.pop().unwrap()
16919 }
16920
16921 async fn sync(
16922 handler: &ControlHandler,
16923 ctx: &RouteCtx,
16924 generation: u64,
16925 scopes: Vec<ScopeRecord>,
16926 ) -> Result<ModuleControlResponseToModule, String> {
16927 let reply = call(
16928 handler,
16929 ctx,
16930 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16931 )
16932 .await;
16933 match reply.header.ty {
16934 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16935 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16936 }
16937 }
16938
16939 async fn describe(
16940 handler: &ControlHandler,
16941 ctx: &RouteCtx,
16942 owner: &str,
16943 scope_ref: &str,
16944 ) -> ModuleControlResponseToModule {
16945 let reply = call(
16946 handler,
16947 ctx,
16948 &ModuleControlRequestFromModule::ScopeDescribe {
16949 owner: Principal::Reserved {
16950 module_id: owner.to_string(),
16951 },
16952 scope_ref: scope_ref.to_string(),
16953 },
16954 )
16955 .await;
16956 assert_eq!(
16957 reply.header.ty,
16958 FrameType::Response,
16959 "{:?}",
16960 parse_error(&reply)
16961 );
16962 serde_json::from_slice(&reply.body).unwrap()
16963 }
16964
16965 #[tokio::test]
16966 async fn scope_describe_never_echoes_bind_proof_or_hash() {
16967 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16968 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16969 let mut rig = rig().await;
16970 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16971 .await;
16972 let _bound = rig
16973 .bound(
16974 Some(AFT),
16975 PLEXUS,
16976 Some(
16977 rig.selector("s", Some(1))
16978 .with_bind_proof(Some(PROOF.into())),
16979 ),
16980 )
16981 .await;
16982 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16983 let output = serde_json::to_string(&reply).unwrap();
16984 assert!(output.contains("\"status\":\"live\""));
16985 assert!(!output.contains(PROOF));
16986 assert!(!output.contains(HASH));
16987 }
16988
16989 async fn module(
16991 handler: &ControlHandler,
16992 connection: u64,
16993 module_id: &str,
16994 nonce: Option<&str>,
16995 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16996 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16997 hello_via_sink(
16998 handler,
16999 &ctx,
17000 &mut rx,
17001 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
17002 )
17003 .await;
17004 (ctx, rx)
17005 }
17006
17007 #[tokio::test]
17010 async fn a_client_connection_cannot_sync_or_describe() {
17011 let handler = ControlHandler::new(Arc::new(Registry::default()));
17012 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
17013 for request in [
17014 ModuleControlRequestFromModule::ScopeSync {
17015 generation: 1,
17016 scopes: vec![head("s", 1)],
17017 },
17018 ModuleControlRequestFromModule::ScopeDescribe {
17019 owner: Principal::Direct,
17020 scope_ref: "s".to_string(),
17021 },
17022 ] {
17023 let reply = call(&handler, &ctx, &request).await;
17024 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
17025 }
17026 assert!(
17027 !handler
17028 .scopes
17029 .read()
17030 .unwrap()
17031 .describe(
17032 &Principal::Reserved {
17033 module_id: OWNER.to_string()
17034 },
17035 "s"
17036 )
17037 .owner_synced
17038 );
17039 }
17040
17041 #[tokio::test]
17044 async fn a_module_without_a_supervised_launch_cannot_sync() {
17045 let handler = ControlHandler::new(Arc::new(Registry::default()));
17046 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
17047 assert_eq!(
17048 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
17049 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17050 );
17051 }
17052
17053 #[tokio::test]
17054 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
17055 let supervisor = SupervisorHandle::new();
17056 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17057 let handler = ControlHandler::new(Arc::new(Registry::default()))
17058 .with_supervisor(supervisor.clone());
17059 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17060 sync(&handler, &incumbent, 1, vec![head("s", 1)])
17061 .await
17062 .expect("the current launch syncs");
17063
17064 supervisor.open_swap(OWNER, "n2".to_string());
17067 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
17068 assert_eq!(
17069 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17070 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17071 );
17072 sync(&handler, &incumbent, 2, vec![head("s", 1)])
17073 .await
17074 .expect("the serving owner syncs during the swap");
17075
17076 supervisor.close_swap(OWNER);
17079 assert_eq!(
17080 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17081 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17082 );
17083 sync(&handler, &incumbent, 3, vec![head("s", 1)])
17084 .await
17085 .expect("the serving owner syncs after the rollback");
17086 handler.cleanup_connection(candidate.connection_id).unwrap();
17087
17088 supervisor.open_swap(OWNER, "n3".to_string());
17093 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
17094 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
17095 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
17096 .await
17097 .expect("the promoted launch takes authority");
17098 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17099 panic!("unexpected reply {reply:?}");
17100 };
17101 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
17102 assert_eq!(
17103 sync(&handler, &incumbent, 4, Vec::new()).await,
17104 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17105 );
17106 }
17107
17108 #[tokio::test]
17111 async fn closing_the_authority_connection_frees_sync_authority() {
17112 let supervisor = SupervisorHandle::new();
17113 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17114 let handler = ControlHandler::new(Arc::new(Registry::default()))
17115 .with_supervisor(supervisor.clone());
17116 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17117 sync(&handler, &first, 10, vec![head("s", 1)])
17118 .await
17119 .unwrap();
17120 handler.cleanup_connection(first.connection_id).unwrap();
17121
17122 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
17123 sync(&handler, &second, 1, vec![head("s", 1)])
17124 .await
17125 .expect("the next connection takes the released authority");
17126 }
17127
17128 #[tokio::test]
17129 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
17130 let supervisor = SupervisorHandle::new();
17131 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17132 let handler =
17133 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
17134 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
17135 sync(&handler, &first, 10, vec![head("s", 1)])
17136 .await
17137 .unwrap();
17138 handler
17139 .handle_control_frame(
17140 &first,
17141 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
17142 )
17143 .await
17144 .unwrap();
17145 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
17146 sync(&handler, &second, 1, vec![head("s", 1)])
17147 .await
17148 .expect("GOODBYE releases authority even if the old socket remains open");
17149 }
17150
17151 #[tokio::test]
17152 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
17153 let registry = Arc::new(Registry::default());
17154 let supervisor_handle = SupervisorHandle::new();
17155 let supervisor =
17156 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17157 .with_handle(supervisor_handle.clone())
17158 .with_daemon_incarnation("incarnation-7".to_string());
17159 supervisor
17162 .supervise_configured(
17163 ModuleSpec {
17164 module_id: OWNER.to_string(),
17165 program: PathBuf::from("/nonexistent/prefrontal-core"),
17166 args: Vec::new(),
17167 env: Vec::new(),
17168 reserved: false,
17169 reserved_prefixes: Vec::new(),
17170 protocol: ModuleProtocol::Subc,
17171 overlap: Default::default(),
17172 },
17173 false,
17174 )
17175 .unwrap();
17176 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
17177 let handler =
17178 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17179 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
17180
17181 let ModuleControlResponseToModule::ScopeDescribe {
17183 status,
17184 daemon_incarnation,
17185 owner_synced,
17186 owner_configured,
17187 scope,
17188 ..
17189 } = describe(&handler, &reader, OWNER, "s").await
17190 else {
17191 panic!("not a describe reply");
17192 };
17193 assert_eq!(status, ScopeStatus::NotLive);
17194 assert_eq!(daemon_incarnation, "incarnation-7");
17195 assert!(!owner_synced);
17196 assert!(owner_configured);
17197 assert!(scope.is_none());
17198
17199 let ModuleControlResponseToModule::ScopeDescribe {
17202 status,
17203 owner_configured,
17204 ..
17205 } = describe(&handler, &reader, "ghost", "s").await
17206 else {
17207 panic!("not a describe reply");
17208 };
17209 assert_eq!(status, ScopeStatus::NotLive);
17210 assert!(!owner_configured);
17211
17212 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
17214 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
17215 let ModuleControlResponseToModule::ScopeDescribe {
17216 status,
17217 scope_epoch,
17218 owner_synced,
17219 scope,
17220 ..
17221 } = describe(&handler, &reader, OWNER, "s").await
17222 else {
17223 panic!("not a describe reply");
17224 };
17225 assert_eq!(status, ScopeStatus::Live);
17226 assert_eq!(scope_epoch, Some(4));
17227 assert!(owner_synced);
17228 let stamp = scope.expect("a live scope carries its stamp");
17229 assert!(
17230 stamp.owner_authorized,
17231 "prefrontal-core is the default authority"
17232 );
17233 assert_eq!(stamp.kind, ScopeKind::Head);
17234 }
17235
17236 #[tokio::test]
17237 async fn scope_authority_owners_decides_owner_authorized() {
17238 let supervisor = SupervisorHandle::new();
17239 supervisor.set_spawn_nonce("broca", "b1".to_string());
17240 let handler = ControlHandler::new(Arc::new(Registry::default()))
17241 .with_supervisor(supervisor)
17242 .with_scope_authority_owners(vec!["broca".to_string()]);
17243 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
17244 let mut gated = head("s", 1);
17245 gated.attributes.agent_id = Some("agent".to_string());
17246 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
17247 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
17248 describe(&handler, &broca, "broca", "s").await
17249 else {
17250 panic!("not a describe reply");
17251 };
17252 assert!(scope.unwrap().owner_authorized);
17253 }
17254
17255 #[tokio::test]
17259 async fn scope_ops_and_the_scopes_capability_are_advertised() {
17260 let handler = ControlHandler::new(Arc::new(Registry::default()));
17261 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
17262 let ack = hello_via_sink(
17263 &handler,
17264 &ctx,
17265 &mut rx,
17266 hello_frame("m", PROTOCOL_VERSION, 1),
17267 )
17268 .await;
17269 let ack = parse_ack(&ack);
17270 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
17271 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
17272 }
17273 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
17274
17275 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
17276 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
17277 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
17278 let reply = handler
17279 .handle_control_frame(&client, frame)
17280 .await
17281 .unwrap()
17282 .pop()
17283 .unwrap();
17284 let ClientControlResponse::ServerDescribe { capabilities, .. } =
17285 serde_json::from_slice(&reply.body).unwrap()
17286 else {
17287 panic!("not a server.describe reply");
17288 };
17289 assert!(
17290 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
17291 "{capabilities:?}"
17292 );
17293 }
17294
17295 const PLEXUS: &str = "plexus";
17298 const OTHER: &str = "other";
17299 const AFT: &str = "aft";
17300 const BROCA: &str = "broca";
17301 const MAGIC: &str = "magic-context";
17302
17303 fn nonce(module_id: &str) -> String {
17304 format!("nonce-{module_id}")
17305 }
17306
17307 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17308 let (tx, rx) = mpsc::channel(64);
17309 (
17310 RouteCtx {
17311 connection_id: ConnectionId::new(connection),
17312 egress: FrameSink::new(tx),
17313 },
17314 rx,
17315 )
17316 }
17317
17318 struct Rig {
17322 handler: ControlHandler,
17323 forwarding: Arc<ForwardingTable>,
17324 owner: RouteCtx,
17325 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17326 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
17327 generation: u64,
17328 next_connection: u64,
17329 _supervisor: Supervisor,
17330 }
17331
17332 async fn rig() -> Rig {
17333 rig_with_flow_support(true).await
17334 }
17335
17336 async fn rig_with_flow_support(flow_support: bool) -> Rig {
17337 rig_with_scope_capabilities(flow_support, false).await
17338 }
17339
17340 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
17341 let mut provides = Vec::new();
17342 if flow_support {
17343 provides.push("flow-scopes/v1");
17344 }
17345 if run_support {
17346 provides.push("agent-run-scopes/v1");
17347 }
17348 rig_with_target_capabilities(&provides).await
17349 }
17350
17351 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
17352 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
17353 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17354 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17357 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
17358 Frame::build(
17359 FrameType::Hello,
17360 control_flags(),
17361 0,
17362 0,
17363 connection,
17364 serde_json::to_vec(&body).unwrap(),
17365 )
17366 .unwrap()
17367 }
17368
17369 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
17370 let registry = Arc::new(Registry::default());
17371 let forwarding = Arc::new(ForwardingTable::default());
17372 let supervisor_handle = SupervisorHandle::new();
17373 let supervisor =
17374 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17375 .with_handle(supervisor_handle.clone());
17376 supervisor
17377 .supervise_configured(
17378 ModuleSpec {
17379 module_id: OWNER.to_string(),
17380 program: PathBuf::from("/nonexistent/prefrontal-core"),
17381 args: Vec::new(),
17382 env: Vec::new(),
17383 reserved: false,
17384 reserved_prefixes: Vec::new(),
17385 protocol: ModuleProtocol::Subc,
17386 overlap: Default::default(),
17387 },
17388 false,
17389 )
17390 .unwrap();
17391 for module_id in [OWNER, AFT, BROCA, MAGIC] {
17392 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
17393 }
17394 let handler =
17395 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17396 .with_supervisor(supervisor_handle);
17397 let (owner, mut owner_rx) = wide_ctx(1);
17398 hello_via_sink(
17399 &handler,
17400 &owner,
17401 &mut owner_rx,
17402 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
17403 )
17404 .await;
17405 let mut modules = BTreeMap::new();
17406 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
17407 let (ctx, mut rx) = wide_ctx(connection);
17408 let hello = hello_with_capabilities(module_id, connection, provides);
17409 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
17410 modules.insert(module_id.to_string(), (ctx, rx));
17411 }
17412 Rig {
17413 handler,
17414 forwarding,
17415 owner,
17416 _owner_rx: owner_rx,
17417 modules,
17418 generation: 0,
17419 next_connection: 100,
17420 _supervisor: supervisor,
17421 }
17422 }
17423
17424 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
17425 ScopeCarrier::new(Principal::Reserved {
17426 module_id: module_id.to_string(),
17427 })
17428 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
17429 }
17430
17431 fn session(scope_epoch: u64) -> ScopeRecord {
17434 let mut record = head("s", scope_epoch);
17435 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
17436 record.attributes.agent_id = Some("agent-1".to_string());
17437 record.attributes.delegates = true;
17438 record
17439 }
17440
17441 impl Rig {
17442 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
17443 self.generation += 1;
17444 sync(&self.handler, &self.owner, self.generation, scopes)
17445 .await
17446 .expect("the owner's sync is accepted");
17447 }
17448
17449 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17450 ScopeSelector::new(
17451 Principal::Reserved {
17452 module_id: OWNER.to_string(),
17453 },
17454 scope_ref.to_string(),
17455 scope_epoch,
17456 )
17457 }
17458
17459 fn open_frame(
17460 &mut self,
17461 opener: Option<&str>,
17462 target: &str,
17463 scope: Option<ScopeSelector>,
17464 ) -> (
17465 RouteCtx,
17466 mpsc::Receiver<crate::router::OutboundFrame>,
17467 Frame,
17468 ) {
17469 self.next_connection += 1;
17470 let (ctx, rx) = wide_ctx(self.next_connection);
17471 let root = unique_project_root("scoped-open");
17472 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
17473 target: RouteTarget::ToolProvider {
17474 module_id: target.to_string(),
17475 },
17476 identity: BindIdentity::new(
17477 root.path().to_path_buf(),
17478 "unit".to_string(),
17479 "session".to_string(),
17480 ),
17481 consumer_identity: opener.map(|module_id| ConsumerIdentity {
17482 module_id: module_id.to_string(),
17483 launch_nonce: nonce(module_id),
17484 }),
17485 consumer_capabilities: None,
17486 role_versions: None,
17487 admission_facts: None,
17488 scope,
17489 })
17490 .unwrap();
17491 let frame = Frame::build(
17492 FrameType::Request,
17493 control_flags(),
17494 0,
17495 0,
17496 self.next_connection,
17497 body,
17498 )
17499 .unwrap();
17500 (ctx, rx, frame)
17501 }
17502
17503 async fn refused(
17505 &mut self,
17506 opener: Option<&str>,
17507 target: &str,
17508 scope: Option<ScopeSelector>,
17509 ) -> String {
17510 self.refusal_body(opener, target, scope).await["code"]
17511 .as_str()
17512 .unwrap()
17513 .to_string()
17514 }
17515
17516 async fn refusal_body(
17517 &mut self,
17518 opener: Option<&str>,
17519 target: &str,
17520 scope: Option<ScopeSelector>,
17521 ) -> Value {
17522 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
17523 let replies = tokio::time::timeout(
17524 Duration::from_secs(2),
17525 self.handler.handle_control_frame(&ctx, frame),
17526 )
17527 .await
17528 .expect("the open must be refused before waiting for a bind ack")
17529 .unwrap();
17530 assert_eq!(replies.len(), 1, "{replies:?}");
17531 assert_eq!(replies[0].header.ty, FrameType::Error);
17532 let (_, module_rx) = self.modules.get_mut(target).unwrap();
17533 assert!(
17534 module_rx.try_recv().is_err(),
17535 "a refused open relays nothing"
17536 );
17537 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
17538 parse_error(&replies[0])
17539 }
17540
17541 async fn relayed(
17543 &mut self,
17544 opener: Option<&str>,
17545 target: &str,
17546 scope: Option<ScopeSelector>,
17547 ) -> Relayed {
17548 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
17549 let handler = self.handler.clone();
17550 let task_ctx = ctx.clone();
17551 let task = tokio::spawn(async move {
17552 handler
17553 .handle_control_frame(&task_ctx, frame)
17554 .await
17555 .unwrap()
17556 });
17557 let (_, module_rx) = self.modules.get_mut(target).unwrap();
17558 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
17559 .await
17560 .expect("the target receives the relayed route.bind")
17561 .unwrap()
17562 .frame;
17563 Relayed {
17564 target: target.to_string(),
17565 client: ctx,
17566 client_rx: rx,
17567 task,
17568 bind,
17569 }
17570 }
17571
17572 async fn ack(&self, relayed: &Relayed) {
17573 let (module, _) = &self.modules[&relayed.target];
17574 self.handler
17575 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
17576 .await
17577 .unwrap();
17578 }
17579
17580 async fn bound(
17582 &mut self,
17583 opener: Option<&str>,
17584 target: &str,
17585 scope: Option<ScopeSelector>,
17586 ) -> Bound {
17587 let relayed = self.relayed(opener, target, scope).await;
17588 self.ack(&relayed).await;
17589 let Relayed {
17590 target,
17591 client,
17592 mut client_rx,
17593 task,
17594 bind,
17595 } = relayed;
17596 assert!(
17597 task.await.unwrap().is_empty(),
17598 "the open is answered by commit"
17599 );
17600 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
17601 Bound {
17602 target,
17603 client,
17604 client_rx,
17605 channel,
17606 epoch,
17607 bind,
17608 }
17609 }
17610
17611 fn live(&self, route: &Bound) -> bool {
17612 matches!(
17613 self.forwarding
17614 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17615 .unwrap(),
17616 DataRoute::Client(DataRouteState::Bound(_))
17617 )
17618 }
17619 }
17620
17621 struct Relayed {
17622 target: String,
17623 client: RouteCtx,
17624 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17625 task: tokio::task::JoinHandle<Vec<Frame>>,
17626 bind: Frame,
17627 }
17628
17629 struct Bound {
17630 target: String,
17631 client: RouteCtx,
17632 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17633 channel: u16,
17634 epoch: u32,
17635 bind: Frame,
17636 }
17637
17638 impl Bound {
17639 fn closed_reason(&mut self) -> RouteCloseReason {
17642 let mut reason = None;
17643 let mut goodbye = false;
17644 while let Ok(outbound) = self.client_rx.try_recv() {
17645 let frame = outbound.frame;
17646 match frame.header.ty {
17647 FrameType::Goodbye => {
17648 assert_eq!(
17649 (frame.header.channel, frame.header.epoch),
17650 (self.channel, self.epoch)
17651 );
17652 goodbye = true;
17653 }
17654 FrameType::Push => {
17655 let ClientControlPush::RouteClosed {
17656 reason: r,
17657 module_id,
17658 ..
17659 } = serde_json::from_slice(&frame.body).unwrap()
17660 else {
17661 panic!("unexpected push");
17662 };
17663 assert_eq!(module_id, self.target);
17664 reason = Some(r);
17665 }
17666 other => panic!("unexpected frame {other:?}"),
17667 }
17668 }
17669 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
17670 reason.expect("the client is told why the route closed")
17671 }
17672
17673 fn untouched(&mut self) -> bool {
17674 self.client_rx.try_recv().is_err()
17675 }
17676
17677 fn stamp(&self) -> Option<ScopeStamp> {
17678 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
17679 ModuleControlRequest::RouteBind { scope, .. } => scope,
17680 other => panic!("expected a route.bind, got {other:?}"),
17681 }
17682 }
17683 }
17684
17685 #[tokio::test]
17686 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
17687 ) {
17688 let mut rig = rig().await;
17689 let mut record = session(1);
17690 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
17691 record.child_owners = vec![Principal::Reserved {
17692 module_id: MAGIC.to_string(),
17693 }];
17694 rig.sync(vec![record]).await;
17695 let scope = || Some(rig_selector("s", Some(1)));
17696
17697 rig.bound(Some(OWNER), PLEXUS, scope()).await;
17700 rig.bound(Some(AFT), OTHER, scope()).await;
17701 rig.bound(Some(BROCA), PLEXUS, scope()).await;
17702
17703 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
17707 assert_eq!(
17708 rig.refused(opener, target, scope()).await,
17709 error_codes::SCOPE_NOT_CARRIER,
17710 "{opener:?} -> {target}"
17711 );
17712 }
17713 }
17714
17715 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17716 ScopeSelector::new(
17717 Principal::Reserved {
17718 module_id: OWNER.to_string(),
17719 },
17720 scope_ref.to_string(),
17721 scope_epoch,
17722 )
17723 }
17724
17725 #[tokio::test]
17726 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
17727 let mut rig = rig().await;
17728 rig.sync(vec![session(1)]).await;
17729 for opener in [OWNER, AFT] {
17730 assert_eq!(
17731 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
17732 .await,
17733 error_codes::SCOPE_EPOCH_REQUIRED,
17734 "{opener}"
17735 );
17736 }
17737 }
17738
17739 #[tokio::test]
17740 async fn admission_separates_not_synced_not_live_and_ended() {
17741 let mut rig = rig().await;
17742 let code = rig
17744 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17745 .await;
17746 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
17747 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
17748
17749 let ghost = ScopeSelector::new(
17751 Principal::Reserved {
17752 module_id: "ghost".to_string(),
17753 },
17754 "s".to_string(),
17755 Some(1),
17756 );
17757 assert_eq!(
17758 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
17759 error_codes::SCOPE_NOT_LIVE
17760 );
17761
17762 rig.sync(vec![session(2)]).await;
17763 assert_eq!(
17764 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
17765 .await,
17766 error_codes::SCOPE_NOT_LIVE
17767 );
17768 for epoch in [1, 3] {
17769 assert_eq!(
17770 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
17771 .await,
17772 error_codes::SCOPE_ENDED,
17773 "epoch {epoch}"
17774 );
17775 }
17776 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
17778 .await;
17779 }
17780
17781 #[tokio::test]
17782 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
17783 let mut rig = rig().await;
17784 rig.sync(vec![session(1)]).await;
17785 let route = rig
17786 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17787 .await;
17788 let stamp = route.stamp().expect("a scoped bind carries the stamp");
17789 assert_eq!(stamp.scope_ref, "s");
17790 assert_eq!(stamp.scope_epoch, 1);
17791 assert_eq!(stamp.kind, ScopeKind::Head);
17792 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
17793 assert!(stamp.attributes.delegates);
17794 assert!(stamp.owner_authorized);
17795 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17796 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
17797
17798 let (broca, mut broca_rx) = wide_ctx(50);
17801 hello_via_sink(
17802 &rig.handler,
17803 &broca,
17804 &mut broca_rx,
17805 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17806 )
17807 .await;
17808 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17809 .await
17810 .unwrap();
17811 let own = ScopeSelector::new(
17812 Principal::Reserved {
17813 module_id: BROCA.to_string(),
17814 },
17815 "b".to_string(),
17816 Some(1),
17817 );
17818 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17819 assert!(!route.stamp().unwrap().owner_authorized);
17820 }
17821
17822 #[tokio::test]
17823 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17824 let mut rig = rig().await;
17825 let mut record = head("s", 1);
17826 record.carriers = vec![carrier(AFT, None)];
17827 let flow_id = "Flow:run-7/step_2!~";
17828 record.attributes.flow_id = Some(flow_id.to_string());
17829 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17830 .await
17831 .unwrap();
17832 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17833 panic!("not a sync reply");
17834 };
17835 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17836 let route = rig
17837 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17838 .await;
17839 assert!(rig.live(&route), "the stamped bind committed");
17840 let stamp = route.stamp().expect("a flow scope carries a stamp");
17841 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17842 assert_eq!(stamp.attributes.agent_id, None);
17843 assert!(!stamp.attributes.delegates);
17844 assert!(stamp.owner_authorized);
17845 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17846 assert_eq!(unscoped.stamp(), None);
17847 }
17848
17849 #[tokio::test]
17850 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17851 let mut rig = rig_with_flow_support(false).await;
17852 let mut record = session(1);
17853 record.attributes.flow_id = Some("flow:7".to_string());
17854 rig.sync(vec![record]).await;
17855 for opener in [OWNER, AFT] {
17856 let body = rig
17857 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17858 .await;
17859 assert_eq!(body["code"], "target_flow_unsupported");
17860 let message = body["message"].as_str().unwrap();
17861 for required in [PLEXUS, "flow-scopes/v1"] {
17862 assert!(message.contains(required), "{message}");
17863 }
17864 }
17865 }
17866
17867 #[tokio::test]
17868 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17869 let mut rig = rig_with_flow_support(true).await;
17870 let mut record = session(1);
17871 record.attributes.flow_id = Some("flow:7".to_string());
17872 rig.sync(vec![record]).await;
17873 for opener in [OWNER, AFT] {
17874 let route = rig
17875 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17876 .await;
17877 assert!(rig.live(&route));
17878 assert_eq!(
17879 route.stamp().unwrap().attributes.flow_id.as_deref(),
17880 Some("flow:7")
17881 );
17882 }
17883 }
17884
17885 #[tokio::test]
17886 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17887 use std::future::Future;
17888
17889 let mut rig = rig_with_flow_support(true).await;
17890 let mut record = session(1);
17891 record.attributes.flow_id = Some("flow:7".to_string());
17892 rig.sync(vec![record]).await;
17893 let (client, mut client_rx, frame) =
17894 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17895 for _ in 0..64 {
17898 client.egress.try_send(route_bind_ack(1)).unwrap();
17899 }
17900 let handler = rig.handler.clone();
17901 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17902 std::future::poll_fn(|cx| {
17903 assert!(open.as_mut().poll(cx).is_pending());
17904 std::task::Poll::Ready(())
17905 })
17906 .await;
17907 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17908
17909 let old_connection = rig.modules[PLEXUS].0.connection_id;
17910 rig.handler.cleanup_connection(old_connection).unwrap();
17911 let (replacement, mut replacement_rx) = wide_ctx(200);
17912 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17913 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17914 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17915 let hello = Frame::build(
17916 FrameType::Hello,
17917 control_flags(),
17918 0,
17919 0,
17920 200,
17921 serde_json::to_vec(&body).unwrap(),
17922 )
17923 .unwrap();
17924 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17925
17926 client_rx.try_recv().unwrap();
17927 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17928 .await
17929 .expect("the replacement is refused without waiting for a bind ack")
17930 .unwrap();
17931 assert_eq!(replies.len(), 1);
17932 let body = parse_error(&replies[0]);
17933 assert_eq!(body["code"], "target_flow_unsupported");
17934 for required in [PLEXUS, "flow-scopes/v1"] {
17935 assert!(body["message"].as_str().unwrap().contains(required));
17936 }
17937 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17938 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17939 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17940 assert!(rig
17941 .handler
17942 .registry
17943 .get_module_by_connection(replacement.connection_id)
17944 .unwrap()
17945 .is_some());
17946 }
17947
17948 #[tokio::test]
17949 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17950 use std::future::Future;
17951 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17952 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17953 const NEXT_HASH: &str =
17954 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17955 let mut rig = rig().await;
17956 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17957 record.carriers.push(ScopeCarrier::new(Principal::Direct));
17958 rig.sync(vec![record.clone()]).await;
17959 let (client, mut rx, frame) = rig.open_frame(
17960 None,
17961 PLEXUS,
17962 Some(
17963 rig.selector("s", Some(1))
17964 .with_bind_proof(Some(PROOF.into())),
17965 ),
17966 );
17967 for _ in 0..64 {
17970 client.egress.try_send(route_bind_ack(1)).unwrap();
17971 }
17972 let handler = rig.handler.clone();
17973 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17974 std::future::poll_fn(|cx| {
17975 assert!(open.as_mut().poll(cx).is_pending());
17976 std::task::Poll::Ready(())
17977 })
17978 .await;
17979 record.bind_proof_sha256 = Some(NEXT_HASH.into());
17980 rig.sync(vec![record]).await;
17981 rx.try_recv().unwrap();
17982 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17983 .await
17984 .unwrap()
17985 .unwrap();
17986 assert_eq!(
17987 parse_error(&replies[0])["code"],
17988 error_codes::SCOPE_BIND_PROOF_MISMATCH
17989 );
17990 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17991 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17992 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17993 }
17994
17995 #[tokio::test]
17996 async fn scope_bind_proof_reserved_openers_ignore_proof() {
17997 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17998 let mut rig = rig().await;
17999 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18000 .await;
18001 for opener in [OWNER, AFT] {
18002 for proof in [None, Some("malformed+proof=")] {
18003 let route = rig
18004 .bound(
18005 Some(opener),
18006 PLEXUS,
18007 Some(
18008 rig.selector("s", Some(1))
18009 .with_bind_proof(proof.map(str::to_string)),
18010 ),
18011 )
18012 .await;
18013 assert!(rig.live(&route));
18014 }
18015 }
18016 }
18017
18018 #[tokio::test]
18019 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
18020 use std::future::Future;
18021 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18022 const NEXT_HASH: &str =
18023 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18024 let mut rig = rig().await;
18025 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18026 .await;
18027 let (client, mut rx, frame) =
18028 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
18029 for _ in 0..64 {
18030 client.egress.try_send(route_bind_ack(1)).unwrap();
18031 }
18032 let handler = rig.handler.clone();
18033 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18034 std::future::poll_fn(|cx| {
18035 assert!(open.as_mut().poll(cx).is_pending());
18036 std::task::Poll::Ready(())
18037 })
18038 .await;
18039 rig.sync(vec![
18040 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
18041 ])
18042 .await;
18043 rx.try_recv().unwrap();
18044 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18045 let bind = std::future::poll_fn(|cx| {
18046 assert!(open.as_mut().poll(cx).is_pending());
18047 module_rx.poll_recv(cx)
18048 })
18049 .await
18050 .unwrap()
18051 .frame;
18052 handler
18053 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
18054 .await
18055 .unwrap();
18056 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18057 .await
18058 .unwrap()
18059 .unwrap();
18060 assert_eq!(replies.len(), 1);
18061 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
18064 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18065 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
18066 let route = rig
18067 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
18068 .await;
18069 assert!(rig.live(&route));
18070 }
18071
18072 #[tokio::test]
18073 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
18074 {
18075 let mut rig = rig_with_flow_support(false).await;
18076 rig.sync(vec![session(1)]).await;
18077 let route = rig
18078 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18079 .await;
18080 assert!(rig.live(&route));
18081 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
18082 let mut record = session(1);
18083 record.attributes.flow_id = Some("flow:7".to_string());
18084 rig.sync(vec![record]).await;
18085 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
18086 assert!(rig.live(&unscoped));
18087 assert_eq!(unscoped.stamp(), None);
18088 }
18089
18090 #[tokio::test]
18091 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
18092 let mut rig = rig().await;
18093 let mut record = session(1);
18094 record.attributes.flow_id = Some("flow:7".to_string());
18095 rig.sync(vec![record.clone()]).await;
18096 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18097 let mut owner_route = rig
18098 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18099 .await;
18100 let mut carrier_route = rig
18101 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
18102 .await;
18103 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18104 rig.sync(vec![record.clone()]).await;
18105 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18106 assert!(rig.live(&owner_route) && owner_route.untouched());
18107 assert!(rig.live(&carrier_route) && carrier_route.untouched());
18108
18109 record.attributes.flow_id = Some("flow:8".to_string());
18110 rig.sync(vec![record]).await;
18111 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
18112 let before = before.unwrap();
18113 assert_eq!(after.scope_epoch, before.scope_epoch);
18114 assert!(after.version > before.version);
18115 for route in [&mut owner_route, &mut carrier_route] {
18116 assert!(!rig.live(route));
18117 assert_eq!(
18118 route.closed_reason(),
18119 RouteCloseReason::ScopeDelegationChanged
18120 );
18121 }
18122 assert!(rig.live(&unscoped) && unscoped.untouched());
18123 for target in [PLEXUS, OTHER] {
18126 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
18127 let goodbye = module_rx
18128 .try_recv()
18129 .expect("the provider sees the drain")
18130 .frame;
18131 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18132 assert!(module_rx.try_recv().is_err());
18133 }
18134 let rebound = rig
18135 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18136 .await;
18137 assert_eq!(
18138 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
18139 Some("flow:8")
18140 );
18141 }
18142
18143 #[tokio::test]
18148 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
18149 let mut rig = rig().await;
18150 rig.sync(vec![session(1)]).await;
18151 let mut cotenant = rig
18152 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18153 .await;
18154
18155 let mut changed = session(1);
18156 changed.child_owners.push(Principal::Reserved {
18157 module_id: MAGIC.to_string(),
18158 });
18159 let mut ended = None;
18160 for (code, next) in [
18161 (error_codes::SCOPE_CHANGED, vec![changed]),
18162 (error_codes::SCOPE_ENDED, Vec::new()),
18163 ] {
18164 let relayed = rig
18165 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18166 .await;
18167 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
18168 ended = Some(next.is_empty());
18169 rig.sync(next).await;
18170 rig.ack(&relayed).await;
18171 let replies = relayed.task.await.unwrap();
18172 assert_eq!(replies.len(), 1, "{replies:?}");
18173 assert_eq!(parse_error(&replies[0])["code"], code);
18174 assert_eq!(
18175 subc_protocol::error_codes::is_retryable_route_open(code),
18176 code == error_codes::SCOPE_CHANGED
18177 );
18178 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18179 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18181 let mut goodbyes = Vec::new();
18184 while let Ok(outbound) = plexus_rx.try_recv() {
18185 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
18186 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
18187 }
18188 assert!(
18189 goodbyes.contains(&(bind_channel, bind_epoch)),
18190 "{goodbyes:?}"
18191 );
18192 assert!(rig
18193 .handler
18194 .registry
18195 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
18196 .unwrap()
18197 .is_some());
18198 }
18199 assert_eq!(ended, Some(true));
18200 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
18203 }
18204
18205 #[tokio::test]
18208 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
18209 struct Case {
18210 name: &'static str,
18211 change: fn(&mut ScopeRecord),
18212 closed: [Option<RouteCloseReason>; 4],
18215 }
18216 use RouteCloseReason::*;
18217 let cases = [
18218 Case {
18219 name: "a carrier entry removed",
18220 change: |r| {
18221 r.carriers.retain(|c| {
18222 c.principal
18223 != Principal::Reserved {
18224 module_id: AFT.to_string(),
18225 }
18226 })
18227 },
18228 closed: [None, Some(ScopeCarrierRemoved), None, None],
18229 },
18230 Case {
18231 name: "a target removed from a carrier",
18232 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
18233 closed: [None, None, None, Some(ScopeCarrierRemoved)],
18234 },
18235 Case {
18236 name: "a bare carrier narrowed to targets",
18237 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
18238 closed: [None, Some(ScopeCarrierRemoved), None, None],
18239 },
18240 Case {
18241 name: "delegates turned off",
18242 change: |r| r.attributes.delegates = false,
18243 closed: [Some(ScopeDelegationChanged); 4],
18244 },
18245 Case {
18246 name: "agent_id changed",
18247 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
18248 closed: [Some(ScopeDelegationChanged); 4],
18249 },
18250 Case {
18251 name: "a carrier added, child owners changed, the record re-sent",
18252 change: |r| {
18253 r.carriers.push(carrier(MAGIC, None));
18254 r.child_owners.push(Principal::Reserved {
18255 module_id: MAGIC.to_string(),
18256 });
18257 },
18258 closed: [None; 4],
18259 },
18260 Case {
18261 name: "a target added",
18262 change: |r| {
18263 r.carriers[1]
18264 .targets
18265 .as_mut()
18266 .unwrap()
18267 .push("third".to_string())
18268 },
18269 closed: [None; 4],
18270 },
18271 Case {
18272 name: "delegates turned on",
18273 change: |r| r.attributes.delegates = true,
18274 closed: [None; 4],
18275 },
18276 ];
18277 for case in cases {
18278 let mut rig = rig().await;
18279 rig.sync(vec![session(1)]).await;
18280 let scope = || Some(rig_selector("s", Some(1)));
18281 let mut routes = [
18282 rig.bound(Some(OWNER), PLEXUS, scope()).await,
18283 rig.bound(Some(AFT), PLEXUS, scope()).await,
18284 rig.bound(Some(BROCA), PLEXUS, scope()).await,
18285 rig.bound(Some(BROCA), OTHER, scope()).await,
18286 ];
18287 let mut record = session(1);
18288 (case.change)(&mut record);
18289 rig.sync(vec![record]).await;
18290 for (index, expected) in case.closed.iter().enumerate() {
18291 let route = &mut routes[index];
18292 match expected {
18293 Some(reason) => {
18294 assert!(!rig.live(route), "{}: route {index} still live", case.name);
18295 assert_eq!(
18296 route.closed_reason(),
18297 *reason,
18298 "{}: route {index}",
18299 case.name
18300 );
18301 }
18302 None => {
18303 assert!(rig.live(route), "{}: route {index} closed", case.name);
18304 assert!(
18305 route.untouched(),
18306 "{}: route {index} was told something",
18307 case.name
18308 );
18309 }
18310 }
18311 }
18312 }
18313 }
18314
18315 #[tokio::test]
18316 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
18317 for next in [Vec::new(), vec![session(2)]] {
18319 let mut rig = rig().await;
18320 rig.sync(vec![session(1)]).await;
18321 let mut route = rig
18322 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18323 .await;
18324 rig.sync(next).await;
18325 assert!(!rig.live(&route));
18326 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
18327 }
18328
18329 let mut rig = rig().await;
18332 let mut child = session(1);
18333 child.scope_ref = "child".to_string();
18334 child.kind = ScopeKind::Worker;
18335 child.parent = Some(ScopeParent::new(
18336 Principal::Reserved {
18337 module_id: OWNER.to_string(),
18338 },
18339 "s".to_string(),
18340 1,
18341 ));
18342 rig.sync(vec![session(1), child.clone()]).await;
18343 let mut child_route = rig
18344 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
18345 .await;
18346 assert_eq!(
18347 child_route.stamp().unwrap().parent_state,
18348 Some(ParentState::Linked)
18349 );
18350 rig.sync(vec![child]).await;
18351 assert!(!rig.live(&child_route));
18352 assert_eq!(
18353 child_route.closed_reason(),
18354 RouteCloseReason::ScopeParentEnded
18355 );
18356 }
18357
18358 #[tokio::test]
18359 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
18360 ) {
18361 let mut rig = rig().await;
18362 rig.sync(vec![session(1)]).await;
18363 let mut route = rig
18364 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18365 .await;
18366 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18367 rig.sync(vec![session(1)]).await;
18368 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18369 assert!(rig.live(&route) && route.untouched());
18370
18371 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
18375 .forwarding
18376 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18377 .unwrap()
18378 else {
18379 panic!("the route is bound");
18380 };
18381 binding.flow.acquire_tagged(9, false).await.unwrap();
18382 let mut widened = session(1);
18383 widened.carriers.push(carrier(MAGIC, None));
18384 rig.sync(vec![widened]).await;
18385 assert!(rig.live(&route) && route.untouched());
18386 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18387 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
18388 assert_eq!(binding.flow.in_flight(), 1);
18391 binding
18392 .flow
18393 .acquire_tagged(10, false)
18394 .await
18395 .expect("the flow is still open");
18396 }
18397
18398 #[tokio::test]
18401 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
18402 let mut rig = rig().await;
18403 rig.sync(vec![session(1)]).await;
18404 let mut on_incumbent = rig
18405 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18406 .await;
18407
18408 let (candidate, _candidate_rx) = wide_ctx(9);
18411 let registration = rig
18412 .handler
18413 .registry
18414 .register_candidate_with_control_ops(
18415 manifest(PLEXUS, PROTOCOL_VERSION),
18416 PROTOCOL_VERSION,
18417 candidate.connection_id,
18418 module_baseline_control_ops(),
18419 )
18420 .unwrap();
18421 rig.forwarding
18422 .register_candidate_module_connection(
18423 candidate.connection_id,
18424 PLEXUS.to_string(),
18425 PROTOCOL_VERSION,
18426 manifest_concurrency(®istration.manifest),
18427 candidate.egress.clone(),
18428 )
18429 .unwrap();
18430 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
18431 rig.handler
18432 .registry
18433 .promote_candidate(PLEXUS)
18434 .unwrap()
18435 .unwrap();
18436 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
18437
18438 rig.sync(Vec::new()).await;
18439 assert!(!rig.live(&on_incumbent));
18440 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
18441 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18442 let goodbye = incumbent_rx
18443 .try_recv()
18444 .expect("the superseded endpoint is told")
18445 .frame;
18446 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18447 }
18448 }
18449}
18450
18451#[cfg(test)]
18452mod concurrency_default_exposure_tests {
18453 use super::*;
18454
18455 fn hello_body(role_json: &str) -> Vec<u8> {
18456 format!(
18457 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
18458 )
18459 .into_bytes()
18460 }
18461
18462 fn manifest_from(body: &[u8]) -> ModuleManifest {
18463 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
18464 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
18465 .expect("manifest parses")
18466 }
18467
18468 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
18469
18470 #[test]
18471 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
18472 let body = hello_body(&format!(
18473 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
18474 ));
18475 let manifest = manifest_from(&body);
18476 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18479 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
18480 }
18481
18482 #[test]
18483 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
18484 let body = hello_body(&format!(
18485 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
18486 ));
18487 let manifest = manifest_from(&body);
18488 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18489 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18490 }
18491
18492 #[test]
18493 fn non_management_roles_are_never_reported() {
18494 let body = hello_body(
18495 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
18496 );
18497 let manifest = manifest_from(&body);
18498 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18499 }
18500}