1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6 fmt,
7 net::SocketAddr,
8 path::{Path, PathBuf},
9 sync::{Arc, Mutex, RwLock},
10 time::{Duration, Instant as StdInstant},
11};
12
13use serde::{Deserialize, Serialize};
14use subc_control::{
15 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
16 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
17 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
18 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
19 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
20 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
21 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
22};
23use subc_protocol::{
24 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
25 error_codes,
26 manifest::{
27 validate_hello_capability_grammar, validate_hello_confirm_operations,
28 validate_hello_display_name, validate_hello_event_declarations,
29 validate_hello_flow_operations, validate_hello_rich_ui_descriptor,
30 validate_hello_role_declarations, validate_hello_self_signal_declarations,
31 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
32 ProviderRole,
33 },
34 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
35 scope::{
36 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
37 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
38 SCOPE_SYNC_OP,
39 },
40 session::{
41 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
42 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
43 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
44 ROLE_VERSIONS_FIELD,
45 },
46 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
47 Priority, RouteTarget, PROTOCOL_VERSION,
48};
49use tokio::time::{timeout_at, Instant};
50use tracing::{debug, info, instrument::WithSubscriber, warn};
51
52use crate::{
53 caller_identity::CallerIdentity,
54 capability_requirements::{
55 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
56 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
57 RequirementStatus, RuntimeModule,
58 },
59 child_roster::DaemonShutdownFlag,
60 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
61 forwarding::{
62 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
63 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
64 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
65 },
66 observability::{
67 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
68 },
69 privileged_prompt::{prompt_text, PrivilegedOperation},
70 provenance::{
71 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
72 },
73 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
74 router::{daemon_error_flags, RouteCtx, RouterError},
75 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
76 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
77 stderr_tail::{CaptureState, TailEntry},
78 supervise::{
79 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
80 SupervisorHandle, SwapHelloAdmission,
81 },
82 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
83};
84
85pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
91
92const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
93const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
94const CAP_PING_PONG: &str = "ping_pong_v1";
95const CAP_SESSION_ATTACH: &str = "session_attach_v1";
96const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
97
98const SUBC_CONTROL_OPS: &[&str] = &[
99 ops::DAEMON_SHUTDOWN_INTENT,
100 ops::ACTIVITY_RECORD,
101 ops::ACTIVITY_QUERY,
102 ops::ACTIVITY_SUBSCRIBE,
103 ops::SERVER_DESCRIBE,
104 ops::CATALOG_LIST,
105 ops::ROUTE_OPEN,
106 ops::ROUTE_POLL,
107 ops::ROUTE_CLOSING,
108 ops::ROUTE_CLOSED,
109 ops::SUPERVISOR_LIST,
110 ops::SUPERVISOR_RESTART,
111 ops::SUPERVISOR_SWAP,
112 ops::SUPERVISOR_RELOAD,
113 ops::SUPERVISOR_RESCAN,
114 ops::SUPERVISOR_RELEASE_RESERVED,
115 ops::SUPERVISOR_SET_ENABLED,
116 ops::SUPERVISOR_HEALTH_PROBE,
117 ops::SUPERVISOR_HEALTH,
118 ops::SUPERVISOR_STDERR_TAIL,
119 ops::SUPERVISOR_TERMINALS,
120 ops::SUPERVISOR_ROUTES,
121 ops::SUPERVISOR_PROVENANCE,
122 ops::SUPERVISOR_SPAWN_SNAPSHOT,
123 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
124];
125
126const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
133
134mod agent_commands;
135mod rescan_checked;
136use rescan_checked::{RescanMode, RescanPlan};
137
138const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
139 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
140 "supervisor.live_roots",
141 "supervisor.agent_commands",
142 SCOPE_SYNC_OP,
143 SCOPE_APPLY_OP,
144 SCOPE_DESCRIBE_OP,
145 "operator.confirm",
146];
147
148const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
152
153const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
154
155pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
163
164pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
179
180pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
198
199const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
200const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
201
202fn reload_verdict(
203 configured: &Path,
204 spawned_from: Option<&Path>,
205 image: subc_control::RunningImageAgreement,
206) -> PendingReloadVerdict {
207 let path = match spawned_from {
208 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
209 ReloadPathAgreement::Match
210 }
211 Some(spawned_from) => ReloadPathAgreement::Mismatch {
212 configured: configured.to_path_buf(),
213 spawned_from: spawned_from.to_path_buf(),
214 },
215 None => ReloadPathAgreement::Unavailable {
216 reason: if matches!(
217 image,
218 subc_control::RunningImageAgreement::Unavailable {
219 reason: subc_control::RunningImageUnavailableReason::NotRunning
220 }
221 ) {
222 ReloadPathUnavailableReason::NotRunning
223 } else {
224 ReloadPathUnavailableReason::SpawnedPathUnavailable
225 },
226 },
227 };
228 PendingReloadVerdict {
229 path,
230 image,
231 spec: None,
232 }
233}
234
235fn launch_spec_agreement(
236 configured: &crate::supervise::ModuleSpec,
237 running: bool,
238 spawned: Option<&crate::supervise::LaunchSpecDigest>,
239) -> subc_control::LaunchSpecAgreement {
240 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
241 if !running {
242 return Agreement::Unavailable {
243 reason: Reason::NotRunning,
244 };
245 }
246 match spawned {
247 None => Agreement::Unavailable {
248 reason: Reason::SpawnedSpecUnavailable,
249 },
250 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
251 Agreement::Match
252 }
253 Some(_) => Agreement::Mismatch,
254 }
255}
256
257fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
258 if configured == spawned_from {
259 return true;
260 }
261 #[cfg(windows)]
262 {
263 match (
269 std::fs::canonicalize(configured),
270 std::fs::canonicalize(spawned_from),
271 ) {
272 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
273 _ => false,
274 }
275 }
276 #[cfg(not(windows))]
277 {
278 false
279 }
280}
281
282#[derive(Clone)]
283struct DaemonProvenanceFacts {
284 build: DaemonBuildProvenance,
285 pid: Option<u32>,
286 started_at_ms: Option<u64>,
287 start_clock: Option<crate::clock::StartClock>,
288 executable_path: Option<PathBuf>,
289 executable_identity: Option<SpawnedFileIdentity>,
290 process_start_time: Option<u64>,
291 probe: ExecutableIdentityProbe,
292}
293
294impl Default for DaemonProvenanceFacts {
295 fn default() -> Self {
296 Self {
297 build: DaemonBuildProvenance {
298 build_git_sha: None,
299 build_lock_digest: None,
300 },
301 pid: None,
302 started_at_ms: None,
303 start_clock: None,
304 executable_path: None,
305 executable_identity: None,
306 process_start_time: None,
307 probe: ExecutableIdentityProbe::default(),
308 }
309 }
310}
311
312#[derive(Debug, Clone)]
313struct SupervisorRescanContext {
314 supervisor: Supervisor,
315 config_path: PathBuf,
316 configured_port: Option<u16>,
317 storage_config: Option<crate::daemon_config::StorageConfig>,
318 admission_facts_carrier_module_id: Option<String>,
319 admission_facts_targets: Option<Vec<String>>,
320 scope_authority_owners: Vec<String>,
321 federated_scope_owner: String,
322 privileged_operator_prompt: PrivilegedOperatorPrompt,
323 generation: Arc<std::sync::atomic::AtomicU64>,
329}
330
331const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
352 "reloading",
353 "supervisor_not_live",
354 "registration_not_active",
355 "no_forwarding_connection",
356 "relay_send_failed",
357];
358
359#[derive(Clone)]
361pub struct ControlHandler {
362 registry: Arc<Registry>,
363 activity: Option<crate::activity::ActivityLedger>,
364 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
365 forwarding: Arc<ForwardingTable>,
366 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
367 supervisor: SupervisorHandle,
368 pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
369 agent_declarations: agent_commands::Queues,
370 list_admission: Arc<tokio::sync::Semaphore>,
371 subc_capabilities: Arc<[String]>,
372 route_bind_relay_timeout: Duration,
376 route_bind_relay_timeouts: BTreeMap<String, Duration>,
380 route_bind_breakers: RouteBindBreakers,
383 route_bind_concurrency: RouteBindConcurrency,
386 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
391 route_bind_breaker_threshold: u32,
393 route_bind_breaker_cooldown: Duration,
395 health_probe_timeout: Duration,
396 storage_config: Option<crate::daemon_config::StorageConfig>,
399 machine_id: Option<crate::machine_id::MachineId>,
403 admission_facts_carrier_module_id: Option<String>,
404 admission_facts_targets: Option<Vec<String>>,
405 scopes: Arc<RwLock<ScopeTable>>,
409 scope_deadline_added: Arc<tokio::sync::Notify>,
413 scope_authority_owners: Vec<String>,
416 federated_scope_owner: String,
417 privileged_operator_prompt: PrivilegedOperatorPrompt,
419 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
422 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
423 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
424 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
425 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
426 pub(crate) operator_apps:
427 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
428 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
433 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
439 #[cfg(test)]
441 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
442 #[cfg(test)]
443 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
444 #[allow(dead_code)]
448 #[cfg(windows)]
449 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
450 #[allow(dead_code)]
451 #[cfg(windows)]
452 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
453 #[allow(dead_code)]
457 #[cfg(windows)]
458 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
459 #[cfg(all(test, windows))]
462 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
463 #[cfg(test)]
464 caller_lookup: Option<TestCallerLookup>,
465 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
468 rescan: Option<SupervisorRescanContext>,
469 connected_clients: ConnectedClients,
470 counters: DaemonCounters,
471 local_endpoint_unavailable: Option<String>,
474 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
475 daemon_shutdown: DaemonShutdownFlag,
480 daemon_provenance: DaemonProvenanceFacts,
481 #[cfg(test)]
482 control_dispatch_delay: Option<Duration>,
483 #[cfg(test)]
484 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
485 #[cfg(test)]
489 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
490 #[cfg(test)]
496 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
497}
498
499impl fmt::Debug for ControlHandler {
500 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
501 f.debug_struct("ControlHandler")
502 .field("registry", &self.registry)
503 .field("forwarding", &self.forwarding)
504 .field("process_liveness", &self.process_liveness.is_some())
505 .field("supervisor", &self.supervisor)
506 .field("subc_capabilities", &self.subc_capabilities)
507 .finish()
508 }
509}
510
511struct RouteOpenRequest {
512 target: RouteTarget,
513 identity: BindIdentity,
514 consumer_identity: Option<ConsumerIdentity>,
515 consumer_capabilities: Option<Vec<String>>,
516 role_versions: Option<BTreeMap<String, String>>,
517 busy_refusals: Option<bool>,
518 admission_facts: Option<serde_json::Value>,
519 scope: Option<ScopeSelector>,
520}
521
522#[derive(Clone, Copy)]
523enum RouteOpenPhase {
524 Admission,
525 ProjectRoot,
526 EgressReservation,
527 BindSend,
528 BindAckWait,
529}
530
531struct RouteOpenTimings {
535 module_id: String,
536 principal: String,
537 admission: Duration,
538 project_root: Duration,
539 egress_reservation: Duration,
540 bind_send: Duration,
541 bind_ack_wait: Duration,
542 phase: Option<RouteOpenPhase>,
543 phase_started_at: Instant,
544}
545
546impl RouteOpenTimings {
547 fn new() -> Self {
548 Self {
549 module_id: String::new(),
550 principal: "unresolved".to_string(),
552 admission: Duration::ZERO,
553 project_root: Duration::ZERO,
554 egress_reservation: Duration::ZERO,
555 bind_send: Duration::ZERO,
556 bind_ack_wait: Duration::ZERO,
557 phase: Some(RouteOpenPhase::Admission),
558 phase_started_at: Instant::now(),
559 }
560 }
561
562 fn finish(&mut self) {
563 let elapsed = self.phase_started_at.elapsed();
564 match self.phase.take() {
565 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
566 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
567 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
568 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
569 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
570 None => {}
571 }
572 }
573
574 fn enter(&mut self, phase: RouteOpenPhase) {
575 self.finish();
576 self.phase = Some(phase);
577 self.phase_started_at = Instant::now();
578 }
579}
580
581struct RouteBindReservationGuard {
582 forwarding: Arc<ForwardingTable>,
583 endpoint: ModuleEndpointId,
584 relay_corr: u64,
585 armed: bool,
586}
587
588struct ModuleControlRpcGuard {
589 forwarding: Arc<ForwardingTable>,
590 endpoint: ModuleEndpointId,
591 corr: u64,
592 armed: bool,
593}
594
595impl ModuleControlRpcGuard {
596 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
597 Self {
598 forwarding,
599 endpoint,
600 corr,
601 armed: true,
602 }
603 }
604
605 fn disarm(&mut self) {
606 self.armed = false;
607 }
608}
609
610impl Drop for ModuleControlRpcGuard {
611 fn drop(&mut self) {
612 if self.armed {
613 let _ = self
614 .forwarding
615 .cancel_module_control_rpc(self.endpoint, self.corr);
616 }
617 }
618}
619
620impl RouteBindReservationGuard {
621 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
622 Self {
623 forwarding,
624 endpoint,
625 relay_corr,
626 armed: true,
627 }
628 }
629
630 fn release_and_disarm(&mut self) {
631 if !self.armed {
632 return;
633 }
634 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
635 self.endpoint,
636 self.relay_corr,
637 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
638 ) {
639 send_goodbye_target_best_effort(
640 &self.forwarding.counters(),
641 &target,
642 "canceled route.bind",
643 );
644 }
645 self.armed = false;
646 }
647
648 fn disarm(&mut self) {
649 self.armed = false;
650 }
651}
652
653impl Drop for RouteBindReservationGuard {
654 fn drop(&mut self) {
655 self.release_and_disarm();
656 }
657}
658
659#[derive(Debug, Clone, Default)]
689pub(crate) struct RouteBindBreakers {
690 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
691}
692
693#[derive(Debug, Clone, Default)]
694pub(crate) struct RouteBindConcurrency {
695 modules: Arc<Mutex<HashMap<String, usize>>>,
696}
697
698struct RouteBindConcurrencyGuard {
699 concurrency: RouteBindConcurrency,
700 module_id: String,
701}
702
703impl RouteBindConcurrency {
704 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
707 let mut modules = self
708 .modules
709 .lock()
710 .expect("route.bind concurrency mutex poisoned");
711 let in_flight = modules.entry(module_id.to_string()).or_default();
712 if *in_flight >= limit {
713 return Err(*in_flight);
714 }
715 *in_flight += 1;
716 Ok(RouteBindConcurrencyGuard {
717 concurrency: self.clone(),
718 module_id: module_id.to_string(),
719 })
720 }
721}
722
723impl Drop for RouteBindConcurrencyGuard {
724 fn drop(&mut self) {
725 let mut modules = self
726 .concurrency
727 .modules
728 .lock()
729 .expect("route.bind concurrency mutex poisoned");
730 let remove = {
731 let in_flight = modules
732 .get_mut(&self.module_id)
733 .expect("admitted route.bind has a concurrency entry");
734 *in_flight -= 1;
735 *in_flight == 0
736 };
737 if remove {
738 modules.remove(&self.module_id);
739 }
740 }
741}
742
743#[derive(Debug, Default)]
744struct ModuleBreakerState {
745 consecutive_timeouts: u32,
747 cooldown_until: Option<Instant>,
750 probe_in_flight: Option<Arc<()>>,
755}
756
757enum RouteBindAdmission<'a> {
759 Admitted {
760 guard: RouteBindBreakerGuard<'a>,
761 probe: bool,
764 },
765 Refused {
766 consecutive_timeouts: u32,
767 retry_in: Duration,
771 probe_in_flight: bool,
772 },
773}
774
775struct RouteBindBreakerGuard<'a> {
782 breakers: RouteBindBreakers,
783 module_id: &'a str,
784 probe_token: Option<Arc<()>>,
785 settled: bool,
786}
787
788impl RouteBindBreakerGuard<'_> {
789 fn record_accepted(&mut self) -> bool {
793 self.settled = true;
794 self.breakers.record_accepted(self.module_id)
795 }
796
797 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
800 self.settled = true;
801 self.breakers.record_timeout(
802 self.module_id,
803 self.probe_token.as_ref(),
804 threshold,
805 cooldown,
806 )
807 }
808
809 fn record_inconclusive(&mut self) {
818 self.settled = true;
819 self.breakers
820 .record_inconclusive(self.module_id, self.probe_token.as_ref());
821 }
822}
823
824impl Drop for RouteBindBreakerGuard<'_> {
825 fn drop(&mut self) {
826 if !self.settled {
827 self.breakers
828 .record_inconclusive(self.module_id, self.probe_token.as_ref());
829 }
830 }
831}
832
833struct BreakerOpened {
837 consecutive_timeouts: u32,
838 reopened_after_probe: bool,
841}
842
843impl RouteBindBreakers {
844 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
845 self.modules
846 .lock()
847 .expect("route.bind breaker mutex poisoned")
848 }
849
850 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
853 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
854 probe: probe_token.is_some(),
855 guard: RouteBindBreakerGuard {
856 breakers: self.clone(),
857 module_id,
858 probe_token,
859 settled: false,
860 },
861 };
862
863 let mut modules = self.lock();
864 let Some(state) = modules.get_mut(module_id) else {
865 return admitted(None);
866 };
867 let Some(cooldown_until) = state.cooldown_until else {
868 return admitted(None);
869 };
870 if state.probe_in_flight.is_some() {
871 return RouteBindAdmission::Refused {
872 consecutive_timeouts: state.consecutive_timeouts,
873 retry_in: Duration::ZERO,
874 probe_in_flight: true,
875 };
876 }
877 let now = Instant::now();
878 if now < cooldown_until {
879 return RouteBindAdmission::Refused {
880 consecutive_timeouts: state.consecutive_timeouts,
881 retry_in: cooldown_until - now,
882 probe_in_flight: false,
883 };
884 }
885 let token = Arc::new(());
886 state.probe_in_flight = Some(Arc::clone(&token));
887 admitted(Some(token))
888 }
889
890 fn record_accepted(&self, module_id: &str) -> bool {
891 self.lock()
892 .remove(module_id)
893 .is_some_and(|state| state.cooldown_until.is_some())
894 }
895
896 fn record_timeout(
897 &self,
898 module_id: &str,
899 probe_token: Option<&Arc<()>>,
900 threshold: u32,
901 cooldown: Duration,
902 ) -> Option<BreakerOpened> {
903 let mut modules = self.lock();
904 let state = modules.entry(module_id.to_string()).or_default();
905 let was_open = state.cooldown_until.is_some();
906 let was_probe = Self::owns_probe(state, probe_token);
907 if was_probe {
908 state.probe_in_flight = None;
909 }
910 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
911 if state.consecutive_timeouts < threshold {
912 return None;
913 }
914 state.cooldown_until = Some(Instant::now() + cooldown);
915 Some(BreakerOpened {
916 consecutive_timeouts: state.consecutive_timeouts,
917 reopened_after_probe: was_open && was_probe,
918 })
919 }
920
921 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
922 state
926 .probe_in_flight
927 .as_ref()
928 .zip(token)
929 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
930 }
931
932 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
933 if let Some(state) = self.lock().get_mut(module_id) {
934 if Self::owns_probe(state, probe_token) {
935 state.probe_in_flight = None;
936 }
937 }
938 }
939
940 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
955 self.lock()
956 .remove(module_id)
957 .map(|state| state.consecutive_timeouts)
958 .filter(|discarded| *discarded > 0)
959 }
960
961 fn open_snapshot(&self) -> Option<serde_json::Value> {
968 let now = Instant::now();
969 let modules = self.lock();
970 let open = modules
971 .iter()
972 .filter_map(|(module_id, state)| {
973 let cooldown_until = state.cooldown_until?;
974 Some((
975 module_id.clone(),
976 serde_json::json!({
977 "consecutive_timeouts": state.consecutive_timeouts,
978 "cooldown_remaining_ms":
979 cooldown_until.saturating_duration_since(now).as_millis() as u64,
980 "probe_in_flight": state.probe_in_flight.is_some(),
981 }),
982 ))
983 })
984 .collect::<serde_json::Map<String, serde_json::Value>>();
985 (!open.is_empty()).then_some(serde_json::Value::Object(open))
986 }
987}
988
989impl ControlHandler {
990 pub fn new(registry: Arc<Registry>) -> Self {
991 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
992 }
993
994 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
995 let counters = forwarding.counters();
996 let route_bind_breakers = forwarding.route_bind_breakers();
1000 let route_bind_concurrency = forwarding.route_bind_concurrency();
1001 let route_outages = forwarding.route_outages();
1002 Self {
1003 registry,
1004 activity: None,
1005 activity_streams: Arc::new(Mutex::new(HashMap::new())),
1006 forwarding,
1007 process_liveness: None,
1008 supervisor: SupervisorHandle::new(),
1009 resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1010 env!("CARGO_PKG_VERSION").to_owned(),
1011 )),
1012 agent_declarations: agent_commands::Queues::default(),
1013 list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1014 subc_capabilities: Arc::from([
1015 CAP_MANIFEST_REGISTRATION.to_string(),
1016 CAP_CHANNEL_LIFECYCLE.to_string(),
1017 CAP_PING_PONG.to_string(),
1018 CAP_SESSION_ATTACH.to_string(),
1019 CAP_ADMISSION_FACTS_RELAY.to_string(),
1020 CAP_SCOPES_V1.to_string(),
1021 subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1022 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1023 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1024 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1025 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1026 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1027 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1028 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1029 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1030 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1031 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1032 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1033 ]),
1034 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1035 route_bind_relay_timeouts: BTreeMap::new(),
1036 route_bind_breakers,
1037 route_bind_concurrency,
1038 route_outages,
1039 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1040 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1041 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1042 storage_config: None,
1043 machine_id: None,
1044 admission_facts_carrier_module_id: None,
1045 admission_facts_targets: None,
1046 scopes: Arc::new(RwLock::new(ScopeTable::new(
1047 crate::daemon_config::default_scope_authority_owners(),
1048 ))),
1049 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1050 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1051 federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1052 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1053 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1054 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1055 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1056 local_connections: Arc::new(Mutex::new(HashSet::new())),
1057 operator_app_policy: Default::default(),
1058 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1059 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1060 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1061 #[cfg(test)]
1062 app_open_test_platform: None,
1063 #[cfg(test)]
1064 app_open_test_launch: None,
1065 #[cfg(windows)]
1066 windows_verify: None,
1067 #[cfg(windows)]
1068 windows_flash: None,
1069 #[cfg(windows)]
1070 windows_track: None,
1071 #[cfg(all(test, windows))]
1072 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1073 #[cfg(test)]
1074 caller_lookup: None,
1075 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1076 rescan: None,
1077 connected_clients: ConnectedClients::new(),
1078 local_endpoint_unavailable: None,
1079 counters,
1080 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1081 daemon_shutdown: DaemonShutdownFlag::default(),
1082 daemon_provenance: DaemonProvenanceFacts::default(),
1083 #[cfg(test)]
1084 control_dispatch_delay: None,
1085 #[cfg(test)]
1086 provenance_probe_override: None,
1087 #[cfg(test)]
1088 route_bind_relay_sent_hold: None,
1089 #[cfg(test)]
1090 hello_registry_to_forwarding_hold: None,
1091 }
1092 }
1093
1094 pub fn with_storage_config(
1097 mut self,
1098 storage_config: Option<crate::daemon_config::StorageConfig>,
1099 ) -> Self {
1100 self.storage_config = storage_config;
1101 self
1102 }
1103
1104 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1107 self.activity = Some(crate::activity::ActivityLedger::open(
1108 run_dir.join("activity.jsonl"),
1109 incarnation,
1110 ));
1111 self.supervisor.set_activity(self.activity.clone());
1112 self
1113 }
1114
1115 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1118 self.machine_id = machine_id;
1119 self
1120 }
1121
1122 pub fn with_admission_facts_config(
1126 mut self,
1127 carrier_module_id: Option<String>,
1128 targets: Option<Vec<String>>,
1129 ) -> Self {
1130 self.admission_facts_carrier_module_id = carrier_module_id;
1131 self.admission_facts_targets = targets;
1132 self
1133 }
1134
1135 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1139 self.scopes = Arc::new(RwLock::new(
1140 ScopeTable::new(owners.iter().cloned())
1141 .with_federated_owner(self.federated_scope_owner.clone()),
1142 ));
1143 self.scope_authority_owners = owners;
1144 self
1145 }
1146
1147 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1149 self.scopes = Arc::new(RwLock::new(
1150 ScopeTable::new(self.scope_authority_owners.iter().cloned())
1151 .with_federated_owner(owner.clone()),
1152 ));
1153 self.federated_scope_owner = owner;
1154 self
1155 }
1156
1157 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1159 self.privileged_operator_prompt = mode;
1160 self
1161 }
1162
1163 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1166 self.route_bind_relay_timeout = timeout;
1167 self
1168 }
1169
1170 pub fn with_route_bind_relay_timeouts(
1176 mut self,
1177 timeouts: impl IntoIterator<Item = (String, Duration)>,
1178 ) -> Self {
1179 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1180 self
1181 }
1182
1183 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1189 self.route_bind_relay_timeouts
1190 .get(module_id)
1191 .copied()
1192 .unwrap_or(self.route_bind_relay_timeout)
1193 }
1194
1195 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1203 self.route_bind_breaker_threshold = threshold.max(1);
1204 self.route_bind_breaker_cooldown = cooldown;
1205 self
1206 }
1207
1208 #[cfg(test)]
1209 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1210 self.health_probe_timeout = timeout;
1211 self
1212 }
1213
1214 #[cfg(test)]
1215 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1216 self.control_dispatch_delay = Some(delay);
1217 self
1218 }
1219
1220 #[cfg(test)]
1224 pub(crate) fn with_route_bind_relay_sent_hold(
1225 mut self,
1226 hold: Arc<tokio::sync::Barrier>,
1227 ) -> Self {
1228 self.route_bind_relay_sent_hold = Some(hold);
1229 self
1230 }
1231
1232 #[cfg(test)]
1237 pub(crate) fn with_hello_registry_to_forwarding_hold(
1238 mut self,
1239 hold: Arc<std::sync::Barrier>,
1240 ) -> Self {
1241 self.hello_registry_to_forwarding_hold = Some(hold);
1242 self
1243 }
1244
1245 #[cfg(test)]
1246 fn hold_between_registry_and_forwarding(&self) {
1247 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1248 hold.wait();
1249 hold.wait();
1250 }
1251 }
1252
1253 pub fn with_process_liveness(
1254 mut self,
1255 process_liveness: Arc<dyn ModuleProcessLiveness>,
1256 ) -> Self {
1257 self.process_liveness = Some(process_liveness);
1258 self
1259 }
1260
1261 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1262 self.supervisor = supervisor;
1263 self.supervisor.set_activity(self.activity.clone());
1264 self
1265 }
1266
1267 pub fn with_daemon_provenance(
1268 mut self,
1269 pid: u32,
1270 started_at_ms: u64,
1271 executable_path: Option<PathBuf>,
1272 build_git_sha: Option<String>,
1273 build_lock_digest: Option<String>,
1274 ) -> Self {
1275 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1276 let process_start_time = process_start_time(pid);
1277 self.daemon_provenance = DaemonProvenanceFacts {
1278 build: DaemonBuildProvenance {
1279 build_git_sha,
1280 build_lock_digest,
1281 },
1282 pid: Some(pid),
1283 started_at_ms: Some(started_at_ms),
1284 start_clock: None,
1285 executable_path,
1286 executable_identity,
1287 process_start_time,
1288 probe: ExecutableIdentityProbe::default(),
1289 };
1290 self
1291 }
1292
1293 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1294 self.daemon_provenance.start_clock = Some(clock);
1295 self
1296 }
1297
1298 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1300 self.scopes
1301 .write()
1302 .unwrap_or_else(|poisoned| poisoned.into_inner())
1303 .set_wall_clock(Arc::new(clock));
1304 self
1305 }
1306
1307 #[cfg(test)]
1308 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1309 self.provenance_probe_override = Some(result);
1310 self
1311 }
1312
1313 pub fn with_capability_config(
1317 self,
1318 modules: impl IntoIterator<Item = (String, bool)>,
1319 reserved_capabilities: BTreeMap<String, String>,
1320 ) -> Self {
1321 self.capability_evaluator
1322 .configure(modules, reserved_capabilities);
1323 self
1324 }
1325
1326 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1330 self.daemon_shutdown = flag;
1331 self
1332 }
1333
1334 pub fn with_supervisor_rescan(
1335 mut self,
1336 supervisor: Supervisor,
1337 config_path: impl Into<PathBuf>,
1338 configured_port: Option<u16>,
1339 ) -> Self {
1340 self.rescan = Some(SupervisorRescanContext {
1341 supervisor,
1342 config_path: config_path.into(),
1343 configured_port,
1344 storage_config: self.storage_config.clone(),
1345 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1346 admission_facts_targets: self.admission_facts_targets.clone(),
1347 scope_authority_owners: self.scope_authority_owners.clone(),
1348 federated_scope_owner: self.federated_scope_owner.clone(),
1349 privileged_operator_prompt: self.privileged_operator_prompt,
1350 generation: Arc::default(),
1351 });
1352 self
1353 }
1354
1355 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1356 self.connected_clients = connected_clients;
1357 self
1358 }
1359
1360 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1363 self.local_endpoint_unavailable = reason;
1364 self
1365 }
1366
1367 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1368 Arc::clone(&self.forwarding)
1369 }
1370
1371 pub(crate) fn counters(&self) -> DaemonCounters {
1372 self.counters.clone()
1373 }
1374
1375 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1378 tokio::spawn(async move {
1379 loop {
1380 self.capability_evaluator
1381 .wait_for_change_or_deadline()
1382 .await;
1383 self.refresh_capability_requirements();
1384 }
1385 });
1386 }
1387
1388 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1394 tokio::spawn(async move {
1395 loop {
1396 let has_deadlines = self
1397 .scopes
1398 .read()
1399 .unwrap_or_else(|poisoned| poisoned.into_inner())
1400 .has_deadlines();
1401 if !has_deadlines {
1402 self.scope_deadline_added.notified().await;
1403 continue;
1404 }
1405 tokio::time::sleep(Duration::from_secs(1)).await;
1406 if let Err(error) = self.sweep_expired_scopes() {
1407 warn!(%error, "scope expiry sweep failed");
1408 }
1409 }
1410 });
1411 }
1412
1413 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1414 let mut table = self
1415 .scopes
1416 .write()
1417 .unwrap_or_else(|poisoned| poisoned.into_inner());
1418 let swept = table.sweep_expired();
1419 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1420 drop(table);
1421 self.close_scope_drained_routes(drained);
1422 Ok(())
1423 }
1424
1425 fn publish_scope_changes(
1426 &self,
1427 changes: &[crate::scopes::ScopeTagChange],
1428 expired: &[crate::scopes::ScopeExpired],
1429 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1430 if changes.is_empty() {
1431 return Ok(Vec::new());
1432 }
1433 let drained = self
1434 .forwarding
1435 .publish_scope_changes(changes)
1436 .map_err(RouterError::Forwarding)?;
1437 let mut counts = HashMap::new();
1438 for route in &drained {
1439 *counts
1440 .entry((
1441 route.scope.owner.as_str(),
1442 route.scope.scope_ref.as_str(),
1443 route.scope.tag.scope_epoch,
1444 ))
1445 .or_insert(0usize) += 1;
1446 }
1447 for scope in expired {
1448 let routes_closed = counts
1449 .get(&(
1450 scope.owner.as_str(),
1451 scope.scope_ref.as_str(),
1452 scope.scope_epoch,
1453 ))
1454 .copied()
1455 .unwrap_or(0);
1456 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1457 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1458 routes_closed, "scope expired");
1459 }
1460 Ok(drained)
1461 }
1462
1463 fn runtime_capability_snapshot(
1464 &self,
1465 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1466 let runtime = self
1467 .supervisor
1468 .list()
1469 .into_iter()
1470 .map(|module| {
1471 let status = module.status().map_err(|err| {
1472 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1473 })?;
1474 Ok(RuntimeModule {
1475 module_id: status.module_id,
1476 state: status.state,
1477 enabled: status.enabled,
1478 })
1479 })
1480 .collect::<Result<Vec<_>, RouterError>>()?;
1481 let registrations = self.registry.list_registered_modules().map_err(|err| {
1487 RouterError::backend(
1488 0,
1489 0,
1490 format!("failed to list capability registrations: {err}"),
1491 )
1492 })?;
1493 let registrations = registrations
1494 .into_iter()
1495 .map(|registration| RegisteredModule {
1496 module_id: registration.manifest.module_id,
1497 module_version: registration.manifest.module_version,
1498 capabilities: registration.manifest.capabilities,
1499 })
1500 .collect();
1501 Ok((runtime, registrations))
1502 }
1503
1504 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1511 let cached_registration = RegisteredModule {
1512 module_id: registration.manifest.module_id.clone(),
1513 module_version: registration.manifest.module_version.clone(),
1514 capabilities: registration.manifest.capabilities.clone(),
1515 };
1516 let change = self.capability_evaluator.record_hello(&cached_registration);
1517 log_manifest_change(&cached_registration.module_id, &change);
1518 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1519 self.enforce_capability_denies();
1520 }
1521 self.refresh_capability_requirements();
1522 }
1523
1524 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1528 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1529 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1530 self.supervisor.set_swap_promotion_observer(observer);
1531 }
1532
1533 pub fn refresh_capability_requirements(&self) {
1534 match self.runtime_capability_snapshot() {
1535 Ok((runtime, registrations)) => {
1536 log_requirement_events(
1537 self.capability_evaluator
1538 .evaluate_now(&runtime, ®istrations),
1539 &self.daemon_shutdown,
1540 );
1541 }
1542 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1543 }
1544 }
1545
1546 fn enforce_capability_denies(&self) {
1550 let registrations = match self.registry.list_registered_modules() {
1551 Ok(snapshot) => snapshot,
1552 Err(err) => {
1553 warn!(error = %err, "failed to read registrations for capability deny census");
1554 return;
1555 }
1556 };
1557 let manifests = registrations
1558 .into_iter()
1559 .map(|registration| {
1560 (
1561 registration.manifest.module_id.clone(),
1562 registration.manifest,
1563 )
1564 })
1565 .collect::<BTreeMap<_, _>>();
1566 let census = match self.forwarding.route_census(None) {
1567 Ok(census) => census,
1568 Err(err) => {
1569 warn!(error = %err, "failed to read route census for capability deny enforcement");
1570 return;
1571 }
1572 };
1573
1574 for (target_module_id, routes) in census {
1575 let Some(target_manifest) = manifests.get(&target_module_id) else {
1576 continue;
1577 };
1578 let mut closed_routes = Vec::new();
1579 let mut goodbyes = Vec::new();
1580 for route in routes {
1581 let Principal::Reserved {
1582 module_id: opening_module_id,
1583 } = &route.principal
1584 else {
1585 continue;
1586 };
1587 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1588 continue;
1589 };
1590 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1591 continue;
1592 };
1593
1594 match self.forwarding.release_client_route(
1595 route.goodbye_target.connection_id,
1596 route.goodbye_target.channel,
1597 route.goodbye_target.epoch,
1598 ) {
1599 Ok(RouteRelease::Removed(module_goodbye)) => {
1600 warn!(
1601 opening_module_id,
1602 target_module_id,
1603 capability,
1604 "force-closing route because an attested capability deny edge now matches"
1605 );
1606 goodbyes.push(module_goodbye);
1607 goodbyes.push(route.goodbye_target.clone());
1608 closed_routes.push(route);
1609 }
1610 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1611 Err(err) => warn!(
1612 opening_module_id,
1613 target_module_id,
1614 capability,
1615 error = %err,
1616 "failed to force-close capability-denied route"
1617 ),
1618 }
1619 }
1620
1621 if closed_routes.is_empty() {
1622 continue;
1623 }
1624 send_route_control_pushes(
1625 &self.forwarding,
1626 closed_routes,
1627 ClientControlPush::RouteClosed {
1628 module_id: target_module_id,
1629 channels: Vec::new(),
1630 reason: RouteCloseReason::CapabilityDenied,
1631 drained: false,
1632 abandoned: 0,
1633 excluded_subscriptions: 0,
1634 terminal: Some(false),
1635 },
1636 );
1637 self.emit_route_goodbyes(goodbyes);
1640 }
1641 }
1642
1643 fn not_ready_reason(
1649 &self,
1650 registration: &crate::registry::ModuleRegistration,
1651 ) -> Option<NotReadyReason> {
1652 if !registration.ready {
1653 return Some(NotReadyReason {
1654 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1655 capability: None,
1656 });
1657 }
1658 self.first_unprovided_required_capability(registration)
1659 .map(|capability| NotReadyReason {
1660 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1661 capability: Some(capability),
1662 })
1663 }
1664
1665 fn first_unprovided_required_capability(
1683 &self,
1684 registration: &crate::registry::ModuleRegistration,
1685 ) -> Option<String> {
1686 let required = registration
1687 .manifest
1688 .capabilities
1689 .iter()
1690 .flat_map(|declarations| declarations.requires.iter())
1691 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1692 .map(|requirement| requirement.capability.as_str())
1693 .collect::<BTreeSet<_>>();
1694 if required.is_empty() {
1695 return None;
1696 }
1697 let module_id = registration.manifest.module_id.as_str();
1698 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1699 if required
1700 .iter()
1701 .any(|capability| verdict(capability).is_none())
1702 {
1703 self.refresh_capability_requirements();
1704 }
1705 required
1706 .into_iter()
1707 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1708 .map(str::to_string)
1709 }
1710
1711 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1712 self.capability_evaluator
1713 .statuses()
1714 .into_iter()
1715 .map(capability_requirement_status)
1716 .collect()
1717 }
1718
1719 fn deregister_connection(
1726 &self,
1727 connection_id: ConnectionId,
1728 reason: RegistrationEndReason,
1729 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1730 self.registry
1731 .deregister_connection_with_reason(connection_id, reason)
1732 }
1733
1734 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1735 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1736 return None;
1737 }
1738 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1739 parse_client_control_request(&frame.body)
1740 else {
1741 return None;
1742 };
1743 Some(target_module_id(&target).to_string())
1744 }
1745
1746 pub(crate) fn route_open_capacity_refusal(
1747 &self,
1748 ctx: &RouteCtx,
1749 frame: &Frame,
1750 target_module_id: &str,
1751 in_flight: usize,
1752 limit: usize,
1753 ) -> Result<Frame, RouterError> {
1754 let caller = self.route_open_caller_from_frame(frame);
1757 self.route_open_admission_refusal_frame(
1758 ctx,
1759 &caller,
1760 frame,
1761 target_module_id,
1762 "open_admission_full",
1763 (in_flight, limit),
1764 format!(
1765 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1766 ),
1767 )
1768 }
1769
1770 fn route_open_target_capacity_refusal(
1771 &self,
1772 ctx: &RouteCtx,
1773 caller: &RouteOpenCaller,
1774 frame: &Frame,
1775 target_module_id: &str,
1776 in_flight: usize,
1777 ) -> Result<Frame, RouterError> {
1778 self.route_open_admission_refusal_frame(
1779 ctx,
1780 caller,
1781 frame,
1782 target_module_id,
1783 "target_binds_full",
1784 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1785 format!(
1786 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1787 ),
1788 )
1789 }
1790
1791 #[allow(clippy::too_many_arguments)]
1799 fn route_open_admission_refusal_frame(
1800 &self,
1801 ctx: &RouteCtx,
1802 caller: &RouteOpenCaller,
1803 frame: &Frame,
1804 target_module_id: &str,
1805 reason: &'static str,
1806 (in_flight, limit): (usize, usize),
1807 message: impl Into<String>,
1808 ) -> Result<Frame, RouterError> {
1809 let code = error_codes::TARGET_UNAVAILABLE;
1810 self.counters.increment_route_open_refused(code);
1811 info!(
1812 target: "control",
1813 code,
1814 reason,
1815 module_id = ?target_module_id,
1816 connection_id = ctx.connection_id.get(),
1817 principal = %caller.principal,
1818 scope = %caller.scope,
1819 harness = %caller.harness,
1820 in_flight,
1821 limit,
1822 "route.open refused"
1823 );
1824 control_error_frame(frame, code, message.into())
1825 }
1826
1827 #[cfg(test)]
1833 pub fn handle_control(
1834 &self,
1835 connection_id: ConnectionId,
1836 frame: Frame,
1837 ) -> Result<Vec<Frame>, RouterError> {
1838 match frame.header.ty {
1839 FrameType::Ping => Ok(vec![pong(&frame)?]),
1840 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1841 FrameType::Goodbye => self.handle_goodbye(connection_id),
1842 ty => Ok(vec![control_error_frame(
1843 &frame,
1844 "unsupported_control_frame",
1845 format!("unsupported channel-0 frame {ty:?}"),
1846 )?]),
1847 }
1848 }
1849
1850 pub async fn handle_control_frame(
1851 &self,
1852 ctx: &RouteCtx,
1853 frame: Frame,
1854 ) -> Result<Vec<Frame>, RouterError> {
1855 self.handle_control_frame_timed(ctx, frame, None).await
1856 }
1857
1858 #[cfg(any(test, feature = "test-support"))]
1859 pub fn with_test_operator_app_policy(
1860 mut self,
1861 policy: crate::operator_apps::TestOperatorAppPolicy,
1862 ) -> Self {
1863 self.operator_app_policy = policy.into();
1864 self
1865 }
1866
1867 #[cfg(all(windows, any(test, feature = "test-support")))]
1871 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1872 self.windows_verify = Some(verify);
1873 self
1874 }
1875
1876 #[cfg(all(windows, any(test, feature = "test-support")))]
1878 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1879 self.windows_flash = Some(flash);
1880 self
1881 }
1882
1883 #[cfg(all(windows, any(test, feature = "test-support")))]
1887 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1888 self.windows_track = Some(track);
1889 self
1890 }
1891
1892 pub(crate) fn record_captured_peer(
1893 &self,
1894 connection: ConnectionId,
1895 peer: subc_os::local_peer::CapturedPeer,
1896 ) {
1897 self.record_observed_peer(connection, peer.observation().clone());
1898 #[cfg(windows)]
1899 if let Some(stamp) = self.stamp_windows_launch(&peer) {
1900 self.operator_apps
1901 .lock()
1902 .unwrap_or_else(|p| p.into_inner())
1903 .insert(connection, stamp);
1904 }
1905 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1906 self.operator_apps
1907 .lock()
1908 .unwrap_or_else(|p| p.into_inner())
1909 .insert(connection, stamp);
1910 }
1911 }
1912
1913 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1914 self.local_connections
1915 .lock()
1916 .unwrap_or_else(|p| p.into_inner())
1917 .insert(connection);
1918 }
1919
1920 fn connection_operator_app(
1923 &self,
1924 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1925 ) -> Option<subc_protocol::peer::OperatorApp> {
1926 stamp
1927 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1928 .map(|stamp| stamp.app.clone())
1929 }
1930
1931 fn handle_connection_describe(
1932 &self,
1933 ctx: &RouteCtx,
1934 frame: Frame,
1935 ) -> Result<Vec<Frame>, RouterError> {
1936 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1937 return Ok(vec![control_error_frame(
1938 &frame,
1939 "invalid_control_body",
1940 "malformed connection.describe body",
1941 )?]);
1942 }
1943 let local = self
1944 .local_connections
1945 .lock()
1946 .unwrap_or_else(|p| p.into_inner())
1947 .contains(&ctx.connection_id);
1948 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1949 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1950 Ok(vec![control_response_body_frame(
1951 &frame,
1952 &subc_control::ConnectionDescribeReply::new(local, operator_app),
1953 ops::CONNECTION_DESCRIBE,
1954 )?])
1955 }
1956
1957 pub(crate) fn record_observed_peer(
1958 &self,
1959 connection: ConnectionId,
1960 peer: subc_os::local_peer::ObservedPeer,
1961 ) {
1962 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1963 let peer = match peer {
1964 OsPeer::Unobserved => ObservedPeer::unobserved(),
1965 OsPeer::Observed(peer) => {
1966 let observed = match peer.user {
1967 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1968 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1969 };
1970 observed
1971 .with_exe(
1972 peer.exe
1973 .and_then(|path| path.into_os_string().into_string().ok()),
1974 )
1975 .with_signing_id(peer.signing_id)
1976 }
1977 };
1978 self.observed_peers
1979 .lock()
1980 .unwrap_or_else(|p| p.into_inner())
1981 .insert(connection, peer);
1982 }
1983
1984 pub(crate) fn record_caller_endpoints(
1985 &self,
1986 connection: ConnectionId,
1987 endpoints: Option<(SocketAddr, SocketAddr)>,
1988 ) {
1989 let mut callers = self
1990 .caller_endpoints
1991 .lock()
1992 .unwrap_or_else(|p| p.into_inner());
1993 callers.remove(&connection);
1994 if let Some(endpoints) = endpoints {
1995 callers.insert(connection, endpoints);
1996 }
1997 }
1998
1999 #[cfg(test)]
2000 pub(crate) fn with_caller_lookup_for_test(
2001 mut self,
2002 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2003 ) -> Self {
2004 self.caller_lookup = Some(Arc::new(lookup));
2005 self
2006 }
2007
2008 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2009 let endpoints = self
2010 .caller_endpoints
2011 .lock()
2012 .unwrap_or_else(|p| p.into_inner())
2013 .get(&connection)
2014 .copied();
2015 #[cfg(test)]
2016 let lookup = self.caller_lookup.clone();
2017 tokio::task::spawn_blocking(move || {
2018 #[cfg(test)]
2019 if let Some(lookup) = lookup {
2020 return lookup(endpoints);
2021 }
2022 endpoints
2023 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2024 .unwrap_or_default()
2025 })
2026 .await
2027 .unwrap_or_default()
2028 }
2029
2030 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2031 frame.header.channel == 0
2032 && frame.header.ty == FrameType::Request
2033 && self
2034 .forwarding
2035 .module_endpoint_for_connection(ctx.connection_id)
2036 .is_ok_and(|endpoint| endpoint.is_none())
2037 && parse_client_control_request(&frame.body)
2038 .ok()
2039 .is_some_and(|request| privileged_operation(&request).is_some())
2040 }
2041
2042 pub(crate) async fn handle_control_frame_timed(
2043 &self,
2044 ctx: &RouteCtx,
2045 frame: Frame,
2046 dispatch_started_at: Option<StdInstant>,
2047 ) -> Result<Vec<Frame>, RouterError> {
2048 match frame.header.ty {
2049 FrameType::Ping => Ok(vec![pong(&frame)?]),
2050 FrameType::Hello => {
2051 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2052 }
2053 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2054 FrameType::Cancel => {
2055 if self
2058 .agent_declarations
2059 .cancel(ctx.connection_id, frame.header.corr)
2060 || self
2061 .forwarding
2062 .operator_confirms()
2063 .cancel(ctx.connection_id, frame.header.corr)
2064 || self
2065 .supervisor
2066 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2067 || self.cancel_activity_subscription(ctx, &frame)?
2068 {
2069 Ok(Vec::new())
2070 } else {
2071 Ok(vec![control_error_frame(
2072 &frame,
2073 "unknown_subscription",
2074 "no subscription has this correlation id",
2075 )?])
2076 }
2077 }
2078 FrameType::Request => {
2079 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2082 if op
2083 .as_ref()
2084 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2085 {
2086 return self.handle_activity_record(ctx, frame).await;
2087 }
2088 if op.as_ref().is_some_and(|probe| {
2089 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2090 }) {
2091 return self.handle_activity_read(ctx, frame).await;
2092 }
2093 if op
2094 .as_ref()
2095 .is_some_and(|probe| probe.op == "operator.confirm")
2096 {
2097 return self.handle_operator_confirm(ctx, frame);
2098 }
2099 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2100 return self
2101 .observe_control_wait(
2102 ctx,
2103 None,
2104 ops::APP_OPEN,
2105 self.handle_app_open(ctx, frame),
2106 )
2107 .await;
2108 }
2109 if op
2110 .as_ref()
2111 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2112 {
2113 return self.handle_connection_describe(ctx, frame);
2114 }
2115 if op
2116 .as_ref()
2117 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2118 {
2119 return self.handle_scope_served_by(ctx.connection_id, frame);
2120 }
2121 if self
2122 .forwarding
2123 .module_endpoint_for_connection(ctx.connection_id)
2124 .map_err(RouterError::Forwarding)?
2125 .is_some()
2126 {
2127 if !is_known_module_request_op(&frame.body) {
2128 return Ok(vec![control_error_frame(
2129 &frame,
2130 "unsupported_control_frame",
2131 "module-originated channel-0 REQUEST is not supported",
2132 )?]);
2133 }
2134 let request = match parse_module_control_request_from_module(&frame.body) {
2135 Ok(request) => request,
2136 Err((err, ControlRequestBodyError::UnknownOp)) => {
2137 return Ok(vec![control_error_frame(
2138 &frame,
2139 "unsupported_control_frame",
2140 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2141 )?])
2142 }
2143 Err((err, ControlRequestBodyError::InvalidBody)) => {
2144 return Ok(vec![control_error_frame(
2145 &frame,
2146 "invalid_control_body",
2147 format!("malformed module control body: {err}"),
2148 )?])
2149 }
2150 };
2151 let op = module_control_request_op(&request);
2152 let corr = frame.header.corr;
2153 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2154 if let ModuleControlRequestFromModule::AgentCommands {
2155 started,
2156 finished,
2157 known,
2158 } = request
2159 {
2160 return self
2161 .agent_declarations
2162 .enqueue(
2163 self,
2164 ctx,
2165 frame,
2166 crate::resource_totals::Declaration {
2167 started,
2168 finished,
2169 known,
2170 },
2171 )
2172 .await;
2173 }
2174 let result =
2175 self.handle_module_control_request(ctx.connection_id, frame, request);
2176 log_slow_control_dispatch(
2177 dispatch_started_at,
2178 op,
2179 ctx.connection_id,
2180 corr,
2181 None,
2182 );
2183 return result;
2184 }
2185
2186 if is_known_module_request_op(&frame.body) {
2187 let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2188 .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2189 return Ok(vec![control_error_frame(
2190 &frame,
2191 if is_agents { "not_supervised" } else { "not_registered" },
2192 "module-originated control requests require the module's own registered connection",
2193 )?]);
2194 }
2195
2196 let request = match parse_client_control_request(&frame.body) {
2197 Ok(request) => request,
2198 Err((err, ControlRequestBodyError::UnknownOp)) => {
2199 return Ok(vec![control_error_frame(
2200 &frame,
2201 "unknown_control_op",
2202 format!("unknown client control op: {err}"),
2203 )?])
2204 }
2205 Err((err, ControlRequestBodyError::InvalidBody)) => {
2206 return Ok(vec![control_error_frame(
2207 &frame,
2208 "invalid_control_body",
2209 format!("malformed client control body: {err}"),
2210 )?])
2211 }
2212 };
2213 let op = client_control_request_op(&request);
2214 let corr = frame.header.corr;
2215 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2216 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2217 #[cfg(test)]
2218 if let Some(delay) = self.control_dispatch_delay {
2219 tokio::time::sleep(delay).await;
2220 }
2221 let result = self
2222 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2223 .await;
2224 if let Some(timings) = route_open_timings.as_mut() {
2225 timings.finish();
2226 }
2227 log_slow_control_dispatch(
2228 dispatch_started_at,
2229 op,
2230 ctx.connection_id,
2231 corr,
2232 route_open_timings.as_ref(),
2233 );
2234 result
2235 }
2236 FrameType::Push => {
2237 let Some(endpoint) = self
2238 .forwarding
2239 .module_endpoint_for_connection(ctx.connection_id)
2240 .map_err(RouterError::Forwarding)?
2241 else {
2242 return Ok(vec![control_error_frame(
2243 &frame,
2244 "unsupported_control_frame",
2245 "client-originated channel-0 PUSH is not supported",
2246 )?]);
2247 };
2248 self.handle_status_update(endpoint, frame)
2249 }
2250 FrameType::Response | FrameType::Error
2251 if self
2252 .forwarding
2253 .module_endpoint_for_connection(ctx.connection_id)
2254 .map_err(RouterError::Forwarding)?
2255 .is_some() =>
2256 {
2257 self.handle_module_relay_response(ctx.connection_id, frame)
2258 }
2259 ty => Ok(vec![control_error_frame(
2260 &frame,
2261 "unsupported_control_frame",
2262 format!("unsupported channel-0 frame {ty:?}"),
2263 )?]),
2264 }
2265 }
2266
2267 pub fn cleanup_connection(
2268 &self,
2269 connection_id: ConnectionId,
2270 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2271 self.local_connections
2272 .lock()
2273 .unwrap_or_else(|p| p.into_inner())
2274 .remove(&connection_id);
2275 self.caller_endpoints
2276 .lock()
2277 .unwrap_or_else(|p| p.into_inner())
2278 .remove(&connection_id);
2279 self.build_class_warned
2280 .lock()
2281 .unwrap_or_else(|p| p.into_inner())
2282 .remove(&connection_id);
2283 self.observed_peers
2284 .lock()
2285 .unwrap_or_else(|p| p.into_inner())
2286 .remove(&connection_id);
2287 self.operator_apps
2288 .lock()
2289 .unwrap_or_else(|p| p.into_inner())
2290 .remove(&connection_id);
2291 self.forwarding
2292 .operator_confirms()
2293 .connection_closed(connection_id);
2294 self.cleanup_connection_with_end_reason(
2295 connection_id,
2296 RegistrationEndReason::ConnectionClosed,
2297 )
2298 }
2299
2300 fn cleanup_connection_with_end_reason(
2301 &self,
2302 connection_id: ConnectionId,
2303 requested_reason: RegistrationEndReason,
2304 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2305 self.agent_declarations.close(connection_id);
2306 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2307 self.registry
2308 .get_module_by_connection(connection_id)?
2309 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2310 .and_then(|module| module.registration_end_reason().ok().flatten())
2311 .unwrap_or(requested_reason)
2312 } else {
2313 requested_reason
2314 };
2315 let crash_closed = self
2316 .registry
2317 .get_module_by_connection(connection_id)?
2318 .and_then(|registration| {
2319 self.forwarding
2320 .module_endpoint_for_connection(connection_id)
2321 .ok()
2322 .flatten()
2323 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2324 .map(|routes| (registration.manifest.module_id, routes))
2325 });
2326 let crash_closed = crash_closed.map(|(module_id, routes)| {
2327 let terminal = match self.supervisor.get(&module_id) {
2328 None => false,
2329 Some(module) => match module.will_recover_after_connection_loss() {
2330 Ok(will_recover) => !will_recover,
2331 Err(err) => {
2332 warn!(
2333 %module_id,
2334 error = %err,
2335 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2336 );
2337 false
2338 }
2339 },
2340 };
2341 let (reason, drained) = match self
2345 .forwarding
2346 .daemon_drain_result_for_connection(connection_id)
2347 {
2348 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2349 Ok(None) => (RouteCloseReason::Crash, false),
2350 Err(err) => {
2351 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2352 (RouteCloseReason::Crash, false)
2353 }
2354 };
2355 (module_id, routes, reason, drained, terminal)
2356 });
2357 let registrations = self.deregister_connection(connection_id, end_reason);
2358 let cleanup = if crash_closed.is_some() {
2359 self.forwarding.cleanup_connection_counted(connection_id)
2360 } else {
2361 self.forwarding
2367 .cleanup_connection(connection_id)
2368 .map(|released| crate::forwarding::ConnectionCleanup {
2369 released,
2370 abandoned_relays: 0,
2371 })
2372 };
2373 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2378 let abandoned = cleanup
2379 .as_ref()
2380 .map_or(0, |cleanup| cleanup.abandoned_relays);
2381 send_route_control_pushes(
2382 &self.forwarding,
2383 routes,
2384 ClientControlPush::RouteClosed {
2385 module_id,
2386 channels: Vec::new(),
2387 reason,
2388 drained,
2389 abandoned,
2390 excluded_subscriptions: 0,
2391 terminal: Some(terminal),
2392 },
2393 );
2394 }
2395 if let Ok(cleanup) = cleanup {
2396 self.emit_route_goodbyes(cleanup.released);
2397 }
2398 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2402 crate::supervise::notify_registration_release();
2403 self.capability_evaluator.wake_deadline_loop();
2404 self.refresh_capability_requirements();
2405 }
2406 self.supervisor.remove_spawn_subscribers(connection_id);
2407 self.activity_streams
2408 .lock()
2409 .unwrap_or_else(|p| p.into_inner())
2410 .retain(|(id, _), task| {
2411 if *id == connection_id {
2412 task.abort();
2413 false
2414 } else {
2415 true
2416 }
2417 });
2418 self.hello_launch_nonces
2421 .lock()
2422 .unwrap_or_else(|poisoned| poisoned.into_inner())
2423 .forget(connection_id);
2424 self.scopes
2425 .write()
2426 .unwrap_or_else(|poisoned| poisoned.into_inner())
2427 .release_connection(connection_id);
2428 registrations
2429 }
2430
2431 pub(crate) fn handle_route_goodbye(
2432 &self,
2433 connection_id: ConnectionId,
2434 route_channel: u16,
2435 route_epoch: u32,
2436 ) -> Result<bool, RouterError> {
2437 debug!(
2438 connection_id = connection_id.get(),
2439 route_channel, route_epoch, "handling route GOODBYE"
2440 );
2441 let RouteRelease::Removed(released_route) = self
2442 .forwarding
2443 .release_client_route(connection_id, route_channel, route_epoch)
2444 .map_err(RouterError::Forwarding)?
2445 else {
2446 return Ok(false);
2447 };
2448 self.emit_route_goodbyes(vec![released_route]);
2449 Ok(true)
2450 }
2451
2452 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2453 for released in released_routes {
2454 let frame = match Frame::build_with_version(
2455 released.negotiated_ver,
2456 FrameType::Goodbye,
2457 control_flags(),
2458 released.channel,
2459 released.epoch,
2460 0,
2461 Vec::new(),
2462 ) {
2463 Ok(frame) => frame,
2464 Err(err) => {
2465 warn!(
2466 route_channel = released.channel,
2467 error = %err,
2468 "failed to build route GOODBYE frame"
2469 );
2470 continue;
2471 }
2472 };
2473 if !released.close_on_delivery_failure() {
2474 crate::forwarding::send_module_route_goodbye(
2475 &self.counters,
2476 &released.sink,
2477 frame,
2478 released.module_id.as_deref(),
2479 "client route released",
2480 );
2481 continue;
2482 }
2483 if let Err(err) = released.sink.try_send(frame) {
2484 warn!(
2485 target_connection_id = released.connection_id.get(),
2486 route_channel = released.channel,
2487 error = %err,
2488 "route GOODBYE was not delivered to client; closing target connection"
2489 );
2490 if self
2491 .forwarding
2492 .escalate_client_delivery_failure(
2493 released.connection_id,
2494 released.channel,
2495 released.epoch,
2496 CloseReason::new(
2497 "route_goodbye_delivery_failed",
2498 format!(
2499 "failed to enqueue route GOODBYE for channel {}: {err}",
2500 released.channel
2501 ),
2502 ),
2503 crate::forwarding::UndeliveredFrame {
2504 module_id: released.module_id.as_deref(),
2505 sink: &released.sink,
2506 },
2507 )
2508 .unwrap_or(false)
2509 {
2510 self.counters.increment_goodbye_relay_client_failed();
2511 }
2512 }
2513 }
2514 }
2515
2516 fn send_abandoned_route_bind_goodbye(
2528 &self,
2529 module_sink: &crate::FrameSink,
2530 negotiated_ver: u8,
2531 module_channel: u16,
2532 module_epoch: u32,
2533 ) {
2534 let frame = match Frame::build_with_version(
2535 negotiated_ver,
2536 FrameType::Goodbye,
2537 control_flags(),
2538 module_channel,
2539 module_epoch,
2540 0,
2541 Vec::new(),
2542 ) {
2543 Ok(frame) => frame,
2544 Err(err) => {
2545 warn!(
2546 route_channel = module_channel,
2547 error = %err,
2548 "failed to build GOODBYE for abandoned route.bind"
2549 );
2550 return;
2551 }
2552 };
2553 crate::forwarding::send_module_route_goodbye(
2554 &self.counters,
2555 module_sink,
2556 frame,
2557 None,
2558 "abandoned route.bind",
2559 );
2560 }
2561
2562 fn handle_hello(
2563 &self,
2564 connection_id: ConnectionId,
2565 sink: Option<crate::FrameSink>,
2566 frame: Frame,
2567 ) -> Result<Vec<Frame>, RouterError> {
2568 debug!(
2569 connection_id = connection_id.get(),
2570 corr = frame.header.corr,
2571 "handling HELLO"
2572 );
2573 if self
2577 .registry
2578 .get_module_by_connection(connection_id)
2579 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2580 .is_some()
2581 {
2582 return Ok(vec![control_error_frame(
2583 &frame,
2584 "invalid_hello",
2585 "connection is already registered as a module",
2586 )?]);
2587 }
2588 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2589 Ok(value) => value,
2590 Err(err) => {
2591 return Ok(vec![control_error_frame(
2592 &frame,
2593 "invalid_hello",
2594 format!("malformed HELLO body: {err}"),
2595 )?])
2596 }
2597 };
2598 if let Err(err) = validate_hello_display_name(&hello_value) {
2599 return Ok(vec![control_error_frame(
2600 &frame,
2601 "invalid_manifest",
2602 err.to_string(),
2603 )?]);
2604 }
2605 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2606 return Ok(vec![control_error_frame(
2607 &frame,
2608 "invalid_capability_grammar",
2609 err.to_string(),
2610 )?]);
2611 }
2612 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2613 return Ok(vec![control_error_frame(
2614 &frame,
2615 "invalid_manifest",
2616 err.to_string(),
2617 )?]);
2618 }
2619 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2620 return Ok(vec![control_error_frame(
2621 &frame,
2622 "invalid_event_declaration",
2623 err.to_string(),
2624 )?]);
2625 }
2626 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2627 return Ok(vec![control_error_frame(
2628 &frame,
2629 "invalid_manifest",
2630 err.to_string(),
2631 )?]);
2632 }
2633 if let Some(provenance) = hello_value
2634 .get("manifest")
2635 .and_then(|manifest| manifest.get("provenance"))
2636 {
2637 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2638 return Ok(vec![control_error_frame(
2639 &frame,
2640 "invalid_manifest",
2641 format!("malformed manifest provenance: {err}"),
2642 )?]);
2643 }
2644 }
2645 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2646 let module_id = hello_value["manifest"]["module_id"]
2647 .as_str()
2648 .unwrap_or("<unknown>");
2649 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2650 }
2651 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2652 let module_id = hello_value["manifest"]["module_id"]
2653 .as_str()
2654 .unwrap_or("<unknown>");
2655 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2656 }
2657 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2658 let module_id = hello_value["manifest"]["module_id"]
2659 .as_str()
2660 .unwrap_or("<unknown>");
2661 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2662 }
2663 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2664 Ok(hello) => hello,
2665 Err(err) => {
2666 return Ok(vec![control_error_frame(
2667 &frame,
2668 "invalid_hello",
2669 format!("malformed HELLO body: {err}"),
2670 )?])
2671 }
2672 };
2673
2674 if hello.protocol_ver != hello.manifest.protocol_ver {
2675 return Ok(vec![control_error_frame(
2676 &frame,
2677 "invalid_manifest",
2678 format!(
2679 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2680 hello.protocol_ver, hello.manifest.protocol_ver
2681 ),
2682 )?]);
2683 }
2684
2685 if hello.manifest.module_id.trim().is_empty() {
2686 return Ok(vec![control_error_frame(
2687 &frame,
2688 "invalid_manifest",
2689 "manifest module_id must not be empty",
2690 )?]);
2691 }
2692
2693 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2694 Ok(negotiated_ver) => negotiated_ver,
2695 Err(message) => {
2696 return Ok(vec![control_error_frame(
2697 &frame,
2698 "version_unsupported",
2699 message,
2700 )?])
2701 }
2702 };
2703
2704 let swap_admission = self
2712 .supervisor
2713 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2714 if swap_admission == SwapHelloAdmission::Refused {
2715 warn!(
2716 module_id = %hello.manifest.module_id,
2717 connection_id = connection_id.get(),
2718 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2719 );
2720 return Ok(vec![control_error_frame(
2721 &frame,
2722 "swap_token_invalid",
2723 format!(
2724 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2725 hello.manifest.module_id
2726 ),
2727 )?]);
2728 }
2729 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2730
2731 let nonce_verified = if swap_candidate {
2736 true
2737 } else {
2738 match self.supervisor.reserved_hello_verification(
2739 &hello.manifest.module_id,
2740 hello.launch_nonce.as_deref(),
2741 ) {
2742 Ok(verified) => verified,
2743 Err(rejection) => {
2744 let message = match rejection {
2745 ReservedHelloRejection::Exact { module_id } => format!(
2746 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2747 ),
2748 ReservedHelloRejection::Prefix {
2749 prefix,
2750 owner_module_id,
2751 } => format!(
2752 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2753 hello.manifest.module_id
2754 ),
2755 };
2756 return Ok(vec![control_error_frame(
2757 &frame,
2758 "reserved_module",
2759 message,
2760 )?]);
2761 }
2762 }
2763 };
2764
2765 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2766 &hello.manifest.module_id,
2767 hello.manifest.capabilities.as_ref(),
2768 );
2769 if let Some(refusal) = reserved_capability_refusals.first() {
2770 let capability = refusal.capability.clone();
2771 let bound_module = refusal.claimants[0].clone();
2772 log_duplicate_claim_events(reserved_capability_refusals);
2773 return Ok(vec![control_error_frame(
2774 &frame,
2775 "reserved_capability",
2776 format!(
2777 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2778 capability, bound_module, hello.manifest.module_id
2779 ),
2780 )?]);
2781 }
2782
2783 if self
2786 .forwarding
2787 .connection_has_client_routes(connection_id)
2788 .map_err(RouterError::Forwarding)?
2789 {
2790 return Ok(vec![control_error_frame(
2791 &frame,
2792 "invalid_hello",
2793 "connection has open client routes and cannot also register as a module",
2794 )?]);
2795 }
2796
2797 self.hello_launch_nonces
2802 .lock()
2803 .unwrap_or_else(|poisoned| poisoned.into_inner())
2804 .record(connection_id, hello.launch_nonce.as_deref());
2805 let control_ops = effective_module_control_ops(hello.control_ops);
2806 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2809 if swap_candidate {
2810 return self.register_swap_candidate(
2811 connection_id,
2812 sink,
2813 &frame,
2814 hello.manifest,
2815 negotiated_ver,
2816 control_ops,
2817 hello_ack,
2818 );
2819 }
2820 let registration = match self
2827 .registry
2828 .register_unpublished_verified_with_control_ops(
2829 hello.manifest,
2830 negotiated_ver,
2831 connection_id,
2832 control_ops,
2833 nonce_verified,
2834 ) {
2835 Ok(registration) => registration,
2836 Err(RegistryError::DuplicateModuleId { module_id }) => {
2837 return Ok(vec![control_error_frame(
2838 &frame,
2839 "duplicate_module_id",
2840 format!(
2841 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2842 ),
2843 )?])
2844 }
2845 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2846 return Ok(vec![control_error_frame(
2847 &frame,
2848 "invalid_module_id",
2849 err.to_string(),
2850 )?])
2851 }
2852 Err(err) => {
2853 return Ok(vec![control_error_frame(
2854 &frame,
2855 "registry_error",
2856 err.to_string(),
2857 )?])
2858 }
2859 };
2860 #[cfg(test)]
2861 self.hold_between_registry_and_forwarding();
2862
2863 let reply = if let Some(sink) = sink {
2864 let concurrency = manifest_concurrency(®istration.manifest);
2880 if let Err(err) = self.forwarding.register_module_connection_acked(
2881 connection_id,
2882 registration.manifest.module_id.clone(),
2883 negotiated_ver,
2884 concurrency,
2885 sink,
2886 hello_ack,
2887 ) {
2888 if matches!(
2893 self.deregister_connection(
2894 connection_id,
2895 RegistrationEndReason::RegistrationFailed,
2896 ),
2897 Ok(r) if !r.is_empty()
2898 ) {
2899 crate::supervise::notify_registration_release();
2900 }
2901 return Ok(vec![control_error_frame(
2902 &frame,
2903 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2904 "invalid_hello"
2905 } else {
2906 forwarding_error_code(&err)
2907 },
2908 err.to_string(),
2909 )?]);
2910 }
2911 Vec::new()
2912 } else {
2913 vec![hello_ack]
2916 };
2917
2918 if !self
2923 .registry
2924 .publish_registration(connection_id)
2925 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2926 {
2927 debug!(
2928 module_id = %registration.manifest.module_id,
2929 connection_id = connection_id.get(),
2930 "registration was removed before it could be published"
2931 );
2932 }
2933
2934 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2943 info!(
2944 module_id = %registration.manifest.module_id,
2945 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2946 );
2947 }
2948
2949 self.apply_registration_capabilities(®istration);
2950
2951 self.supervisor
2952 .record_activity_registration(®istration.manifest.module_id);
2953
2954 info!(
2955 module_id = %registration.manifest.module_id,
2956 module_version = %registration.manifest.module_version,
2957 negotiated_ver,
2958 routable_provider = manifest_provides_routable_role(®istration.manifest),
2959 connection_id = connection_id.get(),
2960 "module registered"
2961 );
2962
2963 Ok(reply)
2964 }
2965
2966 #[allow(clippy::too_many_arguments)]
2983 fn register_swap_candidate(
2984 &self,
2985 connection_id: ConnectionId,
2986 sink: Option<crate::FrameSink>,
2987 frame: &Frame,
2988 manifest: ModuleManifest,
2989 negotiated_ver: u8,
2990 control_ops: Vec<String>,
2991 hello_ack: Frame,
2992 ) -> Result<Vec<Frame>, RouterError> {
2993 let module_id = manifest.module_id.clone();
2994 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2995 manifest,
2996 negotiated_ver,
2997 connection_id,
2998 control_ops,
2999 ) {
3000 Ok(registration) => registration,
3001 Err(RegistryError::DuplicateModuleId { module_id }) => {
3002 return Ok(vec![control_error_frame(
3003 frame,
3004 "duplicate_module_id",
3005 format!(
3006 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
3007 ),
3008 )?])
3009 }
3010 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3011 return Ok(vec![control_error_frame(
3012 frame,
3013 "invalid_module_id",
3014 err.to_string(),
3015 )?])
3016 }
3017 Err(err) => {
3018 return Ok(vec![control_error_frame(
3019 frame,
3020 "registry_error",
3021 err.to_string(),
3022 )?])
3023 }
3024 };
3025 #[cfg(test)]
3026 self.hold_between_registry_and_forwarding();
3027 let reply = if let Some(sink) = sink {
3028 let concurrency = manifest_concurrency(®istration.manifest);
3032 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
3033 connection_id,
3034 module_id.clone(),
3035 negotiated_ver,
3036 concurrency,
3037 sink,
3038 hello_ack,
3039 ) {
3040 if matches!(
3041 self.deregister_connection(
3042 connection_id,
3043 RegistrationEndReason::RegistrationFailed,
3044 ),
3045 Ok(r) if !r.is_empty()
3046 ) {
3047 crate::supervise::notify_registration_release();
3048 }
3049 return Ok(vec![control_error_frame(
3050 frame,
3051 forwarding_error_code(&err),
3052 err.to_string(),
3053 )?]);
3054 }
3055 Vec::new()
3056 } else {
3057 vec![hello_ack]
3058 };
3059 if !self
3060 .registry
3061 .publish_registration(connection_id)
3062 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3063 {
3064 debug!(
3065 module_id = %module_id,
3066 connection_id = connection_id.get(),
3067 "swap candidate registration was removed before it could be published"
3068 );
3069 }
3070 self.supervisor.mark_swap_candidate_admitted(&module_id);
3071 info!(
3072 module_id = %module_id,
3073 module_version = %registration.manifest.module_version,
3074 negotiated_ver,
3075 ready = registration.ready,
3076 connection_id = connection_id.get(),
3077 "swap candidate registered; not routable until cutover"
3078 );
3079 Ok(reply)
3080 }
3081
3082 fn build_hello_ack(
3083 &self,
3084 frame: &Frame,
3085 negotiated_ver: u8,
3086 module_id: &str,
3087 ) -> Result<Frame, RouterError> {
3088 let ack = ModuleHelloAckBody {
3089 negotiated_ver,
3090 subc_ops: module_subc_ops(),
3091 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3092 storage: self
3093 .storage_config
3094 .as_ref()
3095 .map(|cfg| cfg.descriptor_for(module_id)),
3096 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3097 };
3098 let body = serde_json::to_vec(&ack).map_err(|err| {
3099 RouterError::backend(
3100 0,
3101 frame.header.corr,
3102 format!("failed to encode HELLO_ACK: {err}"),
3103 )
3104 })?;
3105
3106 Frame::build_with_version(
3107 negotiated_ver,
3108 FrameType::HelloAck,
3109 control_flags(),
3110 0,
3111 0,
3112 frame.header.corr,
3113 body,
3114 )
3115 .map_err(RouterError::FrameBuild)
3116 }
3117
3118 async fn handle_client_control_request(
3119 &self,
3120 ctx: &RouteCtx,
3121 frame: Frame,
3122 request: ClientControlRequest,
3123 route_open_timings: &mut Option<RouteOpenTimings>,
3124 ) -> Result<Vec<Frame>, RouterError> {
3125 if let Some(operation) = privileged_operation(&request) {
3126 let caller = self.resolve_caller(ctx.connection_id).await;
3127 let op = client_control_request_op(&request);
3128 let text = prompt_text(operation, &caller);
3129 match self.privileged_operator_prompt {
3130 PrivilegedOperatorPrompt::Log => {
3131 info!(target: "subc_daemon::privileged_prompt", operation = op,
3132 target = ?operation, caller = %caller.description(), summary = %text,
3133 "privileged request would prompt the operator");
3134 }
3135 PrivilegedOperatorPrompt::Enforce => {
3136 let key =
3137 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3138 let stamp = self
3139 .operator_apps
3140 .lock()
3141 .unwrap_or_else(|p| p.into_inner())
3142 .get(&ctx.connection_id)
3143 .cloned();
3144 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3145 &self.operator_app_policy,
3146 &self.launch_table,
3147 stamp.as_ref(),
3148 )
3149 .then(|| self.shortcut_context(stamp, None, None));
3150 let outcome = self
3151 .forwarding
3152 .operator_confirms()
3153 .approve_privileged_with_shortcut(
3154 ctx,
3155 &frame,
3156 op,
3157 text,
3158 caller.description(),
3159 key,
3160 shortcut,
3161 )
3162 .await;
3163 if outcome.code != "confirmed" {
3164 return Ok(vec![outcome.frame(&frame)]);
3165 }
3166 }
3167 }
3168 }
3169 match request {
3170 ClientControlRequest::DaemonShutdownIntent { intent } => {
3171 if intent.as_str() != Some("restart") {
3172 return Ok(vec![control_error_frame(
3173 &frame,
3174 "invalid_params",
3175 "intent must be 'restart'",
3176 )?]);
3177 }
3178 let response = ClientControlResponse::DaemonShutdownIntent {
3179 intent: "restart".to_owned(),
3180 expires_at_ms: self.supervisor.announce_daemon_restart(),
3181 };
3182 Ok(vec![control_response_body_frame(
3183 &frame,
3184 &response,
3185 "ClientControlResponse::DaemonShutdownIntent",
3186 )?])
3187 }
3188 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3189 ClientControlRequest::CatalogList { module_id } => {
3190 self.handle_catalog_list(frame, module_id)
3191 }
3192 ClientControlRequest::RouteOpen {
3193 target,
3194 identity,
3195 consumer_identity,
3196 consumer_capabilities,
3197 role_versions,
3198 busy_refusals,
3199 admission_facts,
3200 scope,
3201 ..
3202 } => {
3203 self.handle_route_open(
3204 ctx,
3205 frame,
3206 RouteOpenRequest {
3207 target,
3208 identity,
3209 consumer_identity,
3210 consumer_capabilities,
3211 role_versions,
3212 busy_refusals,
3213 admission_facts,
3214 scope,
3215 },
3216 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3217 )
3218 .await
3219 }
3220 ClientControlRequest::RoutePoll {
3221 route_channel,
3222 route_epoch,
3223 kind,
3224 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3225 ClientControlRequest::SupervisorList {} => {
3226 let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
3227 self.agent_declarations.refuse(
3228 self,
3229 ctx,
3230 control_error_frame(
3231 &frame,
3232 "busy",
3233 "supervisor list work is at capacity; retry",
3234 )?,
3235 );
3236 return Ok(Vec::new());
3237 };
3238 let handler = self.clone();
3240 let sink = ctx.egress.clone();
3241 tokio::spawn(
3242 async move {
3243 let _permit = permit;
3244 let result = handler.handle_supervisor_list(frame.clone()).await;
3245 let replies = match result {
3246 Ok(replies) => replies,
3247 Err(error) => {
3248 control_error_frame(&frame, "backend_error", error.to_string())
3249 .into_iter()
3250 .collect()
3251 }
3252 };
3253 for reply in replies {
3254 if !matches!(
3255 tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
3256 .await,
3257 Ok(Ok(()))
3258 ) {
3259 break;
3260 }
3261 }
3262 }
3263 .with_current_subscriber(),
3264 );
3265 Ok(Vec::new())
3266 }
3267 ClientControlRequest::SupervisorSpawnSnapshot {} => {
3268 self.handle_supervisor_spawn_snapshot(frame)
3269 }
3270 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3271 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3272 }
3273 ClientControlRequest::SupervisorRestart {
3274 module_id,
3275 drain_timeout_ms,
3276 } => {
3277 self.log_supervisor_request_received(
3278 ctx,
3279 frame.header.corr,
3280 ops::SUPERVISOR_RESTART,
3281 Some(&module_id),
3282 None,
3283 )?;
3284 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3285 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3286 .await
3287 }
3288 ClientControlRequest::SupervisorSwap {
3289 module_id,
3290 ready_timeout_ms,
3291 } => {
3292 self.log_supervisor_request_received(
3293 ctx,
3294 frame.header.corr,
3295 ops::SUPERVISOR_SWAP,
3296 Some(&module_id),
3297 None,
3298 )?;
3299 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3300 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3301 .await
3302 }
3303 ClientControlRequest::SupervisorReload { module_id } => {
3304 self.log_supervisor_request_received(
3305 ctx,
3306 frame.header.corr,
3307 ops::SUPERVISOR_RELOAD,
3308 Some(&module_id),
3309 None,
3310 )?;
3311 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3312 self.handle_supervisor_reload(frame, module_id, requester)
3313 .await
3314 }
3315 ClientControlRequest::SupervisorRescan { preview } => {
3316 if !preview {
3317 self.log_supervisor_request_received(
3318 ctx,
3319 frame.header.corr,
3320 ops::SUPERVISOR_RESCAN,
3321 None,
3322 None,
3323 )?;
3324 }
3325 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3326 let operation = self.handle_supervisor_rescan(
3327 frame,
3328 if preview {
3329 RescanMode::Preview
3330 } else {
3331 RescanMode::Execute
3332 },
3333 requester,
3334 );
3335 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3336 .await
3337 }
3338 ClientControlRequest::SupervisorRescanChecked {
3339 preview,
3340 candidate,
3341 expect,
3342 } => {
3343 self.observe_control_wait(
3344 ctx,
3345 None,
3346 ops::SUPERVISOR_RESCAN,
3347 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3348 )
3349 .await
3350 }
3351 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3352 self.log_supervisor_request_received(
3353 ctx,
3354 frame.header.corr,
3355 ops::SUPERVISOR_RELEASE_RESERVED,
3356 Some(&module_id),
3357 None,
3358 )?;
3359 self.handle_supervisor_release_reserved(frame, module_id)
3360 .await
3361 }
3362 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3363 self.log_supervisor_request_received(
3364 ctx,
3365 frame.header.corr,
3366 ops::SUPERVISOR_SET_ENABLED,
3367 Some(&module_id),
3368 Some(enabled),
3369 )?;
3370 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3371 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3372 .await
3373 }
3374 ClientControlRequest::SupervisorHealthProbe { module_id } => {
3375 self.handle_supervisor_health_probe(ctx, frame, module_id)
3376 .await
3377 }
3378 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3379 ClientControlRequest::SupervisorRoutes { module_id } => {
3380 self.handle_supervisor_routes(frame, module_id)
3381 }
3382 ClientControlRequest::SupervisorProvenance { module_id } => {
3383 self.handle_supervisor_provenance(ctx, frame, module_id)
3384 .await
3385 }
3386 ClientControlRequest::SupervisorStderrTail {
3387 module_id,
3388 max_lines,
3389 max_bytes,
3390 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3391 ClientControlRequest::SupervisorTerminals { module_id } => {
3392 self.handle_supervisor_terminals(ctx, frame, module_id)
3393 .await
3394 }
3395 }
3396 }
3397
3398 fn log_supervisor_request_received(
3399 &self,
3400 ctx: &RouteCtx,
3401 corr: u64,
3402 op: &'static str,
3403 module_id: Option<&str>,
3404 enabled: Option<bool>,
3405 ) -> Result<(), RouterError> {
3406 let caller = self.supervisor_requester(ctx, corr)?;
3407 let caller = principal_label(&caller);
3408 info!(
3411 target: "control",
3412 op,
3413 module_id,
3414 enabled,
3415 connection_id = ctx.connection_id.get(),
3416 caller = %caller,
3417 "supervisor request received"
3418 );
3419 Ok(())
3420 }
3421
3422 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3423 Ok(self
3424 .registry
3425 .get_module_by_connection(ctx.connection_id)
3426 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3427 .map(|registration| Principal::Reserved {
3428 module_id: registration.manifest.module_id,
3429 })
3430 .unwrap_or(Principal::Direct))
3431 }
3432
3433 fn handle_module_control_request(
3434 &self,
3435 connection_id: ConnectionId,
3436 frame: Frame,
3437 request: ModuleControlRequestFromModule,
3438 ) -> Result<Vec<Frame>, RouterError> {
3439 match request {
3440 ModuleControlRequestFromModule::CatalogUpdate {
3441 provides,
3442 capabilities,
3443 ready,
3444 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3445 ModuleControlRequestFromModule::LiveRoots {} => {
3446 let registered = self
3447 .registry
3448 .get_module_by_connection(connection_id)
3449 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3450 let Some(registration) = registered else {
3451 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3452 };
3453 let response = self
3454 .forwarding
3455 .live_roots(®istration.manifest.module_id)
3456 .map_err(RouterError::Forwarding)?;
3457 Ok(vec![control_response_body_frame(
3458 &frame,
3459 &response,
3460 "ModuleControlResponseToModule::LiveRoots",
3461 )?])
3462 }
3463 ModuleControlRequestFromModule::AgentCommands {
3464 started,
3465 finished,
3466 known,
3467 } => self.handle_agent_commands(connection_id, frame, started, finished, known),
3468 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3469 self.handle_scope_sync(connection_id, frame, generation, scopes)
3470 }
3471 ModuleControlRequestFromModule::ScopeApply {
3472 generation,
3473 upsert,
3474 end,
3475 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3476 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3477 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3478 }
3479 _ => Ok(vec![control_error_frame(
3480 &frame,
3481 "unsupported_control_frame",
3482 "unsupported module control operation",
3483 )?]),
3484 }
3485 }
3486
3487 fn handle_scope_served_by(
3488 &self,
3489 connection_id: ConnectionId,
3490 frame: Frame,
3491 ) -> Result<Vec<Frame>, RouterError> {
3492 let registered = self
3493 .registry
3494 .get_module_by_connection(connection_id)
3495 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3496 let Some(registered) = registered else {
3497 return Ok(vec![control_error_frame(
3498 &frame,
3499 "not_registered",
3500 "scope.served_by requires an active module registration owned by this connection",
3501 )?]);
3502 };
3503 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3504 {
3505 Ok(request) => request,
3506 Err(err) => {
3507 return Ok(vec![control_error_frame(
3508 &frame,
3509 "invalid_control_body",
3510 format!("malformed scope.served_by body: {err}"),
3511 )?])
3512 }
3513 };
3514 let answer = self
3515 .scopes
3516 .read()
3517 .unwrap_or_else(|poisoned| poisoned.into_inner())
3518 .served_by(
3519 ®istered.manifest.module_id,
3520 &request.scope_ref,
3521 request.scope_epoch,
3522 &request.module_id,
3523 );
3524 let reply = subc_control::ScopeServedByReply::new(
3525 answer,
3526 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3527 );
3528 Ok(vec![control_response_body_frame(
3529 &frame,
3530 &reply,
3531 "ScopeServedByReply",
3532 )?])
3533 }
3534
3535 fn handle_operator_confirm(
3536 &self,
3537 ctx: &RouteCtx,
3538 frame: Frame,
3539 ) -> Result<Vec<Frame>, RouterError> {
3540 use crate::operator_confirm::{audit, Outcome};
3541 let registration = self
3542 .registry
3543 .get_module_by_connection(ctx.connection_id)
3544 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3545 let Some(registration) = registration else {
3546 let outcome = Outcome::refusal("not_registered");
3547 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3548 return Ok(vec![outcome.frame(&frame)]);
3549 };
3550 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3551 Ok(request) => request,
3552 Err(_) => {
3553 let outcome = Outcome::refusal("invalid_control_body");
3554 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3555 return Ok(vec![outcome.frame(&frame)]);
3556 }
3557 };
3558 let declared = matches!(
3559 request.kind,
3560 Some(subc_protocol::session::OperatorConfirmKind::Operation)
3561 ) && request.operation.as_ref().is_some_and(|operation| {
3562 registration
3563 .manifest
3564 .confirm_operations
3565 .as_ref()
3566 .is_some_and(|operations| operations.contains(operation))
3567 });
3568 let module_id = registration.manifest.module_id;
3569 let nonce = self
3573 .hello_launch_nonces
3574 .lock()
3575 .unwrap_or_else(|p| p.into_inner())
3576 .nonce(ctx.connection_id)
3577 .map(str::to_owned);
3578 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3579 self.supervisor
3580 .spawned_consumer_authorized(&module_id, nonce)
3581 });
3582 let confirms = self.forwarding.operator_confirms();
3583 self.forwarding
3584 .with_operator_route(
3585 ctx.connection_id,
3586 request.route_channel,
3587 request.route_epoch,
3588 |binding| {
3589 let shortcut = binding
3590 .filter(|binding| {
3591 nonce_proven
3592 && crate::operator_confirm::valid_summary(&request.summary)
3593 && declared
3594 && crate::operator_confirm::ShortcutContext::route_eligible(
3595 &self.operator_app_policy,
3596 &self.launch_table,
3597 binding,
3598 )
3599 })
3600 .map(|binding| {
3601 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3602 &self.operator_app_policy,
3603 binding,
3604 );
3605 let stamp = if phone.is_some() {
3606 None
3607 } else {
3608 binding.operator_app.clone()
3609 };
3610 self.shortcut_context(
3611 stamp,
3612 phone,
3613 Some((request.route_channel, request.route_epoch)),
3614 )
3615 });
3616 confirms.admit_with_shortcut(
3617 ctx,
3618 frame,
3619 module_id,
3620 nonce_proven,
3621 request,
3622 binding,
3623 shortcut,
3624 )
3625 },
3626 )
3627 .map_err(RouterError::Forwarding)
3628 }
3629
3630 fn shortcut_context(
3631 &self,
3632 stamp: Option<crate::operator_apps::OperatorAppStamp>,
3633 phone_fingerprint: Option<String>,
3634 route: Option<(u16, u32)>,
3635 ) -> crate::operator_confirm::ShortcutContext {
3636 crate::operator_confirm::ShortcutContext {
3637 ledger: self.activity.clone(),
3638 forwarding: self.forwarding.clone(),
3639 launches: self.launch_table.clone(),
3640 stamp,
3641 phone_fingerprint,
3642 route,
3643 connections: self.operator_apps.clone(),
3644 #[cfg(test)]
3645 hooks: Default::default(),
3646 }
3647 }
3648
3649 #[cfg(any(test, feature = "test-support"))]
3651 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3652 self.operator_app_policy.phone_enabled = enabled;
3653 self
3654 }
3655
3656 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3659 let mut scopes = ScopeRecords::from(scopes);
3660 for (index, record) in scopes.records.iter_mut().enumerate() {
3661 if let Some(refusal) = Self::canonicalize_workdir(record) {
3662 scopes.refusals.insert(index, refusal);
3663 }
3664 }
3665 scopes
3666 }
3667
3668 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3669 let workdir = record.workdir.as_mut()?;
3670 let path = std::path::Path::new(workdir);
3671 if !path.is_absolute() {
3672 return Some((
3673 error_codes::INVALID_WORKDIR,
3674 "workdir must be absolute".to_string(),
3675 ));
3676 }
3677 match ProjectRootId::from_path_allowing_missing(path) {
3678 Ok(root) => match root.as_path().to_str() {
3679 Some(canonical) => {
3680 *workdir = canonical.to_string();
3681 None
3682 }
3683 None => Some((
3684 error_codes::INVALID_WORKDIR,
3685 "canonical workdir is not UTF-8".to_string(),
3686 )),
3687 },
3688 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3689 }
3690 }
3691
3692 fn handle_scope_sync(
3696 &self,
3697 connection_id: ConnectionId,
3698 frame: Frame,
3699 generation: u64,
3700 scopes: Vec<ScopeRecord>,
3701 ) -> Result<Vec<Frame>, RouterError> {
3702 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3703 }
3704
3705 fn handle_scope_change(
3706 &self,
3707 connection_id: ConnectionId,
3708 frame: Frame,
3709 generation: u64,
3710 scopes: Vec<ScopeRecord>,
3711 end: Option<Vec<ScopeEnd>>,
3712 ) -> Result<Vec<Frame>, RouterError> {
3713 let is_apply = end.is_some();
3714 let Some(registration) = self
3715 .registry
3716 .get_module_by_connection(connection_id)
3717 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3718 else {
3719 return Ok(vec![control_error_frame(
3720 &frame,
3721 "not_registered",
3722 "scope.sync and scope.apply require an active module registration owned by this connection",
3723 )?]);
3724 };
3725 let owner = registration.manifest.module_id;
3726 let scopes = Self::prepare_scope_records(scopes);
3727 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3728 let is_current_launch = |connection: ConnectionId| {
3729 self.hello_launch_nonces
3730 .lock()
3731 .unwrap_or_else(|poisoned| poisoned.into_inner())
3732 .presented(connection, current_nonce.as_deref())
3733 };
3734 let mut table = self
3739 .scopes
3740 .write()
3741 .unwrap_or_else(|poisoned| poisoned.into_inner());
3742 let outcome = match end {
3743 Some(end) => table.apply(
3744 &owner,
3745 connection_id,
3746 is_current_launch,
3747 generation,
3748 scopes,
3749 end,
3750 ),
3751 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3752 };
3753 let drained = match &outcome {
3754 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3755 Err(_) => Vec::new(),
3756 };
3757 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3758 drop(table);
3759 if has_deadlines {
3760 self.scope_deadline_added.notify_one();
3761 }
3762 match outcome {
3763 Ok(applied) => {
3764 let counts = ScopeOutcomeCounts::of(&applied.results);
3765 info!(
3766 owner = %owner,
3767 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3768 generation,
3769 records = applied.results.len(),
3770 created = counts.created,
3771 replaced = counts.replaced,
3772 updated = counts.updated,
3773 unchanged = counts.unchanged,
3774 refused = counts.refused,
3775 ended = applied.ended.len(),
3776 tag_changes = applied.tag_changes.len(),
3777 routes_closed = drained.len(),
3778 "scope change accepted"
3779 );
3780 for refused in applied
3786 .results
3787 .iter()
3788 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3789 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3790 {
3791 warn!(
3792 owner = %owner,
3793 generation,
3794 scope_ref = %refused.scope_ref,
3795 scope_epoch = refused.scope_epoch,
3796 code = refused.code.as_deref().unwrap_or(""),
3797 "scope record refused"
3798 );
3799 }
3800 self.close_scope_drained_routes(drained);
3801 let response = if is_apply {
3802 ModuleControlResponseToModule::ScopeApply {
3803 generation,
3804 results: applied.results,
3805 end_results: applied.end_results,
3806 ended: applied.ended,
3807 }
3808 } else {
3809 ModuleControlResponseToModule::ScopeSync {
3810 generation,
3811 results: applied.results,
3812 ended: applied.ended,
3813 }
3814 };
3815 Ok(vec![control_response_body_frame(
3816 &frame,
3817 &response,
3818 "scope change response",
3819 )?])
3820 }
3821 Err(refusal) => {
3822 info!(
3823 owner = %owner,
3824 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3825 generation,
3826 code = refusal.code,
3827 "scope change refused"
3828 );
3829 Ok(vec![control_error_frame(
3830 &frame,
3831 refusal.code,
3832 refusal.message,
3833 )?])
3834 }
3835 }
3836 }
3837
3838 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3844 if drained.is_empty() {
3845 return;
3846 }
3847 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3848 BTreeMap::new();
3849 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3850 for route in drained {
3851 warn!(
3852 module_id = %route.module_id,
3853 reason = ?route.reason,
3854 client_connection_id = route.client.connection_id.get(),
3855 route_channel = route.client.channel,
3856 "closing route because its scope changed"
3857 );
3858 pushes
3859 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3860 .or_insert_with(|| (route.reason, Vec::new()))
3861 .1
3862 .push(EndpointRoute {
3863 goodbye_target: route.client.clone(),
3864 principal: Principal::Unverified,
3865 bound_at: Instant::now(),
3866 draining: false,
3867 drain_reason: None,
3868 });
3869 goodbyes.push(route.module);
3870 goodbyes.push(route.client);
3871 }
3872 for ((module_id, _), (reason, routes)) in pushes {
3873 send_route_control_pushes(
3874 &self.forwarding,
3875 routes,
3876 ClientControlPush::RouteClosed {
3877 module_id,
3878 channels: Vec::new(),
3879 reason,
3880 drained: false,
3881 abandoned: 0,
3882 excluded_subscriptions: 0,
3883 terminal: Some(false),
3884 },
3885 );
3886 }
3887 self.emit_route_goodbyes(goodbyes);
3888 }
3889
3890 fn handle_scope_describe(
3893 &self,
3894 connection_id: ConnectionId,
3895 frame: Frame,
3896 owner: Principal,
3897 scope_ref: String,
3898 ) -> Result<Vec<Frame>, RouterError> {
3899 let registered = self
3900 .registry
3901 .get_module_by_connection(connection_id)
3902 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3903 let Some(registered) = registered else {
3904 return Ok(vec![control_error_frame(
3905 &frame,
3906 "not_registered",
3907 "scope.describe requires an active module registration owned by this connection",
3908 )?]);
3909 };
3910 let scopes = self
3911 .scopes
3912 .read()
3913 .unwrap_or_else(|poisoned| poisoned.into_inner());
3914 if scopes.is_federated_owner(&owner)
3915 && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
3916 {
3917 return Ok(vec![control_error_frame(
3918 &frame,
3919 error_codes::SCOPE_FEDERATED_UNSUPPORTED,
3920 "scope.describe of the federated owner requires scope-federated/v1",
3921 )?]);
3922 }
3923 let mut description = scopes.describe(&owner, &scope_ref);
3924 drop(scopes);
3925 if !registered
3928 .manifest
3929 .capabilities
3930 .as_ref()
3931 .is_some_and(|caps| {
3932 caps.provides
3933 .iter()
3934 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3935 })
3936 {
3937 if let Some(stamp) = description.stamp.as_mut() {
3938 stamp.attributes.on_behalf_of_agent = None;
3939 }
3940 }
3941 if !registered
3942 .manifest
3943 .capabilities
3944 .as_ref()
3945 .is_some_and(|caps| {
3946 caps.provides
3947 .iter()
3948 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3949 })
3950 {
3951 if let Some(stamp) = description.stamp.as_mut() {
3952 stamp.attributes.flow_version = None;
3953 stamp.attributes.flow_code_hash = None;
3954 stamp.attributes.flow_install_id = None;
3955 }
3956 }
3957 let owner_configured = match &owner {
3958 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3963 _ => false,
3964 };
3965 let response = ModuleControlResponseToModule::ScopeDescribe {
3966 status: description.status,
3967 scope_epoch: description.scope_epoch,
3968 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3969 owner_synced: description.owner_synced,
3970 owner_configured,
3971 scope: description.stamp,
3972 };
3973 Ok(vec![control_response_body_frame(
3974 &frame,
3975 &response,
3976 "ModuleControlResponseToModule::ScopeDescribe",
3977 )?])
3978 }
3979
3980 fn handle_catalog_update(
3981 &self,
3982 connection_id: ConnectionId,
3983 frame: Frame,
3984 provides: Vec<ProviderRole>,
3985 capabilities: Option<CapabilityDeclarations>,
3986 ready: Option<bool>,
3987 ) -> Result<Vec<Frame>, RouterError> {
3988 self.refresh_capability_requirements();
3989 let Some(registration) = self
3990 .registry
3991 .get_module_by_connection(connection_id)
3992 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3993 else {
3994 return Ok(vec![control_error_frame(
3995 &frame,
3996 "not_registered",
3997 "catalog.update requires an active module registration owned by this connection",
3998 )?]);
3999 };
4000
4001 if let Some(message) =
4002 catalog_update_frozen_field_message(®istration.manifest, &provides)
4003 {
4004 return Ok(vec![control_error_frame(
4005 &frame,
4006 "catalog_update_frozen_field",
4007 message,
4008 )?]);
4009 }
4010
4011 let mut candidate = registration.manifest.clone();
4012 candidate.provides = provides.clone();
4013 candidate.capabilities = capabilities
4014 .clone()
4015 .or_else(|| registration.manifest.capabilities.clone());
4016 if let Some(rich_ui) = &candidate.rich_ui {
4017 if let Err(err) = rich_ui.validate() {
4018 return Ok(vec![control_error_frame(
4019 &frame,
4020 "invalid_manifest",
4021 err.to_string(),
4022 )?]);
4023 }
4024 }
4025 if let Err(err) = candidate.validate_capability_grammar() {
4026 return Ok(vec![control_error_frame(
4027 &frame,
4028 "invalid_capability_grammar",
4029 err.to_string(),
4030 )?]);
4031 }
4032
4033 let mut conflicts = self
4036 .capability_evaluator
4037 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
4038 if let Some(conflict) = conflicts.first() {
4039 let message = format!(
4040 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
4041 conflict.capability, conflict.claimants[0], candidate.module_id
4042 );
4043 for conflict in &mut conflicts {
4044 conflict.source = DuplicateClaimSource::CatalogUpdate;
4045 }
4046 log_duplicate_claim_events(conflicts);
4047 return Ok(vec![control_error_frame(
4048 &frame,
4049 "reserved_capability",
4050 message,
4051 )?]);
4052 }
4053
4054 let updated = match self.registry.replace_catalog_for_connection(connection_id, provides, capabilities, ready) {
4055 Err(crate::registry::RegistryError::FederatedCapabilityChanged) => return Ok(vec![control_error_frame(
4056 &frame, error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
4057 "scope-federated/v1 is fixed at HELLO; re-register on a new connection to change it",
4058 )?]),
4059 result => result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?,
4060 };
4061 if updated.is_none() {
4062 return Ok(vec![control_error_frame(
4063 &frame,
4064 "not_registered",
4065 "catalog.update requires an active module registration owned by this connection",
4066 )?]);
4067 }
4068 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
4069 log_duplicate_claim_events(
4070 self.capability_evaluator
4071 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
4072 );
4073 }
4074 if capability_census_trigger(
4075 registration.manifest.capabilities.as_ref(),
4076 updated
4077 .as_ref()
4078 .and_then(|entry| entry.manifest.capabilities.as_ref()),
4079 ) {
4080 self.enforce_capability_denies();
4081 }
4082 self.refresh_capability_requirements();
4083
4084 let response = ModuleControlResponseToModule::CatalogUpdate {};
4085 control_response_body_frame(
4086 &frame,
4087 &response,
4088 "ModuleControlResponseToModule::CatalogUpdate",
4089 )
4090 .map(|frame| vec![frame])
4091 }
4092
4093 pub(crate) fn with_activity_ledger_config(
4094 mut self,
4095 run_dir: &Path,
4096 incarnation: String,
4097 config: crate::activity::ActivityConfig,
4098 ) -> Self {
4099 self.activity = Some(crate::activity::ActivityLedger::open_config(
4100 run_dir.join("activity.jsonl"),
4101 incarnation,
4102 config,
4103 ));
4104 self.supervisor.set_activity(self.activity.clone());
4105 self
4106 }
4107
4108 pub(crate) async fn configure_activity(
4109 &self,
4110 config: crate::activity::ActivityConfig,
4111 ) -> Result<(), &'static str> {
4112 match &self.activity {
4113 Some(ledger) => ledger.configure(config).await,
4114 None => Ok(()),
4115 }
4116 }
4117
4118 fn cancel_activity_subscription(
4119 &self,
4120 ctx: &RouteCtx,
4121 frame: &Frame,
4122 ) -> Result<bool, RouterError> {
4123 let task = self
4124 .activity_streams
4125 .lock()
4126 .unwrap_or_else(|p| p.into_inner())
4127 .remove(&(ctx.connection_id, frame.header.corr));
4128 let Some(task) = task else {
4129 return Ok(false);
4130 };
4131 task.abort();
4132 let end = Frame::build_with_version(
4133 response_version(frame),
4134 FrameType::StreamEnd,
4135 control_flags(),
4136 0,
4137 0,
4138 frame.header.corr,
4139 Vec::new(),
4140 )
4141 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4142 let sink = ctx.egress.clone();
4143 tokio::spawn(async move {
4144 let _ = sink.send(end).await;
4145 });
4146 Ok(true)
4147 }
4148
4149 async fn handle_activity_read(
4150 &self,
4151 ctx: &RouteCtx,
4152 frame: Frame,
4153 ) -> Result<Vec<Frame>, RouterError> {
4154 use subc_control::{
4155 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4156 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4157 };
4158 #[derive(Deserialize)]
4159 struct Identity {
4160 #[serde(default)]
4161 consumer_identity: Option<ConsumerIdentity>,
4162 }
4163 let principal = match self
4167 .forwarding
4168 .module_id_for_connection(ctx.connection_id)
4169 .map_err(RouterError::Forwarding)?
4170 {
4171 Some(module_id) => Ok(Principal::Reserved { module_id }),
4172 None => {
4173 let identity: Identity = match serde_json::from_slice(&frame.body) {
4174 Ok(identity) => identity,
4175 Err(_) => {
4176 return Ok(vec![control_error_frame(
4177 &frame,
4178 ACTIVITY_MALFORMED,
4179 "malformed activity read",
4180 )?])
4181 }
4182 };
4183 self.route_open_principal(&frame, identity.consumer_identity)?
4184 }
4185 };
4186 let admitted = matches!(principal, Ok(Principal::Direct))
4187 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4188 if !admitted {
4189 return Ok(vec![control_error_frame(
4190 &frame,
4191 ACTIVITY_READ_FORBIDDEN,
4192 "activity reads require an operator or trusted mirror principal",
4193 )?]);
4194 }
4195 let Some(ledger) = &self.activity else {
4196 return Ok(vec![control_error_frame(
4197 &frame,
4198 ACTIVITY_UNAVAILABLE,
4199 "activity ledger unavailable",
4200 )?]);
4201 };
4202 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4203 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4204 {
4205 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4206 Ok(request) => ledger.query(request).await,
4207 Err(_) => Err(ACTIVITY_MALFORMED),
4208 };
4209 return match result {
4210 Ok(reply) => Ok(vec![control_response_body_frame(
4211 &frame,
4212 &reply,
4213 ops::ACTIVITY_QUERY,
4214 )?]),
4215 Err(code) => Ok(vec![control_error_frame(
4216 &frame,
4217 code,
4218 "activity query refused",
4219 )?]),
4220 };
4221 }
4222 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4223 Ok(request) => request,
4224 Err(_) => {
4225 return Ok(vec![control_error_frame(
4226 &frame,
4227 ACTIVITY_MALFORMED,
4228 "malformed activity subscription",
4229 )?])
4230 }
4231 };
4232 let mut subscription = match ledger.subscribe(request.since).await {
4233 Ok(subscription) => subscription,
4234 Err(code) => {
4235 return Ok(vec![control_error_frame(
4236 &frame,
4237 code,
4238 "activity subscription refused",
4239 )?])
4240 }
4241 };
4242 let key = (ctx.connection_id, frame.header.corr);
4243 let sink = ctx.egress.clone();
4244 let mut streams = self
4245 .activity_streams
4246 .lock()
4247 .unwrap_or_else(|p| p.into_inner());
4248 streams.retain(|_, task| !task.is_finished());
4249 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4250 return Ok(vec![control_error_frame(
4251 &frame,
4252 "duplicate_subscription",
4253 "activity subscription correlation id already in use",
4254 )?]);
4255 }
4256 let task = tokio::spawn(async move {
4257 loop {
4258 let entry = if let Some(entry) = subscription.replay.pop_front() {
4259 Some(entry)
4260 } else {
4261 subscription.receiver.recv().await
4262 };
4263 let Some(entry) = entry else {
4264 break;
4265 };
4266 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4267 return;
4268 };
4269 let Ok(next) = Frame::build_with_version(
4270 response_version(&frame),
4271 FrameType::StreamData,
4272 control_flags(),
4273 0,
4274 0,
4275 frame.header.corr,
4276 body,
4277 ) else {
4278 return;
4279 };
4280 if sink.send(next).await.is_err() {
4281 return;
4282 }
4283 }
4284 if let Some(code) = subscription.terminal_code() {
4287 if let Ok(end) =
4288 control_error_frame(&frame, code, "activity subscriber fell behind")
4289 {
4290 let _ = sink.send(end).await;
4291 }
4292 }
4293 });
4294 streams.insert(key, task.abort_handle());
4295 Ok(Vec::new())
4296 }
4297
4298 async fn handle_activity_record(
4299 &self,
4300 ctx: &RouteCtx,
4301 frame: Frame,
4302 ) -> Result<Vec<Frame>, RouterError> {
4303 use subc_control::{
4304 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4305 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4306 };
4307 let result = async {
4308 let module = self
4309 .forwarding
4310 .module_id_for_connection(ctx.connection_id)
4311 .ok()
4312 .flatten()
4313 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4314 let request: ActivityRecordRequest =
4315 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4316 crate::activity::validate(&module, &request)?;
4317 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4318 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4319 let at_ms = std::time::SystemTime::now()
4320 .duration_since(std::time::UNIX_EPOCH)
4321 .unwrap_or_default()
4322 .as_millis() as u64;
4323 ledger.record(module, request, actor, at_ms).await
4324 }
4325 .await;
4326 match result {
4327 Ok(entry) => Ok(vec![control_response_body_frame(
4328 &frame,
4329 &ActivityRecordResponse { seq: entry.seq },
4330 "activity.record",
4331 )?]),
4332 Err(code) => Ok(vec![control_error_frame(
4333 &frame,
4334 code,
4335 "activity recording refused",
4336 )?]),
4337 }
4338 }
4339
4340 fn activity_actor(
4341 &self,
4342 connection: ConnectionId,
4343 module: &str,
4344 on: Option<&subc_control::ActivityOn>,
4345 ) -> Result<subc_control::ActivityActor, &'static str> {
4346 use crate::forwarding::{DataRoute, DataRouteState};
4347 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4348 let principal = |id: &str| Principal::Reserved {
4349 module_id: id.to_owned(),
4350 };
4351 let mut actor = ActivityActor {
4352 principal: principal(module),
4353 scope_owner: None,
4354 agent_id: None,
4355 run_id: None,
4356 flow_id: None,
4357 project_root: None,
4358 workdir: None,
4359 project_id: None,
4360 };
4361 let scope = match on {
4362 None => return Ok(actor),
4363 Some(ActivityOn::Scope(scope)) => {
4364 let selector = subc_protocol::scope::ScopeSelector::new(
4365 principal(module),
4366 scope.scope_ref.clone(),
4367 Some(scope.epoch),
4368 );
4369 self.scopes
4370 .read()
4371 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4372 .attribute(&principal(module), module, &selector, true)
4373 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4374 }
4375 Some(ActivityOn::Route(route)) => {
4376 let DataRoute::Module(DataRouteState::Bound(binding)) = self
4377 .forwarding
4378 .lookup_data_route(connection, route.channel, route.epoch)
4379 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4380 else {
4381 return Err(ACTIVITY_ACTOR_UNKNOWN);
4382 };
4383 if binding.module_id != module {
4384 return Err(ACTIVITY_ACTOR_UNKNOWN);
4385 }
4386 actor.principal = binding.principal.clone();
4387 actor.project_root = binding
4388 .project_root
4389 .as_ref()
4390 .map(|root| root.as_path().to_path_buf());
4391 let Some(bound) = &binding.scope else {
4392 return Ok(actor);
4393 };
4394 let owner = principal(&bound.owner);
4395 let selector = subc_protocol::scope::ScopeSelector::new(
4396 owner.clone(),
4397 bound.scope_ref.clone(),
4398 Some(bound.tag.scope_epoch),
4399 );
4400 let admitted = self
4401 .scopes
4402 .read()
4403 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4404 .attribute(&owner, module, &selector, true)
4405 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4406 if admitted.tag != bound.tag {
4407 return Err(ACTIVITY_ACTOR_UNKNOWN);
4408 }
4409 admitted
4410 }
4411 };
4412 actor.scope_owner = Some(scope.stamp.owner);
4413 actor.agent_id = scope.stamp.attributes.agent_id;
4414 actor.run_id = scope.stamp.attributes.run_id;
4415 actor.flow_id = scope.stamp.attributes.flow_id;
4416 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4417 actor.project_id = scope.stamp.project_id;
4418 Ok(actor)
4419 }
4420
4421 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4422 self.refresh_capability_requirements();
4423 let mut counters = self.counters.snapshot();
4429 let losses = self.resource_tracker.loss_diagnostics();
4430 if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
4431 if let Some(obj) = counters.as_object_mut() {
4432 obj.insert("agent_command_declaration_loss_by_module".into(), losses);
4433 }
4434 }
4435 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4436 obj.insert(
4437 "activity_unparseable_lines".into(),
4438 ledger.read_errors().into(),
4439 );
4440 obj.insert(
4441 "activity_lifecycle_append_failures".into(),
4442 ledger.lifecycle_failures().into(),
4443 );
4444 obj.insert(
4445 "activity_started_digest_skips".into(),
4446 ledger.lifecycle_digest_skips().into(),
4447 );
4448 }
4449 if let (Ok((connections_with_routes, max)), Some(obj)) = (
4450 self.forwarding.client_route_concentration(),
4451 counters.as_object_mut(),
4452 ) {
4453 obj.insert(
4454 "client_connections_with_routes".into(),
4455 connections_with_routes.into(),
4456 );
4457 obj.insert("max_routes_on_one_connection".into(), max.into());
4458 }
4459 if let (Some(open_breakers), Some(obj)) = (
4465 self.route_bind_breakers.open_snapshot(),
4466 counters.as_object_mut(),
4467 ) {
4468 obj.insert("route_bind_breakers_open".into(), open_breakers);
4469 }
4470 if let (Some(reason), Some(obj)) = (
4475 self.local_endpoint_unavailable.clone(),
4476 counters.as_object_mut(),
4477 ) {
4478 obj.insert("local_endpoint_unavailable".into(), reason.into());
4479 }
4480 let response = ClientControlResponse::ServerDescribe {
4481 protocol_ver: PROTOCOL_VERSION,
4482 subc_ops: subc_ops(),
4483 capabilities: self.describe_capabilities(),
4484 connected_clients: self.connected_clients.count(),
4485 counters: Some(counters),
4486 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4487 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4488 capability_requirements: self.capability_requirement_statuses(),
4489 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4490 config_path: self
4491 .rescan
4492 .as_ref()
4493 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4494 };
4495 Ok(vec![control_response_body_frame(
4496 &frame,
4497 &response,
4498 "ClientControlResponse::ServerDescribe",
4499 )?])
4500 }
4501
4502 fn handle_catalog_list(
4503 &self,
4504 frame: Frame,
4505 module_id: Option<String>,
4506 ) -> Result<Vec<Frame>, RouterError> {
4507 let (generation, modules) = self
4508 .registry
4509 .list_modules_with_nonce_verification()
4510 .map_err(|err| {
4511 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4512 })?;
4513 let entries = modules
4514 .into_iter()
4515 .filter(|(registration, _)| {
4516 module_id
4517 .as_deref()
4518 .map(|wanted| registration.manifest.module_id == wanted)
4519 .unwrap_or(true)
4520 })
4521 .map(|(registration, nonce_verified)| {
4522 let not_ready = self.not_ready_reason(®istration);
4523 let roles = registration.manifest.provides;
4524 CatalogEntry::new(
4525 registration.manifest.module_id,
4526 roles,
4527 registration.control_ops,
4528 )
4529 .with_ready(not_ready.is_none())
4530 .with_display_name(registration.manifest.display_name)
4531 .with_not_ready(not_ready)
4532 .with_module_version(Some(registration.manifest.module_version))
4533 .with_provenance(registration.manifest.provenance)
4534 .with_capabilities(registration.manifest.capabilities)
4535 .with_self_signals(registration.manifest.self_signals)
4536 .with_events(registration.manifest.events)
4537 .with_operation_delivery(registration.manifest.operation_delivery)
4538 .with_operation_payload(registration.manifest.operation_payload)
4539 .with_flow_operations(registration.manifest.flow_operations)
4540 .with_confirm_operations(registration.manifest.confirm_operations)
4541 .with_rich_ui(registration.manifest.rich_ui)
4542 .with_role_declarations(registration.manifest.role_declarations)
4543 .with_nonce_verified(nonce_verified)
4544 })
4545 .collect();
4546 let response = ClientControlResponse::CatalogList {
4547 generation,
4548 modules: entries,
4549 subc_ops: subc_ops(),
4550 };
4551 Ok(vec![control_response_body_frame(
4552 &frame,
4553 &response,
4554 "ClientControlResponse::CatalogList",
4555 )?])
4556 }
4557
4558 fn attest_route_open_consumer(
4563 &self,
4564 consumer_identity: Option<&ConsumerIdentity>,
4565 ) -> Option<Principal> {
4566 let Some(consumer_identity) = consumer_identity else {
4567 return Some(Principal::Direct);
4568 };
4569 self.supervisor
4570 .spawned_consumer_authorized(
4571 &consumer_identity.module_id,
4572 &consumer_identity.launch_nonce,
4573 )
4574 .then(|| Principal::Reserved {
4575 module_id: consumer_identity.module_id.clone(),
4576 })
4577 }
4578
4579 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4583 let Ok(ClientControlRequest::RouteOpen {
4584 identity,
4585 consumer_identity,
4586 scope,
4587 ..
4588 }) = parse_client_control_request(&frame.body)
4589 else {
4590 return RouteOpenCaller::unknown();
4591 };
4592 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4593 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4594 }
4595
4596 fn route_open_principal(
4597 &self,
4598 frame: &Frame,
4599 consumer_identity: Option<ConsumerIdentity>,
4600 ) -> Result<Result<Principal, Frame>, RouterError> {
4601 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4602 self.route_open_principal_attested(frame, consumer_identity, attested)
4603 }
4604
4605 fn route_open_principal_attested(
4609 &self,
4610 frame: &Frame,
4611 consumer_identity: Option<ConsumerIdentity>,
4612 attested: Option<Principal>,
4613 ) -> Result<Result<Principal, Frame>, RouterError> {
4614 if let Some(principal) = attested {
4615 return Ok(Ok(principal));
4616 }
4617 let Some(consumer_identity) = consumer_identity else {
4618 return Ok(Ok(Principal::Direct));
4619 };
4620
4621 Ok(Err(control_error_frame(
4622 frame,
4623 "bad_consumer_identity",
4624 format!(
4625 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4626 consumer_identity.module_id
4627 ),
4628 )?))
4629 }
4630
4631 #[allow(clippy::too_many_arguments)]
4638 fn route_open_refusal_frame(
4639 &self,
4640 ctx: &RouteCtx,
4641 caller: &RouteOpenCaller,
4642 frame: &Frame,
4643 module_id: &str,
4644 reason: &'static str,
4645 code: &'static str,
4646 message: impl Into<String>,
4647 ) -> Result<Frame, RouterError> {
4648 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4649 control_error_frame(frame, code, message.into())
4650 }
4651
4652 #[allow(clippy::too_many_arguments)]
4661 fn route_bind_rejection(
4662 &self,
4663 ctx: &RouteCtx,
4664 caller: &RouteOpenCaller,
4665 frame: &Frame,
4666 target_module_id: &str,
4667 body: ErrorBody,
4668 daemon_authored: bool,
4669 ) -> Result<Vec<Frame>, RouterError> {
4670 if daemon_authored {
4676 let scope_code = match body.code.as_str() {
4677 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4678 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4679 _ => None,
4680 };
4681 if let Some(code) = scope_code {
4682 self.observe_route_open_refusal(
4683 ctx,
4684 caller,
4685 target_module_id,
4686 "scope_changed_before_commit",
4687 code,
4688 );
4689 return Ok(vec![control_error_body_frame(frame, body)?]);
4690 }
4691 }
4692 self.counters
4693 .increment_route_open_refused("module_rejected");
4694 info!(
4695 target: "control",
4696 code = "module_rejected",
4697 module_code = ?body.code,
4698 module_id = ?target_module_id,
4699 connection_id = ctx.connection_id.get(),
4700 principal = %caller.principal,
4701 scope = %caller.scope,
4702 harness = %caller.harness,
4703 "route.open refused"
4704 );
4705 let refusal = if daemon_authored {
4706 control_error_body_frame(frame, body)?
4707 } else {
4708 control_relayed_error_body_frame(frame, body)?
4709 };
4710 Ok(vec![refusal])
4711 }
4712
4713 #[allow(clippy::too_many_arguments)]
4731 fn route_open_breaker_refusal_frame(
4732 &self,
4733 ctx: &RouteCtx,
4734 caller: &RouteOpenCaller,
4735 frame: &Frame,
4736 module_id: &str,
4737 consecutive_timeouts: u32,
4738 retry_in: Duration,
4739 probe_in_flight: bool,
4740 ) -> Result<Frame, RouterError> {
4741 self.counters
4742 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4743 debug!(
4744 target: "control",
4745 code = "module_timeout",
4746 module_id = ?module_id,
4747 connection_id = ctx.connection_id.get(),
4748 principal = %caller.principal,
4749 scope = %caller.scope,
4750 harness = %caller.harness,
4751 consecutive_timeouts,
4752 retry_in_ms = retry_in.as_millis() as u64,
4753 probe_in_flight,
4754 "route.open refused by open bind-relay breaker"
4755 );
4756 let detail = if probe_in_flight {
4763 "one test route is already being tried; retry once it settles".to_string()
4764 } else {
4765 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4766 };
4767 control_error_frame(
4768 frame,
4769 "module_timeout",
4770 format!(
4771 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4772 timed out in a row); {detail}; its established routes are unaffected"
4773 ),
4774 )
4775 }
4776
4777 fn observe_route_open_refusal(
4790 &self,
4791 ctx: &RouteCtx,
4792 caller: &RouteOpenCaller,
4793 module_id: &str,
4794 reason: &'static str,
4795 code: &'static str,
4796 ) {
4797 self.counters.increment_route_open_refused(code);
4798 info!(
4799 target: "control",
4800 code,
4801 reason,
4802 module_id = ?module_id,
4803 connection_id = ctx.connection_id.get(),
4804 principal = %caller.principal,
4805 scope = %caller.scope,
4806 harness = %caller.harness,
4807 "route.open refused"
4808 );
4809 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4810 self.route_outages.record_not_serving(module_id, reason);
4811 }
4812 }
4813
4814 fn observe_route_open_accept(
4859 &self,
4860 ctx: &RouteCtx,
4861 module_id: &str,
4862 principal: &str,
4863 scope: &str,
4864 ) {
4865 self.route_outages.record_accepted(module_id);
4866 self.counters.increment_route_open_accepted(principal);
4867 info!(
4868 target: "control",
4869 principal,
4870 module_id,
4871 connection_id = ctx.connection_id.get(),
4872 scope = %scope,
4873 "route.open accepted"
4874 );
4875 }
4876
4877 fn supervised_absent_route_open_refusal_frame(
4878 &self,
4879 ctx: &RouteCtx,
4880 caller: &RouteOpenCaller,
4881 frame: &Frame,
4882 module_id: &str,
4883 code: &'static str,
4884 status: &crate::supervise::ModuleStatus,
4885 ) -> Result<Frame, RouterError> {
4886 self.counters.increment_route_open_refused(code);
4887 info!(
4888 target: "control",
4889 code,
4890 reason = "supervised_not_registered",
4891 module_id = ?module_id,
4892 connection_id = ctx.connection_id.get(),
4893 principal = %caller.principal,
4894 scope = %caller.scope,
4895 harness = %caller.harness,
4896 state = %status.state,
4897 enabled = status.enabled,
4898 live = status.live,
4899 "route.open refused"
4900 );
4901 self.route_outages
4905 .record_not_serving(module_id, "supervised_not_registered");
4906 let message = if let Some(pid) = status.waiting_for_previous_pid {
4907 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4908 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4909 })
4910 } else {
4911 format!(
4912 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4913 status.state, status.enabled, status.live
4914 )
4915 };
4916 control_error_frame(frame, code, message)
4917 }
4918
4919 async fn handle_route_open(
4920 &self,
4921 ctx: &RouteCtx,
4922 frame: Frame,
4923 request: RouteOpenRequest,
4924 timings: &mut RouteOpenTimings,
4925 ) -> Result<Vec<Frame>, RouterError> {
4926 let RouteOpenRequest {
4927 target,
4928 mut identity,
4929 consumer_identity,
4930 consumer_capabilities,
4931 role_versions,
4932 busy_refusals,
4933 admission_facts,
4934 scope,
4935 } = request;
4936 let target_module_id = target_module_id(&target).to_string();
4937 timings.module_id.clone_from(&target_module_id);
4938 if self
4939 .registry
4940 .get_module_by_connection(ctx.connection_id)
4941 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4942 .is_some()
4943 {
4944 return Ok(vec![control_error_frame(
4945 &frame,
4946 "invalid_request",
4947 "module connections cannot open client routes",
4948 )?]);
4949 }
4950 debug!(
4951 connection_id = ctx.connection_id.get(),
4952 corr = frame.header.corr,
4953 module_id = %target_module_id,
4954 "handling route.open"
4955 );
4956
4957 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4962 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4963
4964 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4970 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4971 self.observe_route_open_refusal(
4972 ctx,
4973 &caller,
4974 &target_module_id,
4975 "invalid_role_versions",
4976 error_codes::INVALID_REQUEST,
4977 );
4978 return Ok(vec![control_error_body_frame(
4979 &frame,
4980 ErrorBody {
4981 code: error_codes::INVALID_REQUEST.to_string(),
4982 message: error.to_string(),
4983 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4984 },
4985 )?]);
4986 }
4987
4988 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4989 if status.waiting_for_previous_pid.is_some() {
4990 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4991 ctx,
4992 &caller,
4993 &frame,
4994 &target_module_id,
4995 "module_warming",
4996 &status,
4997 )?]);
4998 }
4999 }
5000
5001 let Some(registration) = self
5020 .registry
5021 .get_module(&target_module_id)
5022 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5023 else {
5024 if let Some((status, warming)) =
5025 self.supervisor_status(&target_module_id, frame.header.corr)?
5026 {
5027 if status.protocol == ModuleProtocol::None {
5034 return Ok(vec![self.route_open_refusal_frame(
5035 ctx,
5036 &caller,
5037 &frame,
5038 &target_module_id,
5039 "protocol_none",
5040 error_codes::MODULE_NO_PROTOCOL,
5041 format!(
5042 "module_id '{target_module_id}' is declared protocol: none; \
5043 it speaks no subc wire and serves no routes"
5044 ),
5045 )?]);
5046 }
5047 let code = if warming {
5048 "module_warming"
5049 } else {
5050 "target_unavailable"
5051 };
5052 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
5053 ctx,
5054 &caller,
5055 &frame,
5056 &target_module_id,
5057 code,
5058 &status,
5059 )?]);
5060 }
5061 if let Some(removed_ago_ms) =
5062 self.supervisor.removal_tombstone_age_ms(&target_module_id)
5063 {
5064 return Ok(vec![self.route_open_refusal_frame(
5065 ctx,
5066 &caller,
5067 &frame,
5068 &target_module_id,
5069 "removed",
5070 error_codes::MODULE_REMOVED,
5071 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
5072 )?]);
5073 }
5074 return Ok(vec![self.route_open_refusal_frame(
5075 ctx,
5076 &caller,
5077 &frame,
5078 &target_module_id,
5079 "not_registered",
5080 error_codes::UNKNOWN_MODULE,
5081 format!("module_id '{target_module_id}' is not registered"),
5082 )?]);
5083 };
5084
5085 if !registration.ready {
5089 self.counters
5090 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
5091 info!(
5092 target: "control",
5093 code = error_codes::MODULE_WARMING,
5094 module_id = ?target_module_id,
5095 connection_id = ctx.connection_id.get(),
5096 principal = %caller.principal,
5097 scope = %caller.scope,
5098 harness = %caller.harness,
5099 reason = "declared_not_ready",
5100 "route.open refused"
5101 );
5102 self.route_outages
5105 .record_not_serving(&target_module_id, "declared_not_ready");
5106 return Ok(vec![control_error_body_frame(
5107 &frame,
5108 ErrorBody {
5109 code: error_codes::MODULE_WARMING.to_string(),
5110 message: format!(
5111 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5112 ),
5113 detail: Some(serde_json::json!({
5114 "reason": "declared_not_ready"
5115 })),
5116 },
5117 )?]);
5118 }
5119
5120 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
5139 self.counters
5140 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5141 info!(
5142 target: "control",
5143 code = error_codes::MODULE_WARMING,
5144 module_id = ?target_module_id,
5145 connection_id = ctx.connection_id.get(),
5146 principal = %caller.principal,
5147 scope = %caller.scope,
5148 harness = %caller.harness,
5149 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5150 capability = %capability,
5151 "route.open refused"
5152 );
5153 return Ok(vec![control_error_body_frame(
5154 &frame,
5155 ErrorBody {
5156 code: error_codes::MODULE_WARMING.to_string(),
5157 message: format!(
5158 "module_id '{target_module_id}' requires capability '{capability}', \
5159 which no registered module provides; retry"
5160 ),
5161 detail: Some(serde_json::json!({
5162 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5163 "capability": capability,
5164 })),
5165 },
5166 )?]);
5167 }
5168
5169 if !target_has_required_role(&target, ®istration.manifest.provides) {
5170 return Ok(vec![self.route_open_refusal_frame(
5171 ctx,
5172 &caller,
5173 &frame,
5174 &target_module_id,
5175 "role_not_provided",
5176 "target_unavailable",
5177 format!("module_id '{target_module_id}' does not provide the requested target"),
5178 )?]);
5179 }
5180
5181 if registration.state != ChannelState::Active {
5182 return Ok(vec![self.route_open_refusal_frame(
5183 ctx,
5184 &caller,
5185 &frame,
5186 &target_module_id,
5187 "registration_not_active",
5188 "target_unavailable",
5189 format!("module_id '{target_module_id}' is not active"),
5190 )?]);
5191 }
5192
5193 if self
5194 .forwarding
5195 .module_is_draining(&target_module_id)
5196 .map_err(RouterError::Forwarding)?
5197 {
5198 return Ok(vec![self.route_open_refusal_frame(
5199 ctx,
5200 &caller,
5201 &frame,
5202 &target_module_id,
5203 "reloading",
5204 "module_reloading",
5205 format!("module_id '{target_module_id}' is reloading"),
5206 )?]);
5207 }
5208
5209 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5210 process_liveness.process_live(&target_module_id) == Some(false)
5211 }) {
5212 if process_liveness.process_replacing(&target_module_id) {
5219 return Ok(vec![self.route_open_refusal_frame(
5220 ctx,
5221 &caller,
5222 &frame,
5223 &target_module_id,
5224 "reloading",
5225 "module_reloading",
5226 format!("module_id '{target_module_id}' is reloading"),
5227 )?]);
5228 }
5229 return Ok(vec![self.route_open_refusal_frame(
5230 ctx,
5231 &caller,
5232 &frame,
5233 &target_module_id,
5234 "supervisor_not_live",
5235 "target_unavailable",
5236 format!("module_id '{target_module_id}' is not live"),
5237 )?]);
5238 }
5239
5240 if !self
5241 .forwarding
5242 .has_live_module_connection(&target_module_id)
5243 .map_err(RouterError::Forwarding)?
5244 {
5245 return Ok(vec![self.route_open_refusal_frame(
5246 ctx,
5247 &caller,
5248 &frame,
5249 &target_module_id,
5250 "no_forwarding_connection",
5251 "target_unavailable",
5252 format!("module_id '{target_module_id}' has no live forwarding connection"),
5253 )?]);
5254 }
5255
5256 if let Some(error) =
5257 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5258 {
5259 self.observe_route_open_refusal(
5260 ctx,
5261 &caller,
5262 &target_module_id,
5263 "op_not_allowed",
5264 "op_not_allowed",
5265 );
5266 return Ok(vec![error]);
5267 }
5268
5269 let principal =
5270 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5271 Ok(principal) => principal,
5272 Err(error) => {
5273 self.observe_route_open_refusal(
5274 ctx,
5275 &caller,
5276 &target_module_id,
5277 "bad_consumer_identity",
5278 "bad_consumer_identity",
5279 );
5280 return Ok(vec![error]);
5281 }
5282 };
5283 timings.principal = principal_label(&principal);
5284
5285 let build_class = identity.build_class.take().filter(|build_class| {
5288 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5289 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5290 } else {
5291 if self
5292 .build_class_warned
5293 .lock()
5294 .unwrap_or_else(|p| p.into_inner())
5295 .insert(ctx.connection_id)
5296 {
5297 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5298 "ignoring build_class from a non-Callosum opener");
5299 }
5300 false
5301 }
5302 });
5303
5304 if let Principal::Reserved {
5308 module_id: opening_module_id,
5309 } = &principal
5310 {
5311 if let Some(opening_registration) = self
5312 .registry
5313 .get_module(opening_module_id)
5314 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5315 {
5316 if let Some(capability) =
5317 denied_capability(&opening_registration.manifest, ®istration.manifest)
5318 {
5319 warn!(
5320 opening_module_id,
5321 target_module_id,
5322 capability,
5323 "refusing route.open because an attested capability deny edge matches"
5324 );
5325 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5326 &frame,
5327 &target_module_id,
5328 "capability_deny_edge",
5329 "capability_forbidden",
5330 format!(
5331 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5332 ),
5333 )?]);
5334 }
5335 }
5336 }
5337
5338 if admission_facts.is_some() {
5339 let carrier_matches = matches!(
5340 &principal,
5341 Principal::Reserved { module_id }
5342 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5343 );
5344 if !carrier_matches {
5345 return Ok(vec![self.route_open_refusal_frame(
5346 ctx,
5347 &caller,
5348 &frame,
5349 &target_module_id,
5350 "admission_facts_carrier_not_permitted",
5351 "admission_facts_not_permitted",
5352 "admission facts may only be carried by the configured reserved module",
5353 )?]);
5354 }
5355
5356 let target_allowed = self
5357 .admission_facts_targets
5358 .as_ref()
5359 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5360 if !target_allowed {
5361 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5362 &frame,
5363 &target_module_id,
5364 "admission_facts_target_not_listed",
5365 "admission_facts_target_not_allowed",
5366 format!(
5367 "admission facts are not permitted for target module_id '{target_module_id}'"
5368 ),
5369 )?]);
5370 }
5371
5372 }
5376
5377 let (bound_scope, mut scope_stamp) = match &scope {
5383 None => (None, None),
5384 Some(selector) => {
5385 let owner_configured = match &selector.owner {
5386 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5392 _ => false,
5393 };
5394 let mut scopes = self
5395 .scopes
5396 .write()
5397 .unwrap_or_else(|poisoned| poisoned.into_inner());
5398 let admitted = scopes
5399 .admit(&principal, &target_module_id, selector, owner_configured)
5400 .and_then(|admission| {
5401 crate::scopes::check_target_flow_support(
5402 &admission.stamp,
5403 &target_module_id,
5404 registration.manifest.capabilities.as_ref(),
5405 )?;
5406 crate::scopes::check_target_agent_run_support(
5407 &admission.stamp,
5408 &target_module_id,
5409 registration.manifest.capabilities.as_ref(),
5410 )?;
5411 let bound = scopes.reserve_bind(
5412 &admission,
5413 &principal,
5414 ctx.connection_id,
5415 selector.hold_presence,
5416 );
5417 Ok((admission, bound))
5418 });
5419 match admitted {
5420 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5421 Err(refusal) => {
5422 return Ok(vec![self.route_open_refusal_frame(
5423 ctx,
5424 &caller,
5425 &frame,
5426 &target_module_id,
5427 refusal.code,
5428 refusal.code,
5429 refusal.message,
5430 )?]);
5431 }
5432 }
5433 }
5434 };
5435 if let Some(stamp) = &scope_stamp {
5438 caller.scope = route_open_scope_label(stamp);
5439 }
5440
5441 timings.enter(RouteOpenPhase::ProjectRoot);
5449 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5450 Ok(project) => project,
5451 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5452 };
5453 timings.enter(RouteOpenPhase::Admission);
5454 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5455 Ok(root) => root,
5456 Err(err) => {
5457 return Ok(vec![control_error_frame(
5458 &frame,
5459 "invalid_project_root",
5460 err.to_string(),
5461 )?])
5462 }
5463 };
5464 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5470
5471 let _concurrency_guard = match self
5482 .route_bind_concurrency
5483 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5484 {
5485 Ok(guard) => guard,
5486 Err(in_flight) => {
5487 return Ok(vec![self.route_open_target_capacity_refusal(
5488 ctx,
5489 &caller,
5490 &frame,
5491 &target_module_id,
5492 in_flight,
5493 )?]);
5494 }
5495 };
5496
5497 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5500 RouteBindAdmission::Admitted { guard, probe } => {
5501 if probe {
5502 info!(
5503 module_id = %target_module_id,
5504 connection_id = ctx.connection_id.get(),
5505 "route.bind breaker half-open: admitting one probe"
5506 );
5507 }
5508 guard
5509 }
5510 RouteBindAdmission::Refused {
5511 consecutive_timeouts,
5512 retry_in,
5513 probe_in_flight,
5514 } => {
5515 return Ok(vec![self.route_open_breaker_refusal_frame(
5516 ctx,
5517 &caller,
5518 &frame,
5519 &target_module_id,
5520 consecutive_timeouts,
5521 retry_in,
5522 probe_in_flight,
5523 )?]);
5524 }
5525 };
5526
5527 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5532 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5533 let operator_app = self
5534 .operator_apps
5535 .lock()
5536 .unwrap_or_else(|p| p.into_inner())
5537 .get(&ctx.connection_id)
5538 .cloned();
5539 timings.enter(RouteOpenPhase::EgressReservation);
5540 let pending = match self
5541 .forwarding
5542 .begin_route_bind_relay_for(
5543 ctx.connection_id,
5544 ctx.egress.clone(),
5545 response_version(&frame),
5546 frame.header.corr,
5547 &target_module_id,
5548 principal.clone(),
5549 bound_scope,
5550 project_root,
5551 identity.harness.clone(),
5552 build_class,
5553 operator_app.clone(),
5554 relay_deadline,
5555 busy_refusals == Some(true),
5556 )
5557 .await
5558 {
5559 Ok(pending) => pending,
5560 Err(err) => {
5561 return Ok(vec![self.route_open_refusal_frame(
5562 ctx,
5563 &caller,
5564 &frame,
5565 &target_module_id,
5566 "relay_reservation_failed",
5567 forwarding_error_code(&err),
5568 err.to_string(),
5569 )?])
5570 }
5571 };
5572 timings.enter(RouteOpenPhase::Admission);
5573 let crate::forwarding::PendingRouteBindRelay {
5574 endpoint,
5575 module_sink,
5576 negotiated_ver,
5577 client_channel,
5578 client_epoch,
5579 module_channel,
5580 module_epoch,
5581 corr: relay_corr,
5582 receiver,
5583 } = pending;
5584 let mut reservation =
5585 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5586
5587 let relay_registration = self
5592 .registry
5593 .get_module_by_connection(endpoint.connection_id)
5594 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5595 let relay_capabilities = relay_registration
5596 .as_ref()
5597 .and_then(|registration| registration.manifest.capabilities.as_ref());
5598 if let Some(selector) = &scope {
5599 if let Err(refusal) = self
5602 .scopes
5603 .read()
5604 .unwrap_or_else(|poisoned| poisoned.into_inner())
5605 .check_bind_proof(&principal, selector)
5606 {
5607 reservation.release_and_disarm();
5608 return Ok(vec![self.route_open_refusal_frame(
5609 ctx,
5610 &caller,
5611 &frame,
5612 &target_module_id,
5613 refusal.code,
5614 refusal.code,
5615 refusal.message,
5616 )?]);
5617 }
5618 }
5619 if let Some(stamp) = scope_stamp.as_ref() {
5620 if let Err(refusal) = crate::scopes::check_target_flow_support(
5621 stamp,
5622 &target_module_id,
5623 relay_capabilities,
5624 )
5625 .and_then(|()| {
5626 crate::scopes::check_target_agent_run_support(
5627 stamp,
5628 &target_module_id,
5629 relay_capabilities,
5630 )
5631 }) {
5632 reservation.release_and_disarm();
5633 return Ok(vec![self.route_open_refusal_frame(
5634 ctx,
5635 &caller,
5636 &frame,
5637 &target_module_id,
5638 refusal.code,
5639 refusal.code,
5640 refusal.message,
5641 )?]);
5642 }
5643 }
5644
5645 if scope_stamp
5648 .as_ref()
5649 .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
5650 && !crate::scopes::target_supports_federated(relay_capabilities)
5651 {
5652 scope_stamp = None;
5654 }
5655 if !relay_capabilities.is_some_and(|caps| {
5656 caps.provides
5657 .iter()
5658 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5659 }) {
5660 if let Some(stamp) = scope_stamp.as_mut() {
5661 stamp.workdir = None;
5662 stamp.project_id = None;
5663 }
5664 }
5665 let worker_run = scope_stamp
5670 .as_ref()
5671 .is_some_and(crate::scopes::is_worker_run)
5672 && relay_capabilities.is_some_and(|caps| {
5673 caps.provides
5674 .iter()
5675 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5676 });
5677 if !worker_run
5678 && !relay_capabilities.is_some_and(|caps| {
5679 caps.provides
5680 .iter()
5681 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5682 })
5683 {
5684 if let Some(stamp) = scope_stamp.as_mut() {
5685 stamp.attributes.on_behalf_of_agent = None;
5686 }
5687 }
5688 if !relay_capabilities.is_some_and(|caps| {
5689 caps.provides
5690 .iter()
5691 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5692 }) {
5693 if let Some(stamp) = scope_stamp.as_mut() {
5694 stamp.attributes.flow_version = None;
5695 stamp.attributes.flow_code_hash = None;
5696 stamp.attributes.flow_install_id = None;
5697 }
5698 }
5699 let call_mode = if worker_run
5700 || relay_capabilities
5701 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5702 {
5703 Some(derive_call_mode(scope_stamp.as_ref()))
5704 } else {
5705 if let Some(stamp) = scope_stamp.as_mut() {
5706 stamp.expires_at_ms = None;
5707 }
5708 None
5709 };
5710
5711 debug!(
5712 connection_id = ctx.connection_id.get(),
5713 client_channel,
5714 client_epoch,
5715 module_channel,
5716 module_epoch,
5717 "reserved route handle pair"
5718 );
5719 let route_principal_label = principal_label(&principal);
5722 let relay = ModuleControlRequest::RouteBind {
5723 route_channel: module_channel,
5724 epoch: module_epoch,
5725 target,
5726 identity,
5727 principal: Some(principal),
5728 consumer_capabilities,
5729 role_versions,
5730 admission_facts,
5731 scope: scope_stamp,
5732 call_mode,
5733 };
5734 let peer = RouteBindPeer::new()
5735 .with_observed_peer(
5736 relay_capabilities
5737 .is_some_and(|caps| {
5738 caps.provides
5739 .iter()
5740 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5741 })
5742 .then(|| {
5743 self.observed_peers
5744 .lock()
5745 .unwrap_or_else(|p| p.into_inner())
5746 .get(&ctx.connection_id)
5747 .cloned()
5748 .unwrap_or_else(ObservedPeer::unobserved)
5749 }),
5750 )
5751 .with_operator_app(
5752 self.connection_operator_app(operator_app.as_ref())
5753 .filter(|_| {
5754 relay_capabilities.is_some_and(|caps| {
5755 caps.provides
5756 .iter()
5757 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5758 })
5759 }),
5760 );
5761 let relay_body = crate::bind_compat::encode_bind(
5762 &relay,
5763 &peer,
5764 &legacy_bind_project,
5765 relay_capabilities,
5766 )
5767 .map_err(|err| {
5768 RouterError::backend(
5769 0,
5770 frame.header.corr,
5771 format!("failed to encode route.bind request: {err}"),
5772 )
5773 })?;
5774 let relay_frame = Frame::build_with_version(
5775 negotiated_ver,
5776 FrameType::Request,
5777 control_flags(),
5778 0,
5779 0,
5780 relay_corr,
5781 relay_body,
5782 )
5783 .map_err(RouterError::FrameBuild)?;
5784
5785 timings.enter(RouteOpenPhase::BindSend);
5786 let sent = module_sink.send(relay_frame).await;
5787 timings.enter(RouteOpenPhase::Admission);
5788 if let Err(err) = sent {
5789 reservation.release_and_disarm();
5790 return Ok(vec![self.route_open_refusal_frame(
5791 ctx,
5792 &caller,
5793 &frame,
5794 &target_module_id,
5795 "relay_send_failed",
5796 "target_unavailable",
5797 err.to_string(),
5798 )?]);
5799 }
5800
5801 #[cfg(test)]
5802 if let Some(hold) = &self.route_bind_relay_sent_hold {
5803 hold.wait().await;
5807 hold.wait().await;
5808 }
5809
5810 if self
5815 .forwarding
5816 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5817 .map_err(RouterError::Forwarding)?
5818 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5819 {
5820 self.send_abandoned_route_bind_goodbye(
5821 &module_sink,
5822 negotiated_ver,
5823 module_channel,
5824 module_epoch,
5825 );
5826 }
5827
5828 timings.enter(RouteOpenPhase::BindAckWait);
5829 let outcome = timeout_at(relay_deadline, receiver).await;
5830 timings.enter(RouteOpenPhase::Admission);
5831 match outcome {
5832 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5833 reservation.disarm();
5834 if breaker.record_accepted() {
5835 info!(
5836 module_id = %target_module_id,
5837 "route.bind breaker closed: the probe was accepted"
5838 );
5839 }
5840 self.observe_route_open_accept(
5841 ctx,
5842 &target_module_id,
5843 &route_principal_label,
5844 &caller.scope,
5845 );
5846 Ok(Vec::new())
5847 }
5848 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5849 reservation.release_and_disarm();
5850 breaker.record_inconclusive();
5854 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5855 }
5856 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5857 reservation.release_and_disarm();
5858 breaker.record_inconclusive();
5859 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5860 }
5861 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5862 reservation.release_and_disarm();
5863 breaker.record_inconclusive();
5864 tracing::warn!(
5871 module_id = %target_module_id,
5872 "route.bind relay abandoned: {message}"
5873 );
5874 Ok(vec![self.route_open_refusal_frame(
5875 ctx,
5876 &caller,
5877 &frame,
5878 &target_module_id,
5879 "relay_abandoned",
5880 "target_unavailable",
5881 message,
5882 )?])
5883 }
5884 Ok(Err(_)) => {
5885 reservation.release_and_disarm();
5886 breaker.record_inconclusive();
5887 Ok(vec![self.route_open_refusal_frame(
5888 ctx,
5889 &caller,
5890 &frame,
5891 &target_module_id,
5892 "relay_waiter_canceled",
5893 "target_unavailable",
5894 "route.bind relay waiter was canceled before the module responded",
5895 )?])
5896 }
5897 Err(_) => {
5898 reservation.release_and_disarm();
5899 if let Some(opened) = breaker.record_timeout(
5903 self.route_bind_breaker_threshold,
5904 self.route_bind_breaker_cooldown,
5905 ) {
5906 warn!(
5907 module_id = %target_module_id,
5908 consecutive_timeouts = opened.consecutive_timeouts,
5909 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5910 reopened_after_probe = opened.reopened_after_probe,
5911 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5912 );
5913 }
5914 tracing::warn!(
5922 module_id = %target_module_id,
5923 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5924 "route.bind relay timed out: module did not ack within budget"
5925 );
5926 Ok(vec![self.route_open_refusal_frame(
5927 ctx,
5928 &caller,
5929 &frame,
5930 &target_module_id,
5931 "relay_timed_out",
5932 "module_timeout",
5933 format!(
5934 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5935 route_bind_relay_timeout
5936 ),
5937 )?])
5938 }
5939 }
5940 }
5941
5942 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5943 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5944 snapshot: self.supervisor.spawn_snapshot(),
5945 };
5946 Ok(vec![control_response_body_frame(
5947 &frame,
5948 &response,
5949 "ClientControlResponse::SupervisorSpawnSnapshot",
5950 )?])
5951 }
5952
5953 fn handle_supervisor_spawn_subscribe(
5954 &self,
5955 ctx: &RouteCtx,
5956 frame: Frame,
5957 since: Option<SpawnCursor>,
5958 ) -> Result<Vec<Frame>, RouterError> {
5959 match self.supervisor.subscribe_spawns(
5960 ctx.connection_id,
5961 frame.header.corr,
5962 response_version(&frame),
5963 since,
5964 ctx.egress.clone(),
5965 ) {
5966 Ok(()) => Ok(Vec::new()),
5967 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5968 Ok(vec![control_error_body_frame(
5969 &frame,
5970 ErrorBody {
5971 code: "spawn_cursor_incarnation_mismatch".to_string(),
5972 message: "spawn cursor belongs to a different daemon incarnation"
5973 .to_string(),
5974 detail: Some(serde_json::json!({
5975 "current_daemon_incarnation": current
5976 })),
5977 },
5978 )?])
5979 }
5980 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5981 &frame,
5982 ErrorBody {
5983 code: "spawn_cursor_too_old".to_string(),
5984 message: "spawn cursor predates the retained event ring".to_string(),
5985 detail: Some(serde_json::json!({
5986 "oldest_retained_cursor": oldest
5987 })),
5988 },
5989 )?]),
5990 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5991 0,
5992 frame.header.corr,
5993 format!("failed to open supervisor spawn subscription: {error}"),
5994 )),
5995 }
5996 }
5997
5998 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5999 let handler = self.clone();
6002 let runtime = tokio::runtime::Handle::current();
6003 let subscriber = tracing::dispatcher::get_default(Clone::clone);
6006 let corr = frame.header.corr;
6007 tokio::task::spawn_blocking(move || {
6008 runtime.block_on(
6009 list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
6010 .with_subscriber(subscriber),
6011 )
6012 })
6013 .await
6014 .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
6015 }
6016
6017 async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6018 let generation = self
6019 .registry
6020 .generation()
6021 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6022 let supervised = self.supervisor.list();
6023 let mut statuses = BTreeMap::new();
6024 let mut not_running = BTreeSet::new();
6025 for module in &supervised {
6026 let status = module.status_for_control("list").map_err(|err| {
6027 RouterError::backend(
6028 0,
6029 frame.header.corr,
6030 format!("failed to read supervisor status: {err}"),
6031 )
6032 })?;
6033 if !status.enabled || !status.process_alive {
6034 not_running.insert(module.module_id().to_owned());
6035 }
6036 statuses.insert(module.module_id().to_owned(), status);
6037 }
6038 let mut process_resources = BTreeMap::new();
6041 for module in &supervised {
6042 process_resources.insert(
6043 module.module_id().to_owned(),
6044 crate::subtree_resources::read_process(module.clone()).await,
6045 );
6046 }
6047 let mut image_readings = BTreeMap::new();
6050 for module in &supervised {
6051 image_readings.insert(
6052 module.module_id().to_owned(),
6053 module.running_image_agreement().await,
6054 );
6055 }
6056 let readings = self
6057 .resource_tracker
6058 .read(supervised.clone(), false, "list")
6059 .await;
6060 let mut modules = Vec::new();
6061 for module in supervised {
6062 let status = statuses
6063 .remove(module.module_id())
6064 .expect("each supervised module has a status snapshot");
6065 let (configured, _) = module.configuration().map_err(|err| {
6066 RouterError::backend(
6067 0,
6068 frame.header.corr,
6069 format!("failed to read module configuration: {err}"),
6070 )
6071 })?;
6072 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
6073 RouterError::backend(
6074 0,
6075 frame.header.corr,
6076 format!("failed to read retry time: {err}"),
6077 )
6078 })?;
6079 let image = image_readings
6080 .remove(module.module_id())
6081 .expect("each supervised module has an image snapshot");
6082 let resources = process_resources.remove(module.module_id());
6083 let mut verdict =
6084 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
6085 verdict.spec = Some(launch_spec_agreement(
6086 &configured,
6087 status.process_alive,
6088 status.launch_spec_digest.as_ref(),
6089 ));
6090 let pending_reload = Some(verdict);
6091 let configured_enabled = self
6092 .capability_evaluator
6093 .configured_enabled(&status.module_id);
6094 modules.push(
6097 SupervisorEntry::new(
6098 status.module_id,
6099 status.state.to_string(),
6100 status.enabled,
6101 status.live,
6102 status.health.status,
6103 )
6104 .with_configured_enabled(configured_enabled)
6105 .with_next_retry_at_ms(next_retry_at_ms)
6106 .with_launch_nonce_env(Some(!cfg!(unix)))
6107 .with_launch_nonce_source(status.launch_nonce_source)
6108 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
6109 .with_protocol(status.protocol)
6110 .with_pending_reload(pending_reload)
6111 .with_last_probe_ms(status.health.last_probe_ms)
6112 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
6113 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
6114 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
6115 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
6116 .with_restart_count(Some(status.restart_count))
6117 .with_max_restarts(Some(status.max_restarts))
6118 .with_lifetime_restarts(Some(status.lifetime_restarts))
6119 .with_spawn_generation(Some(status.spawn_generation))
6120 .with_restart_window_secs(Some(status.restart_window.as_secs()))
6121 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
6122 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
6123 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
6124 .with_resources(resources),
6125 );
6126 }
6127 for module in &mut modules {
6128 if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
6129 module.tree = tree.clone();
6130 module.cpu_last_hour = hour.clone();
6131 }
6132 if module.tree.is_none() {
6133 module.tree_unavailable = Some(
6134 if not_running.contains(&module.module_id) {
6135 "not_running"
6136 } else if readings.modules.contains_key(&module.module_id) {
6137 "tree_unavailable"
6138 } else {
6139 "not_yet_measured"
6140 }
6141 .to_owned(),
6142 );
6143 }
6144 }
6145 let response = ClientControlResponse::SupervisorList {
6146 generation,
6147 modules,
6148 daemon: readings.daemon,
6149 agent_commands: readings.agent_commands,
6150 };
6151 Ok(vec![control_response_body_frame(
6152 &frame,
6153 &response,
6154 "ClientControlResponse::SupervisorList",
6155 )?])
6156 }
6157
6158 fn handle_supervisor_stderr_tail(
6159 &self,
6160 frame: Frame,
6161 module_id: String,
6162 max_lines: Option<u32>,
6163 max_bytes: Option<u32>,
6164 ) -> Result<Vec<Frame>, RouterError> {
6165 let Some(module) = self.supervisor.get(&module_id) else {
6166 return Ok(vec![control_error_frame(
6167 &frame,
6168 "unknown_module",
6169 format!("module_id '{module_id}' is not supervised"),
6170 )?]);
6171 };
6172
6173 let snapshot = module.stderr_tail(
6174 max_lines.map(|value| value as usize),
6175 max_bytes.map(|value| value as usize),
6176 );
6177
6178 let response = ClientControlResponse::SupervisorStderrTail {
6179 module_id,
6180 tail: StderrTail {
6181 capture: match snapshot.capture {
6182 CaptureState::Captured => StderrCaptureState::Captured,
6183 CaptureState::Incomplete { reason } => {
6184 StderrCaptureState::Incomplete { reason }
6185 }
6186 CaptureState::NotCaptured { reason } => {
6187 StderrCaptureState::NotCaptured { reason }
6188 }
6189 },
6190 entries: snapshot
6191 .entries
6192 .into_iter()
6193 .map(|entry| match entry {
6194 TailEntry::Line {
6195 text,
6196 truncated,
6197 at_ms,
6198 } => StderrTailEntry::Line {
6199 text,
6200 truncated,
6201 at_ms,
6202 },
6203 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6204 })
6205 .collect(),
6206 dropped_lines: snapshot.dropped_lines,
6207 },
6208 };
6209 Ok(vec![control_response_body_frame(
6210 &frame,
6211 &response,
6212 "ClientControlResponse::SupervisorStderrTail",
6213 )?])
6214 }
6215
6216 async fn observe_control_wait<F: std::future::Future>(
6217 &self,
6218 ctx: &RouteCtx,
6219 target: Option<&str>,
6220 kind: &'static str,
6221 future: F,
6222 ) -> F::Output {
6223 self.counters
6224 .observe_reader_wait(
6225 || {
6226 let module = self
6227 .forwarding
6228 .module_id_for_connection(ctx.connection_id)
6229 .ok()
6230 .flatten();
6231 let principal = if module.is_some() { "module" } else { "direct" };
6232 (
6233 self.resolve_reader_wait_target(kind, target)
6234 .or(module)
6235 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6236 principal.into(),
6237 )
6238 },
6239 kind,
6240 future,
6241 )
6242 .await
6243 }
6244
6245 pub(crate) fn resolve_reader_wait_target(
6248 &self,
6249 op: &str,
6250 selector: Option<&str>,
6251 ) -> Option<String> {
6252 let selector = selector?;
6253 match op {
6254 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6255 .supervisor
6256 .get(selector)
6257 .map(|module| module.module_id().to_owned()),
6258 ops::SUPERVISOR_HEALTH_PROBE => self
6259 .registry
6260 .get_module(selector)
6261 .ok()
6262 .flatten()
6263 .map(|registration| registration.manifest.module_id),
6264 _ => None,
6265 }
6266 }
6267
6268 async fn handle_supervisor_terminals(
6269 &self,
6270 ctx: &RouteCtx,
6271 frame: Frame,
6272 module_id: String,
6273 ) -> Result<Vec<Frame>, RouterError> {
6274 let Some(module) = self.supervisor.get(&module_id) else {
6275 return Ok(vec![control_error_frame(
6276 &frame,
6277 "unknown_module",
6278 format!("module_id '{module_id}' is not supervised"),
6279 )?]);
6280 };
6281
6282 let terminals = self
6285 .observe_control_wait(
6286 ctx,
6287 Some(module.module_id()),
6288 ops::SUPERVISOR_TERMINALS,
6289 module.read_durable_terminal_history(),
6290 )
6291 .await
6292 .map_err(|error| {
6293 RouterError::backend(
6294 0,
6295 frame.header.corr,
6296 format!("failed to read terminal history: {error}"),
6297 )
6298 })?;
6299 let response = ClientControlResponse::SupervisorTerminals {
6300 module_id,
6301 terminals,
6302 };
6303 Ok(vec![control_response_body_frame(
6304 &frame,
6305 &response,
6306 "ClientControlResponse::SupervisorTerminals",
6307 )?])
6308 }
6309
6310 fn handle_supervisor_routes(
6311 &self,
6312 frame: Frame,
6313 module_id: Option<String>,
6314 ) -> Result<Vec<Frame>, RouterError> {
6315 let modules = self
6316 .forwarding
6317 .route_census(module_id.as_deref())
6318 .map_err(RouterError::Forwarding)?
6319 .into_iter()
6320 .map(|(module_id, routes)| SupervisorRouteModule {
6321 module_id,
6322 routes: routes
6323 .into_iter()
6324 .map(|route| SupervisorRoute {
6325 consumer: match route.principal {
6326 Principal::Reserved { module_id } => {
6327 SupervisorRouteConsumer::Reserved { module_id }
6328 }
6329 Principal::Direct | Principal::Unverified => {
6330 SupervisorRouteConsumer::Direct {
6331 connection_id: route.goodbye_target.connection_id.get(),
6332 }
6333 }
6334 },
6335 age_ms: Instant::now()
6336 .saturating_duration_since(route.bound_at)
6337 .as_millis()
6338 .try_into()
6339 .unwrap_or(u64::MAX),
6340 draining: route.draining,
6341 drain_reason: route.drain_reason,
6342 })
6343 .collect(),
6344 })
6345 .collect();
6346 let response = ClientControlResponse::SupervisorRoutes { modules };
6347 Ok(vec![control_response_body_frame(
6348 &frame,
6349 &response,
6350 "ClientControlResponse::SupervisorRoutes",
6351 )?])
6352 }
6353
6354 async fn handle_supervisor_provenance(
6355 &self,
6356 ctx: &RouteCtx,
6357 frame: Frame,
6358 module_id: Option<String>,
6359 ) -> Result<Vec<Frame>, RouterError> {
6360 let targeted = module_id.is_some();
6361 let selected = if let Some(module_id) = module_id {
6362 let Some(module) = self.supervisor.get(&module_id) else {
6363 return Ok(vec![control_error_frame(
6364 &frame,
6365 "unknown_module",
6366 format!("module_id '{module_id}' is not supervised"),
6367 )?]);
6368 };
6369 vec![module]
6370 } else {
6371 self.supervisor.list()
6372 };
6373
6374 let target = targeted.then(|| selected[0].module_id());
6375 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6376 let mut modules = Vec::with_capacity(selected.len());
6377 for module in &selected {
6378 let (status, observed_image) = module
6379 .status_and_running_image_agreement()
6380 .await
6381 .map_err(|err| {
6382 RouterError::backend(
6383 0,
6384 frame.header.corr,
6385 format!("failed to read supervisor status: {err}"),
6386 )
6387 })?;
6388 let module_declared = self
6389 .registry
6390 .get_module(&status.module_id)
6391 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6392 .and_then(|registration| registration.manifest.provenance)
6393 .map(|build| ModuleDeclaredProvenance::Reported { build })
6394 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6395 #[cfg(test)]
6396 let running_image = match &self.provenance_probe_override {
6397 Some(result) => result.clone(),
6398 None => observed_image,
6399 };
6400 #[cfg(not(test))]
6401 let running_image = observed_image;
6402 modules.push(SupervisorModuleProvenance {
6403 module_id: status.module_id,
6404 module_declared,
6405 daemon_observed: SupervisorObservedProcess {
6406 launch_nonce_source: status.launch_nonce_source,
6407 pid: status.pid,
6408 spawned_at_ms: status.spawned_at_ms,
6409 spawned_from: status.spawned_from,
6410 running_image,
6411 },
6412 });
6413 }
6414 let daemon = SupervisorDaemonProvenance {
6415 daemon_build: self.daemon_provenance.build.clone(),
6416 daemon_observed: DaemonObservedProcess {
6417 pid: self.daemon_provenance.pid,
6418 started_at_ms: self
6419 .daemon_provenance
6420 .start_clock
6421 .map(|clock| clock.started_at_ms())
6422 .or(self.daemon_provenance.started_at_ms),
6423 running_image: self
6424 .daemon_provenance
6425 .probe
6426 .observe(
6427 self.daemon_provenance.pid,
6428 self.daemon_provenance.executable_path.as_deref(),
6429 self.daemon_provenance.executable_identity.clone(),
6430 self.daemon_provenance.process_start_time,
6431 )
6432 .await,
6433 },
6434 };
6435 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6436 Ok(vec![control_response_body_frame(
6437 &frame,
6438 &response,
6439 "ClientControlResponse::SupervisorProvenance",
6440 )?])
6441 })
6442 .await
6443 }
6444
6445 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6446 self.refresh_capability_requirements();
6447 let generation = self
6448 .registry
6449 .generation()
6450 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6451 let modules = self
6452 .supervisor
6453 .list()
6454 .into_iter()
6455 .map(|module| {
6456 let status = module.status_for_control("health").map_err(|err| {
6457 RouterError::backend(
6458 0,
6459 frame.header.corr,
6460 format!("failed to read supervisor health: {err}"),
6461 )
6462 })?;
6463 let module_id = status.module_id;
6464 let capability_detail = self
6465 .capability_evaluator
6466 .required_problem_detail(&module_id);
6467 Ok(SupervisorHealthEntry {
6468 module_id,
6469 status: status.health.status,
6470 detail: append_capability_problem_detail(
6471 status.health.detail,
6472 capability_detail,
6473 ),
6474 metrics: status.health.metrics,
6475 consecutive_failures: status.health.consecutive_failures,
6476 late_answer_count: status.health.late_answer_count,
6477 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6478 last_action: status.health.last_action,
6479 last_action_ms: status.health.last_action_ms,
6480 last_probe_ms: status.health.last_probe_ms,
6481 })
6482 })
6483 .collect::<Result<Vec<_>, RouterError>>()?;
6484 let response = ClientControlResponse::SupervisorHealth {
6485 generation,
6486 modules,
6487 };
6488 Ok(vec![control_response_body_frame(
6489 &frame,
6490 &response,
6491 "ClientControlResponse::SupervisorHealth",
6492 )?])
6493 }
6494
6495 async fn handle_supervisor_restart(
6496 &self,
6497 frame: Frame,
6498 module_id: String,
6499 drain_timeout_ms: Option<u64>,
6500 requester: Principal,
6501 ) -> Result<Vec<Frame>, RouterError> {
6502 let operation_lock = self.supervisor.operation_lock();
6503 let _operation_guard = operation_lock.lock().await;
6504 let Some(module) = self.supervisor.get(&module_id) else {
6505 return Ok(vec![control_error_frame(
6506 &frame,
6507 "unknown_module",
6508 format!("module_id '{module_id}' is not supervised"),
6509 )?]);
6510 };
6511
6512 self.route_outages.mark_operator_action(&module_id);
6513 if let Err(err) = module
6514 .restart_requested(drain_timeout_ms, Some(requester))
6515 .await
6516 {
6517 self.route_outages
6518 .operator_action_ended_unrefused(&module_id);
6519 let (code, message) = match err {
6520 crate::supervise::SuperviseError::Disabled { .. } => {
6521 ("module_disabled", err.to_string())
6522 }
6523 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6524 ("swap_in_progress", err.to_string())
6525 }
6526 _ => (
6527 "target_unavailable",
6528 format!("failed to restart module_id '{module_id}': {err}"),
6529 ),
6530 };
6531 return Ok(vec![control_error_frame(&frame, code, message)?]);
6532 }
6533
6534 let response = ClientControlResponse::SupervisorAck {
6535 module_id,
6536 applied: true,
6537 };
6538 Ok(vec![control_response_body_frame(
6539 &frame,
6540 &response,
6541 "ClientControlResponse::SupervisorAck",
6542 )?])
6543 }
6544
6545 async fn handle_supervisor_swap(
6549 &self,
6550 frame: Frame,
6551 module_id: String,
6552 ready_timeout_ms: Option<u64>,
6553 requester: Principal,
6554 ) -> Result<Vec<Frame>, RouterError> {
6555 let module = {
6562 let operation_lock = self.supervisor.operation_lock();
6563 let _operation_guard = operation_lock.lock().await;
6564 self.supervisor.get(&module_id)
6565 };
6566 let Some(module) = module else {
6567 return Ok(vec![control_error_frame(
6568 &frame,
6569 "unknown_module",
6570 format!("module_id '{module_id}' is not supervised"),
6571 )?]);
6572 };
6573
6574 self.route_outages.mark_operator_action(&module_id);
6575 if let Err(err) = module
6576 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6577 .await
6578 {
6579 self.route_outages
6580 .operator_action_ended_unrefused(&module_id);
6581 use crate::supervise::SuperviseError;
6582 let message = err.to_string();
6583 let error = match err {
6584 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6585 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6586 code: "swap_refused".to_string(),
6587 message,
6588 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6589 },
6590 SuperviseError::SwapFailed {
6591 arm,
6592 candidate_exit,
6593 ..
6594 } => ErrorBody {
6595 code: "swap_failed".to_string(),
6596 message,
6597 detail: Some(serde_json::json!({
6598 "arm": arm.as_str(),
6599 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6600 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6601 })),
6602 },
6603 _ => ErrorBody::new(
6604 "target_unavailable",
6605 format!("failed to swap module_id '{module_id}': {message}"),
6606 ),
6607 };
6608 return Ok(vec![control_error_body_frame(&frame, error)?]);
6609 }
6610 self.route_outages
6614 .operator_action_ended_unrefused(&module_id);
6615
6616 let response = ClientControlResponse::SupervisorAck {
6617 module_id,
6618 applied: true,
6619 };
6620 Ok(vec![control_response_body_frame(
6621 &frame,
6622 &response,
6623 "ClientControlResponse::SupervisorAck",
6624 )?])
6625 }
6626
6627 async fn handle_supervisor_reload(
6628 &self,
6629 frame: Frame,
6630 module_id: String,
6631 requester: Principal,
6632 ) -> Result<Vec<Frame>, RouterError> {
6633 let operation_lock = self.supervisor.operation_lock();
6634 let _operation_guard = operation_lock.lock().await;
6635 let Some(module) = self.supervisor.get(&module_id) else {
6636 return Ok(vec![control_error_frame(
6637 &frame,
6638 "unknown_module",
6639 format!("module_id '{module_id}' is not supervised"),
6640 )?]);
6641 };
6642
6643 self.route_outages.mark_operator_action(&module_id);
6644 if let Err(err) = module.reload_requested(Some(requester)).await {
6645 self.route_outages
6646 .operator_action_ended_unrefused(&module_id);
6647 let (code, message) = match err {
6648 crate::supervise::SuperviseError::Disabled { .. } => {
6649 ("module_disabled", err.to_string())
6650 }
6651 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6652 ("swap_in_progress", err.to_string())
6653 }
6654 _ => (
6655 "reload_failed",
6656 format!("failed to reload module_id '{module_id}': {err}"),
6657 ),
6658 };
6659 return Ok(vec![control_error_frame(&frame, code, message)?]);
6660 }
6661
6662 let response = ClientControlResponse::SupervisorAck {
6663 module_id,
6664 applied: true,
6665 };
6666 Ok(vec![control_response_body_frame(
6667 &frame,
6668 &response,
6669 "ClientControlResponse::SupervisorAck",
6670 )?])
6671 }
6672
6673 async fn handle_supervisor_rescan(
6677 &self,
6678 frame: Frame,
6679 mode: RescanMode,
6680 requester: Principal,
6681 ) -> Result<Vec<Frame>, RouterError> {
6682 let preview = mode.previews();
6683 let Some(context) = self.rescan.clone() else {
6684 return Ok(vec![control_error_frame(
6685 &frame,
6686 "rescan_unavailable",
6687 "the daemon was not started with a reloadable config path".to_string(),
6688 )?]);
6689 };
6690
6691 let operation_lock = self.supervisor.operation_lock();
6692 let _operation_guard = operation_lock.lock().await;
6693 let loaded = match self.load_rescan_config(&mode, &context) {
6697 Ok(config) => config,
6698 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6699 };
6700 let Some((config, activity_config)) = loaded else {
6708 return Ok(vec![control_error_frame(
6709 &frame,
6710 "invalid_daemon_config",
6711 format!(
6712 "daemon config not found at {}; refusing to rescan (an absent config would \
6713 retire every supervised module)",
6714 context.config_path.display()
6715 ),
6716 )?]);
6717 };
6718 let (
6719 configured_port,
6720 storage_config,
6721 admission_facts_carrier_module_id,
6722 admission_facts_targets,
6723 scope_authority_owners,
6724 federated_scope_owner,
6725 privileged_operator_prompt,
6726 modules,
6727 reserved_capabilities,
6728 ) = (
6729 config.port,
6730 config.storage,
6731 config.admission_facts_carrier_module_id,
6732 config.admission_facts_targets,
6733 config.scope_authority_owners,
6734 config.federated_scope_owner,
6735 config.privileged_operator_prompt,
6736 config.modules,
6737 config.reserved_capabilities,
6738 );
6739
6740 let mut restart_required = Vec::new();
6749 for section in RestartRequiredSection::ALL {
6750 let changed = match section {
6751 RestartRequiredSection::Port => configured_port != context.configured_port,
6752 RestartRequiredSection::Storage => storage_config != context.storage_config,
6753 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6754 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6755 }
6756 RestartRequiredSection::AdmissionFactsTargets => {
6757 admission_facts_targets != context.admission_facts_targets
6758 }
6759 RestartRequiredSection::ScopeAuthorityOwners => {
6760 scope_authority_owners != context.scope_authority_owners
6761 }
6762 RestartRequiredSection::FederatedScopeOwner => {
6763 federated_scope_owner != context.federated_scope_owner
6764 }
6765 RestartRequiredSection::PrivilegedOperatorPrompt => {
6766 privileged_operator_prompt != context.privileged_operator_prompt
6767 }
6768 };
6769 if changed {
6770 restart_required.push(section.label().to_string());
6771 }
6772 }
6773 if !restart_required.is_empty() {
6774 warn!(
6775 config_path = %context.config_path.display(),
6776 sections = %restart_required.join(", "),
6777 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6778 );
6779 }
6780
6781 for configured in &modules {
6782 if let Err(err) = validate_spec(&configured.module_spec()) {
6783 return Ok(vec![control_error_frame(
6784 &frame,
6785 "invalid_daemon_config",
6786 format!("supervisor rescan rejected daemon config: {err}"),
6787 )?]);
6788 }
6789 }
6790
6791 let configured_capabilities = modules
6792 .iter()
6793 .map(|module| (module.module_id.clone(), module.enabled))
6794 .collect::<Vec<_>>();
6795 let preview_capability_warnings = if preview {
6796 let (_, registrations) = self.runtime_capability_snapshot()?;
6797 let current_modules = self
6798 .supervisor
6799 .list()
6800 .into_iter()
6801 .map(|module| module.module_id().to_string())
6802 .collect::<BTreeSet<_>>();
6803 let resulting_modules = configured_capabilities.clone();
6804 let removed = current_modules
6805 .into_iter()
6806 .filter(|module_id| {
6807 !resulting_modules
6808 .iter()
6809 .any(|(configured_id, _)| configured_id == module_id)
6810 })
6811 .collect::<Vec<_>>();
6812 let disabled = match &mode {
6815 RescanMode::Candidate(_) => resulting_modules
6816 .iter()
6817 .filter(|(_, enabled)| !enabled)
6818 .map(|(module_id, _)| module_id.clone())
6819 .collect::<Vec<_>>(),
6820 _ => Vec::new(),
6821 };
6822 self.capability_evaluator.preview_warnings(
6823 resulting_modules,
6824 &removed,
6825 &disabled,
6826 ®istrations,
6827 )
6828 } else {
6829 Vec::new()
6830 };
6831 let reconciled = match &mode {
6832 RescanMode::Candidate(candidate) => self
6833 .preview_candidate(&context, modules, candidate)
6834 .map(|result| (result, Vec::new())),
6835 _ => {
6836 self.reconcile_supervised_modules(
6837 &context.supervisor,
6838 modules,
6839 preview,
6840 requester.clone(),
6841 )
6842 .await
6843 }
6844 };
6845 let (result, changed) = match reconciled {
6846 Ok(result) => result,
6847 Err(message) => {
6848 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6849 }
6850 };
6851 if !preview {
6852 if let Some(ledger) = &self.activity {
6853 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6854 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6855 }
6856 if let Err(code) = self.configure_activity(activity_config).await {
6857 warn!(
6858 code,
6859 "activity retention could not be applied during rescan"
6860 );
6861 }
6862 self.capability_evaluator
6863 .configure(configured_capabilities, reserved_capabilities);
6864 self.capability_evaluator.wake_deadline_loop();
6865 self.refresh_capability_requirements();
6866 }
6867 let mut result = result;
6868 result.restart_required = restart_required;
6869 result.capability_warnings = preview_capability_warnings;
6870 let response = ClientControlResponse::SupervisorRescan { result };
6871 Ok(vec![control_response_body_frame(
6872 &frame,
6873 &response,
6874 "ClientControlResponse::SupervisorRescan",
6875 )?])
6876 }
6877
6878 async fn handle_supervisor_release_reserved(
6879 &self,
6880 frame: Frame,
6881 module_id: String,
6882 ) -> Result<Vec<Frame>, RouterError> {
6883 let Some(context) = self.rescan.clone() else {
6884 return Ok(vec![control_error_frame(
6885 &frame,
6886 "release_unavailable",
6887 "reserved-id release requires a daemon started with a reloadable config path",
6888 )?]);
6889 };
6890 let operation_lock = self.supervisor.operation_lock();
6891 let _operation_guard = operation_lock.lock().await;
6892 let loaded = match crate::daemon_config::load(&context.config_path) {
6893 Ok(Some(config)) => config,
6894 Ok(None) => {
6895 return Ok(vec![control_error_frame(
6896 &frame,
6897 "invalid_daemon_config",
6898 format!(
6899 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6900 context.config_path.display()
6901 ),
6902 )?])
6903 }
6904 Err(err) => {
6905 return Ok(vec![control_error_frame(
6906 &frame,
6907 "invalid_daemon_config",
6908 format!("unable to verify reserved-id release against daemon config: {err}"),
6909 )?])
6910 }
6911 };
6912 if loaded
6913 .modules
6914 .iter()
6915 .any(|configured| configured.module_id == module_id)
6916 {
6917 return Ok(vec![control_error_frame(
6918 &frame,
6919 "reserved_module_configured",
6920 format!(
6921 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6922 ),
6923 )?]);
6924 }
6925 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6926 return Ok(vec![control_error_frame(
6927 &frame,
6928 "reserved_gate_not_retained",
6929 format!(
6930 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6931 ),
6932 )?]);
6933 }
6934
6935 let response = ClientControlResponse::SupervisorAck {
6936 module_id,
6937 applied: true,
6938 };
6939 Ok(vec![control_response_body_frame(
6940 &frame,
6941 &response,
6942 "ClientControlResponse::SupervisorAck",
6943 )?])
6944 }
6945
6946 fn plan_rescan(
6950 &self,
6951 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6952 ) -> Result<RescanPlan, String> {
6953 let mut current = BTreeMap::new();
6954 for module in self.supervisor.list() {
6955 let (spec, health) = module.configuration().map_err(|err| {
6956 format!(
6957 "failed to read configuration for module_id '{}': {err}",
6958 module.module_id()
6959 )
6960 })?;
6961 let enabled = module
6962 .status()
6963 .map_err(|err| {
6964 format!(
6965 "failed to read status for module_id '{}': {err}",
6966 module.module_id()
6967 )
6968 })?
6969 .enabled;
6970 current.insert(
6971 module.module_id().to_string(),
6972 (module, spec, health, enabled),
6973 );
6974 }
6975 let configured = configured_modules
6976 .into_iter()
6977 .map(|module| (module.module_id.clone(), module))
6978 .collect::<BTreeMap<_, _>>();
6979
6980 let added = configured
6981 .keys()
6982 .filter(|module_id| !current.contains_key(*module_id))
6983 .cloned()
6984 .collect::<Vec<_>>();
6985 let removed = current
6986 .keys()
6987 .filter(|module_id| !configured.contains_key(*module_id))
6988 .cloned()
6989 .collect::<Vec<_>>();
6990 let mut changed_pending_reload = Vec::new();
6991 let mut configuration_changes = BTreeSet::new();
6992 let mut enabled_changes = BTreeSet::new();
6993 let mut unchanged = 0_u32;
6994
6995 for (module_id, configured_module) in &configured {
6996 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6997 else {
6998 continue;
6999 };
7000 let launch_changed = *current_spec != configured_module.module_spec();
7004 let configuration_changed =
7005 launch_changed || *current_health != configured_module.health;
7006 let enabled_changed = *current_enabled != configured_module.enabled
7009 || self
7010 .capability_evaluator
7011 .configured_enabled(module_id)
7012 .is_some_and(|enabled| enabled != configured_module.enabled);
7013 if configuration_changed {
7014 configuration_changes.insert(module_id.clone());
7015 }
7016 if launch_changed {
7017 changed_pending_reload.push(module_id.clone());
7018 }
7019 if enabled_changed {
7020 enabled_changes.insert(module_id.clone());
7021 }
7022 if !configuration_changed && !enabled_changed {
7023 unchanged = unchanged.saturating_add(1);
7024 }
7025 }
7026 Ok(RescanPlan {
7027 current,
7028 configured,
7029 added,
7030 removed,
7031 changed_pending_reload,
7032 configuration_changes,
7033 enabled_changes,
7034 unchanged,
7035 })
7036 }
7037
7038 async fn reconcile_supervised_modules(
7047 &self,
7048 supervisor: &Supervisor,
7049 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
7050 preview: bool,
7051 requester: Principal,
7052 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
7053 let plan = self.plan_rescan(configured_modules)?;
7054
7055 if preview {
7061 return Ok((plan.preview_result(), Vec::new()));
7062 }
7063 let changed = plan
7064 .configuration_changes
7065 .union(&plan.enabled_changes)
7066 .cloned()
7067 .collect::<Vec<_>>();
7068 let RescanPlan {
7069 current,
7070 configured,
7071 added,
7072 removed,
7073 changed_pending_reload,
7074 configuration_changes,
7075 enabled_changes,
7076 unchanged,
7077 } = plan;
7078 if !(added.is_empty()
7083 && removed.is_empty()
7084 && configuration_changes.is_empty()
7085 && enabled_changes.is_empty())
7086 {
7087 if let Some(context) = &self.rescan {
7088 context
7089 .generation
7090 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7091 }
7092 }
7093
7094 for module_id in &removed {
7095 let module = ¤t
7096 .get(module_id)
7097 .expect("removed module came from current supervisor state")
7098 .0;
7099 module
7100 .retire_requested(Some(requester.clone()))
7101 .await
7102 .map_err(|err| {
7103 format!("failed to retire module_id '{module_id}' during rescan: {err}")
7104 })?;
7105 self.supervisor.record_rescan_removal(module_id);
7139 self.supervisor.retire(module_id);
7140 self.route_outages.forget(module_id);
7141 }
7142
7143 for module_id in configured.keys() {
7144 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
7145 continue;
7146 };
7147 let configured_module = configured
7148 .get(module_id)
7149 .expect("configured module id came from configured map");
7150 if configuration_changes.contains(module_id) {
7151 module
7152 .update_configuration(
7153 configured_module.module_spec(),
7154 configured_module.health.clone(),
7155 configured_module.drain_timeout_ms,
7156 )
7157 .await
7158 .map_err(|err| {
7159 format!(
7160 "failed to update module_id '{module_id}' configuration during rescan: {err}"
7161 )
7162 })?;
7163 }
7164 if *current_enabled != configured_module.enabled {
7167 self.route_outages.mark_operator_action(module_id);
7170 module
7171 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
7172 .await
7173 .map_err(|err| {
7174 self.route_outages.operator_action_ended_unrefused(module_id);
7175 format!(
7176 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
7177 configured_module.enabled
7178 )
7179 })?;
7180 }
7181 }
7182
7183 for module_id in &added {
7184 let configured_module = configured
7185 .get(module_id)
7186 .expect("added module id came from configured map");
7187 supervisor
7188 .supervise_configured_requested(
7189 configured_module.module_spec(),
7190 configured_module.enabled,
7191 configured_module.health.clone(),
7192 configured_module.drain_timeout_ms,
7193 configured_module.restart,
7194 configured_module.register_timeout,
7195 Some(requester.clone()),
7196 )
7197 .map_err(|err| {
7198 format!("failed to add module_id '{module_id}' during rescan: {err}")
7199 })?;
7200 }
7201
7202 Ok((
7203 SupervisorRescanResult {
7204 added,
7205 removed,
7206 changed_pending_reload,
7207 enabled_changes: enabled_changes.iter().cloned().collect(),
7208 unchanged,
7209 preview: false,
7210 restart_required: Vec::new(),
7213 capability_warnings: Vec::new(),
7214 module_actions: Vec::new(),
7215 earlier_pending: Vec::new(),
7216 expect: None,
7217 },
7218 changed,
7219 ))
7220 }
7221
7222 async fn handle_supervisor_set_enabled(
7223 &self,
7224 frame: Frame,
7225 module_id: String,
7226 enabled: bool,
7227 requester: Principal,
7228 ) -> Result<Vec<Frame>, RouterError> {
7229 let operation_lock = self.supervisor.operation_lock();
7230 let _operation_guard = operation_lock.lock().await;
7231 let Some(module) = self.supervisor.get(&module_id) else {
7232 return Ok(vec![control_error_frame(
7233 &frame,
7234 "unknown_module",
7235 format!("module_id '{module_id}' is not supervised"),
7236 )?]);
7237 };
7238
7239 self.route_outages.mark_operator_action(&module_id);
7242 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7243 Ok(applied) => applied,
7244 Err(err) => {
7245 self.route_outages
7246 .operator_action_ended_unrefused(&module_id);
7247 return Ok(vec![control_error_frame(
7248 &frame,
7249 "target_unavailable",
7250 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7251 )?]);
7252 }
7253 };
7254 if !applied {
7255 self.route_outages
7258 .operator_action_ended_unrefused(&module_id);
7259 }
7260
7261 self.capability_evaluator.wake_deadline_loop();
7262 self.refresh_capability_requirements();
7263 let response = ClientControlResponse::SupervisorAck { module_id, applied };
7264 Ok(vec![control_response_body_frame(
7265 &frame,
7266 &response,
7267 "ClientControlResponse::SupervisorAck",
7268 )?])
7269 }
7270
7271 async fn handle_supervisor_health_probe(
7272 &self,
7273 ctx: &RouteCtx,
7274 frame: Frame,
7275 module_id: String,
7276 ) -> Result<Vec<Frame>, RouterError> {
7277 self.refresh_capability_requirements();
7278 let Some(registration) = self
7279 .registry
7280 .get_module(&module_id)
7281 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7282 else {
7283 return Ok(vec![control_error_frame(
7284 &frame,
7285 "unknown_module",
7286 format!("module_id '{module_id}' is not registered"),
7287 )?]);
7288 };
7289
7290 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7303 {
7304 return Ok(vec![control_error_frame(
7305 &frame,
7306 "health_not_advertised",
7307 format!("module_id '{module_id}' did not advertise health.check"),
7308 )?]);
7309 }
7310
7311 self.observe_control_wait(
7312 ctx,
7313 Some(®istration.manifest.module_id),
7314 ops::SUPERVISOR_HEALTH_PROBE,
7315 async {
7316 let deadline = Instant::now() + self.health_probe_timeout;
7317 let pending = match self.forwarding.begin_module_control_rpc_for(
7318 &module_id,
7319 MODULE_CONTROL_OP_HEALTH_CHECK,
7320 deadline,
7321 ) {
7322 Ok(pending) => pending,
7323 Err(err) => {
7324 return Ok(vec![control_error_frame(
7325 &frame,
7326 forwarding_error_code(&err),
7327 err.to_string(),
7328 )?])
7329 }
7330 };
7331
7332 let PendingModuleControlRpc {
7333 endpoint,
7334 module_sink,
7335 negotiated_ver,
7336 corr: probe_corr,
7337 receiver,
7338 } = pending;
7339 let mut guard =
7340 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7341 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7342 .map_err(|err| {
7343 RouterError::backend(
7344 0,
7345 frame.header.corr,
7346 format!("failed to encode health.check request: {err}"),
7347 )
7348 })?;
7349 let probe_frame = Frame::build_with_version(
7350 negotiated_ver,
7351 FrameType::Request,
7352 control_flags(),
7353 0,
7354 0,
7355 probe_corr,
7356 probe_body,
7357 )
7358 .map_err(RouterError::FrameBuild)?;
7359
7360 match timeout_at(deadline, module_sink.send(probe_frame)).await {
7363 Ok(Ok(())) => {}
7364 Ok(Err(err)) => {
7365 return Ok(vec![control_error_frame(
7366 &frame,
7367 "target_unavailable",
7368 err.to_string(),
7369 )?]);
7370 }
7371 Err(_) => {
7372 return Ok(vec![control_error_frame(
7373 &frame,
7374 "module_timeout",
7375 format!(
7376 "module_id '{module_id}' did not answer health.check within {:?}",
7377 self.health_probe_timeout
7378 ),
7379 )?]);
7380 }
7381 }
7382
7383 match timeout_at(deadline, receiver).await {
7384 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7385 guard.disarm();
7386 let Some(report) = response.health_report() else {
7387 return Ok(vec![control_error_frame(
7388 &frame,
7389 "invalid_control_body",
7390 "health.check RPC returned a non-health response",
7391 )?]);
7392 };
7393 let HealthReport {
7398 status,
7399 detail,
7400 metrics,
7401 } = report;
7402 let capability_detail = self
7403 .capability_evaluator
7404 .required_problem_detail(&module_id);
7405 let response = ClientControlResponse::SupervisorHealthProbe {
7406 module_id,
7407 status,
7408 detail: append_capability_problem_detail(detail, capability_detail),
7409 metrics,
7410 };
7411 Ok(vec![control_response_body_frame(
7412 &frame,
7413 &response,
7414 "ClientControlResponse::SupervisorHealthProbe",
7415 )?])
7416 }
7417 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7418 guard.disarm();
7419 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7424 }
7425 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7426 guard.disarm();
7427 Ok(vec![control_error_frame(
7428 &frame,
7429 "target_unavailable",
7430 message,
7431 )?])
7432 }
7433 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7434 guard.disarm();
7435 Ok(vec![control_error_frame(
7436 &frame,
7437 "invalid_control_body",
7438 message,
7439 )?])
7440 }
7441 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7442 guard.disarm();
7443 Ok(vec![control_error_frame(
7444 &frame,
7445 "invalid_control_body",
7446 format!("expected module-control op '{expected}', got '{actual}'"),
7447 )?])
7448 }
7449 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7450 guard.disarm();
7451 Ok(vec![control_error_frame(
7452 &frame,
7453 "module_timeout",
7454 format!(
7455 "module_id '{module_id}' answered health.check after {:?}",
7456 self.health_probe_timeout
7457 ),
7458 )?])
7459 }
7460 Ok(Err(_)) => Ok(vec![control_error_frame(
7461 &frame,
7462 "target_unavailable",
7463 "health.check waiter was canceled before the module responded",
7464 )?]),
7465 Err(_) => Ok(vec![control_error_frame(
7466 &frame,
7467 "module_timeout",
7468 format!(
7469 "module_id '{module_id}' did not answer health.check within {:?}",
7470 self.health_probe_timeout
7471 ),
7472 )?]),
7473 }
7474 },
7475 )
7476 .await
7477 }
7478
7479 fn supervisor_status(
7480 &self,
7481 module_id: &str,
7482 corr: u64,
7483 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7484 self.supervisor
7485 .get(module_id)
7486 .map(|module| {
7487 let warming = module.is_warming_for_control("status").map_err(|err| {
7488 RouterError::backend(
7489 0,
7490 corr,
7491 format!(
7492 "failed to read supervisor warming state for module_id '{module_id}': {err}"
7493 ),
7494 )
7495 })?;
7496 module.status_for_control("status").map_err(|err| {
7497 RouterError::backend(
7498 0,
7499 corr,
7500 format!(
7501 "failed to read supervisor status for module_id '{module_id}': {err}"
7502 ),
7503 )
7504 }).map(|status| (status, warming))
7505 })
7506 .transpose()
7507 }
7508
7509 fn guard_module_control_op(
7510 &self,
7511 frame: &Frame,
7512 module_id: &str,
7513 op: &str,
7514 ) -> Result<Option<Frame>, RouterError> {
7515 if self.module_grants_op(module_id, op, frame.header.corr)? {
7516 return Ok(None);
7517 }
7518
7519 Ok(Some(control_error_frame(
7520 frame,
7521 "op_not_allowed",
7522 format!("module_id '{module_id}' did not grant control op '{op}'"),
7523 )?))
7524 }
7525
7526 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7527 let Some(registration) = self
7528 .registry
7529 .get_module(module_id)
7530 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7531 else {
7532 return Ok(false);
7533 };
7534 Ok(module_registration_grants_op(®istration.control_ops, op))
7535 }
7536
7537 fn handle_status_update(
7538 &self,
7539 endpoint: ModuleEndpointId,
7540 frame: Frame,
7541 ) -> Result<Vec<Frame>, RouterError> {
7542 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7543 Ok(update) => update,
7544 Err(err) => {
7545 if is_known_module_push_op(&frame.body) {
7550 return Ok(vec![control_error_frame(
7551 &frame,
7552 "invalid_control_body",
7553 format!("malformed module control push body: {err}"),
7554 )?]);
7555 }
7556 return Ok(Vec::new());
7557 }
7558 };
7559
7560 match update {
7561 ModuleControlPush::RouteStatus {
7562 route_channel,
7563 route_epoch,
7564 status,
7565 } => {
7566 self.forwarding
7567 .cache_status(endpoint, route_channel, route_epoch, status)
7568 .map_err(RouterError::Forwarding)?;
7569 }
7570 }
7571 Ok(Vec::new())
7572 }
7573
7574 fn handle_route_poll(
7575 &self,
7576 ctx: &RouteCtx,
7577 frame: Frame,
7578 route_channel: u16,
7579 route_epoch: u32,
7580 kind: PollKind,
7581 ) -> Result<Vec<Frame>, RouterError> {
7582 let snapshot = self
7583 .forwarding
7584 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7585 .map_err(RouterError::Forwarding)?;
7586 let response = match (kind, snapshot) {
7587 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7588 ClientControlResponse::RoutePoll {
7589 route_channel,
7590 route_epoch,
7591 status,
7592 live: None,
7593 }
7594 }
7595 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7596 route_channel,
7597 route_epoch,
7598 status: None,
7599 live: None,
7600 },
7601 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7602 let live = self
7628 .process_liveness
7629 .as_ref()
7630 .and_then(|source| source.process_live(&module_id))
7631 .unwrap_or(true);
7632 ClientControlResponse::RoutePoll {
7633 route_channel,
7634 route_epoch,
7635 status: None,
7636 live: Some(live),
7637 }
7638 }
7639 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7640 route_channel,
7641 route_epoch,
7642 status: None,
7643 live: Some(false),
7644 },
7645 };
7646
7647 Ok(vec![control_response_body_frame(
7648 &frame,
7649 &response,
7650 "ClientControlResponse::RoutePoll",
7651 )?])
7652 }
7653
7654 pub(crate) fn observe_module_control_completion(
7655 &self,
7656 completion: ModuleControlRpcCompletion,
7657 ) -> bool {
7658 match completion {
7659 ModuleControlRpcCompletion::Unknown => false,
7660 ModuleControlRpcCompletion::Settled => true,
7661 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7662 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7663 info!(
7664 module_id = %module_id,
7665 latency_ms,
7666 "late health.check answer proves the module is alive"
7667 );
7668 match self
7669 .supervisor
7670 .record_late_health_answer(&module_id, latency_ms)
7671 {
7672 Ok(true) => {}
7673 Ok(false) => debug!(
7674 module_id = %module_id,
7675 latency_ms,
7676 "late health.check answer has no active supervisor snapshot"
7677 ),
7678 Err(err) => warn!(
7679 module_id = %module_id,
7680 latency_ms,
7681 error = %err,
7682 "failed to record late health.check answer"
7683 ),
7684 }
7685 true
7686 }
7687 }
7688 }
7689
7690 fn refuse_to_end_module_connection_for_a_client(
7711 &self,
7712 module_connection_id: ConnectionId,
7713 corr: u64,
7714 err: ForwardingError,
7715 ) -> Result<(), RouterError> {
7716 if let ForwardingError::ConnectionClosing { connection_id } = err {
7717 if connection_id != module_connection_id {
7718 warn!(
7719 module_connection_id = module_connection_id.get(),
7720 client_connection_id = connection_id.get(),
7721 corr,
7722 "dropping a route.bind response for a closing client; the module connection keeps serving"
7723 );
7724 return Ok(());
7725 }
7726 }
7727 Err(RouterError::Forwarding(err))
7728 }
7729
7730 fn handle_module_relay_response(
7731 &self,
7732 connection_id: ConnectionId,
7733 frame: Frame,
7734 ) -> Result<Vec<Frame>, RouterError> {
7735 let mut secondary_error = None;
7736 let outcome = match frame.header.ty {
7737 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7738 Ok(probe) if probe.op == "route.bind" => {
7739 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7740 Ok(ModuleControlResponse::RouteBindAck {}) => {
7741 RouteBindRelayOutcome::Accepted
7742 }
7743 Ok(other) => {
7744 let message =
7745 format!("route.bind response carried unexpected body: {other:?}");
7746 secondary_error = Some(control_error_frame(
7747 &frame,
7748 "invalid_control_body",
7749 message.clone(),
7750 )?);
7751 RouteBindRelayOutcome::ModuleGone(message)
7752 }
7753 Err(err) => {
7754 let message = format!("malformed route.bind response body: {err}");
7755 secondary_error = Some(control_error_frame(
7756 &frame,
7757 "invalid_control_body",
7758 message.clone(),
7759 )?);
7760 RouteBindRelayOutcome::ModuleGone(message)
7761 }
7762 }
7763 }
7764 Ok(probe) => {
7765 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7766 {
7767 Ok(response) => ModuleControlRpcOutcome::Response(response),
7768 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7769 "malformed {} response body: {err}",
7770 probe.op
7771 )),
7772 };
7773 let completion = self
7774 .forwarding
7775 .complete_module_control_rpc(
7776 connection_id,
7777 frame.header.corr,
7778 Some(&probe.op),
7779 outcome,
7780 )
7781 .map_err(RouterError::Forwarding)?;
7782 if !self.observe_module_control_completion(completion) {
7783 debug!(
7784 connection_id = connection_id.get(),
7785 corr = frame.header.corr,
7786 op = %probe.op,
7787 "dropping late or unknown module-control RPC response"
7788 );
7789 }
7790 return Ok(Vec::new());
7791 }
7792 Err(err) => {
7793 if let Some(expected_op) = self
7794 .forwarding
7795 .pending_module_control_op(connection_id, frame.header.corr)
7796 .map_err(RouterError::Forwarding)?
7797 {
7798 let completion = self
7799 .forwarding
7800 .complete_module_control_rpc(
7801 connection_id,
7802 frame.header.corr,
7803 None,
7804 ModuleControlRpcOutcome::MalformedResponse(format!(
7805 "malformed {expected_op} response body: {err}"
7806 )),
7807 )
7808 .map_err(RouterError::Forwarding)?;
7809 if !self.observe_module_control_completion(completion) {
7810 debug!(
7811 connection_id = connection_id.get(),
7812 corr = frame.header.corr,
7813 "dropping late malformed module-control RPC response"
7814 );
7815 }
7816 return Ok(Vec::new());
7817 }
7818 let message = format!("malformed route.bind response body: {err}");
7819 secondary_error = Some(control_error_frame(
7820 &frame,
7821 "invalid_control_body",
7822 message.clone(),
7823 )?);
7824 RouteBindRelayOutcome::ModuleGone(message)
7825 }
7826 },
7827 FrameType::Error => {
7828 if self
7829 .forwarding
7830 .pending_module_control_op(connection_id, frame.header.corr)
7831 .map_err(RouterError::Forwarding)?
7832 .is_some()
7833 {
7834 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7835 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7836 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7837 "malformed module-control ERROR body: {err}"
7838 )),
7839 };
7840 let completion = self
7841 .forwarding
7842 .complete_module_control_rpc(
7843 connection_id,
7844 frame.header.corr,
7845 None,
7846 outcome,
7847 )
7848 .map_err(RouterError::Forwarding)?;
7849 if !self.observe_module_control_completion(completion) {
7850 debug!(
7851 connection_id = connection_id.get(),
7852 corr = frame.header.corr,
7853 "dropping late or unknown module-control RPC error"
7854 );
7855 }
7856 return Ok(Vec::new());
7857 }
7858 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7859 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7860 Err(err) => {
7861 let message = format!("malformed route.bind ERROR body: {err}");
7862 secondary_error = Some(control_error_frame(
7863 &frame,
7864 "invalid_control_body",
7865 message.clone(),
7866 )?);
7867 RouteBindRelayOutcome::ModuleGone(message)
7868 }
7869 }
7870 }
7871 ty => {
7872 return Ok(vec![control_error_frame(
7873 &frame,
7874 "unsupported_control_frame",
7875 format!("unsupported module channel-0 frame {ty:?}"),
7876 )?])
7877 }
7878 };
7879
7880 let settled =
7881 self.forwarding
7882 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7883 let completion = match settled {
7884 Ok(completion) => completion,
7885 Err(err) => {
7886 self.refuse_to_end_module_connection_for_a_client(
7887 connection_id,
7888 frame.header.corr,
7889 err,
7890 )?;
7891 return Ok(secondary_error.into_iter().collect());
7892 }
7893 };
7894 if let Some(target) = completion.abandoned.as_ref() {
7895 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7896 }
7897 if !completion.settled {
7898 debug!(
7899 connection_id = connection_id.get(),
7900 corr = frame.header.corr,
7901 frame_type = ?frame.header.ty,
7902 "dropping late or unknown route.bind relay response"
7903 );
7904 }
7905 Ok(secondary_error.into_iter().collect())
7906 }
7907
7908 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7909 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7910 self.cleanup_connection_with_end_reason(
7914 connection_id,
7915 RegistrationEndReason::ExplicitGoodbye,
7916 )
7917 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7918 Ok(Vec::new())
7919 }
7920}
7921
7922impl Default for ControlHandler {
7923 fn default() -> Self {
7924 Self::new(Arc::new(Registry::default()))
7925 }
7926}
7927
7928impl crate::supervise::SwapPromotionObserver for ControlHandler {
7929 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7930 self.apply_registration_capabilities(registration);
7931 }
7932}
7933
7934fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7935 CapabilityRequirementStatus {
7936 consumer: status.consumer,
7937 capability: status.capability,
7938 need: match status.need {
7939 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7940 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7941 },
7942 verdict: status.verdict.as_str().to_string(),
7943 episode_seq: status.episode_seq,
7944 config_satisfiable: status.config_satisfiable,
7945 runtime_available: status.runtime_available,
7946 detail: status.detail,
7947 }
7948}
7949
7950fn append_capability_problem_detail(
7951 detail: Option<String>,
7952 capability_detail: Option<String>,
7953) -> Option<String> {
7954 match (detail, capability_detail) {
7955 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7956 (Some(detail), None) => Some(detail),
7957 (None, Some(capability_detail)) => Some(capability_detail),
7958 (None, None) => None,
7959 }
7960}
7961
7962fn subc_ops() -> Vec<String> {
7963 SUBC_CONTROL_OPS
7964 .iter()
7965 .map(|op| (*op).to_string())
7966 .collect()
7967}
7968
7969fn module_subc_ops() -> Vec<String> {
7970 SUBC_CONTROL_OPS
7971 .iter()
7972 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7973 .map(|op| (*op).to_string())
7974 .collect()
7975}
7976
7977#[cfg(test)]
7978fn module_baseline_control_ops() -> Vec<String> {
7979 MODULE_BASELINE_CONTROL_OPS
7980 .iter()
7981 .map(|op| (*op).to_string())
7982 .collect()
7983}
7984
7985fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7986 let mut seen = HashSet::new();
7987 let mut effective = Vec::new();
7988 for op in MODULE_BASELINE_CONTROL_OPS {
7989 if seen.insert((*op).to_string()) {
7990 effective.push((*op).to_string());
7991 }
7992 }
7993 for op in declared.unwrap_or_default() {
7994 if seen.insert(op.clone()) {
7995 effective.push(op);
7996 }
7997 }
7998 effective
7999}
8000
8001fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
8002 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
8003}
8004
8005fn target_module_id(target: &RouteTarget) -> &str {
8006 match target {
8007 RouteTarget::ToolProvider { module_id }
8008 | RouteTarget::ManagementSurface { module_id }
8009 | RouteTarget::InternalService { module_id, .. } => module_id,
8010 }
8011}
8012
8013fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
8014 roles.iter().any(|role| match (target, role) {
8015 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
8016 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
8017 (
8018 RouteTarget::InternalService { service_id, .. },
8019 ProviderRole::InternalService {
8020 service_id: provided,
8021 ..
8022 },
8023 ) => service_id == provided,
8024 _ => false,
8025 })
8026}
8027
8028fn is_routable_role(role: &ProviderRole) -> bool {
8029 matches!(
8030 role,
8031 ProviderRole::ToolProvider { .. }
8032 | ProviderRole::ManagementSurface { .. }
8033 | ProviderRole::InternalService { .. }
8034 )
8035}
8036
8037#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8038enum ControlRequestBodyError {
8039 UnknownOp,
8040 InvalidBody,
8041}
8042
8043#[derive(Debug, Deserialize)]
8044struct ControlOpProbe {
8045 op: String,
8046}
8047
8048const MODULE_PUSH_OPS: &[&str] = &["route.status"];
8051
8052fn is_known_module_push_op(body: &[u8]) -> bool {
8053 serde_json::from_slice::<ControlOpProbe>(body)
8054 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
8055 .unwrap_or(false)
8056}
8057
8058fn is_known_module_request_op(body: &[u8]) -> bool {
8059 serde_json::from_slice::<ControlOpProbe>(body)
8060 .map(|probe| is_module_to_subc_op(&probe.op))
8061 .unwrap_or(false)
8062}
8063
8064fn is_module_to_subc_op(op: &str) -> bool {
8065 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
8066}
8067
8068fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
8069 debug!(
8070 op = %op,
8071 connection_id = connection_id.get(),
8072 corr,
8073 "control dispatch"
8074 );
8075}
8076
8077fn log_slow_control_dispatch(
8078 dispatch_started_at: Option<StdInstant>,
8079 op: &'static str,
8080 connection_id: ConnectionId,
8081 corr: u64,
8082 route_open_timings: Option<&RouteOpenTimings>,
8083) {
8084 let Some(dispatch_started_at) = dispatch_started_at else {
8085 return;
8086 };
8087 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
8089 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
8090 if let Some(timings) = route_open_timings {
8091 warn!(
8092 op = %op,
8093 connection_id = connection_id.get(),
8094 corr,
8095 elapsed_ms = elapsed.as_millis() as u64,
8096 module_id = ?timings.module_id,
8097 principal = %timings.principal,
8098 admission_ms = timings.admission.as_millis() as u64,
8099 project_root_ms = timings.project_root.as_millis() as u64,
8100 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
8101 bind_send_ms = timings.bind_send.as_millis() as u64,
8102 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
8103 "slow control dispatch"
8104 );
8105 return;
8106 }
8107 warn!(
8108 op = %op,
8109 connection_id = connection_id.get(),
8110 corr,
8111 elapsed_ms = elapsed.as_millis() as u64,
8112 "slow control dispatch"
8113 );
8114 }
8115}
8116
8117#[cfg(test)]
8119type TestCallerLookup =
8120 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
8121
8122fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
8124 match request {
8125 ClientControlRequest::SupervisorRestart { module_id, .. } => {
8126 Some(PrivilegedOperation::Restart { module_id })
8127 }
8128 ClientControlRequest::SupervisorSwap { module_id, .. } => {
8129 Some(PrivilegedOperation::Swap { module_id })
8130 }
8131 ClientControlRequest::SupervisorReload { module_id } => {
8132 Some(PrivilegedOperation::Reload { module_id })
8133 }
8134 ClientControlRequest::SupervisorRescan { preview } => {
8135 if *preview {
8136 None
8137 } else {
8138 Some(PrivilegedOperation::Rescan)
8139 }
8140 }
8141 ClientControlRequest::SupervisorRescanChecked {
8146 preview,
8147 candidate,
8148 expect,
8149 } => (!*preview && candidate.is_none() && expect.is_some())
8150 .then_some(PrivilegedOperation::Rescan),
8151 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
8152 Some(PrivilegedOperation::SetEnabled {
8153 module_id,
8154 enabled: *enabled,
8155 })
8156 }
8157 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
8158 Some(PrivilegedOperation::ReleaseReserved { module_id })
8159 }
8160 ClientControlRequest::ServerDescribe {}
8161 | ClientControlRequest::DaemonShutdownIntent { .. }
8164 | ClientControlRequest::CatalogList { .. }
8165 | ClientControlRequest::RouteOpen { .. }
8166 | ClientControlRequest::RoutePoll { .. }
8167 | ClientControlRequest::SupervisorList {}
8168 | ClientControlRequest::SupervisorHealth {}
8169 | ClientControlRequest::SupervisorHealthProbe { .. }
8170 | ClientControlRequest::SupervisorRoutes { .. }
8171 | ClientControlRequest::SupervisorProvenance { .. }
8172 | ClientControlRequest::SupervisorStderrTail { .. }
8173 | ClientControlRequest::SupervisorTerminals { .. }
8174 | ClientControlRequest::SupervisorSpawnSnapshot {}
8175 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
8176 }
8177}
8178
8179fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
8180 match request {
8181 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
8182 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
8183 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
8184 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
8185 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
8186 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8187 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8188 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8189 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8190 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8191 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8192 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8193 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8194 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8195 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8196 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8197 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8198 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8199 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8200 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8201 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8202 }
8203}
8204
8205struct RouteOpenCaller {
8213 principal: String,
8217 scope: String,
8220 harness: String,
8223}
8224
8225impl RouteOpenCaller {
8226 fn new(
8227 principal: Option<&Principal>,
8228 scope: Option<&ScopeSelector>,
8229 identity: &BindIdentity,
8230 ) -> Self {
8231 Self {
8232 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8233 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8234 harness: bounded_log_text(&identity.harness),
8235 }
8236 }
8237
8238 fn unknown() -> Self {
8241 Self {
8242 principal: "unknown".to_string(),
8243 scope: "unknown".to_string(),
8244 harness: "unknown".to_string(),
8245 }
8246 }
8247}
8248
8249const MAX_LOGGED_TEXT_CHARS: usize = 64;
8252
8253fn bounded_log_text(value: &str) -> String {
8257 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8258 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8259 bounded.push('…');
8260 }
8261 format!("{bounded:?}")
8262}
8263
8264fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8270 use subc_protocol::scope::ScopeKind;
8271 let kind = match stamp.kind {
8272 ScopeKind::Head => "head",
8273 ScopeKind::Worker => "worker",
8274 ScopeKind::Ephemeral => "ephemeral",
8275 ScopeKind::Federated => "federated",
8276 _ => "unknown",
8277 };
8278 format!(
8279 "{kind}/{}/{}@{}",
8280 principal_label(&stamp.owner),
8281 bounded_log_text(&stamp.scope_ref),
8282 stamp.scope_epoch
8283 )
8284}
8285
8286fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8293 let epoch = selector
8294 .scope_epoch
8295 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8296 format!(
8297 "requested/{}/{}@{epoch}",
8298 bounded_log_text(&principal_label(&selector.owner)),
8299 bounded_log_text(&selector.scope_ref),
8300 )
8301}
8302
8303fn principal_label(principal: &Principal) -> String {
8304 match principal {
8305 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8306 Principal::Direct => "direct".to_string(),
8307 other => format!("{other:?}"),
8308 }
8309}
8310
8311fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8312 match request {
8313 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8314 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8315 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8316 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8317 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8318 ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
8319 _ => "unsupported",
8320 }
8321}
8322
8323fn parse_client_control_request(
8324 body: &[u8],
8325) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8326 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8327 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8328 Ok(probe)
8329 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8330 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8331 {
8332 ControlRequestBodyError::InvalidBody
8333 }
8334 Ok(_) => ControlRequestBodyError::UnknownOp,
8335 Err(_) => ControlRequestBodyError::InvalidBody,
8336 };
8337 (err, classification)
8338 })
8339}
8340
8341fn parse_module_control_request_from_module(
8342 body: &[u8],
8343) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8344 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8345 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8346 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8347 Ok(_) => ControlRequestBodyError::UnknownOp,
8348 Err(_) => ControlRequestBodyError::InvalidBody,
8349 };
8350 (err, classification)
8351 })
8352}
8353
8354#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8355enum ProviderRoleKind {
8356 ToolProvider,
8357 PipelineStage,
8358 ManagementSurface,
8359 InternalService,
8360}
8361
8362fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8363 match role {
8364 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8365 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8366 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8367 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8368 }
8369}
8370
8371fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8372 roles.iter().map(provider_role_kind).collect()
8373}
8374
8375const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8378
8379#[derive(Debug, Default, PartialEq, Eq)]
8381struct ScopeOutcomeCounts {
8382 created: usize,
8383 replaced: usize,
8384 updated: usize,
8385 unchanged: usize,
8386 refused: usize,
8387}
8388
8389impl ScopeOutcomeCounts {
8390 fn of(results: &[ScopeRecordResult]) -> Self {
8391 let mut counts = Self::default();
8392 for result in results {
8393 let slot = match result.outcome {
8394 ScopeRecordOutcome::Created => &mut counts.created,
8395 ScopeRecordOutcome::Replaced => &mut counts.replaced,
8396 ScopeRecordOutcome::Updated => &mut counts.updated,
8397 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8398 ScopeRecordOutcome::Refused => &mut counts.refused,
8399 };
8400 *slot += 1;
8401 }
8402 counts
8403 }
8404}
8405
8406#[cfg(test)]
8407mod scope_outcome_count_tests {
8408 use super::*;
8409
8410 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8411 ScopeRecordResult {
8412 scope_ref: "r".to_string(),
8413 scope_epoch: 1,
8414 outcome,
8415 code: None,
8416 message: None,
8417 version: None,
8418 parent_state: None,
8419 }
8420 }
8421
8422 #[test]
8425 fn every_outcome_is_counted_in_its_own_field() {
8426 let results = [
8427 result(ScopeRecordOutcome::Created),
8428 result(ScopeRecordOutcome::Created),
8429 result(ScopeRecordOutcome::Replaced),
8430 result(ScopeRecordOutcome::Updated),
8431 result(ScopeRecordOutcome::Unchanged),
8432 result(ScopeRecordOutcome::Refused),
8433 result(ScopeRecordOutcome::Refused),
8434 result(ScopeRecordOutcome::Refused),
8435 ];
8436 assert_eq!(
8437 ScopeOutcomeCounts::of(&results),
8438 ScopeOutcomeCounts {
8439 created: 2,
8440 replaced: 1,
8441 updated: 1,
8442 unchanged: 1,
8443 refused: 3,
8444 }
8445 );
8446 }
8447}
8448
8449fn capability_census_trigger(
8453 old: Option<&CapabilityDeclarations>,
8454 new: Option<&CapabilityDeclarations>,
8455) -> bool {
8456 let old_provides = old
8457 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8458 .unwrap_or_default();
8459 let old_denies = old
8460 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8461 .unwrap_or_default();
8462 let new = new.cloned().unwrap_or(CapabilityDeclarations {
8463 provides: Vec::new(),
8464 requires: Vec::new(),
8465 must_never_reach: Vec::new(),
8466 });
8467
8468 new.provides
8469 .iter()
8470 .any(|capability| !old_provides.contains(capability))
8471 || new
8472 .must_never_reach
8473 .iter()
8474 .any(|capability| !old_denies.contains(capability))
8475}
8476
8477fn denied_capability<'a>(
8480 opening_manifest: &'a ModuleManifest,
8481 target_manifest: &ModuleManifest,
8482) -> Option<&'a str> {
8483 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
8484 let target_capabilities = target_manifest.capabilities.as_ref()?;
8485 opening_capabilities
8486 .must_never_reach
8487 .iter()
8488 .find(|denied| {
8489 target_capabilities
8490 .provides
8491 .iter()
8492 .any(|provided| provided == *denied)
8493 })
8494 .map(String::as_str)
8495}
8496
8497fn catalog_update_frozen_field_message(
8498 registered: &ModuleManifest,
8499 provides: &[ProviderRole],
8500) -> Option<String> {
8501 let old_has_provides = !registered.provides.is_empty();
8502 let new_has_provides = !provides.is_empty();
8503 if old_has_provides != new_has_provides {
8504 return Some(format!(
8505 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8506 registered.module_id
8507 ));
8508 }
8509
8510 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
8511 return Some(format!(
8512 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8513 registered.module_id
8514 ));
8515 }
8516
8517 let registered_concurrency = manifest_concurrency(registered);
8518 let mut candidate = registered.clone();
8519 candidate.provides = provides.to_vec();
8520 let candidate_concurrency = manifest_concurrency(&candidate);
8521 if candidate_concurrency != registered_concurrency {
8522 return Some(format!(
8523 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8524 registered.module_id, registered_concurrency, candidate_concurrency
8525 ));
8526 }
8527
8528 None
8531}
8532
8533fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8534 manifest.provides.iter().any(is_routable_role)
8535}
8536
8537fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8543 manifest
8544 .provides
8545 .iter()
8546 .find_map(|provider| match provider {
8547 ProviderRole::ToolProvider { concurrency, .. }
8548 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8549 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8550 })
8551 .unwrap_or(Concurrency::ModuleManaged)
8552}
8553
8554fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8561 let has_management_surface = manifest
8562 .provides
8563 .iter()
8564 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8565 if !has_management_surface {
8566 return false;
8567 }
8568 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8569 return false;
8570 };
8571 let Some(provides) = raw
8572 .get("manifest")
8573 .and_then(|manifest| manifest.get("provides"))
8574 .and_then(serde_json::Value::as_array)
8575 else {
8576 return false;
8577 };
8578 provides.iter().any(|role| {
8583 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8584 && role.get("concurrency").is_none()
8585 })
8586}
8587
8588fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8589 if peer_version != PROTOCOL_VERSION {
8590 return Err(format!(
8591 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8592 ));
8593 }
8594 Ok(PROTOCOL_VERSION)
8595}
8596
8597fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8598 Frame::build_with_version(
8599 response_version(frame),
8600 FrameType::Pong,
8601 frame.header.flags,
8602 0,
8603 0,
8604 frame.header.corr,
8605 Vec::new(),
8606 )
8607 .map_err(RouterError::FrameBuild)
8608}
8609
8610pub(crate) fn control_error_frame(
8611 frame: &Frame,
8612 code: &'static str,
8613 message: impl Into<String>,
8614) -> Result<Frame, RouterError> {
8615 control_error_body_frame(
8616 frame,
8617 ErrorBody {
8618 code: code.to_string(),
8619 message: message.into(),
8620 detail: None,
8621 },
8622 )
8623}
8624
8625fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8633 control_error_frame_with_flags(frame, error, daemon_error_flags())
8634}
8635
8636fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8642 control_error_frame_with_flags(frame, error, control_flags())
8643}
8644
8645fn control_error_frame_with_flags(
8646 frame: &Frame,
8647 error: ErrorBody,
8648 flags: Flags,
8649) -> Result<Frame, RouterError> {
8650 let body = serde_json::to_vec(&error).map_err(|err| {
8651 RouterError::backend(
8652 0,
8653 frame.header.corr,
8654 format!("failed to encode control ERROR: {err}"),
8655 )
8656 })?;
8657
8658 Frame::build_with_version(
8659 response_version(frame),
8660 FrameType::Error,
8661 flags,
8662 0,
8663 0,
8664 frame.header.corr,
8665 body,
8666 )
8667 .map_err(RouterError::FrameBuild)
8668}
8669
8670pub(crate) fn control_response_body_frame<T: Serialize>(
8671 frame: &Frame,
8672 reply: &T,
8673 label: &'static str,
8674) -> Result<Frame, RouterError> {
8675 let body = serde_json::to_vec(reply).map_err(|err| {
8676 RouterError::backend(
8677 0,
8678 frame.header.corr,
8679 format!("failed to encode {label}: {err}"),
8680 )
8681 })?;
8682
8683 Frame::build_with_version(
8684 response_version(frame),
8685 FrameType::Response,
8686 control_flags(),
8687 0,
8688 0,
8689 frame.header.corr,
8690 body,
8691 )
8692 .map_err(RouterError::FrameBuild)
8693}
8694
8695fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8725 match err {
8726 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8727 ForwardingError::NoModuleConnection => "target_unavailable",
8728 ForwardingError::ModuleReloading { .. } => "module_reloading",
8729 ForwardingError::ClientRouteChannelExhausted { .. }
8730 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8731 ForwardingError::StaleModuleEndpoint
8732 | ForwardingError::UnknownReservation { .. }
8733 | ForwardingError::ConnectionClosing { .. }
8734 | ForwardingError::ClientEgressClosed { .. }
8735 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8736 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8739 ForwardingError::RelayCorrelationExhausted
8740 | ForwardingError::RouteOpenBuild(_)
8741 | ForwardingError::Poisoned => "forwarding_error",
8742 }
8743}
8744
8745fn response_version(frame: &Frame) -> u8 {
8746 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8747 frame.header.ver
8748 } else {
8749 PROTOCOL_VERSION
8750 }
8751}
8752
8753fn control_flags() -> Flags {
8754 Flags::new(false, Priority::Passive, false)
8755}
8756
8757fn send_goodbye_target_best_effort(
8762 counters: &DaemonCounters,
8763 target: &GoodbyeTarget,
8764 context: &'static str,
8765) {
8766 let Ok(frame) = Frame::build_with_version(
8767 target.negotiated_ver,
8768 FrameType::Goodbye,
8769 control_flags(),
8770 target.channel,
8771 target.epoch,
8772 0,
8773 Vec::new(),
8774 ) else {
8775 return;
8776 };
8777 crate::forwarding::send_module_route_goodbye(
8778 counters,
8779 &target.sink,
8780 frame,
8781 target.module_id.as_deref(),
8782 context,
8783 );
8784}
8785
8786pub(crate) fn send_route_control_pushes(
8787 forwarding: &ForwardingTable,
8788 routes: Vec<EndpointRoute>,
8789 push: ClientControlPush,
8790) {
8791 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8792 for route in routes {
8793 let target = route.goodbye_target;
8794 if let Some((existing, channels)) = targets
8795 .iter_mut()
8796 .find(|(existing, _)| existing.connection_id == target.connection_id)
8797 {
8798 debug_assert_eq!(
8799 existing.negotiated_ver, target.negotiated_ver,
8800 "one connection cannot negotiate multiple frame versions"
8801 );
8802 if !channels.contains(&target.channel) {
8803 channels.push(target.channel);
8804 }
8805 continue;
8806 }
8807 let channel = target.channel;
8808 targets.push((target, vec![channel]));
8809 }
8810 for (target, mut channels) in targets {
8811 channels.sort_unstable();
8812 let mut push = push.clone();
8813 match &mut push {
8814 ClientControlPush::RouteClosing {
8815 channels: covered, ..
8816 }
8817 | ClientControlPush::RouteClosed {
8818 channels: covered, ..
8819 } => *covered = channels,
8820 }
8821 let body = match serde_json::to_vec(&push) {
8822 Ok(body) => body,
8823 Err(err) => {
8824 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8825 continue;
8826 }
8827 };
8828 let frame = match Frame::build_with_version(
8829 target.negotiated_ver,
8830 FrameType::Push,
8831 control_flags(),
8832 0,
8833 0,
8834 0,
8835 body.clone(),
8836 ) {
8837 Ok(frame) => frame,
8838 Err(err) => {
8839 warn!(
8840 route_channel = target.channel,
8841 error = %err,
8842 "failed to build route lifecycle control PUSH frame"
8843 );
8844 continue;
8845 }
8846 };
8847 if let Err(err) = target.sink.try_send(frame) {
8848 if target.close_on_delivery_failure() {
8849 warn!(
8850 target_connection_id = target.connection_id.get(),
8851 route_channel = target.channel,
8852 error = %err,
8853 "route lifecycle control PUSH was not delivered to client; closing target connection"
8854 );
8855 let _ = forwarding.escalate_client_delivery_failure(
8856 target.connection_id,
8857 target.channel,
8858 target.epoch,
8859 CloseReason::new(
8860 "route_lifecycle_push_delivery_failed",
8861 format!(
8862 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8863 target.channel
8864 ),
8865 ),
8866 crate::forwarding::UndeliveredFrame {
8867 module_id: target.module_id.as_deref(),
8868 sink: &target.sink,
8869 },
8870 );
8871 }
8872 }
8873 }
8874}
8875
8876#[cfg(test)]
8877mod tests {
8878 include!("control/agent_commands_tests.rs");
8879 include!("control/resource_revision_tests.rs");
8880 #[tokio::test]
8881 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
8882 let registry = Arc::new(Registry::default());
8883 let handler = ControlHandler::new(registry);
8884 assert_eq!(
8885 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8886 None
8887 );
8888 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
8889 hello_via_sink(
8890 &handler,
8891 &ctx,
8892 &mut rx,
8893 hello_frame("daemon", PROTOCOL_VERSION, 1),
8894 )
8895 .await;
8896 assert_eq!(
8897 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8898 Some("daemon".into())
8899 );
8900 assert_eq!(
8901 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
8902 None
8903 );
8904 assert_eq!(
8905 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
8906 None,
8907 "a registered module is not necessarily supervised"
8908 );
8909 assert_eq!(
8910 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
8911 None,
8912 "aggregate provenance has no single target"
8913 );
8914 assert_eq!(
8915 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
8916 None
8917 );
8918 }
8919
8920 mod peer_manifest {
8921 include!("peer_manifest_tests.rs");
8922 }
8923 mod activity_read {
8924 include!("activity_read_tests.rs");
8925 }
8926
8927 mod lifecycle_activity {
8928 include!("control_lifecycle_tests.rs");
8929 }
8930 #[cfg(unix)]
8931 #[tokio::test]
8932 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8933 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8934 let path = dir.join("subc.jsonc");
8935 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8936 "program":"/bin/sleep","args":["60"],"protocol":"none",
8937 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8938 }}}).to_string()).unwrap();
8939 let mut configured = crate::daemon_config::load(&path)
8940 .unwrap()
8941 .unwrap()
8942 .modules
8943 .pop()
8944 .unwrap();
8945 let registry = std::sync::Arc::new(crate::Registry::default());
8946 let handle = crate::SupervisorHandle::new();
8947 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8948 .with_handle(handle.clone());
8949 let module = supervisor
8950 .supervise_configured_with_health(
8951 configured.module_spec(),
8952 true,
8953 configured.health.clone(),
8954 None,
8955 configured.restart,
8956 )
8957 .unwrap();
8958 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8959 let before = module.status().unwrap().pid;
8960 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8961 configured.health.cadence = std::time::Duration::from_secs(3600);
8962 let health_only = handler
8963 .reconcile_supervised_modules(
8964 &supervisor,
8965 vec![configured.clone()],
8966 false,
8967 Principal::Direct,
8968 )
8969 .await
8970 .unwrap();
8971 assert!(
8972 health_only.0.changed_pending_reload.is_empty(),
8973 "health policy is already applied live"
8974 );
8975 assert_eq!(module.status().unwrap().pid, before);
8976 assert_eq!(
8977 module.configuration().unwrap().1.http,
8978 configured.health.http
8979 );
8980 configured.args = vec!["61".into()];
8981 let launch = handler
8982 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8983 .await
8984 .unwrap();
8985 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8986 assert_eq!(
8987 module.status().unwrap().pid,
8988 before,
8989 "a launch edit is stored until reload"
8990 );
8991 module.drain().await.unwrap();
8992 }
8993 use cortexkit_test_support::ScratchDir;
8994 use std::{
8995 collections::BTreeMap,
8996 fmt,
8997 path::PathBuf,
8998 sync::{Arc, Mutex},
8999 time::Duration,
9000 };
9001
9002 use serde_json::{json, Value};
9003 use subc_protocol::{
9004 manifest::{
9005 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
9006 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
9007 },
9008 session::HealthStatus,
9009 FrameType,
9010 };
9011
9012 use super::*;
9013 use crate::{
9014 forwarding::{DataRoute, DataRouteState},
9015 registry::ChannelState,
9016 router::FrameSink,
9017 stderr_tail::DEFAULT_MAX_LINE_BYTES,
9018 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
9019 RouteCtx, Router,
9020 };
9021 use tokio::{
9022 sync::mpsc,
9023 time::{sleep, Instant},
9024 };
9025 use tracing::{
9026 field::{Field, Visit},
9027 Event, Subscriber,
9028 };
9029 use tracing_subscriber::{layer::Context, prelude::*, Layer};
9030
9031 fn fake_aft_stub_path() -> PathBuf {
9050 let mut path = std::env::current_exe().expect("current_exe available in tests");
9051 path.pop(); path.pop(); path.push(if cfg!(windows) {
9054 "fake-aft-stub.exe"
9055 } else {
9056 "fake-aft-stub"
9057 });
9058 assert!(
9059 path.exists(),
9060 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
9061 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
9062 path.display()
9063 );
9064 path
9065 }
9066
9067 fn client_retries(code: &str) -> bool {
9072 subc_protocol::error_codes::is_retryable_route_open(code)
9073 }
9074
9075 #[test]
9086 fn retryability_of_forwarding_codes_matches_the_failure() {
9087 let transient = [
9090 ForwardingError::NoModuleConnection,
9091 ForwardingError::ModuleReloading {
9092 module_id: "m".into(),
9093 },
9094 ForwardingError::StaleModuleEndpoint,
9095 ForwardingError::UnknownReservation {
9096 client_channel: 1,
9097 module_channel: 1,
9098 },
9099 ForwardingError::ConnectionClosing {
9100 connection_id: ConnectionId::new(1),
9101 },
9102 ForwardingError::ClientEgressClosed {
9103 connection_id: ConnectionId::new(1),
9104 },
9105 ForwardingError::ModuleEgressUnavailable {
9106 connection_id: ConnectionId::new(1),
9107 },
9108 ];
9109 for err in transient {
9110 let code = forwarding_error_code(&err);
9111 assert!(
9112 client_retries(code),
9113 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
9114 );
9115 }
9116
9117 let permanent = [
9122 ForwardingError::ConnectionRoleConflict {
9123 connection_id: ConnectionId::new(1),
9124 },
9125 ForwardingError::ClientRouteChannelExhausted {
9126 connection_id: ConnectionId::new(1),
9127 },
9128 ForwardingError::ModuleRouteChannelExhausted {
9129 endpoint: ModuleEndpointId {
9130 connection_id: ConnectionId::new(1),
9131 generation: 1,
9132 },
9133 },
9134 ForwardingError::RelayCorrelationExhausted,
9135 ForwardingError::RouteOpenBuild("x".into()),
9136 ForwardingError::Poisoned,
9137 ];
9138 for err in permanent {
9139 let code = forwarding_error_code(&err);
9140 assert!(
9141 !client_retries(code),
9142 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
9143 );
9144 }
9145 }
9146
9147 #[tokio::test]
9175 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
9176 let handler = ControlHandler::default();
9177 let frame =
9178 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
9179
9180 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
9184 assert_eq!(
9185 stamped,
9186 Principal::Direct,
9187 "a caller that proved nothing must not be stamped as a supervised module"
9188 );
9189
9190 let forged = handler
9195 .route_open_principal(
9196 &frame,
9197 Some(ConsumerIdentity {
9198 module_id: "aft".to_string(),
9199 launch_nonce: "not-a-real-nonce".to_string(),
9200 }),
9201 )
9202 .unwrap();
9203 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9204 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9205 }
9206
9207 #[test]
9224 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9225 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9226 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9227 let ClientControlRequest::RouteOpen {
9228 consumer_identity, ..
9229 } = parsed
9230 else {
9231 panic!("route.open body must parse as RouteOpen");
9232 };
9233 assert_eq!(
9234 consumer_identity,
9235 Some(ConsumerIdentity {
9236 module_id: "aft".to_string(),
9237 launch_nonce: "n".to_string(),
9238 }),
9239 "the wire field name must reach the value route_open_principal reads"
9240 );
9241 }
9242
9243 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9244 ModuleManifest::builder(module_id, "0.1.0")
9245 .protocol_ver(protocol_ver)
9246 .provides(vec![ProviderRole::ToolProvider {
9247 tools: vec![Tool {
9248 name: "read".to_string(),
9249 description: None,
9250 execution_mode: ExecutionMode::Pure,
9251 schema: json!({"type": "object"}),
9252 }],
9253 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9254 concurrency: Concurrency::ModuleManaged,
9255 emits_push: true,
9256 sub_supervises: true,
9257 }])
9258 .build()
9259 }
9260
9261 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9262 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9263 }
9264
9265 fn hello_frame_with_control_ops(
9266 module_id: &str,
9267 protocol_ver: u8,
9268 corr: u64,
9269 control_ops: Option<Vec<String>>,
9270 ) -> Frame {
9271 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9272 }
9273
9274 fn hello_frame_with_nonce(
9275 module_id: &str,
9276 protocol_ver: u8,
9277 corr: u64,
9278 launch_nonce: Option<&str>,
9279 ) -> Frame {
9280 hello_frame_full(
9281 module_id,
9282 protocol_ver,
9283 corr,
9284 None,
9285 launch_nonce.map(ToOwned::to_owned),
9286 )
9287 }
9288
9289 fn hello_frame_full(
9290 module_id: &str,
9291 protocol_ver: u8,
9292 corr: u64,
9293 control_ops: Option<Vec<String>>,
9294 launch_nonce: Option<String>,
9295 ) -> Frame {
9296 let body = serde_json::to_vec(&ModuleHelloBody {
9297 manifest: manifest(module_id, protocol_ver),
9298 protocol_ver,
9299 control_ops,
9300 launch_nonce,
9301 })
9302 .unwrap();
9303 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9304 }
9305
9306 fn hello_frame_with_rich_ui(
9307 module_id: &str,
9308 protocol_ver: u8,
9309 corr: u64,
9310 launch_nonce: Option<&str>,
9311 rich_ui: Value,
9312 ) -> Frame {
9313 let mut body = serde_json::to_value(ModuleHelloBody {
9314 manifest: manifest(module_id, protocol_ver),
9315 protocol_ver,
9316 control_ops: None,
9317 launch_nonce: launch_nonce.map(str::to_string),
9318 })
9319 .unwrap();
9320 body["manifest"]["rich_ui"] = rich_ui;
9321 Frame::build(
9322 FrameType::Hello,
9323 control_flags(),
9324 0,
9325 0,
9326 corr,
9327 serde_json::to_vec(&body).unwrap(),
9328 )
9329 .unwrap()
9330 }
9331
9332 fn non_routable_hello_frame_with_control_ops(
9333 module_id: &str,
9334 corr: u64,
9335 control_ops: Option<Vec<String>>,
9336 ) -> Frame {
9337 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9338 manifest.provides.clear();
9339 let body = serde_json::to_vec(&ModuleHelloBody {
9340 manifest,
9341 protocol_ver: PROTOCOL_VERSION,
9342 control_ops,
9343 launch_nonce: None,
9344 })
9345 .unwrap();
9346 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9347 }
9348
9349 fn capability_grammar_hello_frame(
9350 capabilities: Value,
9351 runtime_computed: Option<Value>,
9352 corr: u64,
9353 ) -> Frame {
9354 let mut body = serde_json::to_value(ModuleHelloBody {
9355 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9356 protocol_ver: PROTOCOL_VERSION,
9357 control_ops: None,
9358 launch_nonce: None,
9359 })
9360 .expect("HELLO body serializes");
9361 body["manifest"]["capabilities"] = capabilities;
9362 if let Some(runtime_computed) = runtime_computed {
9363 body["runtime_computed"] = runtime_computed;
9364 }
9365 Frame::build(
9366 FrameType::Hello,
9367 control_flags(),
9368 0,
9369 0,
9370 corr,
9371 serde_json::to_vec(&body).expect("HELLO body reserializes"),
9372 )
9373 .expect("HELLO frame builds")
9374 }
9375
9376 fn channel_request(channel: u16, corr: u64) -> Frame {
9377 Frame::build(
9378 FrameType::Request,
9379 Flags::new(true, Priority::Interactive, false),
9380 channel,
9381 0,
9382 corr,
9383 b"opaque".to_vec(),
9384 )
9385 .unwrap()
9386 }
9387
9388 fn route_ctx(
9389 connection_id: ConnectionId,
9390 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9391 let (tx, rx) = mpsc::channel(8);
9392 (
9393 RouteCtx {
9394 connection_id,
9395 egress: FrameSink::new(tx),
9396 },
9397 rx,
9398 )
9399 }
9400
9401 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9402 serde_json::from_slice(&frame.body).unwrap()
9403 }
9404
9405 async fn hello_via_sink(
9410 handler: &ControlHandler,
9411 ctx: &RouteCtx,
9412 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9413 hello: Frame,
9414 ) -> Frame {
9415 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9416 assert!(
9417 replies.is_empty(),
9418 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9419 );
9420 let ack = rx
9421 .try_recv()
9422 .expect("HELLO_ACK is queued on the module sink")
9423 .frame;
9424 assert_eq!(ack.header.ty, FrameType::HelloAck);
9425 ack
9426 }
9427
9428 fn parse_error(frame: &Frame) -> Value {
9429 serde_json::from_slice(&frame.body).unwrap()
9430 }
9431
9432 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9433 serde_json::from_slice(&frame.body).unwrap()
9434 }
9435
9436 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9437 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9438 route_channel,
9439 route_epoch: 0,
9440 kind,
9441 })
9442 .unwrap();
9443 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9444 }
9445
9446 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9447 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9448 module_id: module_id.to_string(),
9449 })
9450 .unwrap();
9451 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9452 }
9453
9454 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9455 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9456 }
9457
9458 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9460 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9461 let mut frame = route_open_frame(corr, module_id, root);
9462 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9463 body["identity"]["project_root"] = serde_json::json!(path.as_path());
9464 frame.body = serde_json::to_vec(&body).unwrap();
9465 frame
9466 }
9467
9468 fn route_open_frame_with_consumer_capabilities(
9469 corr: u64,
9470 module_id: &str,
9471 _project_root: ScratchDir,
9472 consumer_capabilities: Option<Vec<String>>,
9473 ) -> Frame {
9474 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9475 busy_refusals: None,
9476 target: RouteTarget::ToolProvider {
9477 module_id: module_id.to_string(),
9478 },
9479 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9480 consumer_identity: None,
9481 consumer_capabilities,
9482 role_versions: None,
9483 admission_facts: None,
9484 scope: None,
9485 })
9486 .unwrap();
9487 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9488 }
9489
9490 fn route_open_frame_with_role_versions(
9491 corr: u64,
9492 module_id: &str,
9493 _project_root: ScratchDir,
9494 role_versions: Option<BTreeMap<String, String>>,
9495 ) -> Frame {
9496 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9497 busy_refusals: None,
9498 target: RouteTarget::ToolProvider {
9499 module_id: module_id.to_string(),
9500 },
9501 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9502 consumer_identity: None,
9503 consumer_capabilities: None,
9504 role_versions,
9505 admission_facts: None,
9506 scope: None,
9507 })
9508 .unwrap();
9509 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9510 }
9511
9512 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9513 entries
9514 .iter()
9515 .map(|(role, version)| (role.to_string(), version.to_string()))
9516 .collect()
9517 }
9518
9519 fn route_open_frame_with_admission_facts(
9520 corr: u64,
9521 module_id: &str,
9522 _project_root: ScratchDir,
9523 consumer_identity: Option<subc_control::ConsumerIdentity>,
9524 facts: Option<Value>,
9525 ) -> Frame {
9526 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9527 busy_refusals: None,
9528 target: RouteTarget::ToolProvider {
9529 module_id: module_id.to_string(),
9530 },
9531 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9532 consumer_identity,
9533 consumer_capabilities: None,
9534 role_versions: None,
9535 admission_facts: facts,
9536 scope: None,
9537 })
9538 .unwrap();
9539 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9540 }
9541
9542 #[derive(Clone, Default)]
9543 struct EventCapture {
9544 events: Arc<Mutex<Vec<CapturedEvent>>>,
9545 }
9546
9547 #[derive(Clone, Debug)]
9548 struct CapturedEvent {
9549 target: String,
9550 level: tracing::Level,
9551 fields: BTreeMap<String, String>,
9552 }
9553
9554 impl EventCapture {
9555 fn events(&self) -> Vec<CapturedEvent> {
9556 self.events.lock().unwrap().clone()
9557 }
9558 }
9559
9560 impl<S> Layer<S> for EventCapture
9561 where
9562 S: Subscriber,
9563 {
9564 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9565 let mut visitor = EventFieldVisitor::default();
9566 event.record(&mut visitor);
9567 self.events.lock().unwrap().push(CapturedEvent {
9568 target: event.metadata().target().to_string(),
9569 level: *event.metadata().level(),
9570 fields: visitor.fields,
9571 });
9572 }
9573 }
9574
9575 #[derive(Default)]
9576 struct EventFieldVisitor {
9577 fields: BTreeMap<String, String>,
9578 }
9579
9580 impl Visit for EventFieldVisitor {
9581 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9582 self.fields
9583 .insert(field.name().to_string(), format!("{value:?}"));
9584 }
9585 }
9586
9587 fn health_response(corr: u64, status: HealthStatus) -> Frame {
9588 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9589 status,
9590 detail: Some("warming".to_string()),
9591 metrics: Some(json!({"queue_depth": 3})),
9592 })
9593 .unwrap();
9594 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9595 }
9596
9597 fn route_bind_ack(corr: u64) -> Frame {
9598 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9599 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9600 }
9601
9602 fn unique_project_root(label: &str) -> ScratchDir {
9603 ScratchDir::new(label)
9604 }
9605
9606 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9607 match parse_route_poll(frame) {
9608 ClientControlResponse::RoutePoll {
9609 status: None,
9610 live: Some(live),
9611 ..
9612 } => assert_eq!(live, expected_live),
9613 other => panic!("unexpected route.poll response: {other:?}"),
9614 }
9615 }
9616
9617 fn bind_liveness_route(
9618 registry: &Registry,
9619 forwarding: &ForwardingTable,
9620 module_id: &str,
9621 ) -> (RouteCtx, u16, u32) {
9622 let module_connection = ConnectionId::new(101);
9623 let client_connection = ConnectionId::new(202);
9624 let registration = registry
9625 .register_with_control_ops(
9626 manifest(module_id, PROTOCOL_VERSION),
9627 PROTOCOL_VERSION,
9628 module_connection,
9629 module_baseline_control_ops(),
9630 )
9631 .unwrap();
9632 let (module_tx, _module_rx) = mpsc::channel(8);
9633 let endpoint = forwarding
9634 .register_module_connection(
9635 module_connection,
9636 module_id.to_string(),
9637 PROTOCOL_VERSION,
9638 manifest_concurrency(®istration.manifest),
9639 FrameSink::new(module_tx),
9640 )
9641 .unwrap();
9642 let (client_ctx, _client_rx) = route_ctx(client_connection);
9643 let pending = forwarding
9644 .begin_route_bind_relay_for_test(
9645 client_connection,
9646 client_ctx.egress.clone(),
9647 1,
9648 module_id,
9649 )
9650 .unwrap();
9651 assert_eq!(pending.endpoint, endpoint);
9652 let route_channel = pending.client_channel;
9653 let route_epoch = pending.client_epoch;
9654 forwarding
9655 .complete_pending_relay(
9656 module_connection,
9657 pending.corr,
9658 RouteBindRelayOutcome::Accepted,
9659 )
9660 .unwrap();
9661 (client_ctx, route_channel, route_epoch)
9662 }
9663
9664 struct FakeProcessLiveness {
9665 live: Option<bool>,
9666 }
9667
9668 impl ModuleProcessLiveness for FakeProcessLiveness {
9669 fn process_live(&self, _module_id: &str) -> Option<bool> {
9670 self.live
9671 }
9672 }
9673
9674 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9675 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9676 {
9677 let registry = Arc::new(Registry::default());
9678 let supervisor_handle = SupervisorHandle::new();
9679 let supervisor = Supervisor::new_for_test(
9680 Arc::clone(®istry),
9681 RestartPolicy::new(1, Duration::from_millis(10)),
9682 )
9683 .with_handle(supervisor_handle.clone());
9684 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9685 let module = supervisor
9686 .spawn(ModuleSpec {
9687 carry_containment: false,
9688 module_id: "stderr-tail-wire".to_string(),
9689 program: fake_aft_stub_path(),
9690 args: Vec::new(),
9691 env: vec![
9692 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9693 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9694 ],
9695 reserved: false,
9696 reserved_prefixes: Vec::new(),
9697 protocol: ModuleProtocol::Subc,
9698 overlap: Default::default(),
9699 })
9700 .unwrap();
9701
9702 let deadline = Instant::now() + Duration::from_secs(5);
9703 loop {
9704 let tail = module.stderr_tail(None, None);
9705 if tail
9706 .entries
9707 .iter()
9708 .any(|entry| matches!(entry, TailEntry::ProcessStart))
9709 && tail.entries.iter().any(|entry| {
9710 matches!(
9711 entry,
9712 TailEntry::Line {
9713 truncated: true,
9714 ..
9715 }
9716 )
9717 })
9718 {
9719 break;
9720 }
9721 assert!(
9722 Instant::now() < deadline,
9723 "module did not produce a truncated line and restart boundary: {tail:?}"
9724 );
9725 sleep(Duration::from_millis(10)).await;
9726 }
9727
9728 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9729 let request = ClientControlRequest::SupervisorStderrTail {
9730 module_id: "stderr-tail-wire".to_string(),
9731 max_lines: None,
9732 max_bytes: None,
9733 };
9734 let frame = Frame::build(
9735 FrameType::Request,
9736 control_flags(),
9737 0,
9738 0,
9739 1,
9740 serde_json::to_vec(&request).unwrap(),
9741 )
9742 .unwrap();
9743 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9744 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9745 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9746 serde_json::from_slice(&responses[0].body).unwrap()
9747 else {
9748 panic!("expected supervisor.stderr_tail response");
9749 };
9750
9751 assert!(
9752 tail.entries
9753 .iter()
9754 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9755 "the control response lost the restart boundary"
9756 );
9757 let Some(StderrTailEntry::Line {
9758 text,
9759 truncated,
9760 at_ms,
9761 }) = tail.entries.iter().find(|entry| {
9762 matches!(
9763 entry,
9764 StderrTailEntry::Line {
9765 truncated: true,
9766 ..
9767 }
9768 )
9769 })
9770 else {
9771 panic!("the control response lost the truncated line");
9772 };
9773 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9774 assert!(*truncated);
9775 assert!(
9776 at_ms.is_some(),
9777 "the control response lost the line's capture time"
9778 );
9779 }
9780
9781 #[tokio::test(flavor = "current_thread")]
9786 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9787 let dir = ScratchDir::new("terminals-off-worker");
9788 let journal_path = dir.join("terminals.jsonl");
9789 let registry = Arc::new(Registry::default());
9790 let supervisor_handle = SupervisorHandle::new();
9791 let supervisor =
9792 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9793 .with_handle(supervisor_handle.clone())
9794 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9795 let module = supervisor
9796 .spawn(ModuleSpec {
9797 carry_containment: false,
9798 module_id: "terminal-off-worker".to_string(),
9799 program: fake_aft_stub_path(),
9800 args: Vec::new(),
9801 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9802 reserved: false,
9803 reserved_prefixes: Vec::new(),
9804 protocol: ModuleProtocol::Subc,
9805 overlap: Default::default(),
9806 })
9807 .unwrap();
9808 let deadline = Instant::now() + Duration::from_secs(5);
9809 while module.terminal_history().entries.len() != 2 {
9810 assert!(Instant::now() < deadline, "module did not record two exits");
9811 sleep(Duration::from_millis(10)).await;
9812 }
9813
9814 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9815 let handler =
9816 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
9817 let frame = Frame::build(
9818 FrameType::Request,
9819 control_flags(),
9820 0,
9821 0,
9822 1,
9823 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9824 module_id: "terminal-off-worker".to_string(),
9825 })
9826 .unwrap(),
9827 )
9828 .unwrap();
9829 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9830 let spawned_at = std::time::Instant::now();
9831 let read = tokio::spawn({
9832 let handler = Arc::clone(&handler);
9833 async move { handler.handle_control_frame(&ctx, frame).await }
9834 });
9835 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9838 .await
9839 .unwrap()
9840 .expect("the history read reached its pause");
9841 let elapsed = spawned_at.elapsed();
9842 assert!(
9843 elapsed < Duration::from_secs(2) && !read.is_finished(),
9844 "this task could not run while the history read was paused \
9845 (resumed after {elapsed:?}, read finished: {})",
9846 read.is_finished()
9847 );
9848
9849 drop(release);
9850 let responses = read.await.unwrap().unwrap();
9851 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9852 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9853 panic!("expected supervisor.terminals response");
9854 };
9855 assert_eq!(terminals.entries.len(), 2);
9856 assert_eq!(terminals.journal_skipped_lines, 0);
9857 assert_eq!(terminals.journal_read_errors, 0);
9858 }
9859
9860 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9861 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9862 let registry = Arc::new(Registry::default());
9863 let supervisor_handle = SupervisorHandle::new();
9864 let supervisor =
9865 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9866 .with_handle(supervisor_handle.clone());
9867 let module = supervisor
9868 .spawn(ModuleSpec {
9869 carry_containment: false,
9870 module_id: "terminal-golden".to_string(),
9871 program: fake_aft_stub_path(),
9872 args: Vec::new(),
9873 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9874 reserved: false,
9875 reserved_prefixes: Vec::new(),
9876 protocol: ModuleProtocol::Subc,
9877 overlap: Default::default(),
9878 })
9879 .unwrap();
9880
9881 let deadline = Instant::now() + Duration::from_secs(5);
9882 while module.terminal_history().entries.len() != 2 {
9883 assert!(
9884 Instant::now() < deadline,
9885 "module did not retain two terminal exits: {:?}",
9886 module.terminal_history()
9887 );
9888 sleep(Duration::from_millis(10)).await;
9889 }
9890
9891 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9892 let request = ClientControlRequest::SupervisorTerminals {
9893 module_id: "terminal-golden".to_string(),
9894 };
9895 let frame = Frame::build(
9896 FrameType::Request,
9897 control_flags(),
9898 0,
9899 0,
9900 1,
9901 serde_json::to_vec(&request).unwrap(),
9902 )
9903 .unwrap();
9904 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9905 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9906 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9907 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9908 panic!("expected supervisor.terminals response");
9909 };
9910 assert_eq!(terminals.entries.len(), 2);
9911 assert_eq!(terminals.dropped, 0);
9912
9913 let mut rendered = serde_json::to_value(response).unwrap();
9914 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9917 for (index, entry) in rendered["entries"]
9918 .as_array_mut()
9919 .expect("terminal response entries array")
9920 .iter_mut()
9921 .enumerate()
9922 {
9923 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9924 }
9925
9926 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9927 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9928 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9929 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9930 std::fs::write(&golden_path, &serialized).unwrap();
9931 }
9932 let expected: Value =
9933 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9934 assert_eq!(rendered, expected);
9935 }
9936
9937 #[test]
9938 fn hello_registers_manifest_and_returns_ack() {
9939 let registry = Arc::new(Registry::default());
9940 let handler = ControlHandler::new(Arc::clone(®istry));
9941 let conn = ConnectionId::new(1);
9942
9943 let responses = handler
9944 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9945 .unwrap();
9946
9947 assert_eq!(responses.len(), 1);
9948 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9949 assert_eq!(responses[0].header.channel, 0);
9950 assert_eq!(responses[0].header.corr, 7);
9951 let ack = parse_ack(&responses[0]);
9952 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9953 assert!(ack
9954 .subc_capabilities
9955 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9956 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9957 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9958 assert!(ack
9959 .subc_ops
9960 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9961 assert!(ack
9962 .subc_ops
9963 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9964
9965 let registration = registry.get_module("aft").unwrap().unwrap();
9966 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9967 assert_eq!(registration.state, ChannelState::Active);
9968 assert_eq!(registration.connection_id, conn);
9969 assert_eq!(registration.control_ops, module_baseline_control_ops());
9970 assert!(registration.manifest.display_name.is_none());
9971 }
9972
9973 #[test]
9974 fn hello_display_name_refuses_invalid_manifest_by_field() {
9975 let registry = Arc::new(Registry::default());
9976 let handler = ControlHandler::new(Arc::clone(®istry));
9977 for (index, invalid) in [
9978 json!(""),
9979 json!("a".repeat(41)),
9980 json!(" Routing"),
9981 json!("a\nb"),
9982 json!("a\u{202e}b"),
9983 json!(7),
9984 ]
9985 .into_iter()
9986 .enumerate()
9987 {
9988 let module_id = format!("bad-display-name-{index}");
9989 let mut body = serde_json::to_value(ModuleHelloBody {
9990 manifest: manifest(&module_id, PROTOCOL_VERSION),
9991 protocol_ver: PROTOCOL_VERSION,
9992 control_ops: None,
9993 launch_nonce: None,
9994 })
9995 .unwrap();
9996 body["manifest"]["display_name"] = invalid;
9997 let hello = Frame::build(
9998 FrameType::Hello,
9999 control_flags(),
10000 0,
10001 0,
10002 7,
10003 serde_json::to_vec(&body).unwrap(),
10004 )
10005 .unwrap();
10006 let response = handler
10007 .handle_control(ConnectionId::new(100 + index as u64), hello)
10008 .unwrap();
10009 assert_eq!(response[0].header.ty, FrameType::Error);
10010 let error = parse_error(&response[0]);
10011 assert_eq!(error["code"], "invalid_manifest");
10012 assert!(error["message"].as_str().unwrap().contains("display_name"));
10013 assert!(registry.get_module(&module_id).unwrap().is_none());
10014 }
10015 }
10016
10017 #[test]
10018 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
10019 let invalid_identifiers = [
10020 ("case_change", "credentials-Provider/v1"),
10021 ("leading_zero", "credentials-provider/v01"),
10022 ("trailing_hyphen", "credentials-provider-/v1"),
10023 ("consecutive_hyphens", "credentials--provider/v1"),
10024 ("uppercase", "Credentials-provider/v1"),
10025 ("missing_v", "credentials-provider/1"),
10026 ("whitespace", "credentials provider/v1"),
10027 ("zero_version", "credentials-provider/v0"),
10028 ("out_of_range_version", "credentials-provider/v4294967296"),
10029 (
10030 "overlength_name",
10031 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
10032 ),
10033 ];
10034 let mut cases = invalid_identifiers
10035 .into_iter()
10036 .map(|(name, identifier)| {
10037 (
10038 format!("identifier_{name}"),
10039 "capabilities.provides[0]".to_string(),
10040 identifier.to_string(),
10041 json!({ "provides": [identifier] }),
10042 None,
10043 )
10044 })
10045 .collect::<Vec<_>>();
10046 cases.extend([
10047 (
10048 "unknown_need".to_string(),
10049 "capabilities.requires[0].need".to_string(),
10050 "deferred".to_string(),
10051 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
10052 None,
10053 ),
10054 (
10055 "duplicate_provides".to_string(),
10056 "capabilities.provides[1]".to_string(),
10057 "credentials-provider/v1".to_string(),
10058 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
10059 None,
10060 ),
10061 (
10062 "duplicate_must_never_reach".to_string(),
10063 "capabilities.must_never_reach[1]".to_string(),
10064 "credentials-provider/v1".to_string(),
10065 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
10066 None,
10067 ),
10068 (
10069 "duplicate_requires_same_need".to_string(),
10070 "capabilities.requires[1]".to_string(),
10071 "credentials-provider/v1".to_string(),
10072 json!({ "requires": [
10073 { "capability": "credentials-provider/v1", "need": "required" },
10074 { "capability": "credentials-provider/v1", "need": "required" }
10075 ] }),
10076 None,
10077 ),
10078 (
10079 "duplicate_requires_conflicting_need".to_string(),
10080 "capabilities.requires[1]".to_string(),
10081 "credentials-provider/v1".to_string(),
10082 json!({ "requires": [
10083 { "capability": "credentials-provider/v1", "need": "required" },
10084 { "capability": "credentials-provider/v1", "need": "optional" }
10085 ] }),
10086 None,
10087 ),
10088 (
10089 "capabilities_root_pointer".to_string(),
10090 "runtime_computed[0]".to_string(),
10091 "/capabilities".to_string(),
10092 json!({}),
10093 Some(json!(["/capabilities"])),
10094 ),
10095 (
10096 "capabilities_descendant_pointer".to_string(),
10097 "runtime_computed[0]".to_string(),
10098 "/capabilities/provides".to_string(),
10099 json!({}),
10100 Some(json!(["/capabilities/provides"])),
10101 ),
10102 (
10103 "malformed_pointer_without_leading_slash".to_string(),
10104 "runtime_computed[0]".to_string(),
10105 "capabilities".to_string(),
10106 json!({}),
10107 Some(json!(["capabilities"])),
10108 ),
10109 (
10110 "malformed_pointer_escape".to_string(),
10111 "runtime_computed[0]".to_string(),
10112 "/roles/~2/tools".to_string(),
10113 json!({}),
10114 Some(json!(["/roles/~2/tools"])),
10115 ),
10116 (
10117 "unknown_capabilities_field".to_string(),
10118 "capabilities.future".to_string(),
10119 "<array>".to_string(),
10120 json!({ "future": [] }),
10121 None,
10122 ),
10123 ]);
10124
10125 for (index, (name, field, value, capabilities, runtime_computed)) in
10126 cases.into_iter().enumerate()
10127 {
10128 let registry = Arc::new(Registry::default());
10129 let handler = ControlHandler::new(Arc::clone(®istry));
10130 let response = handler
10131 .handle_control(
10132 ConnectionId::new((index + 1) as u64),
10133 capability_grammar_hello_frame(
10134 capabilities,
10135 runtime_computed,
10136 index as u64 + 1,
10137 ),
10138 )
10139 .expect("invalid HELLO returns a refusal");
10140
10141 assert_eq!(response.len(), 1, "{name} must emit one refusal");
10142 let error = parse_error(&response[0]);
10143 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
10144 let message = error["message"]
10145 .as_str()
10146 .expect("error message is a string");
10147 assert!(
10148 message.contains(&field),
10149 "{name}: field missing from {message}"
10150 );
10151 assert!(
10152 message.contains(&value),
10153 "{name}: value missing from {message}"
10154 );
10155 assert_eq!(
10156 registry
10157 .active_registration_count()
10158 .expect("registry reads"),
10159 0,
10160 "{name}: refused HELLO must not create a catalog entry"
10161 );
10162 }
10163 }
10164
10165 #[test]
10166 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
10167 let registry = Arc::new(Registry::default());
10168 let handler = ControlHandler::new(Arc::clone(®istry));
10169 let capabilities = json!({
10170 "provides": ["credentials-provider/v1"],
10171 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
10172 "must_never_reach": ["federation-transport/v1"]
10173 });
10174 let response = handler
10175 .handle_control(
10176 ConnectionId::new(99),
10177 capability_grammar_hello_frame(
10178 capabilities.clone(),
10179 Some(json!(["/roles/0/tools"])),
10180 99,
10181 ),
10182 )
10183 .expect("valid HELLO registers");
10184 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10185
10186 let request = Frame::build(
10187 FrameType::Request,
10188 control_flags(),
10189 0,
10190 0,
10191 100,
10192 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10193 .expect("catalog request serializes"),
10194 )
10195 .expect("catalog request frame builds");
10196 let response = handler
10197 .handle_catalog_list(request, None)
10198 .expect("catalog list succeeds");
10199 let ClientControlResponse::CatalogList { modules, .. } =
10200 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
10201 else {
10202 panic!("catalog request must return catalog.list");
10203 };
10204 assert_eq!(modules.len(), 1);
10205 assert_eq!(
10206 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
10207 capabilities
10208 );
10209 }
10210
10211 #[test]
10212 fn catalog_list_mirrors_management_operation_description() {
10213 let registry = Arc::new(Registry::default());
10214 let handler = ControlHandler::new(Arc::clone(®istry));
10215 let description = "List managed records and return their identifiers and metadata.";
10216 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
10217 manifest.provides = vec![ProviderRole::ManagementSurface {
10218 operations: vec![ManagementOperation {
10219 name: "records.list".to_string(),
10220 kind: ManagementOperationKind::Query,
10221 description: Some(description.to_string()),
10222 }],
10223 config_schema: json!({"type": "object"}),
10224 observability: vec![ObservabilitySurface {
10225 name: "records.stats".to_string(),
10226 kind: ObservabilityKind::Snapshot,
10227 }],
10228 identity_scope: vec![IdentityScope::Project],
10229 concurrency: Concurrency::ModuleManaged,
10230 }];
10231 registry
10232 .register_with_control_ops(
10233 manifest,
10234 PROTOCOL_VERSION,
10235 ConnectionId::new(99),
10236 Vec::new(),
10237 )
10238 .expect("described management manifest registers");
10239
10240 let request = Frame::build(
10241 FrameType::Request,
10242 control_flags(),
10243 0,
10244 0,
10245 100,
10246 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10247 .expect("catalog request serializes"),
10248 )
10249 .expect("catalog request frame builds");
10250 let response = handler
10251 .handle_catalog_list(request, None)
10252 .expect("catalog list succeeds");
10253 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10254 assert_eq!(
10255 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10256 "catalog.list must preserve the declared operation description verbatim"
10257 );
10258 }
10259
10260 #[test]
10261 fn catalog_list_mirrors_registered_operation_delivery() {
10262 let registry = Arc::new(Registry::default());
10263 let handler = ControlHandler::new(Arc::clone(®istry));
10264 let delivery =
10265 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10266 let mut body = serde_json::to_value(ModuleHelloBody {
10267 manifest: manifest("watch-management", PROTOCOL_VERSION),
10268 protocol_ver: PROTOCOL_VERSION,
10269 control_ops: None,
10270 launch_nonce: None,
10271 })
10272 .unwrap();
10273 body["manifest"]["provides"] = json!([{
10274 "role": "management_surface",
10275 "operations": [
10276 {"name": "status", "kind": "query"},
10277 {"name": "future", "kind": "query"},
10278 {"name": "reset", "kind": "mutate"}
10279 ],
10280 "config_schema": {}, "observability": [], "identity_scope": []
10281 }]);
10282 body["manifest"]["operation_delivery"] = delivery.clone();
10283 let hello = Frame::build(
10284 FrameType::Hello,
10285 control_flags(),
10286 0,
10287 0,
10288 99,
10289 serde_json::to_vec(&body).unwrap(),
10290 )
10291 .unwrap();
10292 let response = handler
10293 .handle_control(ConnectionId::new(99), hello)
10294 .unwrap();
10295 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10296
10297 let (_, registrations) = registry.list_modules().unwrap();
10298 let registered = ®istrations[0].manifest;
10299 assert!(registered.is_watch_operation("status"));
10300 assert!(!registered.is_watch_operation("future"));
10301 assert!(!registered.is_watch_operation("reset"));
10302 assert!(!registered.is_watch_operation("missing"));
10303
10304 let request = Frame::build(
10305 FrameType::Request,
10306 control_flags(),
10307 0,
10308 0,
10309 100,
10310 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10311 )
10312 .unwrap();
10313 let response = handler.handle_catalog_list(request, None).unwrap();
10314 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10315 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10316 let ClientControlResponse::CatalogList { modules, .. } =
10317 serde_json::from_slice(&response[0].body).unwrap()
10318 else {
10319 panic!("expected catalog.list");
10320 };
10321 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10322 }
10323
10324 #[test]
10325 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10326 let registry = Arc::new(Registry::default());
10327 let supervisor = SupervisorHandle::new();
10328 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10329 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10330 let descriptor = json!({
10331 "schema": "ckios.richui.manifest/1",
10332 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10333 "bytes": 4096,
10334 "chunks": 4
10335 });
10336 let connection = ConnectionId::new(201);
10337 let hello = handler
10338 .handle_control(
10339 connection,
10340 hello_frame_with_rich_ui(
10341 "vault",
10342 PROTOCOL_VERSION,
10343 201,
10344 Some("the-real-nonce"),
10345 descriptor.clone(),
10346 ),
10347 )
10348 .unwrap();
10349 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10350
10351 let request =
10352 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10353 let provides = registry
10354 .get_module_by_connection(connection)
10355 .unwrap()
10356 .unwrap()
10357 .manifest
10358 .provides;
10359 handler
10360 .handle_catalog_update(connection, request, provides, None, Some(false))
10361 .unwrap();
10362
10363 let request = Frame::build(
10364 FrameType::Request,
10365 control_flags(),
10366 0,
10367 0,
10368 203,
10369 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10370 )
10371 .unwrap();
10372 let response = handler.handle_catalog_list(request, None).unwrap();
10373 let ClientControlResponse::CatalogList { modules, .. } =
10374 serde_json::from_slice(&response[0].body).unwrap()
10375 else {
10376 panic!("expected catalog.list");
10377 };
10378 assert_eq!(
10379 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10380 descriptor
10381 );
10382 assert!(modules[0].nonce_verified);
10383 }
10384
10385 fn hello_with_role_declarations(declarations: Value) -> Frame {
10386 let mut body = serde_json::to_value(ModuleHelloBody {
10387 manifest: manifest("role-test", PROTOCOL_VERSION),
10388 protocol_ver: PROTOCOL_VERSION,
10389 control_ops: None,
10390 launch_nonce: None,
10391 })
10392 .unwrap();
10393 body["manifest"]["role_declarations"] = declarations;
10394 Frame::build(
10395 FrameType::Hello,
10396 control_flags(),
10397 0,
10398 0,
10399 301,
10400 serde_json::to_vec(&body).unwrap(),
10401 )
10402 .unwrap()
10403 }
10404
10405 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10406 let request = Frame::build(
10407 FrameType::Request,
10408 control_flags(),
10409 0,
10410 0,
10411 302,
10412 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10413 )
10414 .unwrap();
10415 let response = handler.handle_catalog_list(request, None).unwrap();
10416 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10417 let ClientControlResponse::CatalogList { mut modules, .. } =
10418 serde_json::from_slice(&response[0].body).unwrap()
10419 else {
10420 panic!("expected catalog.list");
10421 };
10422 (raw["modules"][0].clone(), modules.remove(0))
10423 }
10424
10425 #[test]
10426 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10427 let registry = Arc::new(Registry::default());
10428 let handler = ControlHandler::new(Arc::clone(®istry));
10429 let declarations = json!({
10430 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10431 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10432 });
10433 let connection = ConnectionId::new(301);
10434 let response = handler
10435 .handle_control(
10436 connection,
10437 hello_with_role_declarations(declarations.clone()),
10438 )
10439 .unwrap();
10440 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10441 let registered = registry
10442 .get_module_by_connection(connection)
10443 .unwrap()
10444 .unwrap()
10445 .manifest;
10446 assert_eq!(
10450 serde_json::to_value(®istered.role_declarations).unwrap(),
10451 declarations
10452 );
10453 let (raw, entry) = role_declarations_catalog(&handler);
10454 assert_eq!(raw["role_declarations"], declarations);
10455 assert_eq!(entry.role_declarations, registered.role_declarations);
10456
10457 let request =
10458 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10459 handler
10460 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10461 .unwrap();
10462 let (raw, entry) = role_declarations_catalog(&handler);
10463 assert_eq!(raw["role_declarations"], declarations);
10464 assert_eq!(entry.role_declarations, registered.role_declarations);
10465 }
10466
10467 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10468 let registry = Arc::new(Registry::default());
10469 let handler = ControlHandler::new(Arc::clone(®istry));
10470 let capture = EventCapture::default();
10471 let _subscriber =
10472 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10473 let response = handler
10474 .handle_control(
10475 ConnectionId::new(301),
10476 hello_with_role_declarations(declarations),
10477 )
10478 .unwrap();
10479 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10480 assert!(registry
10481 .get_module_by_connection(ConnectionId::new(301))
10482 .unwrap()
10483 .unwrap()
10484 .manifest
10485 .role_declarations
10486 .is_empty());
10487 let (raw, entry) = role_declarations_catalog(&handler);
10488 assert!(raw.get("role_declarations").is_none());
10489 assert!(entry.role_declarations.is_empty());
10490 let warnings: Vec<_> = capture
10491 .events()
10492 .into_iter()
10493 .filter(|event| {
10494 event.fields.get("message").map(String::as_str)
10495 == Some("dropping whole role_declarations map from HELLO")
10496 })
10497 .collect();
10498 assert_eq!(warnings.len(), 1);
10499 assert_eq!(warnings[0].level, tracing::Level::WARN);
10500 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10501 assert_eq!(warnings[0].fields["reason"], reason);
10502 }
10503
10504 #[test]
10505 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10506 for value in [
10507 json!(null),
10508 json!(false),
10509 json!(0),
10510 json!("opaque"),
10511 json!([]),
10512 ] {
10513 assert_hello_drops_role_declarations_with_warning(
10514 json!({"routing/v1": {}, "broken/v1": value}),
10515 "role_declarations values must be objects",
10516 );
10517 }
10518 assert_hello_drops_role_declarations_with_warning(
10519 json!([]),
10520 "role_declarations must be an object",
10521 );
10522 }
10523
10524 #[test]
10525 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10526 assert_hello_drops_role_declarations_with_warning(
10527 json!({"routing/v1": {}, "bad key": {}}),
10528 "role_declarations key must use capability-name grammar",
10529 );
10530 }
10531
10532 #[test]
10533 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10534 assert_hello_drops_role_declarations_with_warning(
10535 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10536 "role_declarations exceeds 16384 compact JSON bytes",
10537 );
10538 }
10539
10540 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10541 let mut body = serde_json::to_value(ModuleHelloBody {
10542 manifest: manifest("flow-ops", PROTOCOL_VERSION),
10543 protocol_ver: PROTOCOL_VERSION,
10544 control_ops: None,
10545 launch_nonce: None,
10546 })
10547 .unwrap();
10548 body["manifest"]["provides"] = json!([{
10549 "role": "management_surface",
10550 "operations": [
10551 {"name": "diagnostics.findings", "kind": "query"},
10552 {"name": "reset", "kind": "mutate"}
10553 ],
10554 "config_schema": {}, "observability": [], "identity_scope": []
10555 }]);
10556 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10557 if let Some(flow_operations) = flow_operations {
10558 body["manifest"]["flow_operations"] = flow_operations;
10559 }
10560 Frame::build(
10561 FrameType::Hello,
10562 control_flags(),
10563 0,
10564 0,
10565 401,
10566 serde_json::to_vec(&body).unwrap(),
10567 )
10568 .unwrap()
10569 }
10570
10571 #[test]
10572 fn catalog_list_mirrors_flow_operations_unchanged() {
10573 for (capability, flow_operations) in [
10576 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10577 ("call-modes/v1", json!(["diagnostics.findings"])),
10578 ("flow-scopes/v1", json!([])),
10579 ] {
10580 let registry = Arc::new(Registry::default());
10581 let handler = ControlHandler::new(Arc::clone(®istry));
10582 let connection = ConnectionId::new(401);
10583 let response = handler
10584 .handle_control(
10585 connection,
10586 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10587 )
10588 .unwrap();
10589 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10590 let registered = registry
10591 .get_module_by_connection(connection)
10592 .unwrap()
10593 .unwrap()
10594 .manifest;
10595 assert_eq!(
10596 serde_json::to_value(®istered.flow_operations).unwrap(),
10597 flow_operations
10598 );
10599 let (raw, entry) = role_declarations_catalog(&handler);
10600 assert_eq!(raw["flow_operations"], flow_operations);
10601 assert_eq!(entry.flow_operations, registered.flow_operations);
10602 }
10603
10604 let registry = Arc::new(Registry::default());
10605 let handler = ControlHandler::new(Arc::clone(®istry));
10606 let response = handler
10607 .handle_control(
10608 ConnectionId::new(401),
10609 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10610 )
10611 .unwrap();
10612 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10613 let (raw, entry) = role_declarations_catalog(&handler);
10614 assert!(raw.get("flow_operations").is_none());
10615 assert_eq!(entry.flow_operations, None);
10616 }
10617
10618 fn assert_hello_drops_flow_operations_with_warning(
10619 capabilities: Value,
10620 flow_operations: Value,
10621 reason: &str,
10622 ) {
10623 let registry = Arc::new(Registry::default());
10624 let handler = ControlHandler::new(Arc::clone(®istry));
10625 let capture = EventCapture::default();
10626 let _subscriber =
10627 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10628 let response = handler
10629 .handle_control(
10630 ConnectionId::new(401),
10631 hello_with_flow_operations(capabilities, Some(flow_operations)),
10632 )
10633 .unwrap();
10634 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10635 let registered = registry
10636 .get_module_by_connection(ConnectionId::new(401))
10637 .unwrap()
10638 .expect("the module still registers")
10639 .manifest;
10640 assert_eq!(registered.module_id, "flow-ops");
10641 assert_eq!(registered.flow_operations, None);
10642 let (raw, entry) = role_declarations_catalog(&handler);
10643 assert_eq!(raw["module_id"], "flow-ops");
10644 assert!(raw.get("flow_operations").is_none());
10645 assert_eq!(entry.flow_operations, None);
10646 let warnings: Vec<_> = capture
10647 .events()
10648 .into_iter()
10649 .filter(|event| {
10650 event.fields.get("message").map(String::as_str)
10651 == Some("dropping flow_operations from HELLO")
10652 })
10653 .collect();
10654 assert_eq!(warnings.len(), 1);
10655 assert_eq!(warnings[0].level, tracing::Level::WARN);
10656 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10657 assert_eq!(warnings[0].fields["reason"], reason);
10658 }
10659
10660 #[test]
10661 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10662 assert_hello_drops_flow_operations_with_warning(
10663 json!(["flow-scopes/v1"]),
10664 json!(["diagnostics.findings", "missing"]),
10665 "flow_operations names 'missing', which is not a declared management operation",
10666 );
10667 }
10668
10669 #[test]
10670 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10671 assert_hello_drops_flow_operations_with_warning(
10672 json!([]),
10673 json!(["diagnostics.findings"]),
10674 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10675 );
10676 assert_hello_drops_flow_operations_with_warning(
10678 json!(["agent-run-scopes/v1"]),
10679 json!([]),
10680 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10681 );
10682 }
10683
10684 #[test]
10685 fn hello_malformed_flow_operations_is_dropped_with_warning() {
10686 assert_hello_drops_flow_operations_with_warning(
10687 json!(["flow-scopes/v1"]),
10688 json!(["reset", "reset"]),
10689 "flow_operations names 'reset' more than once",
10690 );
10691 assert_hello_drops_flow_operations_with_warning(
10692 json!(["flow-scopes/v1"]),
10693 json!(vec!["reset"; 257]),
10694 "flow_operations lists 257 names, more than 256",
10695 );
10696 assert_hello_drops_flow_operations_with_warning(
10697 json!(["flow-scopes/v1"]),
10698 json!("reset"),
10699 "flow_operations must be an array of operation names",
10700 );
10701 }
10702
10703 #[test]
10704 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10705 let registry = Arc::new(Registry::default());
10706 let handler = ControlHandler::new(registry.clone());
10707 let descriptor = json!({
10708 "schema": "ckios.richui.manifest/1",
10709 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10710 "bytes": 128,
10711 "chunks": 2
10712 });
10713 let malformed = [
10714 ("schema", json!("")),
10715 ("schema", json!("a".repeat(65))),
10716 ("schema", json!("Bad schema")),
10717 (
10718 "manifest_digest",
10719 json!(format!("sha256:{}", "A".repeat(64))),
10720 ),
10721 ("manifest_digest", json!("sha256:abc")),
10722 ("bytes", json!(0)),
10723 ("bytes", json!(16_777_217)),
10724 ("bytes", json!("128")),
10725 ("chunks", json!(0)),
10726 ("chunks", json!(1025)),
10727 ("chunks", json!("2")),
10728 ];
10729 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10730 let mut invalid = descriptor.clone();
10731 invalid[field] = invalid_value;
10732 let response = handler
10733 .handle_control(
10734 ConnectionId::new(210 + index as u64),
10735 hello_frame_with_rich_ui(
10736 &format!("bad-rich-ui-{index}"),
10737 PROTOCOL_VERSION,
10738 210 + index as u64,
10739 None,
10740 invalid,
10741 ),
10742 )
10743 .unwrap();
10744 assert_eq!(response[0].header.ty, FrameType::Error);
10745 let error = parse_error(&response[0]);
10746 assert_eq!(error["code"], "invalid_manifest");
10747 assert!(error["message"]
10748 .as_str()
10749 .unwrap()
10750 .contains(&format!("rich_ui.{field}")));
10751 assert!(registry
10752 .get_module(&format!("bad-rich-ui-{index}"))
10753 .unwrap()
10754 .is_none());
10755 }
10756 }
10757
10758 #[test]
10759 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10760 let registry = Arc::new(Registry::default());
10761 let handler = ControlHandler::new(Arc::clone(®istry));
10762 let connection = ConnectionId::new(219);
10763 registry
10764 .register_with_control_ops(
10765 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10766 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10767 "bad schema",
10768 format!("sha256:{}", "a".repeat(64)),
10769 128,
10770 2,
10771 )))
10772 .build(),
10773 PROTOCOL_VERSION,
10774 connection,
10775 Vec::new(),
10776 )
10777 .unwrap();
10778 let request =
10779 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10780 let response = handler
10781 .handle_catalog_update(connection, request, Vec::new(), None, None)
10782 .unwrap();
10783 assert_eq!(response[0].header.ty, FrameType::Error);
10784 let error = parse_error(&response[0]);
10785 assert_eq!(error["code"], "invalid_manifest");
10786 assert!(error["message"]
10787 .as_str()
10788 .unwrap()
10789 .contains("rich_ui.schema"));
10790 }
10791
10792 #[test]
10793 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10794 let registry = Arc::new(Registry::default());
10795 let supervisor = SupervisorHandle::new();
10796 supervisor.open_swap("provider", "candidate-nonce".to_string());
10797 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10798 let connection = ConnectionId::new(218);
10799 let hello = handler
10800 .handle_control(
10801 connection,
10802 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10803 )
10804 .unwrap();
10805 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10806 assert!(registry.list_modules().unwrap().1.is_empty());
10807
10808 registry.promote_candidate("provider").unwrap().unwrap();
10809 let request = Frame::build(
10810 FrameType::Request,
10811 control_flags(),
10812 0,
10813 0,
10814 219,
10815 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10816 )
10817 .unwrap();
10818 let response = handler.handle_catalog_list(request, None).unwrap();
10819 let ClientControlResponse::CatalogList { modules, .. } =
10820 serde_json::from_slice(&response[0].body).unwrap()
10821 else {
10822 panic!("expected catalog.list");
10823 };
10824 assert_eq!(modules.len(), 1);
10825 assert!(modules[0].nonce_verified);
10826 }
10827
10828 #[test]
10829 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
10830 for provenance_json in [
10831 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
10832 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
10833 ] {
10834 let registry = Arc::new(Registry::default());
10835 let handler = ControlHandler::new(registry);
10836 let hello = Frame::build(
10837 FrameType::Hello,
10838 control_flags(),
10839 0,
10840 0,
10841 220,
10842 serde_json::to_vec(&ModuleHelloBody {
10843 manifest: ModuleManifest::builder("built", "0.1.0")
10844 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
10845 .build(),
10846 protocol_ver: PROTOCOL_VERSION,
10847 control_ops: None,
10848 launch_nonce: None,
10849 })
10850 .unwrap(),
10851 )
10852 .unwrap();
10853 assert_eq!(
10854 handler
10855 .handle_control(ConnectionId::new(220), hello)
10856 .unwrap()[0]
10857 .header
10858 .ty,
10859 FrameType::HelloAck
10860 );
10861 let request = Frame::build(
10862 FrameType::Request,
10863 control_flags(),
10864 0,
10865 0,
10866 221,
10867 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10868 )
10869 .unwrap();
10870 let response = handler.handle_catalog_list(request, None).unwrap();
10871 let ClientControlResponse::CatalogList { modules, .. } =
10872 serde_json::from_slice(&response[0].body).unwrap()
10873 else {
10874 panic!("expected catalog.list");
10875 };
10876 assert_eq!(modules.len(), 1);
10877 assert_eq!(
10878 serde_json::to_vec(&modules[0]).unwrap(),
10879 format!(
10880 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
10881 )
10882 .as_bytes()
10883 );
10884 }
10885 }
10886
10887 #[test]
10888 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10889 let registry = Arc::new(Registry::default());
10890 let handler = ControlHandler::new(Arc::clone(®istry));
10891 let hello = Frame::build(
10892 FrameType::Hello,
10893 control_flags(),
10894 0,
10895 0,
10896 220,
10897 serde_json::to_vec(&ModuleHelloBody {
10898 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10899 protocol_ver: PROTOCOL_VERSION,
10900 control_ops: None,
10901 launch_nonce: Some("unverified".to_string()),
10902 })
10903 .unwrap(),
10904 )
10905 .unwrap();
10906 assert_eq!(
10907 handler
10908 .handle_control(ConnectionId::new(220), hello)
10909 .unwrap()[0]
10910 .header
10911 .ty,
10912 FrameType::HelloAck
10913 );
10914 let request = Frame::build(
10915 FrameType::Request,
10916 control_flags(),
10917 0,
10918 0,
10919 221,
10920 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10921 )
10922 .unwrap();
10923 let response = handler.handle_catalog_list(request, None).unwrap();
10924 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
10925 assert!(wire["modules"][0].get("provenance").is_none());
10926 let ClientControlResponse::CatalogList { modules, .. } =
10927 serde_json::from_slice(&response[0].body).unwrap()
10928 else {
10929 panic!("expected catalog.list");
10930 };
10931 assert_eq!(
10932 serde_json::to_vec(&modules[0]).unwrap(),
10933 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10934 );
10935 assert!(!modules[0].nonce_verified);
10936 assert!(modules[0].provenance.is_none());
10937 }
10938
10939 #[test]
10940 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10941 let registry = Arc::new(Registry::default());
10942 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10943 [("vault".to_string(), true), ("squatter".to_string(), true)],
10944 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10945 );
10946 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10947 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10948 provides: vec!["credentials-provider/v1".to_string()],
10949 requires: Vec::new(),
10950 must_never_reach: Vec::new(),
10951 });
10952 let frame = Frame::build(
10953 FrameType::Hello,
10954 control_flags(),
10955 0,
10956 0,
10957 77,
10958 serde_json::to_vec(&ModuleHelloBody {
10959 manifest: squatter,
10960 protocol_ver: PROTOCOL_VERSION,
10961 control_ops: None,
10962 launch_nonce: None,
10963 })
10964 .expect("HELLO serializes"),
10965 )
10966 .expect("HELLO frame builds");
10967 let response = handler
10968 .handle_control(ConnectionId::new(77), frame)
10969 .expect("reserved claim receives a typed refusal");
10970 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10971 assert_eq!(
10972 registry
10973 .active_registration_count()
10974 .expect("registry reads"),
10975 0,
10976 "a reserved capability refusal must not leave a catalog entry"
10977 );
10978 }
10979
10980 #[test]
10981 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10982 for settlement in ["timeout", "inconclusive", "drop"] {
10983 let breakers = RouteBindBreakers::default();
10984 let RouteBindAdmission::Admitted {
10985 guard: mut old,
10986 probe: false,
10987 } = breakers.admit("prov")
10988 else {
10989 panic!("ordinary relay admitted")
10990 };
10991 let RouteBindAdmission::Admitted {
10992 guard: mut opener, ..
10993 } = breakers.admit("prov")
10994 else {
10995 panic!("second relay admitted")
10996 };
10997 assert!(
10998 !opener
10999 .record_timeout(1, Duration::ZERO)
11000 .unwrap()
11001 .reopened_after_probe
11002 );
11003 let RouteBindAdmission::Admitted {
11004 guard: mut probe,
11005 probe: true,
11006 } = breakers.admit("prov")
11007 else {
11008 panic!("one cooldown probe admitted")
11009 };
11010 match settlement {
11011 "timeout" => assert!(
11012 !old.record_timeout(1, Duration::ZERO)
11013 .unwrap()
11014 .reopened_after_probe
11015 ),
11016 "inconclusive" => old.record_inconclusive(),
11017 "drop" => drop(old),
11018 _ => unreachable!(),
11019 }
11020 assert!(
11021 matches!(
11022 breakers.admit("prov"),
11023 RouteBindAdmission::Refused {
11024 probe_in_flight: true,
11025 ..
11026 }
11027 ),
11028 "{settlement} of a pre-open relay cannot release the real probe"
11029 );
11030 assert!(
11031 probe
11032 .record_timeout(1, Duration::ZERO)
11033 .unwrap()
11034 .reopened_after_probe
11035 );
11036 assert!(matches!(
11037 breakers.admit("prov"),
11038 RouteBindAdmission::Admitted { probe: true, .. }
11039 ));
11040 }
11041 let breakers = RouteBindBreakers::default();
11042 let admit = || match breakers.admit("prov") {
11043 RouteBindAdmission::Admitted { guard, .. } => guard,
11044 _ => panic!("relay admitted"),
11045 };
11046 admit().record_timeout(1, Duration::ZERO);
11047 let mut old_probe = admit();
11048 breakers.reset_for_new_module_connection("prov");
11049 admit().record_timeout(1, Duration::ZERO);
11050 let _new_probe = admit();
11051 old_probe.record_inconclusive();
11052 assert!(matches!(
11053 breakers.admit("prov"),
11054 RouteBindAdmission::Refused {
11055 probe_in_flight: true,
11056 ..
11057 }
11058 ));
11059 }
11060
11061 #[tokio::test]
11062 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
11063 for candidate in [false, true] {
11064 let registry = Arc::new(Registry::default());
11065 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11066 [("vault".to_string(), true), ("squatter".to_string(), true)],
11067 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
11068 );
11069 let conn = ConnectionId::new(77);
11070 let (ctx, mut rx) = route_ctx(conn);
11071 let initial = capability_manifest("squatter", &[], &[]);
11072 if candidate {
11073 registry
11074 .register_candidate_with_control_ops(
11075 initial.clone(),
11076 PROTOCOL_VERSION,
11077 conn,
11078 module_baseline_control_ops(),
11079 )
11080 .unwrap();
11081 handler
11082 .forwarding
11083 .register_candidate_module_connection(
11084 conn,
11085 "squatter".to_string(),
11086 PROTOCOL_VERSION,
11087 manifest_concurrency(&initial),
11088 ctx.egress.clone(),
11089 )
11090 .unwrap();
11091 } else {
11092 hello_via_sink(
11093 &handler,
11094 &ctx,
11095 &mut rx,
11096 hello_frame_with_manifest(initial.clone(), 1),
11097 )
11098 .await;
11099 }
11100 let response = handler
11101 .handle_control_frame(
11102 &ctx,
11103 catalog_update_with_capabilities_frame(
11104 2,
11105 capability_manifest("squatter", &["credentials-provider/v1"], &[])
11106 .capabilities
11107 .unwrap(),
11108 ),
11109 )
11110 .await
11111 .unwrap();
11112 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
11113 assert_eq!(
11114 registry
11115 .get_module_by_connection(conn)
11116 .unwrap()
11117 .unwrap()
11118 .manifest,
11119 initial
11120 );
11121 }
11122 }
11123
11124 #[test]
11125 fn server_describe_surfaces_required_capability_verdict_fields() {
11126 let registry = Arc::new(Registry::default());
11127 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11128 [
11129 ("consumer".to_string(), true),
11130 ("provider".to_string(), false),
11131 ],
11132 BTreeMap::new(),
11133 );
11134 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11135 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
11136 provides: Vec::new(),
11137 requires: vec![subc_protocol::manifest::CapabilityRequirement {
11138 capability: "credentials-provider/v1".to_string(),
11139 need: subc_protocol::manifest::CapabilityNeed::Required,
11140 }],
11141 must_never_reach: Vec::new(),
11142 });
11143 let hello = Frame::build(
11144 FrameType::Hello,
11145 control_flags(),
11146 0,
11147 0,
11148 78,
11149 serde_json::to_vec(&ModuleHelloBody {
11150 manifest: consumer,
11151 protocol_ver: PROTOCOL_VERSION,
11152 control_ops: None,
11153 launch_nonce: None,
11154 })
11155 .expect("HELLO serializes"),
11156 )
11157 .expect("HELLO frame builds");
11158 handler
11159 .handle_control(ConnectionId::new(78), hello)
11160 .expect("consumer registers");
11161 let describe = Frame::build(
11162 FrameType::Request,
11163 control_flags(),
11164 0,
11165 0,
11166 79,
11167 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
11168 .expect("request serializes"),
11169 )
11170 .expect("describe frame builds");
11171 let response = handler
11172 .handle_server_describe(describe)
11173 .expect("server.describe succeeds");
11174 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
11175 let requirement = &rendered["capability_requirements"][0];
11176 assert_eq!(requirement["consumer"], "consumer");
11177 assert_eq!(requirement["verdict"], "never_provided");
11178 assert_eq!(requirement["episode_seq"], 1);
11179 assert_eq!(requirement["config_satisfiable"], false);
11180 assert_eq!(requirement["runtime_available"], false);
11181 assert!(requirement["detail"]
11182 .as_str()
11183 .expect("detail string")
11184 .contains("credentials-provider/v1"));
11185 }
11186
11187 fn deregister_required_provider(
11193 daemon_shutdown_begun: bool,
11194 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
11195 const CAPABILITY: &str = "project-identity/v1";
11196 let registry = Arc::new(Registry::default());
11197 let daemon_shutdown = DaemonShutdownFlag::default();
11198 let handler = ControlHandler::new(Arc::clone(®istry))
11199 .with_capability_config(
11200 [
11201 ("consumer".to_string(), true),
11202 ("provider".to_string(), true),
11203 ],
11204 BTreeMap::new(),
11205 )
11206 .with_daemon_shutdown_flag(daemon_shutdown.clone());
11207 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11208 consumer.capabilities = Some(CapabilityDeclarations {
11209 provides: Vec::new(),
11210 requires: vec![subc_protocol::manifest::CapabilityRequirement {
11211 capability: CAPABILITY.to_string(),
11212 need: subc_protocol::manifest::CapabilityNeed::Required,
11213 }],
11214 must_never_reach: Vec::new(),
11215 });
11216 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
11217 let provider_conn = ConnectionId::new(182);
11218 for (conn, manifest, corr) in [
11219 (ConnectionId::new(181), consumer, 181),
11220 (provider_conn, provider, 182),
11221 ] {
11222 let hello = handler
11223 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
11224 .expect("module registers");
11225 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11226 }
11227 assert_eq!(
11228 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11229 Some(CapabilityVerdict::Provided),
11230 "the registered provider must satisfy the requirement first"
11231 );
11232 if daemon_shutdown_begun {
11233 daemon_shutdown.set_for_test();
11234 }
11235
11236 let capture = EventCapture::default();
11237 let _subscriber =
11238 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11239 handler
11240 .cleanup_connection(provider_conn)
11241 .expect("provider deregisters");
11242 let events = capture
11243 .events()
11244 .into_iter()
11245 .filter(|event| {
11246 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11247 })
11248 .collect();
11249 (
11250 events,
11251 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11252 )
11253 }
11254
11255 #[test]
11256 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11257 let (events, verdict) = deregister_required_provider(true);
11258 assert_eq!(
11259 verdict,
11260 Some(CapabilityVerdict::NeverProvided),
11261 "shutdown must not change the verdict route.open refusals read"
11262 );
11263 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11264 let event = &events[0];
11265 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11266 assert_eq!(
11267 event.fields.get("cause").map(String::as_str),
11268 Some("\"daemon_shutdown\""),
11269 "{event:?}"
11270 );
11271 assert_eq!(
11272 event.fields.get("verdict").map(String::as_str),
11273 Some("\"never_provided\""),
11274 "{event:?}"
11275 );
11276 }
11277
11278 #[test]
11279 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11280 let (events, verdict) = deregister_required_provider(false);
11281 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11282 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11283 let event = &events[0];
11284 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11285 assert_eq!(event.fields.get("cause"), None, "{event:?}");
11286 assert_eq!(
11287 event.fields.get("verdict").map(String::as_str),
11288 Some("\"never_provided\""),
11289 "{event:?}"
11290 );
11291 }
11292
11293 #[test]
11294 fn catalog_list_omits_capabilities_for_legacy_manifest() {
11295 let registry = Arc::new(Registry::default());
11296 let handler = ControlHandler::new(Arc::clone(®istry));
11297 let hello = handler
11298 .handle_control(
11299 ConnectionId::new(101),
11300 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11301 )
11302 .expect("legacy HELLO registers");
11303 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11304
11305 let request = Frame::build(
11306 FrameType::Request,
11307 control_flags(),
11308 0,
11309 0,
11310 102,
11311 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11312 .expect("catalog request serializes"),
11313 )
11314 .expect("catalog request frame builds");
11315 let response = handler
11316 .handle_catalog_list(request, None)
11317 .expect("catalog list succeeds");
11318 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11319 assert!(
11320 body["modules"][0].get("capabilities").is_none(),
11321 "legacy manifest must retain an absent capabilities field on catalog.list"
11322 );
11323 }
11324
11325 #[test]
11326 fn hello_ack_omits_storage_when_no_storage_config() {
11327 let registry = Arc::new(Registry::default());
11328 let handler = ControlHandler::new(Arc::clone(®istry));
11329 let responses = handler
11330 .handle_control(
11331 ConnectionId::new(1),
11332 hello_frame("aft", PROTOCOL_VERSION, 7),
11333 )
11334 .unwrap();
11335 let ack = parse_ack(&responses[0]);
11336 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11337 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11338 }
11339
11340 #[tokio::test]
11341 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11342 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11343 let registry = Arc::new(Registry::default());
11344 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
11345 let responses = handler
11346 .handle_control(
11347 ConnectionId::new(1),
11348 hello_frame("aft", PROTOCOL_VERSION, 7),
11349 )
11350 .unwrap();
11351 let ack = parse_ack(&responses[0]);
11352 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11353
11354 let described = handler
11355 .handle_control_frame(
11356 &route_ctx(ConnectionId::new(2)).0,
11357 Frame::build(
11358 FrameType::Request,
11359 control_flags(),
11360 0,
11361 0,
11362 9,
11363 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11364 )
11365 .unwrap(),
11366 )
11367 .await
11368 .unwrap();
11369 let ClientControlResponse::ServerDescribe { machine_id, .. } =
11370 serde_json::from_slice(&described[0].body).unwrap()
11371 else {
11372 panic!("server.describe answered with another shape");
11373 };
11374 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11375 }
11376
11377 #[test]
11378 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11379 let registry = Arc::new(Registry::default());
11382 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
11383 crate::daemon_config::StorageConfig::Sqlite {
11384 data_home: std::path::PathBuf::from("/data"),
11385 },
11386 ));
11387
11388 let responses = handler
11389 .handle_control(
11390 ConnectionId::new(1),
11391 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11392 )
11393 .unwrap();
11394 let ack = parse_ack(&responses[0]);
11395 assert_eq!(
11396 ack.storage,
11397 Some(serde_json::json!({
11398 "module_id": "alfonso-routing",
11399 "storage_namespace": "default",
11400 "isolation": { "kind": "module" },
11401 "backend": {
11402 "backend": "sqlite",
11403 "path": "/data/cortexkit/alfonso-routing/store.db"
11404 }
11405 })),
11406 "the delivered descriptor is the module's own sqlite store path"
11407 );
11408 }
11409
11410 #[test]
11411 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11412 let registry = Arc::new(Registry::default());
11413 let handler = ControlHandler::new(Arc::clone(®istry));
11414 let conn = ConnectionId::new(1);
11415 let responses = handler
11416 .handle_control(
11417 conn,
11418 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11419 )
11420 .unwrap();
11421 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11422 let registration = registry.get_module("aft").unwrap().unwrap();
11423 assert_eq!(registration.control_ops, module_baseline_control_ops());
11424
11425 let frame =
11426 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11427 assert!(handler
11428 .guard_module_control_op(&frame, "aft", "route.bind")
11429 .unwrap()
11430 .is_none());
11431 let error = handler
11432 .guard_module_control_op(&frame, "aft", "test.synthetic")
11433 .unwrap()
11434 .expect("synthetic ungranted op should be rejected");
11435 assert_eq!(error.header.ty, FrameType::Error);
11436 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11437 }
11438
11439 #[test]
11440 fn hello_control_ops_some_adds_optional_grants() {
11441 let registry = Arc::new(Registry::default());
11442 let handler = ControlHandler::new(Arc::clone(®istry));
11443 handler
11444 .handle_control(
11445 ConnectionId::new(1),
11446 hello_frame_with_control_ops(
11447 "aft",
11448 PROTOCOL_VERSION,
11449 7,
11450 Some(vec![
11451 "future.synthetic".to_string(),
11452 "route.bind".to_string(),
11453 ]),
11454 ),
11455 )
11456 .unwrap();
11457 let registration = registry.get_module("aft").unwrap().unwrap();
11458 assert_eq!(
11459 registration.control_ops,
11460 vec![
11461 "route.bind".to_string(),
11462 "route.status".to_string(),
11463 "future.synthetic".to_string(),
11464 ]
11465 );
11466 let frame =
11467 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11468 assert!(handler
11469 .guard_module_control_op(&frame, "aft", "future.synthetic")
11470 .unwrap()
11471 .is_none());
11472 }
11473
11474 #[tokio::test]
11475 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11476 let registry = Arc::new(Registry::default());
11477 let forwarding = Arc::new(ForwardingTable::default());
11478 let handler =
11479 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11480 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11481 hello_via_sink(
11482 &handler,
11483 &module_ctx,
11484 &mut module_rx,
11485 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11486 )
11487 .await;
11488
11489 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11490 let responses = handler
11491 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11492 .await
11493 .unwrap();
11494 assert_eq!(responses.len(), 1);
11495 assert_eq!(responses[0].header.ty, FrameType::Error);
11496 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11497 assert!(module_rx.try_recv().is_err());
11498 }
11499
11500 #[tokio::test]
11501 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11502 let registry = Arc::new(Registry::default());
11503 let forwarding = Arc::new(ForwardingTable::default());
11504 let handler =
11505 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11506 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11507 hello_via_sink(
11508 &handler,
11509 &module_ctx,
11510 &mut module_rx,
11511 hello_frame_with_control_ops(
11512 "aft",
11513 PROTOCOL_VERSION,
11514 7,
11515 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11516 ),
11517 )
11518 .await;
11519
11520 let project_root = unique_project_root("demux");
11521 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11522 let route_handler = handler.clone();
11523 let route_task = tokio::spawn(async move {
11524 route_handler
11525 .handle_control_frame(
11526 &route_client_ctx,
11527 route_open_frame(100, "aft", project_root),
11528 )
11529 .await
11530 .unwrap()
11531 });
11532 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11533 .await
11534 .unwrap()
11535 .unwrap();
11536 assert!(matches!(
11537 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11538 ModuleControlRequest::RouteBind { .. }
11539 ));
11540
11541 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11542 let health_handler = handler.clone();
11543 let health_task = tokio::spawn(async move {
11544 health_handler
11545 .handle_control_frame(
11546 &health_client_ctx,
11547 supervisor_health_probe_frame(101, "aft"),
11548 )
11549 .await
11550 .unwrap()
11551 });
11552 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11553 .await
11554 .unwrap()
11555 .unwrap();
11556 assert_eq!(
11557 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11558 ModuleControlRequest::HealthCheck {}
11559 );
11560
11561 handler
11562 .handle_control_frame(
11563 &module_ctx,
11564 health_response(health_frame.header.corr, HealthStatus::Degraded),
11565 )
11566 .await
11567 .unwrap();
11568 let health_response = health_task.await.unwrap();
11569 assert_eq!(health_response.len(), 1);
11570 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11571 ClientControlResponse::SupervisorHealthProbe {
11572 module_id,
11573 status,
11574 detail,
11575 metrics,
11576 } => {
11577 assert_eq!(module_id, "aft");
11578 assert_eq!(status, HealthStatus::Degraded);
11579 assert_eq!(detail.as_deref(), Some("warming"));
11580 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11581 }
11582 other => panic!("unexpected health response: {other:?}"),
11583 }
11584
11585 handler
11586 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11587 .await
11588 .unwrap();
11589 let route_response = route_task.await.unwrap();
11590 assert!(route_response.is_empty());
11591 let published = route_client_rx.recv().await.unwrap();
11592 assert!(matches!(
11593 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11594 ClientControlResponse::RouteOpen { .. }
11595 ));
11596 }
11597
11598 async fn relay_route_open(
11603 handler: &ControlHandler,
11604 client_connection: ConnectionId,
11605 client_egress: &FrameSink,
11606 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11607 corr: u64,
11608 module_id: &str,
11609 project_root_label: &str,
11610 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11611 let ctx = RouteCtx {
11612 connection_id: client_connection,
11613 egress: client_egress.clone(),
11614 };
11615 let handler = handler.clone();
11616 let project_root = unique_project_root(project_root_label);
11617 let module_id = module_id.to_string();
11618 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11619 let task = tokio::spawn(async move {
11620 let _guard = tracing::dispatcher::set_default(&dispatch);
11621 handler
11622 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11623 .await
11624 .unwrap()
11625 });
11626 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11627 .await
11628 .expect("module receives the relayed route.bind")
11629 .expect("module egress is open");
11630 (task, bind.frame)
11631 }
11632
11633 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11634 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11635 ModuleControlRequest::RouteBind {
11636 route_channel,
11637 epoch,
11638 ..
11639 } => (route_channel, epoch),
11640 other => panic!("expected a route.bind request, got {other:?}"),
11641 }
11642 }
11643
11644 fn published_route(frame: &Frame) -> (u16, u32) {
11645 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11646 ClientControlResponse::RouteOpen {
11647 route_channel,
11648 route_epoch,
11649 ..
11650 } => (route_channel, route_epoch),
11651 other => panic!("expected a route.open response, got {other:?}"),
11652 }
11653 }
11654
11655 #[tokio::test]
11675 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11676 let registry = Arc::new(Registry::default());
11677 let forwarding = Arc::new(ForwardingTable::default());
11678 let handler =
11679 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11680
11681 let module_connection = ConnectionId::new(30);
11682 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11683 hello_via_sink(
11684 &handler,
11685 &module_ctx,
11686 &mut module_rx,
11687 hello_frame("aft", PROTOCOL_VERSION, 7),
11688 )
11689 .await;
11690
11691 let dying_client = ConnectionId::new(31);
11692 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11693
11694 let (first_task, first_bind) = relay_route_open(
11697 &handler,
11698 dying_client,
11699 &dying_ctx.egress,
11700 &mut module_rx,
11701 100,
11702 "aft",
11703 "closing-first",
11704 )
11705 .await;
11706 handler
11707 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11708 .await
11709 .unwrap();
11710 assert!(first_task.await.unwrap().is_empty());
11711 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11712
11713 let (second_task, second_bind) = relay_route_open(
11715 &handler,
11716 dying_client,
11717 &dying_ctx.egress,
11718 &mut module_rx,
11719 101,
11720 "aft",
11721 "closing-second",
11722 )
11723 .await;
11724 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11725
11726 assert!(forwarding
11729 .escalate_client_delivery_failure(
11730 dying_client,
11731 first_channel,
11732 first_epoch,
11733 CloseReason::new(
11734 "module_to_client_delivery_failed",
11735 "client egress refused a module frame",
11736 ),
11737 crate::forwarding::UndeliveredFrame {
11738 module_id: None,
11739 sink: &dying_ctx.egress,
11740 },
11741 )
11742 .unwrap());
11743 assert!(!dying_ctx.egress.is_closed());
11744
11745 let ack = handler
11747 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11748 .await;
11749 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11750 if module_loop_error.is_some() {
11751 handler.cleanup_connection(module_connection).unwrap();
11755 }
11756 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11760 .await
11761 .ok()
11762 .flatten();
11763
11764 assert!(
11766 registry
11767 .get_module_by_connection(module_connection)
11768 .unwrap()
11769 .is_some(),
11770 "one client's closing connection ended the shared module connection: \
11771 {module_loop_error:?}"
11772 );
11773 let cotenant = ConnectionId::new(32);
11775 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11776 let (cotenant_task, cotenant_bind) = relay_route_open(
11777 &handler,
11778 cotenant,
11779 &cotenant_ctx.egress,
11780 &mut module_rx,
11781 102,
11782 "aft",
11783 "closing-cotenant",
11784 )
11785 .await;
11786 handler
11787 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11788 .await
11789 .unwrap();
11790 assert!(cotenant_task.await.unwrap().is_empty());
11791 let (cotenant_channel, cotenant_epoch) =
11792 published_route(&cotenant_rx.recv().await.unwrap());
11793 assert!(matches!(
11794 forwarding
11795 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11796 .unwrap(),
11797 DataRoute::Client(DataRouteState::Bound(_))
11798 ));
11799
11800 let goodbye = post_ack_module_frame
11803 .expect("module receives a GOODBYE for the abandoned route channel");
11804 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11805 assert_eq!(goodbye.header.channel, abandoned_channel);
11806 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11807
11808 assert!(dying_rx.try_recv().is_err());
11812 let second_response = second_task.await.unwrap();
11813 assert_eq!(second_response.len(), 1);
11814 assert_eq!(
11815 parse_error(&second_response[0])["code"],
11816 "target_unavailable"
11817 );
11818 }
11819
11820 async fn catalog_snapshot(
11823 handler: &ControlHandler,
11824 client_ctx: &RouteCtx,
11825 corr: u64,
11826 ) -> (u64, Vec<(String, bool)>) {
11827 let body =
11828 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11829 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11830 let reply = handler
11831 .handle_control_frame(client_ctx, request)
11832 .await
11833 .unwrap()
11834 .pop()
11835 .unwrap();
11836 let ClientControlResponse::CatalogList {
11837 generation,
11838 modules,
11839 ..
11840 } = serde_json::from_slice(&reply.body).unwrap()
11841 else {
11842 panic!("not a catalog.list reply");
11843 };
11844 (
11845 generation,
11846 modules
11847 .into_iter()
11848 .map(|entry| (entry.module_id, entry.ready))
11849 .collect(),
11850 )
11851 }
11852
11853 fn hello_held_before_forwarding() -> (
11859 ControlHandler,
11860 Arc<std::sync::Barrier>,
11861 std::thread::JoinHandle<Vec<Frame>>,
11862 RouteCtx,
11863 mpsc::Receiver<crate::router::OutboundFrame>,
11864 ) {
11865 let hold = Arc::new(std::sync::Barrier::new(2));
11866 let handler = ControlHandler::with_forwarding(
11867 Arc::new(Registry::default()),
11868 Arc::new(ForwardingTable::default()),
11869 )
11870 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11871 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11872 let hello = {
11873 let handler = handler.clone();
11874 let module_ctx = module_ctx.clone();
11875 std::thread::spawn(move || {
11876 handler
11877 .handle_hello(
11878 module_ctx.connection_id,
11879 Some(module_ctx.egress.clone()),
11880 hello_frame("aft", PROTOCOL_VERSION, 7),
11881 )
11882 .unwrap()
11883 })
11884 };
11885 (handler, hold, hello, module_ctx, module_rx)
11886 }
11887
11888 #[tokio::test]
11897 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11898 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11899 let client = ConnectionId::new(41);
11900 let (client_ctx, mut client_rx) = route_ctx(client);
11901
11902 hold.wait();
11903 assert!(handler.registry.get_module("aft").unwrap().is_some());
11905 assert!(!handler
11906 .forwarding
11907 .has_live_module_connection("aft")
11908 .unwrap());
11909 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11910 assert!(
11911 listed.is_empty(),
11912 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11913 );
11914 let refused = handler
11915 .handle_control_frame(
11916 &client_ctx,
11917 route_open_frame(301, "aft", unique_project_root("listed-window")),
11918 )
11919 .await
11920 .unwrap();
11921 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11922 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11923 let duplicate = handler
11924 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11925 .await
11926 .unwrap();
11927 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11928
11929 hold.wait();
11930 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11931 let ack = module_rx.try_recv().unwrap().frame;
11932 assert_eq!(ack.header.ty, FrameType::HelloAck);
11933 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11934 assert_eq!(listed, vec![("aft".to_string(), true)]);
11935
11936 let (open_task, bind) = relay_route_open(
11937 &handler,
11938 client,
11939 &client_ctx.egress,
11940 &mut module_rx,
11941 303,
11942 "aft",
11943 "listed-after",
11944 )
11945 .await;
11946 assert!(handler
11947 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11948 .await
11949 .unwrap()
11950 .is_empty());
11951 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11952 published_route(&client_rx.try_recv().unwrap().frame);
11953 }
11954
11955 #[tokio::test]
11962 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11963 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11964 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11965 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11966
11967 hold.wait();
11968 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11969 assert!(listed.is_empty());
11970 assert_eq!(
11971 during, before,
11972 "the generation moved while the module was not listed"
11973 );
11974
11975 hold.wait();
11976 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11977 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11978 assert_eq!(listed, vec![("aft".to_string(), true)]);
11979 assert_eq!(
11980 after,
11981 before + 1,
11982 "the module appeared without exactly one generation bump"
11983 );
11984 }
11985
11986 #[tokio::test]
11991 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11992 let registry = Arc::new(Registry::default());
11993 let handler = ControlHandler::with_forwarding(
11994 Arc::clone(®istry),
11995 Arc::new(ForwardingTable::default()),
11996 );
11997 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11998 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11999 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
12000 drop(module_rx);
12001
12002 let replies = handler
12003 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
12004 .await
12005 .unwrap();
12006 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
12007 assert!(registry.get_module("aft").unwrap().is_none());
12008 assert!(registry
12009 .get_module_by_connection(module_ctx.connection_id)
12010 .unwrap()
12011 .is_none());
12012 assert_eq!(
12013 catalog_snapshot(&handler, &client_ctx, 301).await,
12014 (before, Vec::new())
12015 );
12016 }
12017
12018 async fn route_open_held_after_bind_send() -> (
12022 ControlHandler,
12023 Arc<tokio::sync::Barrier>,
12024 RouteCtx,
12025 mpsc::Receiver<crate::router::OutboundFrame>,
12026 ) {
12027 let hold = Arc::new(tokio::sync::Barrier::new(2));
12028 let handler = ControlHandler::with_forwarding(
12029 Arc::new(Registry::default()),
12030 Arc::new(ForwardingTable::default()),
12031 )
12032 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
12033 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12034 hello_via_sink(
12035 &handler,
12036 &module_ctx,
12037 &mut module_rx,
12038 hello_frame("aft", PROTOCOL_VERSION, 7),
12039 )
12040 .await;
12041 (handler, hold, module_ctx, module_rx)
12042 }
12043
12044 #[tokio::test]
12051 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
12052 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12053 let client = ConnectionId::new(41);
12054 let (client_ctx, mut client_rx) = route_ctx(client);
12055
12056 let (open_task, bind) = relay_route_open(
12057 &handler,
12058 client,
12059 &client_ctx.egress,
12060 &mut module_rx,
12061 200,
12062 "aft",
12063 "acked-before-mark",
12064 )
12065 .await;
12066 let (module_channel, module_epoch) = route_bind_channel(&bind);
12067 hold.wait().await;
12069
12070 assert!(handler
12072 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12073 .await
12074 .unwrap()
12075 .is_empty());
12076 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
12077
12078 hold.wait().await;
12079 assert!(
12080 open_task.await.unwrap().is_empty(),
12081 "route.open answers accepted"
12082 );
12083
12084 match module_rx.try_recv() {
12087 Err(mpsc::error::TryRecvError::Empty) => {}
12088 Ok(outbound) => panic!(
12089 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
12090 outbound.frame.header.ty,
12091 outbound.frame.header.channel,
12092 outbound.frame.header.epoch
12093 ),
12094 Err(err) => panic!("module egress closed: {err}"),
12095 }
12096
12097 let router = Router::with_control_handler(Arc::new(handler.clone()));
12100 let request = Frame::build(
12101 FrameType::Request,
12102 Flags::new(true, Priority::Interactive, false),
12103 client_channel,
12104 client_epoch,
12105 201,
12106 b"first request".to_vec(),
12107 )
12108 .unwrap();
12109 router
12110 .route_for_connection(&client_ctx, request)
12111 .await
12112 .unwrap();
12113 let delivered = module_rx.try_recv().unwrap().frame;
12114 assert_eq!(delivered.header.ty, FrameType::Request);
12115 assert_eq!(
12116 (delivered.header.channel, delivered.header.epoch),
12117 (module_channel, module_epoch)
12118 );
12119 assert_eq!(delivered.body, b"first request");
12120 }
12121
12122 #[tokio::test]
12128 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
12129 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12130 let client = ConnectionId::new(42);
12131 let (client_ctx, _client_rx) = route_ctx(client);
12132
12133 let (open_task, bind) = relay_route_open(
12134 &handler,
12135 client,
12136 &client_ctx.egress,
12137 &mut module_rx,
12138 300,
12139 "aft",
12140 "released-before-mark",
12141 )
12142 .await;
12143 let (module_channel, module_epoch) = route_bind_channel(&bind);
12144 hold.wait().await;
12145
12146 handler.cleanup_connection(client).unwrap();
12147 assert!(
12148 module_rx.try_recv().is_err(),
12149 "client teardown must not GOODBYE a bind route.open is still queueing"
12150 );
12151
12152 hold.wait().await;
12153 let response = open_task.await.unwrap();
12154 assert_eq!(response.len(), 1);
12155 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12156
12157 let goodbye = module_rx
12158 .try_recv()
12159 .expect("module is told to drop the binding the daemon abandoned")
12160 .frame;
12161 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12162 assert_eq!(
12163 (goodbye.header.channel, goodbye.header.epoch),
12164 (module_channel, module_epoch)
12165 );
12166 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12167 }
12168
12169 #[tokio::test]
12174 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
12175 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12176 let client = ConnectionId::new(43);
12177 let (client_ctx, client_rx) = route_ctx(client);
12178
12179 let (open_task, bind) = relay_route_open(
12180 &handler,
12181 client,
12182 &client_ctx.egress,
12183 &mut module_rx,
12184 400,
12185 "aft",
12186 "gone-client-before-mark",
12187 )
12188 .await;
12189 let (module_channel, module_epoch) = route_bind_channel(&bind);
12190 hold.wait().await;
12191
12192 drop(client_rx);
12193 assert!(client_ctx.egress.is_closed());
12194 handler
12195 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12196 .await
12197 .unwrap();
12198
12199 hold.wait().await;
12200 let response = open_task.await.unwrap();
12201 assert_eq!(response.len(), 1);
12202 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12203
12204 let goodbye = module_rx
12205 .try_recv()
12206 .expect("module is told to drop the binding it acked")
12207 .frame;
12208 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12209 assert_eq!(
12210 (goodbye.header.channel, goodbye.header.epoch),
12211 (module_channel, module_epoch)
12212 );
12213 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12214 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12215 }
12216
12217 #[test]
12224 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
12225 let handler = ControlHandler::default();
12226 let module_connection = ConnectionId::new(30);
12227 let client_connection = ConnectionId::new(31);
12228
12229 handler
12230 .refuse_to_end_module_connection_for_a_client(
12231 module_connection,
12232 77,
12233 ForwardingError::ConnectionClosing {
12234 connection_id: client_connection,
12235 },
12236 )
12237 .expect("a closing client must never end the module connection");
12238
12239 assert!(matches!(
12240 handler.refuse_to_end_module_connection_for_a_client(
12241 module_connection,
12242 78,
12243 ForwardingError::ConnectionClosing {
12244 connection_id: module_connection,
12245 },
12246 ),
12247 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12248 connection_id
12249 })) if connection_id == module_connection
12250 ));
12251 assert!(matches!(
12252 handler.refuse_to_end_module_connection_for_a_client(
12253 module_connection,
12254 79,
12255 ForwardingError::Poisoned,
12256 ),
12257 Err(RouterError::Forwarding(ForwardingError::Poisoned))
12258 ));
12259 assert!(matches!(
12260 handler.refuse_to_end_module_connection_for_a_client(
12261 module_connection,
12262 80,
12263 ForwardingError::StaleModuleEndpoint,
12264 ),
12265 Err(RouterError::Forwarding(
12266 ForwardingError::StaleModuleEndpoint
12267 ))
12268 ));
12269 }
12270
12271 #[tokio::test]
12284 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12285 let registry = Arc::new(Registry::default());
12286 let forwarding = Arc::new(ForwardingTable::default());
12287 let supervisor = SupervisorHandle::new();
12288 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12289 let handler =
12290 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12291 .with_supervisor(supervisor);
12292
12293 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12294 hello_via_sink(
12295 &handler,
12296 &target_ctx,
12297 &mut target_rx,
12298 hello_frame("target", PROTOCOL_VERSION, 1),
12299 )
12300 .await;
12301
12302 let wrong_nonce = handler
12306 .handle_control_frame(
12307 &route_ctx(ConnectionId::new(91)).0,
12308 route_open_frame_with_admission_facts(
12309 20,
12310 "target",
12311 unique_project_root("admission-facts"),
12312 Some(subc_control::ConsumerIdentity {
12313 module_id: "fed".to_string(),
12314 launch_nonce: "not-the-real-nonce".to_string(),
12315 }),
12316 None,
12317 ),
12318 )
12319 .await
12320 .unwrap();
12321 assert_eq!(
12322 parse_error(&wrong_nonce[0])["code"],
12323 "bad_consumer_identity",
12324 "a mismatched launch nonce must be refused, not stamped Reserved"
12325 );
12326
12327 let never_spawned = handler
12331 .handle_control_frame(
12332 &route_ctx(ConnectionId::new(92)).0,
12333 route_open_frame_with_admission_facts(
12334 21,
12335 "target",
12336 unique_project_root("admission-facts"),
12337 Some(subc_control::ConsumerIdentity {
12338 module_id: "never-spawned".to_string(),
12339 launch_nonce: "any-nonce".to_string(),
12340 }),
12341 None,
12342 ),
12343 )
12344 .await
12345 .unwrap();
12346 assert_eq!(
12347 parse_error(&never_spawned[0])["code"],
12348 "bad_consumer_identity",
12349 "an unspawned module_id must be refused rather than accepted for lack of a record"
12350 );
12351 }
12352
12353 #[tokio::test]
12376 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12377 let registry = Arc::new(Registry::default());
12378 let forwarding = Arc::new(ForwardingTable::default());
12379 let supervisor = SupervisorHandle::new();
12380 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12381 let handler =
12382 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12383 .with_supervisor(supervisor);
12384
12385 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12386 hello_via_sink(
12387 &handler,
12388 &target_ctx,
12389 &mut target_rx,
12390 hello_frame("target", PROTOCOL_VERSION, 1),
12391 )
12392 .await;
12393
12394 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12395 let route_handler = handler.clone();
12396 let route_task = tokio::spawn(async move {
12397 route_handler
12398 .handle_control_frame(
12399 &client_ctx,
12400 route_open_frame_with_admission_facts(
12401 30,
12402 "target",
12403 unique_project_root("admission-facts"),
12404 Some(subc_control::ConsumerIdentity {
12405 module_id: "fed".to_string(),
12406 launch_nonce: "fed-nonce".to_string(),
12407 }),
12408 None,
12409 ),
12410 )
12411 .await
12412 .unwrap()
12413 });
12414
12415 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12422 .await
12423 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12424 .expect("module control channel closed before route.bind");
12425 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12426 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12427 panic!("expected route.bind")
12428 };
12429 assert_eq!(
12430 principal,
12431 Some(Principal::Reserved {
12432 module_id: "fed".to_string()
12433 }),
12434 "a correctly attested consumer must be stamped Reserved for its own id"
12435 );
12436
12437 handler
12438 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12439 .await
12440 .unwrap();
12441 assert!(route_task.await.unwrap().is_empty());
12442 assert!(
12443 matches!(
12444 serde_json::from_slice::<ClientControlResponse>(
12445 &client_rx.recv().await.unwrap().body
12446 )
12447 .unwrap(),
12448 ClientControlResponse::RouteOpen { .. }
12449 ),
12450 "the route must actually open, not merely avoid an error"
12451 );
12452 }
12453
12454 #[tokio::test(start_paused = true)]
12455 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12456 let registry = Arc::new(Registry::default());
12457 let forwarding = Arc::new(ForwardingTable::default());
12458 let supervisor = SupervisorHandle::new();
12459 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12460 let handler =
12461 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12462 .with_supervisor(supervisor);
12463
12464 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12465 hello_via_sink(
12466 &handler,
12467 &target_ctx,
12468 &mut target_rx,
12469 hello_frame("target", PROTOCOL_VERSION, 1),
12470 )
12471 .await;
12472
12473 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12474 let direct_handler = handler.clone();
12475 let direct_open = tokio::spawn(async move {
12476 direct_handler
12477 .handle_control_frame(
12478 &direct_ctx,
12479 route_open_frame(2, "target", unique_project_root("route-census-direct")),
12480 )
12481 .await
12482 .unwrap()
12483 });
12484 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12485 .await
12486 .expect("no direct route.bind within 5s")
12487 .expect("target control channel closed before direct route.bind");
12488 handler
12489 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12490 .await
12491 .unwrap();
12492 assert!(direct_open.await.unwrap().is_empty());
12493 let _ = direct_rx.recv().await.unwrap();
12494
12495 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12496 let reserved_handler = handler.clone();
12497 let reserved_open = tokio::spawn(async move {
12498 reserved_handler
12499 .handle_control_frame(
12500 &reserved_ctx,
12501 route_open_frame_with_admission_facts(
12502 3,
12503 "target",
12504 unique_project_root("admission-facts"),
12505 Some(ConsumerIdentity {
12506 module_id: "fed".to_string(),
12507 launch_nonce: "fed-nonce".to_string(),
12508 }),
12509 None,
12510 ),
12511 )
12512 .await
12513 .unwrap()
12514 });
12515 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12516 .await
12517 .expect("no reserved route.bind within 5s")
12518 .expect("target control channel closed before reserved route.bind");
12519 handler
12520 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12521 .await
12522 .unwrap();
12523 assert!(reserved_open.await.unwrap().is_empty());
12524 let _ = reserved_rx.recv().await.unwrap();
12525
12526 forwarding
12527 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12528 .unwrap();
12529 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12530 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12531 module_id: Some("target".to_string()),
12532 })
12533 .unwrap();
12534 let census_frame =
12535 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12536 let response = handler
12537 .handle_control_frame(&census_ctx, census_frame)
12538 .await
12539 .unwrap()
12540 .pop()
12541 .unwrap();
12542 let actual: Value = serde_json::from_slice(&response.body).unwrap();
12543 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12544 assert!(matches!(
12545 decoded,
12546 ClientControlResponse::SupervisorRoutes { .. }
12547 ));
12548 let routes = actual["modules"][0]["routes"].as_array().unwrap();
12549 assert_eq!(routes.len(), 2);
12550 assert!(routes.iter().all(|route| route["draining"] == true));
12551 assert!(
12554 routes.iter().all(|route| route["drain_reason"] == "reload"),
12555 "draining routes must name the drain's reason: {routes:?}"
12556 );
12557 assert!(routes.iter().any(|route| {
12558 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12559 }));
12560 assert!(routes.iter().any(|route| {
12561 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12562 }));
12563
12564 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12565 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12566 if std::env::var_os("UPDATE_GOLDEN").is_some() {
12567 std::fs::write(
12568 &golden_path,
12569 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12570 )
12571 .unwrap();
12572 }
12573 let expected: Value =
12574 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12575 assert_eq!(actual, expected);
12576 }
12577
12578 async fn query_live_roots(
12579 handler: &ControlHandler,
12580 module_ctx: &RouteCtx,
12581 ) -> ModuleControlResponseToModule {
12582 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12583 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12584 let response = handler
12585 .handle_control_frame(module_ctx, frame)
12586 .await
12587 .unwrap()
12588 .pop()
12589 .unwrap();
12590 serde_json::from_slice(&response.body).unwrap()
12591 }
12592
12593 #[tokio::test(start_paused = true)]
12594 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12595 let registry = Arc::new(Registry::default());
12596 let forwarding = Arc::new(ForwardingTable::default());
12597 let handler = ControlHandler::with_forwarding(registry, forwarding);
12598 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12599 hello_via_sink(
12600 &handler,
12601 &target_ctx,
12602 &mut target_rx,
12603 hello_frame("target", PROTOCOL_VERSION, 1),
12604 )
12605 .await;
12606 let root = unique_project_root("live-roots-known");
12607 let path = ProjectRootId::from_path_allowing_missing(root.path())
12608 .unwrap()
12609 .as_path()
12610 .to_path_buf();
12611 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12612 let open_handler = handler.clone();
12613 let opened = tokio::spawn(async move {
12614 open_handler
12615 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
12616 .await
12617 .unwrap()
12618 });
12619 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12620 .await
12621 .unwrap()
12622 .unwrap();
12623 handler
12624 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12625 .await
12626 .unwrap();
12627 assert!(opened.await.unwrap().is_empty());
12628 let _ = client_rx.recv().await.unwrap();
12629
12630 let root = unique_project_root("live-roots-pending");
12631 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12632 .unwrap()
12633 .as_path()
12634 .to_path_buf();
12635 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12636 let open_handler = handler.clone();
12637 let pending = tokio::spawn(async move {
12638 open_handler
12639 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
12640 .await
12641 .unwrap()
12642 });
12643 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12644 .await
12645 .unwrap()
12646 .unwrap();
12647 let actual = query_live_roots(&handler, &target_ctx).await;
12648 let ModuleControlResponseToModule::LiveRoots {
12649 roots,
12650 unknown_root_bindings,
12651 total_bindings,
12652 } = actual
12653 else {
12654 panic!("expected live roots")
12655 };
12656 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12657 assert_eq!(unknown_root_bindings, 0);
12658 assert_eq!(
12659 roots.len(),
12660 2,
12661 "root-known arm must retain each canonical root"
12662 );
12663 assert_eq!(
12664 total_bindings,
12665 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12666 );
12667 let counts = roots
12668 .iter()
12669 .map(|root| (root.project_root.clone(), root.bound, root.pending))
12670 .collect::<Vec<_>>();
12671 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12672 expected.sort_by(|a, b| a.0.cmp(&b.0));
12673 assert_eq!(
12674 counts, expected,
12675 "roots must sort by path and count pending separately"
12676 );
12677 handler
12678 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12679 .await
12680 .unwrap();
12681 assert!(pending.await.unwrap().is_empty());
12682 }
12683
12684 #[tokio::test(start_paused = true)]
12685 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12686 let forwarding = Arc::new(ForwardingTable::default());
12687 let handler =
12688 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12689 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12690 hello_via_sink(
12691 &handler,
12692 &target_ctx,
12693 &mut target_rx,
12694 hello_frame("target", PROTOCOL_VERSION, 1),
12695 )
12696 .await;
12697 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12698 let pending = forwarding
12699 .begin_route_bind_relay_for_test(
12700 client_ctx.connection_id,
12701 client_ctx.egress.clone(),
12702 2,
12703 "target",
12704 )
12705 .unwrap();
12706 forwarding
12707 .complete_pending_relay(
12708 target_ctx.connection_id,
12709 pending.corr,
12710 RouteBindRelayOutcome::Accepted,
12711 )
12712 .unwrap();
12713 let actual = query_live_roots(&handler, &target_ctx).await;
12714 let ModuleControlResponseToModule::LiveRoots {
12715 roots,
12716 unknown_root_bindings,
12717 total_bindings,
12718 } = actual
12719 else {
12720 panic!("expected live roots")
12721 };
12722 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12723 assert_eq!(
12724 unknown_root_bindings, 1,
12725 "unknown-root arm must not read as no bindings"
12726 );
12727 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12728 assert_eq!(
12729 total_bindings,
12730 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12731 );
12732 }
12733
12734 #[tokio::test(start_paused = true)]
12742 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12743 let forwarding = Arc::new(ForwardingTable::default());
12744 let handler =
12745 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12746 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12747 let replies = handler
12748 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12749 .await
12750 .unwrap();
12751 let queued_by_hello = module_rx.len();
12752
12753 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12754 let open_handler = handler.clone();
12755 let open = tokio::spawn(async move {
12756 open_handler
12757 .handle_control_frame(
12758 &client_ctx,
12759 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12760 )
12761 .await
12762 .unwrap()
12763 });
12764 let mut spins = 0;
12766 while module_rx.len() == queued_by_hello {
12767 spins += 1;
12768 assert!(spins < 10_000, "route.open never queued a route.bind");
12769 tokio::task::yield_now().await;
12770 }
12771
12772 for reply in replies {
12774 module_ctx.egress.send(reply).await.unwrap();
12775 }
12776
12777 let first = module_rx.recv().await.unwrap().frame;
12778 assert_eq!(
12779 first.header.ty,
12780 FrameType::HelloAck,
12781 "the first frame a registering module reads must be its HELLO_ACK"
12782 );
12783 assert_eq!(first.header.corr, 7);
12784 let second = module_rx.recv().await.unwrap().frame;
12785 assert_eq!(second.header.ty, FrameType::Request);
12786 assert!(
12787 matches!(
12788 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12789 ModuleControlRequest::RouteBind { .. }
12790 ),
12791 "the route.bind follows the ack"
12792 );
12793 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12794
12795 handler
12796 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12797 .await
12798 .unwrap();
12799 assert!(open.await.unwrap().is_empty());
12800 let _ = client_rx.recv().await.unwrap();
12801 }
12802
12803 #[tokio::test(start_paused = true)]
12804 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12805 let handler = ControlHandler::with_forwarding(
12806 Arc::new(Registry::default()),
12807 Arc::new(ForwardingTable::default()),
12808 );
12809 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12810 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12811 hello_via_sink(
12812 &handler,
12813 &first_ctx,
12814 &mut first_rx,
12815 hello_frame("first", PROTOCOL_VERSION, 1),
12816 )
12817 .await;
12818 hello_via_sink(
12819 &handler,
12820 &second_ctx,
12821 &mut second_rx,
12822 hello_frame("second", PROTOCOL_VERSION, 2),
12823 )
12824 .await;
12825 let root = unique_project_root("second-only");
12826 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12827 let cloned = handler.clone();
12828 let open = tokio::spawn(async move {
12829 cloned
12830 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12831 .await
12832 .unwrap()
12833 });
12834 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12835 .await
12836 .unwrap()
12837 .unwrap();
12838 let first = query_live_roots(&handler, &first_ctx).await;
12839 let second = query_live_roots(&handler, &second_ctx).await;
12840 assert!(
12841 matches!(
12842 first,
12843 ModuleControlResponseToModule::LiveRoots {
12844 total_bindings: 0,
12845 ..
12846 }
12847 ),
12848 "cross-module scope must not expose another module's roots"
12849 );
12850 assert!(
12851 matches!(
12852 second,
12853 ModuleControlResponseToModule::LiveRoots {
12854 total_bindings: 1,
12855 ..
12856 }
12857 ),
12858 "second module must see its pending route"
12859 );
12860 handler
12861 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12862 .await
12863 .unwrap();
12864 assert!(open.await.unwrap().is_empty());
12865 }
12866
12867 #[tokio::test(start_paused = true)]
12868 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12869 let handler = ControlHandler::with_forwarding(
12870 Arc::new(Registry::default()),
12871 Arc::new(ForwardingTable::default()),
12872 );
12873 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12874 hello_via_sink(
12875 &handler,
12876 &target_ctx,
12877 &mut target_rx,
12878 hello_frame("target", PROTOCOL_VERSION, 1),
12879 )
12880 .await;
12881 let actual = query_live_roots(&handler, &target_ctx).await;
12882 let ModuleControlResponseToModule::LiveRoots {
12883 roots,
12884 unknown_root_bindings,
12885 total_bindings,
12886 } = actual
12887 else {
12888 panic!("expected live roots")
12889 };
12890 assert!(roots.is_empty());
12891 assert_eq!(unknown_root_bindings, 0);
12892 assert_eq!(total_bindings, 0);
12893 assert_eq!(
12894 total_bindings,
12895 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12896 );
12897 }
12898
12899 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12906 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12907 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12908 let text = std::fs::read_to_string(&path)
12909 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12910 let vectors: Vec<(String, Value)> = text
12911 .lines()
12912 .filter(|line| !line.trim().is_empty())
12913 .map(|line| {
12914 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12915 let id = entry["corpus_id"]
12916 .as_str()
12917 .expect("every vector carries a corpus_id")
12918 .to_string();
12919 (id, entry["package"].clone())
12920 })
12921 .collect();
12922 assert_eq!(
12926 vectors.len(),
12927 3,
12928 "vendored fed corpus changed size; re-sync from subc-federation"
12929 );
12930
12931 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12946 let richest = vectors
12947 .iter()
12948 .filter_map(|(_, package)| package.as_object())
12949 .map(|object| {
12950 object
12951 .keys()
12952 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12953 .count()
12954 })
12955 .max()
12956 .unwrap_or(0);
12957 assert!(
12958 richest >= 2,
12959 "vendored corpus no longer carries a package with unmodelled fields, \
12960 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12961 );
12962
12963 vectors
12964 }
12965
12966 #[tokio::test]
12974 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12975 for (corpus_id, package) in fed_admission_facts_vectors() {
12976 let registry = Arc::new(Registry::default());
12977 let forwarding = Arc::new(ForwardingTable::default());
12978 let supervisor = SupervisorHandle::new();
12979 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12980 let handler =
12981 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12982 .with_supervisor(supervisor)
12983 .with_admission_facts_config(
12984 Some("fed".to_string()),
12985 Some(vec!["target".to_string()]),
12986 );
12987
12988 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12989 hello_via_sink(
12990 &handler,
12991 &target_ctx,
12992 &mut target_rx,
12993 hello_frame("target", PROTOCOL_VERSION, 1),
12994 )
12995 .await;
12996
12997 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12998 let route_handler = handler.clone();
12999 let expected = package.clone();
13000 let route_task = tokio::spawn(async move {
13001 route_handler
13002 .handle_control_frame(
13003 &client_ctx,
13004 route_open_frame_with_admission_facts(
13005 20,
13006 "target",
13007 unique_project_root("admission-facts"),
13008 Some(subc_control::ConsumerIdentity {
13009 module_id: "fed".to_string(),
13010 launch_nonce: "fed-nonce".to_string(),
13011 }),
13012 Some(package),
13013 ),
13014 )
13015 .await
13016 .unwrap()
13017 });
13018
13019 let bind_frame = target_rx.recv().await.unwrap();
13020 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13021 let ModuleControlRequest::RouteBind {
13022 admission_facts, ..
13023 } = bind
13024 else {
13025 panic!("{corpus_id}: expected route.bind")
13026 };
13027 assert_eq!(
13028 admission_facts,
13029 Some(expected),
13030 "{corpus_id}: relay must not add, drop or reshape any field"
13031 );
13032
13033 handler
13034 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13035 .await
13036 .unwrap();
13037 route_task.await.unwrap();
13038 }
13039 }
13040
13041 #[tokio::test]
13042 async fn route_bind_never_relays_the_openers_build_class() {
13043 let registry = Arc::new(Registry::default());
13044 let forwarding = Arc::new(ForwardingTable::default());
13045 let handler =
13046 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13047 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
13048 hello_via_sink(
13049 &handler,
13050 &target_ctx,
13051 &mut target_rx,
13052 hello_frame("target", PROTOCOL_VERSION, 1),
13053 )
13054 .await;
13055
13056 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
13057 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
13058 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13059 busy_refusals: None,
13060 target: RouteTarget::ToolProvider {
13061 module_id: "target".to_string(),
13062 },
13063 identity,
13064 consumer_identity: None,
13065 consumer_capabilities: None,
13066 role_versions: None,
13067 admission_facts: None,
13068 scope: None,
13069 })
13070 .unwrap();
13071 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
13072
13073 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
13074 let route_handler = handler.clone();
13075 let route_task = tokio::spawn(async move {
13076 route_handler
13077 .handle_control_frame(&client_ctx, open)
13078 .await
13079 .unwrap()
13080 });
13081 let bind_frame = target_rx.recv().await.unwrap();
13082 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13083 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
13084 panic!("expected route.bind")
13085 };
13086 assert_eq!(
13087 identity.build_class, None,
13088 "route.bind carried the build class"
13089 );
13090 assert!(
13091 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
13092 "the route.bind bytes still name build_class"
13093 );
13094 handler
13095 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13096 .await
13097 .unwrap();
13098 route_task.await.unwrap();
13099 }
13100
13101 #[tokio::test]
13102 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
13103 let registry = Arc::new(Registry::default());
13104 let forwarding = Arc::new(ForwardingTable::default());
13105 let supervisor = SupervisorHandle::new();
13106 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
13107 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
13108 let handler =
13109 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13110 .with_supervisor(supervisor)
13111 .with_admission_facts_config(
13112 Some("fed".to_string()),
13113 Some(vec!["target".to_string()]),
13114 );
13115
13116 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
13117 hello_via_sink(
13118 &handler,
13119 &target_ctx,
13120 &mut target_rx,
13121 hello_frame("target", PROTOCOL_VERSION, 1),
13122 )
13123 .await;
13124 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
13125 hello_via_sink(
13126 &handler,
13127 &other_ctx,
13128 &mut other_rx,
13129 hello_frame("other", PROTOCOL_VERSION, 2),
13130 )
13131 .await;
13132
13133 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
13134 let expected_facts = facts.clone();
13135 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
13136 let route_handler = handler.clone();
13137 let route_task = tokio::spawn(async move {
13138 route_handler
13139 .handle_control_frame(
13140 &client_ctx,
13141 route_open_frame_with_admission_facts(
13142 10,
13143 "target",
13144 unique_project_root("admission-facts"),
13145 Some(subc_control::ConsumerIdentity {
13146 module_id: "fed".to_string(),
13147 launch_nonce: "fed-nonce".to_string(),
13148 }),
13149 Some(facts.clone()),
13150 ),
13151 )
13152 .await
13153 .unwrap()
13154 });
13155 let bind_frame = target_rx.recv().await.unwrap();
13156 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13157 let ModuleControlRequest::RouteBind {
13158 admission_facts, ..
13159 } = bind
13160 else {
13161 panic!("expected route.bind")
13162 };
13163 assert_eq!(admission_facts, Some(expected_facts));
13164 handler
13165 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13166 .await
13167 .unwrap();
13168 assert!(route_task.await.unwrap().is_empty());
13169 assert!(matches!(
13170 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
13171 .unwrap(),
13172 ClientControlResponse::RouteOpen { .. }
13173 ));
13174
13175 let direct = handler
13176 .handle_control_frame(
13177 &route_ctx(ConnectionId::new(73)).0,
13178 route_open_frame_with_admission_facts(
13179 11,
13180 "target",
13181 unique_project_root("admission-facts"),
13182 None,
13183 Some(json!({"x": 1})),
13184 ),
13185 )
13186 .await
13187 .unwrap();
13188 assert_eq!(
13189 parse_error(&direct[0])["code"],
13190 "admission_facts_not_permitted"
13191 );
13192
13193 let different_reserved = handler
13194 .handle_control_frame(
13195 &route_ctx(ConnectionId::new(77)).0,
13196 route_open_frame_with_admission_facts(
13197 15,
13198 "target",
13199 unique_project_root("admission-facts"),
13200 Some(subc_control::ConsumerIdentity {
13201 module_id: "other".to_string(),
13202 launch_nonce: "other-nonce".to_string(),
13203 }),
13204 Some(json!({"x": 1})),
13205 ),
13206 )
13207 .await
13208 .unwrap();
13209 assert_eq!(
13210 parse_error(&different_reserved[0])["code"],
13211 "admission_facts_not_permitted"
13212 );
13213
13214 let other_target = handler
13215 .handle_control_frame(
13216 &route_ctx(ConnectionId::new(74)).0,
13217 route_open_frame_with_admission_facts(
13218 12,
13219 "other",
13220 unique_project_root("admission-facts"),
13221 Some(subc_control::ConsumerIdentity {
13222 module_id: "fed".to_string(),
13223 launch_nonce: "fed-nonce".to_string(),
13224 }),
13225 Some(json!({"x": 1})),
13226 ),
13227 )
13228 .await
13229 .unwrap();
13230 assert_eq!(
13231 parse_error(&other_target[0])["code"],
13232 "admission_facts_target_not_allowed"
13233 );
13234
13235 let nonexistent = handler
13236 .handle_control_frame(
13237 &route_ctx(ConnectionId::new(75)).0,
13238 route_open_frame_with_admission_facts(
13239 13,
13240 "missing",
13241 unique_project_root("admission-facts"),
13242 None,
13243 Some(json!({"x": 1})),
13244 ),
13245 )
13246 .await
13247 .unwrap();
13248 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13249
13250 let described = handler
13251 .handle_control_frame(
13252 &route_ctx(ConnectionId::new(76)).0,
13253 Frame::build(
13254 FrameType::Request,
13255 control_flags(),
13256 0,
13257 0,
13258 14,
13259 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13260 )
13261 .unwrap(),
13262 )
13263 .await
13264 .unwrap();
13265 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13266 serde_json::from_slice(&described[0].body).unwrap()
13267 else {
13268 panic!("expected server.describe response")
13269 };
13270 assert!(capabilities
13271 .iter()
13272 .any(|cap| cap == "admission_facts_relay_v1"));
13273 }
13274
13275 #[tokio::test]
13276 async fn admission_facts_without_configured_carrier_are_rejected() {
13277 let registry = Arc::new(Registry::default());
13278 let forwarding = Arc::new(ForwardingTable::default());
13279 let handler = ControlHandler::with_forwarding(registry, forwarding);
13280 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13281 hello_via_sink(
13282 &handler,
13283 &target_ctx,
13284 &mut target_rx,
13285 hello_frame("target", PROTOCOL_VERSION, 1),
13286 )
13287 .await;
13288
13289 let responses = handler
13290 .handle_control_frame(
13291 &route_ctx(ConnectionId::new(79)).0,
13292 route_open_frame_with_admission_facts(
13293 16,
13294 "target",
13295 unique_project_root("admission-facts"),
13296 None,
13297 Some(json!({"x": 1})),
13298 ),
13299 )
13300 .await
13301 .unwrap();
13302 assert_eq!(
13303 parse_error(&responses[0])["code"],
13304 "admission_facts_not_permitted"
13305 );
13306 }
13307
13308 #[tokio::test]
13309 async fn route_open_relays_consumer_capabilities_verbatim() {
13310 let registry = Arc::new(Registry::default());
13311 let forwarding = Arc::new(ForwardingTable::default());
13312 let handler =
13313 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13314 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13315 hello_via_sink(
13316 &handler,
13317 &module_ctx,
13318 &mut module_rx,
13319 hello_frame("aft", PROTOCOL_VERSION, 7),
13320 )
13321 .await;
13322
13323 let expected = vec!["elicitation".to_string(), "roots".to_string()];
13324 let expected_for_request = expected.clone();
13325 let project_root = unique_project_root("consumer-capabilities-present");
13326 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13327 let route_handler = handler.clone();
13328 let route_task = tokio::spawn(async move {
13329 route_handler
13330 .handle_control_frame(
13331 &client_ctx,
13332 route_open_frame_with_consumer_capabilities(
13333 401,
13334 "aft",
13335 project_root,
13336 Some(expected_for_request),
13337 ),
13338 )
13339 .await
13340 .unwrap()
13341 });
13342 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13343 .await
13344 .unwrap()
13345 .unwrap();
13346 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13347 let ModuleControlRequest::RouteBind {
13348 consumer_capabilities,
13349 ..
13350 } = bind
13351 else {
13352 panic!("expected route.bind request, got {bind:?}");
13353 };
13354 assert_eq!(consumer_capabilities, Some(expected.clone()));
13355
13356 handler
13357 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13358 .await
13359 .unwrap();
13360 let route_response = route_task.await.unwrap();
13361 assert!(route_response.is_empty());
13362 let published = client_rx.recv().await.unwrap();
13363 assert!(matches!(
13364 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13365 ClientControlResponse::RouteOpen { .. }
13366 ));
13367 }
13368
13369 #[tokio::test]
13370 async fn route_open_without_consumer_capabilities_relays_none() {
13371 let registry = Arc::new(Registry::default());
13372 let forwarding = Arc::new(ForwardingTable::default());
13373 let handler =
13374 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13375 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13376 hello_via_sink(
13377 &handler,
13378 &module_ctx,
13379 &mut module_rx,
13380 hello_frame("aft", PROTOCOL_VERSION, 7),
13381 )
13382 .await;
13383
13384 let project_root = unique_project_root("consumer-capabilities-absent");
13385 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13386 let route_handler = handler.clone();
13387 let route_task = tokio::spawn(async move {
13388 route_handler
13389 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13390 .await
13391 .unwrap()
13392 });
13393 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13394 .await
13395 .unwrap()
13396 .unwrap();
13397 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13398 let ModuleControlRequest::RouteBind {
13399 consumer_capabilities,
13400 ..
13401 } = bind
13402 else {
13403 panic!("expected route.bind request, got {bind:?}");
13404 };
13405 assert_eq!(consumer_capabilities, None);
13406
13407 handler
13408 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13409 .await
13410 .unwrap();
13411 let route_response = route_task.await.unwrap();
13412 assert!(route_response.is_empty());
13413 let published = client_rx.recv().await.unwrap();
13414 assert!(matches!(
13415 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13416 ClientControlResponse::RouteOpen { .. }
13417 ));
13418 }
13419
13420 async fn bind_role_versions_for(
13424 sent: Option<BTreeMap<String, String>>,
13425 connection: u64,
13426 ) -> Option<BTreeMap<String, String>> {
13427 let registry = Arc::new(Registry::default());
13428 let forwarding = Arc::new(ForwardingTable::default());
13429 let handler =
13430 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13431 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13432 hello_via_sink(
13433 &handler,
13434 &module_ctx,
13435 &mut module_rx,
13436 hello_frame("aft", PROTOCOL_VERSION, 7),
13437 )
13438 .await;
13439 let project_root = unique_project_root("role-versions");
13440 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13441 let route_handler = handler.clone();
13442 let route_task = tokio::spawn(async move {
13443 route_handler
13444 .handle_control_frame(
13445 &client_ctx,
13446 route_open_frame_with_role_versions(403, "aft", project_root, sent),
13447 )
13448 .await
13449 .unwrap()
13450 });
13451 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13452 .await
13453 .expect("a well-formed route.open reaches the module as a bind")
13454 .unwrap();
13455 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13456 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13457 panic!("expected route.bind request, got {bind:?}");
13458 };
13459 handler
13460 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13461 .await
13462 .unwrap();
13463 assert!(route_task.await.unwrap().is_empty());
13464 let published = client_rx.recv().await.unwrap();
13465 assert!(matches!(
13466 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13467 ClientControlResponse::RouteOpen { .. }
13468 ));
13469 role_versions
13470 }
13471
13472 #[tokio::test]
13473 async fn route_open_relays_role_versions_verbatim() {
13474 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13475 assert_eq!(
13476 bind_role_versions_for(Some(sent.clone()), 141).await,
13477 Some(sent)
13478 );
13479 }
13480
13481 #[tokio::test]
13484 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13485 assert_eq!(bind_role_versions_for(None, 143).await, None);
13486 assert_eq!(
13487 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13488 None
13489 );
13490 }
13491
13492 #[tokio::test]
13493 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13494 let registry = Arc::new(Registry::default());
13495 let forwarding = Arc::new(ForwardingTable::default());
13496 let handler =
13497 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13498 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13499 hello_via_sink(
13500 &handler,
13501 &module_ctx,
13502 &mut module_rx,
13503 hello_frame("aft", PROTOCOL_VERSION, 7),
13504 )
13505 .await;
13506
13507 let nine: BTreeMap<String, String> = (0..9)
13508 .map(|index| (format!("role-{index}"), "v1".to_string()))
13509 .collect();
13510 for (label, malformed) in [
13511 (
13512 "invalid role name",
13513 role_versions(&[("Tool_Provider", "v1")]),
13514 ),
13515 ("invalid version", role_versions(&[("tool-provider", "v0")])),
13516 ("nine entries", nine),
13517 ] {
13518 let responses = tokio::time::timeout(
13521 Duration::from_secs(1),
13522 handler.handle_control_frame(
13523 &route_ctx(ConnectionId::new(148)).0,
13524 route_open_frame_with_role_versions(
13525 404,
13526 "aft",
13527 unique_project_root("role-versions-malformed"),
13528 Some(malformed),
13529 ),
13530 ),
13531 )
13532 .await
13533 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13534 .unwrap();
13535 assert_eq!(responses.len(), 1, "{label}");
13536 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13537 let error = parse_error(&responses[0]);
13538 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13539 assert_eq!(
13540 error["detail"]["field"], "role_versions",
13541 "{label}: {error}"
13542 );
13543 assert!(
13544 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13545 "{label}: a malformed declaration is terminal"
13546 );
13547 assert!(
13548 module_rx.try_recv().is_err(),
13549 "{label}: the module must never see a bind"
13550 );
13551 }
13552 }
13553
13554 #[tokio::test]
13558 async fn route_role_versions_capability_is_advertised() {
13559 let handler = ControlHandler::new(Arc::new(Registry::default()));
13560 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13561 let ack = hello_via_sink(
13562 &handler,
13563 &ctx,
13564 &mut rx,
13565 hello_frame("m", PROTOCOL_VERSION, 1),
13566 )
13567 .await;
13568 let ack = parse_ack(&ack);
13569 assert!(
13570 ack.subc_capabilities
13571 .iter()
13572 .any(|c| c == "route-role-versions/v1"),
13573 "{:?}",
13574 ack.subc_capabilities
13575 );
13576
13577 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13578 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13579 let reply = handler
13580 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13581 .await
13582 .unwrap()
13583 .pop()
13584 .unwrap();
13585 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13586 serde_json::from_slice(&reply.body).unwrap()
13587 else {
13588 panic!("not a server.describe reply");
13589 };
13590 assert!(
13591 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13592 "{capabilities:?}"
13593 );
13594 }
13595
13596 #[tokio::test]
13600 async fn catalog_events_capability_is_advertised() {
13601 let handler = ControlHandler::new(Arc::new(Registry::default()));
13602 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13603 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13604 let reply = handler
13605 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13606 .await
13607 .unwrap()
13608 .pop()
13609 .unwrap();
13610 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13611 serde_json::from_slice(&reply.body).unwrap()
13612 else {
13613 panic!("not a server.describe reply");
13614 };
13615 assert!(
13616 capabilities
13617 .iter()
13618 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13619 "{capabilities:?}"
13620 );
13621 }
13622
13623 #[tokio::test]
13624 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13625 let handler = ControlHandler::new(Arc::new(Registry::default()));
13626 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13627 let ack = hello_via_sink(
13628 &handler,
13629 &module,
13630 &mut module_rx,
13631 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13632 )
13633 .await;
13634 let ack = parse_ack(&ack);
13635 assert!(
13636 ack.subc_capabilities
13637 .iter()
13638 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13639 "{:?}",
13640 ack.subc_capabilities
13641 );
13642 assert!(ack
13643 .subc_capabilities
13644 .iter()
13645 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13646
13647 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13648 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13649 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13650 let reply = handler
13651 .handle_control_frame(&client, frame)
13652 .await
13653 .unwrap()
13654 .pop()
13655 .unwrap();
13656 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13657 serde_json::from_slice(&reply.body).unwrap()
13658 else {
13659 panic!("not a server.describe reply");
13660 };
13661 assert!(
13662 capabilities
13663 .iter()
13664 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13665 "{capabilities:?}"
13666 );
13667 assert!(capabilities
13668 .iter()
13669 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13670 }
13671
13672 #[tokio::test]
13673 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13674 let handler = ControlHandler::new(Arc::new(Registry::default()));
13675 let (module, mut rx) = route_ctx(ConnectionId::new(156));
13676 let ack = hello_via_sink(
13677 &handler,
13678 &module,
13679 &mut rx,
13680 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13681 )
13682 .await;
13683 assert!(parse_ack(&ack)
13684 .subc_capabilities
13685 .iter()
13686 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13687 assert!(parse_ack(&ack)
13688 .subc_capabilities
13689 .iter()
13690 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13691 let (client, _rx) = route_ctx(ConnectionId::new(157));
13692 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13693 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13694 let reply = handler
13695 .handle_control_frame(&client, frame)
13696 .await
13697 .unwrap()
13698 .pop()
13699 .unwrap();
13700 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13701 serde_json::from_slice(&reply.body).unwrap()
13702 else {
13703 panic!("server.describe reply")
13704 };
13705 assert!(capabilities
13706 .iter()
13707 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13708 assert!(capabilities
13709 .iter()
13710 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13711 }
13712
13713 #[tokio::test]
13714 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13715 let handler = ControlHandler::new(Arc::new(Registry::default()));
13716 let (module, mut rx) = route_ctx(ConnectionId::new(158));
13717 let ack = hello_via_sink(
13718 &handler,
13719 &module,
13720 &mut rx,
13721 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13722 )
13723 .await;
13724 assert!(parse_ack(&ack)
13725 .subc_capabilities
13726 .iter()
13727 .any(|cap| cap == "scope-workdir/v1"));
13728 let (client, _rx) = route_ctx(ConnectionId::new(159));
13729 let frame = Frame::build(
13730 FrameType::Request,
13731 control_flags(),
13732 0,
13733 0,
13734 5,
13735 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13736 )
13737 .unwrap();
13738 let reply = handler
13739 .handle_control_frame(&client, frame)
13740 .await
13741 .unwrap()
13742 .remove(0);
13743 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13744 serde_json::from_slice(&reply.body).unwrap()
13745 else {
13746 panic!("server.describe reply")
13747 };
13748 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13749 }
13750
13751 #[tokio::test]
13752 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13753 let handler = ControlHandler::new(Arc::new(Registry::default()));
13754 let (module, mut rx) = route_ctx(ConnectionId::new(160));
13755 let ack = hello_via_sink(
13756 &handler,
13757 &module,
13758 &mut rx,
13759 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13760 )
13761 .await;
13762 assert!(parse_ack(&ack)
13763 .subc_capabilities
13764 .iter()
13765 .any(|cap| cap == "worker-runs/v1"));
13766 let (client, _rx) = route_ctx(ConnectionId::new(161));
13767 let frame = Frame::build(
13768 FrameType::Request,
13769 control_flags(),
13770 0,
13771 0,
13772 5,
13773 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13774 )
13775 .unwrap();
13776 let reply = handler
13777 .handle_control_frame(&client, frame)
13778 .await
13779 .unwrap()
13780 .remove(0);
13781 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13782 serde_json::from_slice(&reply.body).unwrap()
13783 else {
13784 panic!("server.describe reply")
13785 };
13786 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13787 }
13788
13789 #[tokio::test]
13790 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13791 let handler = ControlHandler::new(Arc::new(Registry::default()));
13792 let (module, mut rx) = route_ctx(ConnectionId::new(154));
13793 let ack = hello_via_sink(
13794 &handler,
13795 &module,
13796 &mut rx,
13797 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13798 )
13799 .await;
13800 assert!(parse_ack(&ack)
13801 .subc_capabilities
13802 .iter()
13803 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13804 let (client, _rx) = route_ctx(ConnectionId::new(155));
13805 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13806 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13807 let reply = handler
13808 .handle_control_frame(&client, frame)
13809 .await
13810 .unwrap()
13811 .pop()
13812 .unwrap();
13813 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13814 serde_json::from_slice(&reply.body).unwrap()
13815 else {
13816 panic!("not a server.describe reply")
13817 };
13818 assert!(capabilities
13819 .iter()
13820 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13821 }
13822
13823 #[tokio::test]
13824 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13825 let registry = Arc::new(Registry::default());
13826 let forwarding = Arc::new(ForwardingTable::default());
13827 let handler =
13828 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13829 .with_health_probe_timeout(Duration::from_secs(5));
13830 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13831 hello_via_sink(
13832 &handler,
13833 &module_ctx,
13834 &mut module_rx,
13835 non_routable_hello_frame_with_control_ops(
13836 "mcp",
13837 300,
13838 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13839 ),
13840 )
13841 .await;
13842 assert!(registry
13843 .get_module("mcp")
13844 .unwrap()
13845 .unwrap()
13846 .manifest
13847 .provides
13848 .is_empty());
13849
13850 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13851 let route_response = handler
13852 .handle_control_frame(
13853 &route_client_ctx,
13854 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13855 )
13856 .await
13857 .unwrap();
13858 assert_eq!(route_response[0].header.ty, FrameType::Error);
13859 assert_eq!(
13860 parse_error(&route_response[0])["code"],
13861 "target_unavailable"
13862 );
13863 assert!(parse_error(&route_response[0])["message"]
13864 .as_str()
13865 .unwrap()
13866 .contains("does not provide the requested target"));
13867 assert!(module_rx.try_recv().is_err());
13868
13869 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13870 let health_handler = handler.clone();
13871 let health_task = tokio::spawn(async move {
13872 health_handler
13873 .handle_control_frame(
13874 &health_client_ctx,
13875 supervisor_health_probe_frame(302, "mcp"),
13876 )
13877 .await
13878 .unwrap()
13879 });
13880 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13881 .await
13882 .unwrap()
13883 .unwrap();
13884 assert_eq!(
13885 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13886 ModuleControlRequest::HealthCheck {}
13887 );
13888 handler
13889 .handle_control_frame(
13890 &module_ctx,
13891 health_response(health_frame.header.corr, HealthStatus::Ok),
13892 )
13893 .await
13894 .unwrap();
13895 let health_response = health_task.await.unwrap();
13896 assert_eq!(health_response[0].header.ty, FrameType::Response);
13897 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13898 ClientControlResponse::SupervisorHealthProbe {
13899 module_id, status, ..
13900 } => {
13901 assert_eq!(module_id, "mcp");
13902 assert_eq!(status, HealthStatus::Ok);
13903 }
13904 other => panic!("unexpected health response: {other:?}"),
13905 }
13906
13907 forwarding
13912 .cleanup_connection(module_ctx.connection_id)
13913 .unwrap();
13914 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13915 let cleanup_response = tokio::time::timeout(
13916 Duration::from_millis(200),
13917 handler.handle_control_frame(
13918 &cleanup_probe_ctx,
13919 supervisor_health_probe_frame(303, "mcp"),
13920 ),
13921 )
13922 .await
13923 .expect("probe should fail immediately when the control lane is gone")
13924 .unwrap();
13925 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13926 assert_eq!(
13927 parse_error(&cleanup_response[0])["code"],
13928 "target_unavailable"
13929 );
13930 assert!(parse_error(&cleanup_response[0])["message"]
13931 .as_str()
13932 .unwrap()
13933 .contains("no module connection"));
13934
13935 handler
13936 .cleanup_connection(module_ctx.connection_id)
13937 .unwrap();
13938 }
13939
13940 #[tokio::test]
13941 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13942 let registry = Arc::new(Registry::default());
13943 let supervisor_handle = SupervisorHandle::new();
13944 let supervisor =
13945 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13946 .with_handle(supervisor_handle.clone())
13947 .with_connection_file_path(
13948 std::env::temp_dir()
13949 .join(format!("subc-route-open-warming-{}", std::process::id())),
13950 );
13951 let module = supervisor
13952 .supervise_configured(
13953 ModuleSpec {
13954 carry_containment: false,
13955 module_id: "warming".to_string(),
13956 program: fake_aft_stub_path(),
13957 args: Vec::new(),
13958 env: Vec::new(),
13959 reserved: false,
13960 reserved_prefixes: Vec::new(),
13961 protocol: ModuleProtocol::Subc,
13962 overlap: Default::default(),
13963 },
13964 true,
13965 )
13966 .unwrap();
13967 assert_eq!(module.state().unwrap(), ModuleState::Running);
13968
13969 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13970 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13971 let response = handler
13972 .handle_control_frame(
13973 &ctx,
13974 route_open_frame(304, "warming", unique_project_root("warming")),
13975 )
13976 .await
13977 .unwrap();
13978 module.stop().await.unwrap();
13979
13980 assert_eq!(response[0].header.ty, FrameType::Error);
13981 let error = parse_error(&response[0]);
13982 assert_eq!(error["code"], "module_warming");
13983 assert!(error["message"]
13984 .as_str()
13985 .unwrap()
13986 .contains("state=running, enabled=true, live=false"));
13987 }
13988
13989 #[test]
13990 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13991 let handler = ControlHandler::new(Arc::new(Registry::default()));
13992 let capture = EventCapture::default();
13993 let _subscriber =
13994 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13995 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13996 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13997 let pending = (0..limit).collect::<Vec<_>>();
13998 let response = handler
13999 .route_open_capacity_refusal(
14000 &ctx,
14001 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
14002 "busy",
14003 pending.len(),
14004 limit,
14005 )
14006 .unwrap();
14007 assert_eq!(parse_error(&response)["code"], "target_unavailable");
14008 let event = capture
14009 .events()
14010 .into_iter()
14011 .find(|event| {
14012 event.target == "control"
14013 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
14014 })
14015 .expect("connection admission refusal event");
14016 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14017 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14018 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14021 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14022 }
14023
14024 #[test]
14025 fn route_open_target_cap_logs_admission_reason_and_capacity() {
14026 let handler = ControlHandler::new(Arc::new(Registry::default()));
14027 let capture = EventCapture::default();
14028 let _subscriber =
14029 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14030 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
14031 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
14032 let guards = (0..limit)
14033 .map(|_| {
14034 handler
14035 .route_bind_concurrency
14036 .try_admit("busy", limit)
14037 .unwrap()
14038 })
14039 .collect::<Vec<_>>();
14040 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
14041 Err(in_flight) => in_flight,
14042 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
14043 };
14044 let response = handler
14045 .route_open_target_capacity_refusal(
14046 &ctx,
14047 &RouteOpenCaller::unknown(),
14048 &route_open_frame(397, "busy", unique_project_root("target-cap")),
14049 "busy",
14050 in_flight,
14051 )
14052 .unwrap();
14053 assert_eq!(parse_error(&response)["code"], "target_unavailable");
14054 let event = capture
14055 .events()
14056 .into_iter()
14057 .find(|event| {
14058 event.target == "control"
14059 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
14060 })
14061 .expect("target admission refusal event");
14062 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14063 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14064 drop(guards);
14065 }
14066
14067 #[cfg(unix)]
14068 #[tokio::test(start_paused = true)]
14069 async fn supervisor_list_shows_a_failed_modules_next_retry() {
14070 let registry = Arc::new(Registry::default());
14071 let handle = SupervisorHandle::new();
14072 let supervisor =
14073 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
14074 .with_handle(handle.clone());
14075 let module = supervisor
14076 .spawn(crate::ModuleSpec {
14077 carry_containment: false,
14078 module_id: "retry-list".into(),
14079 program: PathBuf::from("/bin/sh"),
14080 args: vec!["-c".into(), "exit 17".into()],
14081 env: vec![],
14082 reserved: false,
14083 reserved_prefixes: vec![],
14084 protocol: ModuleProtocol::None,
14085 overlap: Default::default(),
14086 })
14087 .unwrap();
14088 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14089 while module.state().unwrap() != crate::ModuleState::Failed {
14090 assert!(
14091 std::time::Instant::now() < deadline,
14092 "waiting for budget failure"
14093 );
14094 tokio::task::yield_now().await;
14095 }
14096 let handler = ControlHandler::new(registry).with_supervisor(handle);
14097 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14098 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14099 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14100 assert_eq!(body["modules"][0]["state"], "failed");
14101 let retry = body["modules"][0]["next_retry_at_ms"]
14102 .as_u64()
14103 .expect("scheduled retry on the wire");
14104 let now = std::time::SystemTime::now()
14105 .duration_since(std::time::UNIX_EPOCH)
14106 .unwrap()
14107 .as_millis() as u64;
14108 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
14109 module.set_enabled(false).await.unwrap();
14110 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14111 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14112 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14113 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
14114 }
14115
14116 #[tokio::test]
14117 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
14118 let _guard = crate::subtree_resources::fault_read_guard().await;
14119 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
14120 let supervisor =
14121 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14122 let module = supervisor
14123 .spawn(crate::ModuleSpec {
14124 carry_containment: false,
14125 module_id: "pid-read-budget".into(),
14126 program: std::env::current_exe().unwrap(),
14127 args: vec![
14128 "--exact".into(),
14129 "control::tests::pid_resource_read_fixture".into(),
14130 ],
14131 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14132 .into_iter()
14133 .map(|key| (key.into(), dir.display().to_string()))
14134 .chain(std::iter::once((
14135 "SUBC_SUBTREE_PID_FIXTURE".into(),
14136 "1".into(),
14137 )))
14138 .collect(),
14139 reserved: false,
14140 reserved_prefixes: Vec::new(),
14141 protocol: ModuleProtocol::None,
14142 overlap: Default::default(),
14143 })
14144 .unwrap();
14145 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14146 while !matches!(
14147 module.child_resource_usage(),
14148 subc_control::ChildResourceUsage::Measured(_)
14149 ) {
14150 assert!(
14151 tokio::time::Instant::now() < ready_deadline,
14152 "fixture process was not measurable"
14153 );
14154 tokio::time::sleep(Duration::from_millis(10)).await;
14155 }
14156 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
14157 let reading = crate::subtree_resources::read_process(module.clone()).await;
14158 module.stop().await.unwrap();
14159 assert!(
14160 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
14161 "{reading:?}"
14162 );
14163 }
14164
14165 #[test]
14166 fn pid_resource_read_fixture() {
14167 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
14168 loop {
14169 std::thread::park_timeout(Duration::from_secs(600));
14170 }
14171 }
14172 }
14173
14174 #[tokio::test]
14175 async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
14176 let registry = Arc::new(Registry::default());
14177 let handle = SupervisorHandle::new();
14178 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14179 .with_handle(handle.clone());
14180 supervisor
14181 .supervise_configured(
14182 crate::ModuleSpec {
14183 module_id: "disabled-resource-reading".into(),
14184 program: std::env::current_exe().unwrap(),
14185 args: Vec::new(),
14186 env: Vec::new(),
14187 reserved: false,
14188 reserved_prefixes: Vec::new(),
14189 protocol: ModuleProtocol::None,
14190 overlap: Default::default(),
14191 carry_containment: false,
14192 },
14193 false,
14194 )
14195 .unwrap();
14196 let handler = ControlHandler::new(registry).with_supervisor(handle);
14197 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14198 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14199 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14200 assert!(body["modules"][0].get("tree").is_none());
14201 assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
14202 }
14203
14204 #[tokio::test]
14205 async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
14206 let _guard = crate::subtree_resources::fault_read_guard().await;
14207 let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
14208 let registry = Arc::new(Registry::default());
14209 let handle = SupervisorHandle::new();
14210 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14211 .with_handle(handle.clone());
14212 let module = supervisor
14213 .spawn(crate::ModuleSpec {
14214 carry_containment: false,
14215 module_id: "list-process-priority".into(),
14216 program: std::env::current_exe().unwrap(),
14217 args: vec![
14218 "--exact".into(),
14219 "control::tests::pid_resource_read_fixture".into(),
14220 ],
14221 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14222 .into_iter()
14223 .map(|key| (key.into(), dir.display().to_string()))
14224 .chain(std::iter::once((
14225 "SUBC_SUBTREE_PID_FIXTURE".into(),
14226 "1".into(),
14227 )))
14228 .collect(),
14229 reserved: false,
14230 reserved_prefixes: Vec::new(),
14231 protocol: ModuleProtocol::None,
14232 overlap: Default::default(),
14233 })
14234 .unwrap();
14235 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14236 loop {
14237 let image = module.running_image_agreement().await;
14238 if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
14239 break;
14240 }
14241 assert!(
14242 tokio::time::Instant::now() < ready_deadline,
14243 "the fixture process never became observable: {image:?}"
14244 );
14245 tokio::time::sleep(Duration::from_millis(10)).await;
14246 }
14247 let (target, release, _) = crate::subtree_resources::held();
14248 module.set_subtree_target_for_test(target);
14249 let handler = ControlHandler::new(registry).with_supervisor(handle);
14250 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14251 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14252 drop(release);
14253 module.stop().await.unwrap();
14254 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14255 assert_eq!(
14256 body["modules"][0]["resources"]["status"], "measured",
14257 "the optional tree timeout must not discard the process reading: {body}"
14258 );
14259 assert!(body["modules"][0].get("tree").is_none());
14260 assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
14261 assert_eq!(
14262 body["modules"][0]["pending_reload"]["image"]["status"], "match",
14263 "the tree timeout must not hide a completed image observation"
14264 );
14265 }
14266
14267 #[cfg(any(target_os = "linux", target_os = "macos"))]
14268 #[tokio::test]
14269 async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
14270 let mut handler = ControlHandler::new(Arc::new(Registry::default()))
14271 .with_daemon_provenance(
14272 std::process::id(),
14273 0,
14274 Some(std::env::current_exe().unwrap()),
14275 None,
14276 None,
14277 );
14278 let (probe, release, began) = ExecutableIdentityProbe::held();
14279 handler.daemon_provenance.probe = probe;
14280 let observe = |handler: ControlHandler, corr| {
14281 tokio::spawn(async move {
14282 let (ctx, _) = route_ctx(ConnectionId::new(corr));
14283 let frame =
14284 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
14285 .unwrap();
14286 handler
14287 .handle_supervisor_provenance(&ctx, frame, None)
14288 .await
14289 .unwrap()
14290 })
14291 };
14292 let explicit = observe(handler.clone(), 114);
14293 began.await.unwrap();
14294 let concurrent = observe(handler.clone(), 115);
14295 tokio::time::sleep(Duration::from_millis(300)).await;
14298 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14299 let list = tokio::time::timeout(
14300 Duration::from_secs(1),
14301 handler.handle_supervisor_list(frame),
14302 )
14303 .await;
14304 let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
14305 drop(release);
14306 let replies = explicit.await.unwrap();
14307 let concurrent_replies = concurrent.await.unwrap();
14308 assert!(
14309 explicit_was_pending,
14310 "explicit provenance must await the full observation"
14311 );
14312 let list = list
14313 .expect("a concurrent explicit probe must not stall list")
14314 .unwrap();
14315 let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
14316 assert_eq!(list_body["op"], "supervisor.list");
14317 for replies in [replies, concurrent_replies] {
14318 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14319 assert_eq!(
14320 body["daemon"]["daemon_observed"]["running_image"]["status"],
14321 "match"
14322 );
14323 }
14324 }
14325
14326 #[tokio::test]
14327 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
14328 let _guard = crate::subtree_resources::fault_read_guard().await;
14329 let registry = Arc::new(Registry::default());
14330 let handle = SupervisorHandle::new();
14331 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14332 .with_handle(handle.clone());
14333 let module = supervisor
14334 .supervise_configured(
14335 crate::ModuleSpec {
14336 carry_containment: false,
14337 module_id: "memory-read".into(),
14338 program: PathBuf::from("disabled-module"),
14339 args: Vec::new(),
14340 env: Vec::new(),
14341 reserved: false,
14342 reserved_prefixes: Vec::new(),
14343 protocol: ModuleProtocol::Subc,
14344 overlap: Default::default(),
14345 },
14346 true,
14347 )
14348 .unwrap();
14349 let (target, release, began) = crate::subtree_resources::held();
14350 let stalled_target = target.clone();
14353 let stalled = tokio::task::spawn_blocking(move || {
14354 stalled_target.tree(&subc_os::ProcessTable::default())
14355 });
14356 began.await.unwrap();
14357 module.set_subtree_target_for_test(target);
14358 let handler = ControlHandler::new(registry).with_supervisor(handle);
14359 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
14360 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14361 let replies = handler
14362 .handle_client_control_request(
14363 &ctx,
14364 frame,
14365 ClientControlRequest::SupervisorList {},
14366 &mut None,
14367 )
14368 .await
14369 .unwrap();
14370 assert!(
14371 replies.is_empty(),
14372 "list reply must be detached from the connection reader"
14373 );
14374 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14375 let replies = handler
14376 .handle_client_control_request(
14377 &ctx,
14378 frame,
14379 ClientControlRequest::SupervisorSpawnSnapshot {},
14380 &mut None,
14381 )
14382 .await
14383 .unwrap();
14384 assert_eq!(
14385 replies.len(),
14386 1,
14387 "another request must complete while the subtree read stalls"
14388 );
14389 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14390 .await
14391 .unwrap()
14392 .unwrap();
14393 release.send(()).unwrap();
14394 drop(release);
14395 stalled.await.unwrap();
14396 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
14397 assert!(
14398 body["modules"][0].get("tree").is_none(),
14399 "an unknown tree reading must be omitted after the bounded read expires"
14400 );
14401 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14402 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14403 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14404 assert_eq!(
14405 body["modules"][0]["tree"]["source"], "process_tree",
14406 "the next request must read afresh"
14407 );
14408 assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
14409 }
14410
14411 #[tokio::test]
14412 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14413 let registry = Arc::new(Registry::default());
14414 let handle = SupervisorHandle::new();
14415 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14416 .with_handle(handle.clone());
14417 supervisor
14418 .supervise_configured(
14419 crate::ModuleSpec {
14420 carry_containment: false,
14421 module_id: "drift".into(),
14422 program: PathBuf::from("disabled-module"),
14423 args: Vec::new(),
14424 env: Vec::new(),
14425 reserved: false,
14426 reserved_prefixes: Vec::new(),
14427 protocol: ModuleProtocol::Subc,
14428 overlap: Default::default(),
14429 },
14430 false,
14431 )
14432 .unwrap();
14433 let handler = ControlHandler::new(registry).with_supervisor(handle);
14434 for configured in [None, Some(false), Some(true)] {
14435 handler.capability_evaluator.configure(
14436 configured.map(|enabled| ("drift".to_string(), enabled)),
14437 BTreeMap::new(),
14438 );
14439 let frame =
14440 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14441 let replies = handler
14442 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14443 .await
14444 .unwrap();
14445 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14446 assert!(body.get("configured_enabled").is_none());
14447 assert_eq!(body["op"], "supervisor.ack");
14448 let frame =
14449 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14450 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14451 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14452 assert_eq!(
14453 body["modules"][0].get("configured_enabled"),
14454 configured.map(Value::Bool).as_ref()
14455 );
14456 assert!(body["modules"][0].get("tree").is_none());
14457 assert!(body["modules"][0].get("subtree").is_none());
14458 }
14459 }
14460
14461 #[tokio::test(flavor = "current_thread")]
14462 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
14463 let handler = ControlHandler::new(Arc::new(Registry::default()));
14464 let capture = EventCapture::default();
14465 let _subscriber =
14466 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14467 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
14468 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14469
14470 let _ = handler
14471 .handle_client_control_request(
14472 &ctx,
14473 frame,
14474 ClientControlRequest::SupervisorSetEnabled {
14475 module_id: "broca".to_string(),
14476 enabled: false,
14477 },
14478 &mut None,
14479 )
14480 .await
14481 .unwrap();
14482
14483 let events = capture
14484 .events()
14485 .into_iter()
14486 .filter(|event| {
14487 event.target == "control"
14488 && event.fields.get("message").map(String::as_str)
14489 == Some("supervisor request received")
14490 })
14491 .collect::<Vec<_>>();
14492 assert_eq!(events.len(), 1, "one supervisor request log line");
14493 let event = &events[0];
14494 assert_eq!(event.level, tracing::Level::INFO);
14495 assert_eq!(
14496 event.fields.get("op"),
14497 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
14498 );
14499 assert_eq!(
14500 event.fields.get("module_id"),
14501 Some(&"\"broca\"".to_string())
14502 );
14503 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
14504 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
14505 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
14506 }
14507
14508 #[tokio::test(flavor = "current_thread")]
14509 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
14510 let registry = Arc::new(Registry::default());
14511 let module_connection = ConnectionId::new(303);
14512 registry
14513 .register_with_control_ops(
14514 manifest("aft", PROTOCOL_VERSION),
14515 PROTOCOL_VERSION,
14516 module_connection,
14517 module_baseline_control_ops(),
14518 )
14519 .unwrap();
14520 let handler = ControlHandler::new(registry);
14521 let capture = EventCapture::default();
14522 let _subscriber =
14523 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14524 let (ctx, _rx) = route_ctx(module_connection);
14525 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14526
14527 let _ = handler
14528 .handle_client_control_request(
14529 &ctx,
14530 frame,
14531 ClientControlRequest::SupervisorSetEnabled {
14532 module_id: "broca".to_string(),
14533 enabled: false,
14534 },
14535 &mut None,
14536 )
14537 .await
14538 .unwrap();
14539
14540 let callers = capture
14541 .events()
14542 .into_iter()
14543 .filter(|event| {
14544 event.fields.get("message").map(String::as_str)
14545 == Some("supervisor request received")
14546 })
14547 .map(|event| event.fields.get("caller").cloned())
14548 .collect::<Vec<_>>();
14549 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
14550 }
14551
14552 #[tokio::test(flavor = "current_thread")]
14553 async fn supervisor_list_does_not_log_request_received() {
14554 let handler = ControlHandler::new(Arc::new(Registry::default()));
14555 let capture = EventCapture::default();
14556 let _subscriber =
14557 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14558 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
14559 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14560
14561 handler
14562 .handle_client_control_request(
14563 &ctx,
14564 frame,
14565 ClientControlRequest::SupervisorList {},
14566 &mut None,
14567 )
14568 .await
14569 .unwrap();
14570
14571 assert!(capture.events().into_iter().all(|event| {
14572 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14573 }));
14574 }
14575
14576 #[tokio::test(flavor = "current_thread")]
14577 async fn supervisor_rescan_preview_does_not_log_request_received() {
14578 let handler = ControlHandler::new(Arc::new(Registry::default()));
14579 let capture = EventCapture::default();
14580 let _subscriber =
14581 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14582 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
14583 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14584
14585 handler
14586 .handle_client_control_request(
14587 &ctx,
14588 frame,
14589 ClientControlRequest::SupervisorRescan { preview: true },
14590 &mut None,
14591 )
14592 .await
14593 .unwrap();
14594
14595 assert!(capture.events().into_iter().all(|event| {
14596 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14597 }));
14598 }
14599
14600 struct ApproveOperator;
14601 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
14602 fn prompt(
14603 &self,
14604 _: &str,
14605 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
14606 ) -> crate::operator_confirm::ProviderResult {
14607 crate::operator_confirm::ProviderResult::Approved
14608 }
14609 }
14610
14611 #[tokio::test]
14612 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
14613 assert_eq!(
14614 ControlHandler::default().privileged_operator_prompt,
14615 PrivilegedOperatorPrompt::Log
14616 );
14617 for (initial, changed, running) in [
14618 ("log", "enforce", PrivilegedOperatorPrompt::Log),
14619 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
14620 ] {
14621 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
14622 let path = dir.join("subc.jsonc");
14623 std::fs::write(
14624 &path,
14625 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
14626 )
14627 .unwrap();
14628 let config = crate::daemon_config::load(&path).unwrap().unwrap();
14629 let registry = Arc::new(Registry::default());
14630 let handle = SupervisorHandle::new();
14631 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
14632 .with_handle(handle.clone());
14633 let handler = ControlHandler::new(registry)
14634 .with_supervisor(handle)
14635 .with_privileged_operator_prompt(config.privileged_operator_prompt)
14636 .with_supervisor_rescan(supervisor, &path, config.port);
14637 handler.forwarding.operator_confirms().configure(
14638 Arc::new(ApproveOperator),
14639 crate::operator_confirm::OperatorLimits::default(),
14640 );
14641 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
14642
14643 for (disk_mode, needs_restart) in [
14644 (Some(initial), false),
14645 (Some(changed), true),
14646 (Some(changed), true),
14647 (None, running == PrivilegedOperatorPrompt::Enforce),
14648 (Some(initial), false),
14649 ] {
14650 let mut doc = json!({"version":1});
14651 if let Some(mode) = disk_mode {
14652 doc["privileged_operator_prompt"] = json!(mode);
14653 }
14654 std::fs::write(&path, doc.to_string()).unwrap();
14655 for preview in [true, false] {
14656 let request = ClientControlRequest::SupervisorRescan { preview };
14657 let frame = Frame::build(
14658 FrameType::Request,
14659 control_flags(),
14660 0,
14661 0,
14662 1,
14663 serde_json::to_vec(&request).unwrap(),
14664 )
14665 .unwrap();
14666 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14667 assert_eq!(frames.len(), 1);
14668 let ClientControlResponse::SupervisorRescan { result } =
14669 serde_json::from_slice(&frames[0].body).unwrap()
14670 else {
14671 panic!("expected a rescan response");
14672 };
14673 let expected = if needs_restart {
14674 vec!["privileged_operator_prompt"]
14675 } else {
14676 Vec::new()
14677 };
14678 assert_eq!(result.restart_required, expected);
14679 assert_eq!(handler.privileged_operator_prompt, running);
14680 assert_eq!(
14681 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
14682 running
14683 );
14684 }
14685 }
14686 }
14687 }
14688
14689 #[tokio::test]
14690 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
14691 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
14692 let path = dir.join("subc.jsonc");
14693 let registry = Arc::new(Registry::default());
14694 let handle = SupervisorHandle::new();
14695 let supervisor =
14696 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
14697 let handler = ControlHandler::new(registry)
14698 .with_supervisor(handle)
14699 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
14700 .with_supervisor_rescan(supervisor, &path, None);
14701 handler.forwarding.operator_confirms().configure(
14702 Arc::new(ApproveOperator),
14703 crate::operator_confirm::OperatorLimits::default(),
14704 );
14705 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
14706 for preview in [true, false] {
14707 std::fs::write(
14708 &path,
14709 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
14710 )
14711 .unwrap();
14712 let request = ClientControlRequest::SupervisorRescan { preview };
14713 let frame = Frame::build(
14714 FrameType::Request,
14715 control_flags(),
14716 0,
14717 0,
14718 1,
14719 serde_json::to_vec(&request).unwrap(),
14720 )
14721 .unwrap();
14722 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14723 assert_eq!(frames.len(), 1);
14724 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
14725 assert_eq!(
14726 handler.privileged_operator_prompt,
14727 PrivilegedOperatorPrompt::Enforce
14728 );
14729 }
14730 }
14731
14732 #[tokio::test]
14737 async fn route_open_refusal_names_the_check_that_refused() {
14738 let handler = ControlHandler::new(Arc::new(Registry::default()));
14739 let capture = EventCapture::default();
14740 let _subscriber =
14741 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14742 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14743 let response = handler
14744 .handle_control_frame(
14745 &ctx,
14746 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
14747 )
14748 .await
14749 .unwrap();
14750
14751 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14752 let event = capture
14753 .events()
14754 .into_iter()
14755 .find(|event| {
14756 event.target == "control"
14757 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14758 })
14759 .expect("route.open refusal event");
14760 assert_eq!(
14761 event.fields.get("reason"),
14762 Some(&"\"not_registered\"".to_string())
14763 );
14764 }
14765
14766 #[tokio::test]
14767 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
14768 let registry = Arc::new(Registry::default());
14769 let supervisor_handle = SupervisorHandle::new();
14770 let supervisor =
14771 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14772 .with_handle(supervisor_handle.clone())
14773 .with_connection_file_path(std::env::temp_dir().join(format!(
14774 "subc-route-open-refusal-info-{}",
14775 std::process::id()
14776 )));
14777 let module = supervisor
14778 .supervise_configured(
14779 ModuleSpec {
14780 carry_containment: false,
14781 module_id: "warming".to_string(),
14782 program: fake_aft_stub_path(),
14783 args: Vec::new(),
14784 env: Vec::new(),
14785 reserved: false,
14786 reserved_prefixes: Vec::new(),
14787 protocol: ModuleProtocol::Subc,
14788 overlap: Default::default(),
14789 },
14790 true,
14791 )
14792 .unwrap();
14793 assert_eq!(module.state().unwrap(), ModuleState::Running);
14794
14795 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14796 assert!(handler
14797 .counters()
14798 .snapshot()
14799 .get("route_open_refused_by_code")
14800 .is_none());
14801 let capture = EventCapture::default();
14802 let _subscriber =
14803 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14804 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
14805 let response = handler
14806 .handle_control_frame(
14807 &ctx,
14808 route_open_frame(394, "warming", unique_project_root("refusal-info")),
14809 )
14810 .await
14811 .unwrap();
14812 module.stop().await.unwrap();
14813
14814 assert_eq!(parse_error(&response[0])["code"], "module_warming");
14815 let event = capture
14816 .events()
14817 .into_iter()
14818 .find(|event| {
14819 event.target == "control"
14820 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
14821 })
14822 .expect("route.open refusal event");
14823 assert_eq!(
14824 event.fields.get("module_id"),
14825 Some(&"\"warming\"".to_string())
14826 );
14827 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
14828 assert_eq!(
14829 event.fields.get("reason"),
14830 Some(&"\"supervised_not_registered\"".to_string())
14831 );
14832 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
14833 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
14834 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
14835 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14836 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14837 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
14838 assert_eq!(
14839 handler.counters().snapshot()["route_open_refused_by_code"],
14840 json!({ "module_warming": 1 })
14841 );
14842 }
14843
14844 const OUTAGE_START: &str = "route.open refusing module: not serving";
14845 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
14846
14847 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
14848 capture
14849 .events()
14850 .into_iter()
14851 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
14852 .collect()
14853 }
14854
14855 fn supervise_stub(
14856 registry: &Arc<Registry>,
14857 module_id: &str,
14858 enabled: bool,
14859 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
14860 let supervisor_handle = SupervisorHandle::new();
14861 let supervisor =
14862 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
14863 .with_handle(supervisor_handle.clone())
14864 .with_connection_file_path(std::env::temp_dir().join(format!(
14865 "subc-route-outage-{module_id}-{}",
14866 std::process::id()
14867 )));
14868 let module = supervisor
14869 .supervise_configured(
14870 ModuleSpec {
14871 carry_containment: false,
14872 module_id: module_id.to_string(),
14873 program: fake_aft_stub_path(),
14874 args: Vec::new(),
14875 env: Vec::new(),
14876 reserved: false,
14877 reserved_prefixes: Vec::new(),
14878 protocol: ModuleProtocol::Subc,
14879 overlap: Default::default(),
14880 },
14881 enabled,
14882 )
14883 .unwrap();
14884 (supervisor_handle, module)
14885 }
14886
14887 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
14888 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
14889 module_id: module_id.to_string(),
14890 drain_timeout_ms: Some(50),
14891 })
14892 .unwrap();
14893 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
14894 }
14895
14896 #[test]
14900 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
14901 let registry = Arc::new(Registry::default());
14902 let forwarding = Arc::new(ForwardingTable::default());
14903 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14904 let second = ControlHandler::with_forwarding(registry, forwarding);
14905 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
14906 }
14907
14908 #[tokio::test(flavor = "current_thread")]
14912 async fn route_open_unknown_module_refusals_add_no_outage_state() {
14913 let handler = ControlHandler::new(Arc::new(Registry::default()));
14914 let capture = EventCapture::default();
14915 let _subscriber =
14916 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14917 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
14918 for corr in 0..8 {
14919 let response = handler
14920 .handle_control_frame(
14921 &ctx,
14922 route_open_frame(
14923 380 + corr,
14924 &format!("nobody-{corr}"),
14925 unique_project_root("outage-unknown"),
14926 ),
14927 )
14928 .await
14929 .unwrap();
14930 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14931 }
14932
14933 assert_eq!(handler.route_outages.tracked_module_count(), 0);
14934 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
14935 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
14936 }
14937
14938 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14943 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
14944 let registry = Arc::new(Registry::default());
14945 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
14946 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14947 let capture = EventCapture::default();
14948 let _subscriber =
14949 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14950 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
14951 handler.route_outages.record_accepted("outage-restart");
14954
14955 let response = handler
14956 .handle_control_frame(
14957 &ctx,
14958 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
14959 )
14960 .await
14961 .unwrap();
14962 assert_eq!(response[0].header.ty, FrameType::Error);
14963 let starts = outage_lines(&capture, OUTAGE_START);
14964 assert_eq!(starts.len(), 1, "{starts:?}");
14965 assert_eq!(starts[0].level, tracing::Level::WARN);
14966 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
14967 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
14968 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
14969 handler.route_outages.record_accepted("outage-restart");
14970 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
14971
14972 let restart = handler
14973 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
14974 .await
14975 .unwrap();
14976 assert_eq!(
14977 restart[0].header.ty,
14978 FrameType::Response,
14979 "{:?}",
14980 parse_error(&restart[0])
14981 );
14982 handler
14983 .handle_control_frame(
14984 &ctx,
14985 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
14986 )
14987 .await
14988 .unwrap();
14989 module.stop().await.unwrap();
14990
14991 let starts = outage_lines(&capture, OUTAGE_START);
14992 assert_eq!(starts.len(), 2, "{starts:?}");
14993 assert_eq!(starts[1].level, tracing::Level::INFO);
14994 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
14995 }
14996
14997 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15001 async fn failed_operator_restart_leaves_no_operator_mark() {
15002 let registry = Arc::new(Registry::default());
15003 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
15004 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15005 let capture = EventCapture::default();
15006 let _subscriber =
15007 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15008 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
15009 handler.route_outages.record_accepted("outage-disabled");
15010
15011 let restart = handler
15012 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
15013 .await
15014 .unwrap();
15015 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
15016 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
15017
15018 handler
15019 .handle_control_frame(
15020 &ctx,
15021 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
15022 )
15023 .await
15024 .unwrap();
15025 let starts = outage_lines(&capture, OUTAGE_START);
15026 assert_eq!(starts.len(), 1, "{starts:?}");
15027 assert_eq!(starts[0].level, tracing::Level::WARN);
15028 }
15029
15030 #[tokio::test(flavor = "current_thread")]
15031 async fn route_open_unknown_module_escapes_target_module_id() {
15032 let handler = ControlHandler::new(Arc::new(Registry::default()));
15033 let capture = EventCapture::default();
15034 let _subscriber =
15035 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15036 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
15037 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15038 let response = handler
15039 .handle_control_frame(
15040 &ctx,
15041 route_open_frame(
15042 395,
15043 hostile_module_id,
15044 unique_project_root("hostile-target-module-id"),
15045 ),
15046 )
15047 .await
15048 .unwrap();
15049
15050 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15051 let event = capture
15052 .events()
15053 .into_iter()
15054 .find(|event| {
15055 event.target == "control"
15056 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
15057 })
15058 .expect("route.open unknown-module refusal event");
15059 let logged = event.fields.get("module_id").expect("module_id field");
15060 assert!(!logged.bytes().any(|byte| byte < 0x20));
15061 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15062 }
15063
15064 #[tokio::test(flavor = "current_thread")]
15067 async fn route_open_refusal_carries_daemon_origin() {
15068 let handler = ControlHandler::new(Arc::new(Registry::default()));
15069 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15070 let response = handler
15071 .handle_control_frame(
15072 &ctx,
15073 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
15074 )
15075 .await
15076 .unwrap();
15077
15078 assert_eq!(response.len(), 1);
15079 assert_eq!(response[0].header.ty, FrameType::Error);
15080 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15081 assert!(
15082 response[0].header.flags.is_daemon_origin(),
15083 "route.open refusal must carry daemon origin: {:?}",
15084 response[0].header.flags
15085 );
15086 }
15087
15088 #[tokio::test(flavor = "current_thread")]
15092 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
15093 let registry = Arc::new(Registry::default());
15094 let forwarding = Arc::new(ForwardingTable::default());
15095 let handler =
15096 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15097 let module_connection = ConnectionId::new(95);
15098 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15099 hello_via_sink(
15100 &handler,
15101 &module_ctx,
15102 &mut module_rx,
15103 hello_frame("aft", PROTOCOL_VERSION, 395),
15104 )
15105 .await;
15106 forwarding
15107 .begin_module_drain("aft", RouteCloseReason::Restart)
15108 .unwrap()
15109 .expect("the module endpoint drains");
15110
15111 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
15112 let response = handler
15113 .handle_control_frame(
15114 &client_ctx,
15115 route_open_frame(396, "aft", unique_project_root("drain-origin")),
15116 )
15117 .await
15118 .unwrap();
15119
15120 assert_eq!(response.len(), 1);
15121 assert_eq!(response[0].header.ty, FrameType::Error);
15122 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15123 assert!(
15124 response[0].header.flags.is_daemon_origin(),
15125 "drain module_reloading must carry daemon origin: {:?}",
15126 response[0].header.flags
15127 );
15128 }
15129
15130 #[tokio::test(flavor = "current_thread")]
15134 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
15135 let registry = Arc::new(Registry::default());
15136 let forwarding = Arc::new(ForwardingTable::default());
15137 let handler =
15138 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15139 let module_connection = ConnectionId::new(95);
15140 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15141 hello_via_sink(
15142 &handler,
15143 &module_ctx,
15144 &mut module_rx,
15145 hello_frame("aft", PROTOCOL_VERSION, 395),
15146 )
15147 .await;
15148
15149 let client_connection = ConnectionId::new(96);
15150 let (client_ctx, _client_rx) = route_ctx(client_connection);
15151 let (route_task, bind) = relay_route_open(
15152 &handler,
15153 client_connection,
15154 &client_ctx.egress,
15155 &mut module_rx,
15156 396,
15157 "aft",
15158 "spoofed-reloading",
15159 )
15160 .await;
15161 let rejection = Frame::build(
15164 FrameType::Error,
15165 crate::router::daemon_error_flags(),
15166 0,
15167 0,
15168 bind.header.corr,
15169 serde_json::to_vec(&ErrorBody::new(
15170 "module_reloading",
15171 "module refused route.bind",
15172 ))
15173 .unwrap(),
15174 )
15175 .unwrap();
15176 handler
15177 .handle_control_frame(&module_ctx, rejection)
15178 .await
15179 .unwrap();
15180
15181 let response = route_task.await.unwrap();
15182 assert_eq!(response.len(), 1);
15183 assert_eq!(response[0].header.ty, FrameType::Error);
15184 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15185 assert!(
15186 !response[0].header.flags.is_daemon_origin(),
15187 "a module's own rejection must not carry daemon origin: {:?}",
15188 response[0].header.flags
15189 );
15190 }
15191
15192 #[tokio::test(flavor = "current_thread")]
15193 async fn route_open_module_rejection_uses_daemon_counter_key() {
15194 let registry = Arc::new(Registry::default());
15195 let forwarding = Arc::new(ForwardingTable::default());
15196 let handler =
15197 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15198 let module_connection = ConnectionId::new(95);
15199 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15200 hello_via_sink(
15201 &handler,
15202 &module_ctx,
15203 &mut module_rx,
15204 hello_frame("aft", PROTOCOL_VERSION, 395),
15205 )
15206 .await;
15207
15208 let client_connection = ConnectionId::new(96);
15209 let (client_ctx, _client_rx) = route_ctx(client_connection);
15210 let capture = EventCapture::default();
15211 let _subscriber =
15212 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15213 let (route_task, bind) = relay_route_open(
15214 &handler,
15215 client_connection,
15216 &client_ctx.egress,
15217 &mut module_rx,
15218 396,
15219 "aft",
15220 "hostile-module-code",
15221 )
15222 .await;
15223 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
15224 let rejection = Frame::build(
15225 FrameType::Error,
15226 control_flags(),
15227 0,
15228 0,
15229 bind.header.corr,
15230 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
15231 )
15232 .unwrap();
15233 handler
15234 .handle_control_frame(&module_ctx, rejection)
15235 .await
15236 .unwrap();
15237
15238 let response = route_task.await.unwrap();
15239 assert_eq!(parse_error(&response[0])["code"], hostile_code);
15240 let counters = handler.counters().snapshot();
15241 assert_eq!(
15242 counters["route_open_refused_by_code"],
15243 json!({ "module_rejected": 1 })
15244 );
15245 assert!(counters["route_open_refused_by_code"]
15246 .get(hostile_code)
15247 .is_none());
15248
15249 let event = capture
15250 .events()
15251 .into_iter()
15252 .find(|event| {
15253 event.target == "control"
15254 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
15255 })
15256 .expect("route.open module-rejection refusal event");
15257 let logged = event.fields.get("module_code").expect("module_code field");
15258 assert!(!logged.bytes().any(|byte| byte < 0x20));
15259 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15260 }
15261
15262 #[tokio::test]
15263 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
15264 let registry = Arc::new(Registry::default());
15265 let supervisor_handle = SupervisorHandle::new();
15266 let missing_program = std::env::temp_dir().join(format!(
15267 "subc-route-open-missing-program-{}",
15268 std::process::id()
15269 ));
15270 let supervisor =
15271 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
15272 .with_handle(supervisor_handle.clone());
15273 let module = supervisor
15274 .supervise_configured(
15275 ModuleSpec {
15276 carry_containment: false,
15277 module_id: "failed".to_string(),
15278 program: missing_program,
15279 args: Vec::new(),
15280 env: Vec::new(),
15281 reserved: false,
15282 reserved_prefixes: Vec::new(),
15283 protocol: ModuleProtocol::Subc,
15284 overlap: Default::default(),
15285 },
15286 true,
15287 )
15288 .unwrap();
15289 assert_eq!(module.state().unwrap(), ModuleState::Failed);
15290
15291 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15292 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
15293 let response = handler
15294 .handle_control_frame(
15295 &ctx,
15296 route_open_frame(305, "failed", unique_project_root("failed")),
15297 )
15298 .await
15299 .unwrap();
15300
15301 assert_eq!(response[0].header.ty, FrameType::Error);
15302 let error = parse_error(&response[0]);
15303 assert_eq!(error["code"], "target_unavailable");
15304 assert!(error["message"]
15305 .as_str()
15306 .unwrap()
15307 .contains("state=failed, enabled=true, live=false"));
15308 }
15309
15310 #[tokio::test]
15311 async fn route_open_role_mismatch_remains_target_unavailable() {
15312 let registry = Arc::new(Registry::default());
15313 let handler = ControlHandler::new(Arc::clone(®istry));
15314 handler
15315 .handle_control(
15316 ConnectionId::new(41),
15317 non_routable_hello_frame_with_control_ops("health-only", 306, None),
15318 )
15319 .unwrap();
15320
15321 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
15322 let response = handler
15323 .handle_control_frame(
15324 &ctx,
15325 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
15326 )
15327 .await
15328 .unwrap();
15329
15330 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15331 assert!(parse_error(&response[0])["message"]
15332 .as_str()
15333 .unwrap()
15334 .contains("does not provide the requested target"));
15335 }
15336
15337 #[tokio::test]
15338 async fn route_open_inactive_registration_remains_target_unavailable() {
15339 let registry = Arc::new(Registry::default());
15340 let handler = ControlHandler::new(Arc::clone(®istry));
15341 handler
15342 .handle_control(
15343 ConnectionId::new(43),
15344 hello_frame("inactive", PROTOCOL_VERSION, 308),
15345 )
15346 .unwrap();
15347 assert!(registry
15348 .set_module_state_for_test("inactive", ChannelState::Closed)
15349 .unwrap());
15350
15351 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
15352 let response = handler
15353 .handle_control_frame(
15354 &ctx,
15355 route_open_frame(309, "inactive", unique_project_root("inactive")),
15356 )
15357 .await
15358 .unwrap();
15359
15360 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15361 assert!(parse_error(&response[0])["message"]
15362 .as_str()
15363 .unwrap()
15364 .contains("is not active"));
15365 }
15366
15367 #[tokio::test]
15368 async fn late_health_reply_is_recorded_through_the_module_response_path() {
15369 let registry = Arc::new(Registry::default());
15370 let forwarding = Arc::new(ForwardingTable::default());
15371 let supervisor_handle = SupervisorHandle::new();
15372 let supervisor =
15373 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
15374 .with_forwarding(Arc::clone(&forwarding))
15375 .with_handle(supervisor_handle.clone());
15376 let module = supervisor
15377 .supervise_configured(
15378 crate::ModuleSpec {
15379 carry_containment: false,
15380 module_id: "late-health-response".to_string(),
15381 program: PathBuf::from("disabled-module"),
15382 args: Vec::new(),
15383 env: Vec::new(),
15384 reserved: false,
15385 reserved_prefixes: Vec::new(),
15386 protocol: ModuleProtocol::Subc,
15387 overlap: Default::default(),
15388 },
15389 false,
15390 )
15391 .unwrap();
15392 let handler =
15393 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15394 .with_supervisor(supervisor_handle);
15395 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
15396 handler
15397 .handle_control_frame(
15398 &module_ctx,
15399 hello_frame_with_control_ops(
15400 "late-health-response",
15401 PROTOCOL_VERSION,
15402 7,
15403 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15404 ),
15405 )
15406 .await
15407 .unwrap();
15408 let probe_started_at = Instant::now() - Duration::from_millis(80);
15409 let pending = forwarding
15410 .begin_health_probe_rpc_for(
15411 "late-health-response",
15412 MODULE_CONTROL_OP_HEALTH_CHECK,
15413 probe_started_at,
15414 Instant::now() - Duration::from_millis(1),
15415 )
15416 .unwrap();
15417 assert!(forwarding
15418 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15419 .unwrap());
15420
15421 let responses = handler
15422 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15423 .await
15424 .unwrap();
15425
15426 assert!(responses.is_empty());
15427 let health = module.status().unwrap().health;
15428 assert_eq!(health.late_answer_count, 1);
15429 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15430 }
15431
15432 #[tokio::test]
15433 async fn health_probe_timeout_and_module_death_are_typed() {
15434 let registry = Arc::new(Registry::default());
15435 let forwarding = Arc::new(ForwardingTable::default());
15436 let handler =
15437 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15438 .with_health_probe_timeout(Duration::from_millis(50));
15439 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15440 hello_via_sink(
15441 &handler,
15442 &module_ctx,
15443 &mut module_rx,
15444 hello_frame_with_control_ops(
15445 "aft",
15446 PROTOCOL_VERSION,
15447 7,
15448 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15449 ),
15450 )
15451 .await;
15452
15453 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
15454 let responses = handler
15455 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
15456 .await
15457 .unwrap();
15458 assert_eq!(responses[0].header.ty, FrameType::Error);
15459 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
15460 let _ = module_rx.try_recv();
15461
15462 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
15463 let health_handler = handler.clone();
15464 let death_task = tokio::spawn(async move {
15465 health_handler
15466 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
15467 .await
15468 .unwrap()
15469 });
15470 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15471 .await
15472 .unwrap()
15473 .unwrap();
15474 handler
15475 .cleanup_connection(module_ctx.connection_id)
15476 .unwrap();
15477 let responses = death_task.await.unwrap();
15478 assert_eq!(responses[0].header.ty, FrameType::Error);
15479 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
15480 }
15481
15482 #[test]
15483 fn hello_requires_exact_protocol_version() {
15484 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
15485 let registry = Arc::new(Registry::default());
15486 let handler = ControlHandler::new(Arc::clone(®istry));
15487 let responses = handler
15488 .handle_control(
15489 ConnectionId::new(connection),
15490 hello_frame("aft", offered, 9),
15491 )
15492 .unwrap();
15493
15494 assert_eq!(responses.len(), 1);
15495 assert_eq!(responses[0].header.ty, FrameType::Error);
15496 let error = parse_error(&responses[0]);
15497 assert_eq!(error["code"], "version_unsupported");
15498 assert!(registry.get_module("aft").unwrap().is_none());
15499 assert_eq!(registry.active_registration_count().unwrap(), 0);
15500 }
15501 }
15502
15503 #[test]
15504 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
15505 let registry = Arc::new(Registry::default());
15506 let forwarding = Arc::new(ForwardingTable::default());
15507 let handler =
15508 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15509 let module_connection = ConnectionId::new(301);
15510 let registration = registry
15511 .register_with_control_ops(
15512 manifest("aft-push", PROTOCOL_VERSION),
15513 PROTOCOL_VERSION,
15514 module_connection,
15515 module_baseline_control_ops(),
15516 )
15517 .unwrap();
15518 let (module_tx, _module_rx) = mpsc::channel(8);
15519 let endpoint = forwarding
15520 .register_module_connection(
15521 module_connection,
15522 "aft-push".to_string(),
15523 PROTOCOL_VERSION,
15524 manifest_concurrency(®istration.manifest),
15525 FrameSink::new(module_tx),
15526 )
15527 .unwrap();
15528
15529 let unknown = Frame::build(
15531 FrameType::Push,
15532 control_flags(),
15533 0,
15534 0,
15535 5,
15536 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
15537 )
15538 .unwrap();
15539 let out = handler.handle_status_update(endpoint, unknown).unwrap();
15540 assert!(
15541 out.is_empty(),
15542 "unknown push op must be ignored, got {out:?}"
15543 );
15544
15545 let malformed = Frame::build(
15547 FrameType::Push,
15548 control_flags(),
15549 0,
15550 0,
15551 6,
15552 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
15553 )
15554 .unwrap();
15555 let out = handler.handle_status_update(endpoint, malformed).unwrap();
15556 assert_eq!(out.len(), 1);
15557 assert_eq!(out[0].header.ty, FrameType::Error);
15558 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
15559 }
15560
15561 #[test]
15562 fn hello_rejected_when_connection_already_owns_client_routes() {
15563 let registry = Arc::new(Registry::default());
15564 let forwarding = Arc::new(ForwardingTable::default());
15565 let handler =
15566 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15567 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
15569 let client_connection = ConnectionId::new(202);
15570
15571 let responses = handler
15574 .handle_control(
15575 client_connection,
15576 hello_frame("aft-second", PROTOCOL_VERSION, 9),
15577 )
15578 .unwrap();
15579 assert_eq!(responses[0].header.ty, FrameType::Error);
15580 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
15581 assert!(registry.get_module("aft-second").unwrap().is_none());
15582 }
15583
15584 #[tokio::test]
15585 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
15586 let registry = Arc::new(Registry::default());
15587 let forwarding = Arc::new(ForwardingTable::default());
15588 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
15589 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
15590 hello_via_sink(
15591 &handler,
15592 &module_ctx,
15593 &mut module_rx,
15594 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
15595 )
15596 .await;
15597 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
15598 let pending = forwarding
15599 .begin_route_bind_relay_for_test(
15600 client_ctx.connection_id,
15601 client_ctx.egress.clone(),
15602 2,
15603 "alpha",
15604 )
15605 .unwrap();
15606 forwarding
15607 .complete_pending_relay(
15608 module_ctx.connection_id,
15609 pending.corr,
15610 RouteBindRelayOutcome::Accepted,
15611 )
15612 .unwrap();
15613 client_rx.try_recv().unwrap();
15614 for module_id in ["beta", "alpha"] {
15615 let replies = handler
15616 .handle_control_frame(
15617 &module_ctx,
15618 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
15619 )
15620 .await
15621 .unwrap();
15622 assert_eq!(replies.len(), 1, "second HELLO must be refused");
15623 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
15624 }
15625 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
15626 assert!(registry.get_module("beta").unwrap().is_none());
15627 assert!(matches!(
15628 forwarding
15629 .lookup_data_route(
15630 client_ctx.connection_id,
15631 pending.client_channel,
15632 pending.client_epoch,
15633 )
15634 .unwrap(),
15635 DataRoute::Client(DataRouteState::Bound(_))
15636 ));
15637 assert!(handler
15638 .hello_launch_nonces
15639 .lock()
15640 .unwrap()
15641 .presented(module_ctx.connection_id, Some("alpha-nonce")));
15642 assert!(module_rx.try_recv().is_err());
15643 }
15644
15645 #[test]
15646 fn reserved_module_hello_requires_matching_launch_nonce() {
15647 let registry = Arc::new(Registry::default());
15648 let supervisor = SupervisorHandle::new();
15649 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
15652 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15653
15654 let no_nonce = handler
15656 .handle_control(
15657 ConnectionId::new(1),
15658 hello_frame("vault", PROTOCOL_VERSION, 1),
15659 )
15660 .unwrap();
15661 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
15662 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
15663 assert!(registry.get_module("vault").unwrap().is_none());
15664
15665 let wrong = handler
15667 .handle_control(
15668 ConnectionId::new(2),
15669 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
15670 )
15671 .unwrap();
15672 assert_eq!(wrong[0].header.ty, FrameType::Error);
15673 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
15674 assert!(registry.get_module("vault").unwrap().is_none());
15675
15676 let ok = handler
15678 .handle_control(
15679 ConnectionId::new(3),
15680 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
15681 )
15682 .unwrap();
15683 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
15684 assert!(registry.get_module("vault").unwrap().is_some());
15685 }
15686
15687 #[test]
15688 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
15689 let registry = Arc::new(Registry::default());
15690 let supervisor = SupervisorHandle::new();
15691 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15692 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15693 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15694
15695 let squat = handler
15696 .handle_control(
15697 ConnectionId::new(1),
15698 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
15699 )
15700 .unwrap();
15701 assert_eq!(squat[0].header.ty, FrameType::Error);
15702 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
15703 assert!(parse_error(&squat[0])["message"]
15704 .as_str()
15705 .unwrap()
15706 .contains("fed:"));
15707
15708 let accepted_peer = handler
15709 .handle_control(
15710 ConnectionId::new(2),
15711 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
15712 )
15713 .unwrap();
15714 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
15715
15716 let accepted_short = handler
15717 .handle_control(
15718 ConnectionId::new(3),
15719 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
15720 )
15721 .unwrap();
15722 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
15723
15724 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
15725 let response = handler
15726 .handle_control(
15727 ConnectionId::new(conn),
15728 hello_frame(module_id, PROTOCOL_VERSION, conn),
15729 )
15730 .unwrap();
15731 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
15732 }
15733 }
15734
15735 #[test]
15736 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
15737 let registry = Arc::new(Registry::default());
15738 let supervisor = SupervisorHandle::new();
15739 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15740 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15741 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
15742 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15743
15744 let owner_nonce = handler
15745 .handle_control(
15746 ConnectionId::new(1),
15747 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
15748 )
15749 .unwrap();
15750 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
15751 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
15752 assert!(registry.get_module("fed:special").unwrap().is_none());
15753
15754 let exact_nonce = handler
15755 .handle_control(
15756 ConnectionId::new(2),
15757 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
15758 )
15759 .unwrap();
15760 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
15761 assert!(registry.get_module("fed:special").unwrap().is_some());
15762 }
15763
15764 #[test]
15765 fn non_reserved_module_ignores_launch_nonce() {
15766 let registry = Arc::new(Registry::default());
15767 let handler = ControlHandler::new(Arc::clone(®istry));
15770 let no_nonce = handler
15771 .handle_control(
15772 ConnectionId::new(1),
15773 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
15774 )
15775 .unwrap();
15776 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
15777 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
15778
15779 let echoed_nonce = handler
15780 .handle_control(
15781 ConnectionId::new(2),
15782 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
15783 )
15784 .unwrap();
15785 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
15786 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
15787 }
15788
15789 #[test]
15790 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
15791 let handler = ControlHandler::default();
15792 let conn = ConnectionId::new(1);
15793 let malformed = Frame::build(
15794 FrameType::Hello,
15795 control_flags(),
15796 0,
15797 0,
15798 3,
15799 b"{not json".to_vec(),
15800 )
15801 .unwrap();
15802
15803 let error = handler.handle_control(conn, malformed).unwrap();
15804 assert_eq!(error[0].header.ty, FrameType::Error);
15805 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
15806
15807 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15808 let pong = handler.handle_control(conn, ping).unwrap();
15809 assert_eq!(pong[0].header.ty, FrameType::Pong);
15810 assert_eq!(pong[0].header.corr, 4);
15811 }
15812
15813 #[test]
15814 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
15815 let registry = Arc::new(Registry::default());
15816 let handler = ControlHandler::new(Arc::clone(®istry));
15817
15818 handler
15819 .handle_control(
15820 ConnectionId::new(1),
15821 hello_frame("aft", PROTOCOL_VERSION, 1),
15822 )
15823 .unwrap();
15824 let duplicate = handler
15825 .handle_control(
15826 ConnectionId::new(2),
15827 hello_frame("aft", PROTOCOL_VERSION, 2),
15828 )
15829 .unwrap();
15830
15831 assert_eq!(duplicate[0].header.ty, FrameType::Error);
15832 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15833 let registration = registry.get_module("aft").unwrap().unwrap();
15834 assert_eq!(registration.connection_id, ConnectionId::new(1));
15835 }
15836
15837 #[test]
15838 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
15839 let registry = Arc::new(Registry::default());
15840 let forwarding = Arc::new(ForwardingTable::default());
15841 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
15842 let handler =
15843 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15844 .with_process_liveness(process_liveness);
15845 let (ctx, route_channel, route_epoch) =
15846 bind_liveness_route(®istry, &forwarding, "aft-dead");
15847 let responses = handler
15848 .handle_route_poll(
15849 &ctx,
15850 route_poll_frame(41, PollKind::Liveness, route_channel),
15851 route_channel,
15852 route_epoch,
15853 PollKind::Liveness,
15854 )
15855 .unwrap();
15856
15857 assert_eq!(responses.len(), 1);
15858 assert_eq!(responses[0].header.ty, FrameType::Response);
15859 assert_route_poll_liveness(&responses[0], false);
15860 }
15861
15862 #[test]
15863 fn liveness_poll_without_process_source_uses_bound_route() {
15864 let registry = Arc::new(Registry::default());
15865 let forwarding = Arc::new(ForwardingTable::default());
15866 let handler =
15867 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15868 let (ctx, route_channel, route_epoch) =
15869 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
15870 let responses = handler
15871 .handle_route_poll(
15872 &ctx,
15873 route_poll_frame(42, PollKind::Liveness, route_channel),
15874 route_channel,
15875 route_epoch,
15876 PollKind::Liveness,
15877 )
15878 .unwrap();
15879
15880 assert_route_poll_liveness(&responses[0], true);
15881 }
15882
15883 #[test]
15884 fn liveness_poll_untracked_process_source_uses_bound_route() {
15885 let registry = Arc::new(Registry::default());
15886 let forwarding = Arc::new(ForwardingTable::default());
15887 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
15888 let handler =
15889 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15890 .with_process_liveness(process_liveness);
15891 let (ctx, route_channel, route_epoch) =
15892 bind_liveness_route(®istry, &forwarding, "aft-untracked");
15893 let responses = handler
15894 .handle_route_poll(
15895 &ctx,
15896 route_poll_frame(43, PollKind::Liveness, route_channel),
15897 route_channel,
15898 route_epoch,
15899 PollKind::Liveness,
15900 )
15901 .unwrap();
15902
15903 assert_route_poll_liveness(&responses[0], true);
15904 }
15905
15906 #[tokio::test]
15907 async fn unknown_op_returns_unknown_control_op() {
15908 let handler = ControlHandler::default();
15909 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
15910 let request = Frame::build(
15911 FrameType::Request,
15912 control_flags(),
15913 0,
15914 0,
15915 55,
15916 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
15917 )
15918 .unwrap();
15919
15920 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15921
15922 assert_eq!(response.len(), 1);
15923 assert_eq!(response[0].header.ty, FrameType::Error);
15924 assert_eq!(response[0].header.corr, 55);
15925 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
15926 }
15927
15928 #[tokio::test]
15929 async fn supervisor_provenance_rejects_unknown_exact_module() {
15930 let handler = ControlHandler::default();
15931 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
15932 let request = Frame::build(
15933 FrameType::Request,
15934 control_flags(),
15935 0,
15936 0,
15937 57,
15938 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
15939 )
15940 .unwrap();
15941
15942 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15943
15944 assert_eq!(response.len(), 1);
15945 assert_eq!(response[0].header.ty, FrameType::Error);
15946 assert_eq!(response[0].header.corr, 57);
15947 let error = parse_error(&response[0]);
15948 assert_eq!(error["code"], "unknown_module");
15949 assert_eq!(error["message"], "module_id 'missing' is not supervised");
15950 }
15951
15952 #[test]
15953 fn provenance_probe_override_keeps_handler_tests_deterministic() {
15954 let expected = subc_control::RunningImageAgreement::Unavailable {
15955 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15956 };
15957 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
15958 assert_eq!(handler.provenance_probe_override, Some(expected));
15959 }
15960
15961 #[test]
15962 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
15963 let verdict = reload_verdict(
15964 std::path::Path::new("/bin/new"),
15965 Some(std::path::Path::new("/bin/old")),
15966 subc_control::RunningImageAgreement::Unavailable {
15967 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15968 },
15969 );
15970 assert!(matches!(
15971 verdict.path,
15972 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
15973 if configured == std::path::Path::new("/bin/new")
15974 && spawned_from == std::path::Path::new("/bin/old")
15975 ));
15976 }
15977
15978 #[test]
15979 fn launch_spec_without_running_digest_is_unavailable_not_match() {
15980 use subc_control::{
15981 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
15982 };
15983 let configured = crate::supervise::ModuleSpec {
15984 carry_containment: false,
15985 module_id: "legacy".into(),
15986 program: "/module".into(),
15987 args: Vec::new(),
15988 env: Vec::new(),
15989 reserved: false,
15990 reserved_prefixes: Vec::new(),
15991 protocol: subc_control::ModuleProtocol::Subc,
15992 overlap: Default::default(),
15993 };
15994 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
15995 assert_eq!(
15996 launch_spec_agreement(&configured, false, Some(&digest)),
15997 Agreement::Unavailable {
15998 reason: Reason::NotRunning
15999 }
16000 );
16001 assert_eq!(
16002 launch_spec_agreement(&configured, true, None),
16003 Agreement::Unavailable {
16004 reason: Reason::SpawnedSpecUnavailable
16005 }
16006 );
16007 assert_eq!(
16008 launch_spec_agreement(&configured, true, Some(&digest)),
16009 Agreement::Match
16010 );
16011 }
16012
16013 #[cfg(windows)]
16014 #[test]
16015 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
16016 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
16017 let configured = dir.join("module.exe");
16018 std::fs::write(&configured, b"executable path fixture").unwrap();
16019 let spawned = std::fs::canonicalize(&configured).unwrap();
16020 assert_ne!(
16021 configured, spawned,
16022 "fixture must exercise different path spellings"
16023 );
16024 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
16025 let lower_case = configured.to_string_lossy().to_lowercase();
16026 for spelling in [
16027 &configured,
16028 Path::new(&forward_slashes),
16029 Path::new(&lower_case),
16030 ] {
16031 let verdict = reload_verdict(
16032 spelling,
16033 Some(&spawned),
16034 subc_control::RunningImageAgreement::Unavailable {
16035 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16036 },
16037 );
16038 assert_eq!(
16039 verdict.path,
16040 ReloadPathAgreement::Match,
16041 "configured {spelling:?}, spawned {spawned:?}"
16042 );
16043 }
16044 }
16045
16046 #[cfg(windows)]
16047 #[test]
16048 fn windows_reload_path_keeps_different_existing_program_pending() {
16049 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
16050 let old = dir.join("old.exe");
16051 let new = dir.join("new.exe");
16052 std::fs::write(&old, b"same executable content").unwrap();
16053 std::fs::copy(&old, &new).unwrap();
16054 let spawned = std::fs::canonicalize(&old).unwrap();
16055 let image = subc_control::RunningImageAgreement::Unavailable {
16056 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16057 };
16058 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
16059 assert!(
16060 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
16061 if configured == new && spawned_from == spawned)
16062 );
16063 let missing = dir.join("missing.exe");
16064 assert!(matches!(
16065 reload_verdict(&missing, Some(&spawned), image).path,
16066 ReloadPathAgreement::Mismatch { .. }
16067 ));
16068 }
16069
16070 #[test]
16071 fn reload_verdict_detects_replaced_image_at_same_path() {
16072 let image = subc_control::RunningImageAgreement::Mismatch {
16073 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
16074 digest: "old".into(),
16075 },
16076 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
16077 digest: "new".into(),
16078 },
16079 };
16080 let verdict = reload_verdict(
16081 std::path::Path::new("/bin/same"),
16082 Some(std::path::Path::new("/bin/same")),
16083 image.clone(),
16084 );
16085 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16086 assert_eq!(verdict.image, image);
16087 }
16088
16089 #[test]
16090 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
16091 let image = subc_control::RunningImageAgreement::Unavailable {
16092 reason: subc_control::RunningImageUnavailableReason::NotRunning,
16093 };
16094 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
16095 assert_eq!(
16096 verdict.path,
16097 subc_control::ReloadPathAgreement::Unavailable {
16098 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
16099 }
16100 );
16101 assert_eq!(verdict.image, image);
16102
16103 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
16104 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16105 };
16106 let verdict = reload_verdict(
16107 std::path::Path::new("/bin/same"),
16108 Some(std::path::Path::new("/bin/same")),
16109 unconfirmed.clone(),
16110 );
16111 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16112 assert_eq!(verdict.image, unconfirmed);
16113 }
16114
16115 #[test]
16116 fn reload_verdict_preserves_each_image_unavailability_reason() {
16117 use subc_control::RunningImageUnavailableReason as Reason;
16118
16119 for reason in [
16120 Reason::NotRunning,
16121 Reason::UnsupportedPlatform,
16122 Reason::RunningExecutableUnreadable,
16123 Reason::SpawnedPathUnreadable,
16124 Reason::HashFailed,
16125 Reason::ProcessIdentityUnconfirmed,
16126 Reason::Unknown("future_probe_reason".to_string()),
16127 ] {
16128 let image = subc_control::RunningImageAgreement::Unavailable {
16129 reason: reason.clone(),
16130 };
16131 let verdict = reload_verdict(
16132 std::path::Path::new("/bin/same"),
16133 Some(std::path::Path::new("/bin/same")),
16134 image.clone(),
16135 );
16136 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16137 assert_eq!(verdict.image, image, "{reason:?}");
16138 }
16139 }
16140
16141 #[tokio::test]
16142 async fn malformed_control_bodies_return_invalid_control_body() {
16143 let handler = ControlHandler::default();
16144 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
16145
16146 for (corr, body) in [
16147 (56, br#"{"route_channel":1}"#.as_slice()),
16148 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
16149 (
16150 58,
16151 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
16152 ),
16153 ] {
16154 let request = Frame::build(
16155 FrameType::Request,
16156 control_flags(),
16157 0,
16158 0,
16159 corr,
16160 body.to_vec(),
16161 )
16162 .unwrap();
16163 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16164
16165 assert_eq!(response.len(), 1);
16166 assert_eq!(response[0].header.ty, FrameType::Error);
16167 assert_eq!(response[0].header.corr, corr);
16168 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
16169 }
16170 }
16171
16172 #[tokio::test]
16173 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
16174 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16175 let registry = Arc::new(Registry::default());
16176 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16177 let router = Router::with_control_handler(Arc::clone(&control));
16178 let connection = router.begin_connection();
16179 let (ctx, mut rx) = route_ctx(connection.id());
16180
16181 router
16182 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
16183 .await
16184 .unwrap();
16185 let response = rx.recv().await.unwrap();
16186 let ack = parse_ack(&response);
16187 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16188 let channel = 1;
16189
16190 let goodbye =
16191 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
16192 router.route_for_connection(&ctx, goodbye).await.unwrap();
16193 assert!(rx.try_recv().is_err());
16194 assert!(registry.get_module("aft").unwrap().is_none());
16195
16196 router
16197 .route_for_connection(&ctx, channel_request(channel, 13))
16198 .await
16199 .unwrap();
16200 let error_frame = rx.recv().await.unwrap();
16201 assert_eq!(error_frame.header.ty, FrameType::Error);
16202 assert_eq!(error_frame.header.channel, channel);
16203 let captured = crate::router::test_log::captured_logs(&logs);
16204 assert_eq!(
16205 captured
16206 .lines()
16207 .filter(|line| {
16208 line.contains("module_id=aft")
16209 && line.contains("reason=explicit_goodbye")
16210 && line.contains("module registration ended")
16211 })
16212 .count(),
16213 1,
16214 "unexpected GOODBYE registry log: {captured}"
16215 );
16216 }
16217
16218 #[tokio::test]
16219 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
16220 let registry = Arc::new(Registry::default());
16221 let handler = ControlHandler::new(registry).with_capability_config(
16222 [("prov".to_string(), true), ("cons".to_string(), true)],
16223 BTreeMap::new(),
16224 );
16225 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
16226 register_capability_manifest(
16227 &handler,
16228 &provider_ctx,
16229 &mut provider_rx,
16230 capability_manifest("prov", &["thing/v1"], &[]),
16231 1,
16232 )
16233 .await;
16234 let mut consumer = capability_manifest("cons", &[], &[]);
16235 consumer.capabilities.as_mut().unwrap().requires.push(
16236 subc_protocol::manifest::CapabilityRequirement {
16237 capability: "thing/v1".to_string(),
16238 need: subc_protocol::manifest::CapabilityNeed::Required,
16239 },
16240 );
16241 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
16242 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
16243 assert_eq!(
16244 handler.capability_evaluator.verdict("cons", "thing/v1"),
16245 Some(CapabilityVerdict::Provided)
16246 );
16247 let (mut client_rx, _) = open_route_for_capability_test(
16248 &handler,
16249 &provider_ctx,
16250 &mut provider_rx,
16251 703,
16252 3,
16253 "prov",
16254 None,
16255 )
16256 .await;
16257 let goodbye =
16258 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
16259 handler
16260 .handle_control_frame(&provider_ctx, goodbye)
16261 .await
16262 .unwrap();
16263 assert_eq!(
16264 handler.capability_evaluator.verdict("cons", "thing/v1"),
16265 Some(CapabilityVerdict::NeverProvided)
16266 );
16267 let closed = client_rx
16268 .try_recv()
16269 .expect("GOODBYE pushes route.closed before route GOODBYE");
16270 assert!(
16271 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
16272 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
16273 );
16274 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
16275 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
16276 }
16277
16278 struct RouteClosedFixture {
16279 handler: ControlHandler,
16280 #[cfg(unix)]
16281 supervisor: Supervisor,
16282 module_ctx: RouteCtx,
16283 #[cfg(unix)]
16284 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16285 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16286 route: Arc<crate::forwarding::RouteBinding>,
16287 }
16288
16289 async fn route_closed_fixture() -> RouteClosedFixture {
16290 let registry = Arc::new(Registry::default());
16291 let handler = ControlHandler::new(Arc::clone(®istry));
16292 #[cfg(unix)]
16293 let supervisor = Supervisor::new(registry, RestartPolicy::default())
16294 .with_forwarding(Arc::clone(&handler.forwarding));
16295 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
16296 register_capability_manifest(
16297 &handler,
16298 &module_ctx,
16299 &mut module_rx,
16300 capability_manifest("prov", &[], &[]),
16301 1,
16302 )
16303 .await;
16304 let (client_rx, bind) = open_route_for_capability_test(
16305 &handler,
16306 &module_ctx,
16307 &mut module_rx,
16308 702,
16309 2,
16310 "prov",
16311 None,
16312 )
16313 .await;
16314 let ModuleControlRequest::RouteBind {
16315 route_channel: channel,
16316 epoch: route_epoch,
16317 ..
16318 } = bind
16319 else {
16320 panic!("expected route.bind");
16321 };
16322 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
16323 handler
16324 .forwarding
16325 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
16326 .unwrap()
16327 else {
16328 panic!("the acknowledged route must be bound");
16329 };
16330 RouteClosedFixture {
16331 handler,
16332 #[cfg(unix)]
16333 supervisor,
16334 module_ctx,
16335 #[cfg(unix)]
16336 module_rx,
16337 client_rx,
16338 route,
16339 }
16340 }
16341
16342 impl RouteClosedFixture {
16343 #[cfg(unix)]
16344 async fn cut(&mut self) {
16345 let notices = async {
16346 let mut draining = self.module_rx.recv().await.unwrap();
16347 assert!(matches!(
16348 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
16349 &draining.body
16350 )
16351 .unwrap(),
16352 subc_protocol::session::ModuleControlCommand::Draining {
16353 reason: RouteCloseReason::Disable,
16354 ..
16355 }
16356 ));
16357 draining.flushed.take().unwrap().send(()).unwrap();
16358 let mut closing = self.client_rx.recv().await.unwrap();
16359 assert!(matches!(
16360 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
16361 ClientControlPush::RouteClosing {
16362 reason: RouteCloseReason::Restart,
16363 ..
16364 }
16365 ));
16366 closing.flushed.take().unwrap().send(()).unwrap();
16367 };
16368 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
16369 result.unwrap();
16370 }
16371
16372 fn disconnect(&mut self) -> ClientControlPush {
16373 self.handler
16374 .cleanup_connection(self.module_ctx.connection_id)
16375 .unwrap();
16376 let closed = self.client_rx.try_recv().unwrap();
16377 assert_eq!(closed.header.ty, FrameType::Push);
16378 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
16379 assert!(matches!(
16380 &push,
16381 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
16382 if channels == &[self.route.client_channel]
16383 ));
16384 assert_eq!(
16385 self.client_rx.try_recv().unwrap().header.ty,
16386 FrameType::Goodbye
16387 );
16388 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
16389 push
16390 }
16391 }
16392
16393 #[cfg(unix)]
16394 #[tokio::test(start_paused = true)]
16395 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
16396 let mut fixture = route_closed_fixture().await;
16397 fixture.cut().await;
16398 assert!(matches!(
16399 fixture.disconnect(),
16400 ClientControlPush::RouteClosed {
16401 reason: RouteCloseReason::Restart,
16402 drained: true,
16403 ..
16404 }
16405 ));
16406 }
16407
16408 #[cfg(unix)]
16409 #[tokio::test(start_paused = true)]
16410 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16411 let mut fixture = route_closed_fixture().await;
16412 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16413 let started = tokio::time::Instant::now();
16414 fixture.cut().await;
16415 assert!(started.elapsed() >= Duration::from_secs(2));
16416 assert_eq!(fixture.route.flow.in_flight(), 1);
16417 assert!(matches!(
16418 fixture.disconnect(),
16419 ClientControlPush::RouteClosed {
16420 reason: RouteCloseReason::Restart,
16421 drained: false,
16422 ..
16423 }
16424 ));
16425 }
16426
16427 #[tokio::test]
16428 async fn module_connection_crash_reports_crash_and_not_drained() {
16429 let mut fixture = route_closed_fixture().await;
16430 assert!(matches!(
16434 fixture.disconnect(),
16435 ClientControlPush::RouteClosed {
16436 reason: RouteCloseReason::Crash,
16437 drained: false,
16438 ..
16439 }
16440 ));
16441 }
16442
16443 #[tokio::test]
16444 async fn dropping_router_connection_releases_registration() {
16445 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16446 let registry = Arc::new(Registry::default());
16447 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16448 let router = Router::with_control_handler(Arc::clone(&control));
16449 let connection = router.begin_connection();
16450 let connection_id = connection.id();
16451 let (ctx, mut rx) = route_ctx(connection_id);
16452
16453 router
16454 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
16455 .await
16456 .unwrap();
16457 let response = rx.recv().await.unwrap();
16458 let ack = parse_ack(&response);
16459 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16460 assert!(registry.get_module("aft").unwrap().is_some());
16461
16462 drop(connection);
16463
16464 assert!(registry.get_module("aft").unwrap().is_none());
16465 assert_eq!(registry.active_registration_count().unwrap(), 0);
16466
16467 control
16468 .cleanup_connection(ConnectionId::new(u64::MAX))
16469 .unwrap();
16470 let captured = crate::router::test_log::captured_logs(&logs);
16471 println!("captured registry lifecycle logs:\n{captured}");
16472 let events: Vec<_> = captured
16473 .lines()
16474 .filter(|line| {
16475 line.contains("module registered module_id=aft ")
16476 || line.contains("module registration ended")
16477 })
16478 .collect();
16479 assert_eq!(
16480 events.len(),
16481 2,
16482 "unexpected registry lifecycle logs: {captured}"
16483 );
16484 assert!(events[0].contains("module registered module_id=aft "));
16485 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
16486 assert!(events[1].contains(&format!(
16487 "module_id=aft connection_id={}",
16488 connection_id.get()
16489 )));
16490 assert!(events[1].contains("reason=connection_closed"));
16491 assert!(events[1].contains("module registration ended"));
16492 assert_eq!(
16493 captured
16494 .lines()
16495 .filter(|line| line.contains("module registered module_id=aft "))
16496 .count(),
16497 1,
16498 "legacy registration admission line must appear once: {captured}"
16499 );
16500 }
16501
16502 #[tokio::test]
16503 async fn hello_registration_keeps_legacy_module_registered_line_once() {
16504 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16505 let registry = Arc::new(Registry::default());
16506 let control = ControlHandler::new(Arc::clone(®istry));
16507 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
16508 hello_via_sink(
16509 &control,
16510 &ctx,
16511 &mut rx,
16512 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
16513 )
16514 .await;
16515
16516 let captured = crate::router::test_log::captured_logs(&logs);
16517 let admissions: Vec<_> = captured
16518 .lines()
16519 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
16520 .collect();
16521 assert_eq!(
16522 admissions.len(),
16523 1,
16524 "legacy admission line must remain exactly once: {captured}"
16525 );
16526 assert!(admissions[0].contains("routable_provider=true"));
16527 assert!(admissions[0].contains("connection_id=777"));
16528 }
16529
16530 fn capability_manifest(
16531 module_id: &str,
16532 provides: &[&str],
16533 must_never_reach: &[&str],
16534 ) -> ModuleManifest {
16535 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
16536 manifest.capabilities = Some(CapabilityDeclarations {
16537 provides: provides
16538 .iter()
16539 .map(|capability| (*capability).to_string())
16540 .collect(),
16541 requires: Vec::new(),
16542 must_never_reach: must_never_reach
16543 .iter()
16544 .map(|capability| (*capability).to_string())
16545 .collect(),
16546 });
16547 manifest
16548 }
16549
16550 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
16551 Frame::build(
16552 FrameType::Hello,
16553 control_flags(),
16554 0,
16555 0,
16556 corr,
16557 serde_json::to_vec(&ModuleHelloBody {
16558 protocol_ver: manifest.protocol_ver,
16559 manifest,
16560 control_ops: None,
16561 launch_nonce: None,
16562 })
16563 .expect("capability test HELLO serializes"),
16564 )
16565 .expect("capability test HELLO frame builds")
16566 }
16567
16568 fn catalog_update_with_capabilities_frame(
16569 corr: u64,
16570 capabilities: CapabilityDeclarations,
16571 ) -> Frame {
16572 Frame::build(
16573 FrameType::Request,
16574 control_flags(),
16575 0,
16576 0,
16577 corr,
16578 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16579 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
16580 capabilities: Some(capabilities),
16581 ready: None,
16582 })
16583 .expect("capability catalog.update serializes"),
16584 )
16585 .expect("capability catalog.update frame builds")
16586 }
16587
16588 async fn register_capability_manifest(
16589 handler: &ControlHandler,
16590 ctx: &RouteCtx,
16591 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16592 manifest: ModuleManifest,
16593 corr: u64,
16594 ) {
16595 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
16596 }
16597
16598 async fn open_route_for_capability_test(
16599 handler: &ControlHandler,
16600 target_ctx: &RouteCtx,
16601 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16602 client_connection_id: u64,
16603 corr: u64,
16604 target_module_id: &str,
16605 consumer_identity: Option<ConsumerIdentity>,
16606 ) -> (
16607 mpsc::Receiver<crate::router::OutboundFrame>,
16608 ModuleControlRequest,
16609 ) {
16610 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
16611 let route_handler = handler.clone();
16612 let target_module_id = target_module_id.to_string();
16613 let route_task = tokio::spawn(async move {
16614 route_handler
16615 .handle_control_frame(
16616 &client_ctx,
16617 route_open_frame_with_admission_facts(
16618 corr,
16619 &target_module_id,
16620 unique_project_root("admission-facts"),
16621 consumer_identity,
16622 None,
16623 ),
16624 )
16625 .await
16626 .expect("capability test route.open succeeds")
16627 });
16628 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
16629 .await
16630 .expect("capability test route.open must reach route.bind")
16631 .expect("target control receiver stays open");
16632 let bind_request: ModuleControlRequest =
16633 serde_json::from_slice(&bind.body).expect("route.bind decodes");
16634 handler
16635 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
16636 .await
16637 .expect("capability test route.bind ACK succeeds");
16638 assert!(route_task.await.expect("route.open task joins").is_empty());
16639 let opened = client_rx
16640 .recv()
16641 .await
16642 .expect("successful route.open publishes a response");
16643 assert!(matches!(
16644 serde_json::from_slice::<ClientControlResponse>(&opened.body),
16645 Ok(ClientControlResponse::RouteOpen { .. })
16646 ));
16647 (client_rx, bind_request)
16648 }
16649
16650 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
16651 assert_eq!(frame.header.ty, FrameType::Push);
16652 assert_eq!(frame.header.channel, 0);
16653 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
16654 .expect("route.closed control push decodes");
16655 let ClientControlPush::RouteClosed { channels, .. } = &push else {
16656 panic!("expected route.closed");
16657 };
16658 assert_eq!(channels.len(), 1, "exactly one violating route closed");
16659 let closed_channel = channels[0];
16660 let channels = channels.clone();
16661 assert_eq!(
16662 push,
16663 ClientControlPush::RouteClosed {
16664 module_id: target_module_id.to_string(),
16665 channels,
16666 reason: RouteCloseReason::CapabilityDenied,
16667 drained: false,
16668 abandoned: 0,
16669 excluded_subscriptions: 0,
16670 terminal: Some(false),
16671 }
16672 );
16673 closed_channel
16674 }
16675
16676 #[tokio::test]
16677 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
16678 let registry = Arc::new(Registry::default());
16679 let forwarding = Arc::new(ForwardingTable::default());
16680 let supervisor = SupervisorHandle::new();
16681 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16682 let handler =
16683 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16684 .with_supervisor(supervisor);
16685 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
16686 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
16687 register_capability_manifest(
16688 &handler,
16689 &target_ctx,
16690 &mut target_rx,
16691 capability_manifest("target", &["credentials-provider/v1"], &[]),
16692 1,
16693 )
16694 .await;
16695 register_capability_manifest(
16696 &handler,
16697 &opener_ctx,
16698 &mut opener_rx,
16699 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16700 2,
16701 )
16702 .await;
16703
16704 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
16705 let replies = handler
16706 .handle_control_frame(
16707 &client_ctx,
16708 route_open_frame_with_admission_facts(
16709 3,
16710 "target",
16711 unique_project_root("admission-facts"),
16712 Some(ConsumerIdentity {
16713 module_id: "opener".to_string(),
16714 launch_nonce: "opener-nonce".to_string(),
16715 }),
16716 None,
16717 ),
16718 )
16719 .await
16720 .expect("denied route.open returns a typed frame");
16721 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16722 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16723 assert!(
16724 target_rx.try_recv().is_err(),
16725 "forbidden route.open must not relay route.bind"
16726 );
16727 }
16728
16729 #[tokio::test]
16730 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
16731 let registry = Arc::new(Registry::default());
16732 let forwarding = Arc::new(ForwardingTable::default());
16733 let supervisor = SupervisorHandle::new();
16734 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16735 let handler =
16736 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16737 .with_supervisor(supervisor);
16738 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
16739 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
16740 register_capability_manifest(
16741 &handler,
16742 &target_ctx,
16743 &mut target_rx,
16744 capability_manifest("target", &["credentials-provider/v1"], &[]),
16745 1,
16746 )
16747 .await;
16748 register_capability_manifest(
16749 &handler,
16750 &old_opener_ctx,
16751 &mut old_opener_rx,
16752 capability_manifest("opener", &[], &[]),
16753 2,
16754 )
16755 .await;
16756 let (mut client_rx, _) = open_route_for_capability_test(
16757 &handler,
16758 &target_ctx,
16759 &mut target_rx,
16760 712,
16761 3,
16762 "target",
16763 Some(ConsumerIdentity {
16764 module_id: "opener".to_string(),
16765 launch_nonce: "opener-nonce".to_string(),
16766 }),
16767 )
16768 .await;
16769 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16770
16771 handler
16772 .cleanup_connection(old_opener_ctx.connection_id)
16773 .expect("old opener registration cleans up");
16774 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
16775 register_capability_manifest(
16776 &handler,
16777 &new_opener_ctx,
16778 &mut new_opener_rx,
16779 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16780 4,
16781 )
16782 .await;
16783
16784 let closed_channel = assert_capability_denied_push(
16785 client_rx
16786 .try_recv()
16787 .expect("HELLO deny addition must emit route.closed")
16788 .frame,
16789 "target",
16790 );
16791 let goodbye = client_rx
16792 .try_recv()
16793 .expect("HELLO deny addition must send the client a route GOODBYE")
16794 .frame;
16795 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16796 assert_eq!(goodbye.header.channel, closed_channel);
16797 assert_ne!(goodbye.header.channel, 0);
16798 assert_ne!(goodbye.header.epoch, 0);
16799 assert!(client_rx.try_recv().is_err());
16800 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16801 assert!(matches!(
16802 target_rx.try_recv(),
16803 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16804 ));
16805 }
16806
16807 #[tokio::test]
16808 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
16809 let registry = Arc::new(Registry::default());
16810 let forwarding = Arc::new(ForwardingTable::default());
16811 let supervisor = SupervisorHandle::new();
16812 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16813 let handler =
16814 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16815 .with_supervisor(supervisor);
16816 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
16817 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
16818 register_capability_manifest(
16819 &handler,
16820 &target_ctx,
16821 &mut target_rx,
16822 capability_manifest("target", &[], &[]),
16823 1,
16824 )
16825 .await;
16826 register_capability_manifest(
16827 &handler,
16828 &opener_ctx,
16829 &mut opener_rx,
16830 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16831 2,
16832 )
16833 .await;
16834 let (mut client_rx, _) = open_route_for_capability_test(
16835 &handler,
16836 &target_ctx,
16837 &mut target_rx,
16838 722,
16839 3,
16840 "target",
16841 Some(ConsumerIdentity {
16842 module_id: "opener".to_string(),
16843 launch_nonce: "opener-nonce".to_string(),
16844 }),
16845 )
16846 .await;
16847 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16848
16849 let replies = handler
16850 .handle_control_frame(
16851 &target_ctx,
16852 catalog_update_with_capabilities_frame(
16853 4,
16854 CapabilityDeclarations {
16855 provides: vec!["credentials-provider/v1".to_string()],
16856 requires: Vec::new(),
16857 must_never_reach: Vec::new(),
16858 },
16859 ),
16860 )
16861 .await
16862 .expect("claim catalog.update succeeds");
16863 assert!(matches!(
16864 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
16865 Ok(ModuleControlResponseToModule::CatalogUpdate {})
16866 ));
16867 let closed_channel = assert_capability_denied_push(
16868 client_rx
16869 .try_recv()
16870 .expect("claim addition must emit route.closed")
16871 .frame,
16872 "target",
16873 );
16874 let goodbye = client_rx
16875 .try_recv()
16876 .expect("claim addition must send the client a route GOODBYE")
16877 .frame;
16878 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16879 assert_eq!(goodbye.header.channel, closed_channel);
16880 assert_ne!(goodbye.header.channel, 0);
16881 assert_ne!(goodbye.header.epoch, 0);
16882 assert!(client_rx.try_recv().is_err());
16883 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16884 assert!(matches!(
16885 target_rx.try_recv(),
16886 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16887 ));
16888 }
16889
16890 #[tokio::test]
16891 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
16892 let registry = Arc::new(Registry::default());
16893 let forwarding = Arc::new(ForwardingTable::default());
16894 let supervisor = SupervisorHandle::new();
16895 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16896 let handler =
16897 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16898 .with_supervisor(supervisor);
16899 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
16900 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
16901 register_capability_manifest(
16902 &handler,
16903 &target_ctx,
16904 &mut target_rx,
16905 capability_manifest("target", &["credentials-provider/v1"], &[]),
16906 1,
16907 )
16908 .await;
16909 register_capability_manifest(
16910 &handler,
16911 &opener_ctx,
16912 &mut opener_rx,
16913 capability_manifest("opener", &[], &[]),
16914 2,
16915 )
16916 .await;
16917 let (mut client_rx, _) = open_route_for_capability_test(
16918 &handler,
16919 &target_ctx,
16920 &mut target_rx,
16921 732,
16922 3,
16923 "target",
16924 Some(ConsumerIdentity {
16925 module_id: "opener".to_string(),
16926 launch_nonce: "opener-nonce".to_string(),
16927 }),
16928 )
16929 .await;
16930 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16931
16932 handler
16933 .handle_control_frame(
16934 &target_ctx,
16935 catalog_update_with_capabilities_frame(
16936 4,
16937 CapabilityDeclarations {
16938 provides: Vec::new(),
16939 requires: Vec::new(),
16940 must_never_reach: Vec::new(),
16941 },
16942 ),
16943 )
16944 .await
16945 .expect("claim removal catalog.update succeeds");
16946 assert_eq!(
16947 forwarding.active_binding_count().unwrap(),
16948 1,
16949 "removing an attested target claim must leave the route census unchanged"
16950 );
16951 assert!(
16952 client_rx.try_recv().is_err(),
16953 "claim removal must not emit route.closed or a client route GOODBYE"
16954 );
16955 assert!(
16956 target_rx.try_recv().is_err(),
16957 "claim removal must not send the target a route GOODBYE"
16958 );
16959 }
16960
16961 #[tokio::test]
16964 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
16965 let registry = Arc::new(Registry::default());
16966 let forwarding = Arc::new(ForwardingTable::default());
16967 let supervisor = SupervisorHandle::new();
16968 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16969 let handler =
16970 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16971 .with_supervisor(supervisor);
16972 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
16973 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
16974 register_capability_manifest(
16975 &handler,
16976 &target_ctx,
16977 &mut target_rx,
16978 capability_manifest("target", &["credentials-provider/v1"], &[]),
16979 1,
16980 )
16981 .await;
16982 register_capability_manifest(
16983 &handler,
16984 &opener_ctx,
16985 &mut opener_rx,
16986 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16987 2,
16988 )
16989 .await;
16990
16991 let (_client_rx, bind) = open_route_for_capability_test(
16992 &handler,
16993 &target_ctx,
16994 &mut target_rx,
16995 742,
16996 3,
16997 "target",
16998 None,
16999 )
17000 .await;
17001 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
17002 panic!("direct scope-honesty route must bind");
17003 };
17004 assert_eq!(principal, Some(Principal::Direct));
17005 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17006 }
17007
17008 #[tokio::test]
17011 async fn must_never_reach_self_route_is_capability_forbidden() {
17012 let registry = Arc::new(Registry::default());
17013 let forwarding = Arc::new(ForwardingTable::default());
17014 let supervisor = SupervisorHandle::new();
17015 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
17016 let handler =
17017 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17018 .with_supervisor(supervisor);
17019 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
17020 register_capability_manifest(
17021 &handler,
17022 &self_ctx,
17023 &mut self_rx,
17024 capability_manifest(
17025 "self-provider",
17026 &["credentials-provider/v1"],
17027 &["credentials-provider/v1"],
17028 ),
17029 1,
17030 )
17031 .await;
17032
17033 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
17034 let replies = handler
17035 .handle_control_frame(
17036 &client_ctx,
17037 route_open_frame_with_admission_facts(
17038 2,
17039 "self-provider",
17040 unique_project_root("admission-facts"),
17041 Some(ConsumerIdentity {
17042 module_id: "self-provider".to_string(),
17043 launch_nonce: "self-nonce".to_string(),
17044 }),
17045 None,
17046 ),
17047 )
17048 .await
17049 .expect("self-route refusal returns a typed frame");
17050 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
17051 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17052 assert!(
17053 self_rx.try_recv().is_err(),
17054 "self denial must not relay route.bind"
17055 );
17056 }
17057
17058 #[test]
17059 fn unsupported_channel_zero_frame_returns_error() {
17060 let handler = ControlHandler::default();
17061 let request = Frame::build(
17062 FrameType::Request,
17063 control_flags(),
17064 0,
17065 0,
17066 21,
17067 b"opaque".to_vec(),
17068 )
17069 .unwrap();
17070
17071 let response = handler
17072 .handle_control(ConnectionId::new(1), request)
17073 .unwrap();
17074
17075 assert_eq!(response[0].header.ty, FrameType::Error);
17076 assert_eq!(
17077 parse_error(&response[0])["code"],
17078 "unsupported_control_frame"
17079 );
17080 }
17081
17082 mod swap {
17087 use super::*;
17088
17089 const INCUMBENT: ConnectionId = ConnectionId::new(30);
17090 const CANDIDATE: ConnectionId = ConnectionId::new(40);
17091
17092 struct Swap {
17093 registry: Arc<Registry>,
17094 forwarding: Arc<ForwardingTable>,
17095 handler: ControlHandler,
17096 incumbent_ctx: RouteCtx,
17097 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17098 candidate_ctx: RouteCtx,
17099 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17100 }
17101
17102 async fn swap_with_incumbent() -> Swap {
17103 let registry = Arc::new(Registry::default());
17104 let forwarding = Arc::new(ForwardingTable::default());
17105 let handler =
17106 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17107 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
17108 hello_via_sink(
17109 &handler,
17110 &incumbent_ctx,
17111 &mut incumbent_rx,
17112 hello_frame("aft", PROTOCOL_VERSION, 7),
17113 )
17114 .await;
17115 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
17116 Swap {
17117 registry,
17118 forwarding,
17119 handler,
17120 incumbent_ctx,
17121 incumbent_rx,
17122 candidate_ctx,
17123 candidate_rx,
17124 }
17125 }
17126
17127 fn register_candidate(swap: &Swap, ready: Option<bool>) {
17128 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
17129 candidate_manifest.ready = ready;
17130 let registration = swap
17131 .registry
17132 .register_candidate_with_control_ops(
17133 candidate_manifest,
17134 PROTOCOL_VERSION,
17135 CANDIDATE,
17136 module_baseline_control_ops(),
17137 )
17138 .unwrap();
17139 swap.forwarding
17140 .register_candidate_module_connection(
17141 CANDIDATE,
17142 "aft".to_string(),
17143 PROTOCOL_VERSION,
17144 manifest_concurrency(®istration.manifest),
17145 swap.candidate_ctx.egress.clone(),
17146 )
17147 .unwrap();
17148 }
17149
17150 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
17151 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
17152 swap.registry.promote_candidate("aft").unwrap().unwrap();
17153 cutover.incumbent.unwrap()
17154 }
17155
17156 #[tokio::test]
17165 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
17166 let mut swap = swap_with_incumbent().await;
17167 let hold = Arc::new(std::sync::Barrier::new(2));
17168 let handler = swap
17169 .handler
17170 .clone()
17171 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
17172 handler
17173 .supervisor
17174 .open_swap("aft", "candidate-nonce".to_string());
17175 let hello = {
17176 let handler = handler.clone();
17177 let candidate_ctx = swap.candidate_ctx.clone();
17178 std::thread::spawn(move || {
17179 handler
17180 .handle_hello(
17181 candidate_ctx.connection_id,
17182 Some(candidate_ctx.egress.clone()),
17183 hello_frame_with_nonce(
17184 "aft",
17185 PROTOCOL_VERSION,
17186 8,
17187 Some("candidate-nonce"),
17188 ),
17189 )
17190 .unwrap()
17191 })
17192 };
17193
17194 hold.wait();
17195 assert!(swap
17198 .registry
17199 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
17200 .unwrap()
17201 .is_some());
17202 assert!(swap
17203 .registry
17204 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17205 .unwrap()
17206 .is_none());
17207 assert!(swap
17208 .forwarding
17209 .module_endpoint_for_connection(CANDIDATE)
17210 .unwrap()
17211 .is_none());
17212
17213 hold.wait();
17214 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
17215 assert_eq!(
17216 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
17217 FrameType::HelloAck
17218 );
17219 let published = swap
17220 .registry
17221 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17222 .unwrap()
17223 .expect("candidate is published once its HELLO completes");
17224 assert_eq!(published.connection_id, CANDIDATE);
17225 assert!(swap
17226 .forwarding
17227 .module_endpoint_for_connection(CANDIDATE)
17228 .unwrap()
17229 .is_some());
17230 }
17231
17232 fn keyed_total(counters: &Value, key: &str) -> u64 {
17233 counters[key]
17234 .as_object()
17235 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
17236 .unwrap_or(0)
17237 }
17238
17239 #[tokio::test]
17240 async fn replacement_logs_old_end_and_new_admission_once() {
17241 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
17242 let mut swap = swap_with_incumbent().await;
17243 swap.handler
17244 .supervisor
17245 .open_swap("aft", "candidate-nonce".to_string());
17246 hello_via_sink(
17247 &swap.handler,
17248 &swap.candidate_ctx,
17249 &mut swap.candidate_rx,
17250 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
17251 )
17252 .await;
17253 cutover(&swap);
17254 swap.handler.cleanup_connection(INCUMBENT).unwrap();
17255
17256 let captured = crate::router::test_log::captured_logs(&logs);
17257 println!("captured replacement registry logs:\n{captured}");
17258 let new_admissions: Vec<_> = captured
17259 .lines()
17260 .filter(|line| {
17261 line.contains("connection_id=40")
17262 && line.contains("swap candidate registered; not routable until cutover")
17263 })
17264 .collect();
17265 assert_eq!(
17266 new_admissions.len(),
17267 1,
17268 "unexpected admission logs: {captured}"
17269 );
17270 assert!(new_admissions[0].contains("module_id=aft"));
17271 assert!(new_admissions[0].contains("connection_id=40"));
17272 assert!(
17273 new_admissions[0].contains("swap candidate registered; not routable until cutover")
17274 );
17275 assert!(new_admissions[0].contains("ready=true"));
17276
17277 let old_admissions: Vec<_> = captured
17278 .lines()
17279 .filter(|line| {
17280 line.contains("module registered module_id=aft ")
17281 && line.contains("connection_id=30")
17282 })
17283 .collect();
17284 assert_eq!(
17285 old_admissions.len(),
17286 1,
17287 "unexpected incumbent admission: {captured}"
17288 );
17289
17290 let old_ends: Vec<_> = captured
17291 .lines()
17292 .filter(|line| {
17293 line.contains("connection_id=30") && line.contains("module registration ended")
17294 })
17295 .collect();
17296 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
17297 assert!(old_ends[0].contains("module_id=aft"));
17298 assert!(old_ends[0].contains("connection_id=30"));
17299 assert!(old_ends[0].contains("reason=replaced"));
17300 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
17301 assert_eq!(
17302 captured
17303 .lines()
17304 .filter(|line| line.contains("module registration promoted"))
17305 .count(),
17306 1,
17307 "unexpected promotion logs: {captured}"
17308 );
17309 }
17310
17311 #[tokio::test]
17317 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
17318 let mut swap = swap_with_incumbent().await;
17319 let handler = swap.handler.clone();
17320
17321 let cotenant = ConnectionId::new(31);
17323 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
17324 let (cotenant_task, cotenant_bind) = relay_route_open(
17325 &handler,
17326 cotenant,
17327 &cotenant_ctx.egress,
17328 &mut swap.incumbent_rx,
17329 100,
17330 "aft",
17331 "swap-cotenant",
17332 )
17333 .await;
17334 handler
17335 .handle_control_frame(
17336 &swap.incumbent_ctx,
17337 route_bind_ack(cotenant_bind.header.corr),
17338 )
17339 .await
17340 .unwrap();
17341 assert!(cotenant_task.await.unwrap().is_empty());
17342 let (cotenant_channel, cotenant_epoch) =
17343 published_route(&cotenant_rx.recv().await.unwrap());
17344
17345 let caller = ConnectionId::new(32);
17347 let (caller_ctx, mut caller_rx) = route_ctx(caller);
17348 let (caller_task, caller_bind) = relay_route_open(
17349 &handler,
17350 caller,
17351 &caller_ctx.egress,
17352 &mut swap.incumbent_rx,
17353 101,
17354 "aft",
17355 "swap-caller",
17356 )
17357 .await;
17358 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
17359
17360 register_candidate(&swap, None);
17361 cutover(&swap);
17362
17363 let ack = handler
17365 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
17366 .await;
17367 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
17368 if module_loop_error.is_some() {
17369 handler.cleanup_connection(INCUMBENT).unwrap();
17372 }
17373
17374 assert!(
17376 cotenant_rx.try_recv().is_err(),
17377 "the co-tenant route on the incumbent was torn down by one late ack: \
17378 {module_loop_error:?}"
17379 );
17380 assert!(matches!(
17381 swap.forwarding
17382 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
17383 .unwrap(),
17384 DataRoute::Client(DataRouteState::Bound(_))
17385 ));
17386 assert_eq!(module_loop_error, None);
17387 assert!(swap
17388 .registry
17389 .get_module_by_connection(INCUMBENT)
17390 .unwrap()
17391 .is_some());
17392
17393 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
17395 .await
17396 .expect("the incumbent is told to drop the abandoned binding")
17397 .unwrap()
17398 .frame;
17399 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17400 assert_eq!(goodbye.header.channel, abandoned_channel);
17401 assert_eq!(goodbye.header.epoch, abandoned_epoch);
17402 assert!(swap.incumbent_rx.try_recv().is_err());
17403
17404 let response = caller_task.await.unwrap();
17406 assert_eq!(response.len(), 1);
17407 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17408 assert!(caller_rx.try_recv().is_err());
17409
17410 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17414 let counters = handler.counters().snapshot();
17415 assert_eq!(
17416 keyed_total(&counters, "route_open_accepted_by_principal"),
17417 1
17418 );
17419 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17420 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17421 }
17422
17423 #[tokio::test]
17427 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17428 let mut swap = swap_with_incumbent().await;
17429 register_candidate(&swap, None);
17430 let incumbent = cutover(&swap);
17431 swap.forwarding
17432 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17433 .unwrap()
17434 .expect("the incumbent is still registered");
17435
17436 let client = ConnectionId::new(33);
17437 let (client_ctx, mut client_rx) = route_ctx(client);
17438 let route_handler = swap.handler.clone();
17439 let open_ctx = RouteCtx {
17440 connection_id: client,
17441 egress: client_ctx.egress.clone(),
17442 };
17443 let mut route_task = tokio::spawn(async move {
17444 route_handler
17445 .handle_control_frame(
17446 &open_ctx,
17447 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
17448 )
17449 .await
17450 .unwrap()
17451 });
17452 let bind = tokio::select! {
17453 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
17454 response = &mut route_task => {
17455 let response = response.unwrap();
17456 panic!(
17457 "post-cutover route.open was refused instead of relayed to the candidate: {}",
17458 parse_error(&response[0])["code"]
17459 );
17460 }
17461 };
17462 swap.handler
17463 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
17464 .await
17465 .unwrap();
17466 assert!(route_task.await.unwrap().is_empty());
17467 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
17468 match swap
17469 .forwarding
17470 .lookup_data_route(client, channel, epoch)
17471 .unwrap()
17472 {
17473 DataRoute::Client(DataRouteState::Bound(route)) => {
17474 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
17475 }
17476 other => panic!("expected a bound route on the candidate, got {other:?}"),
17477 }
17478 assert!(swap.incumbent_rx.try_recv().is_err());
17479 }
17480
17481 #[tokio::test]
17486 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
17487 let swap = swap_with_incumbent().await;
17488 register_candidate(&swap, Some(false));
17489 let update = Frame::build(
17490 FrameType::Request,
17491 control_flags(),
17492 0,
17493 0,
17494 55,
17495 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17496 provides: manifest("aft", PROTOCOL_VERSION).provides,
17497 capabilities: None,
17498 ready: Some(true),
17499 })
17500 .unwrap(),
17501 )
17502 .unwrap();
17503
17504 let replies = swap
17505 .handler
17506 .handle_control_frame(&swap.candidate_ctx, update)
17507 .await
17508 .unwrap();
17509
17510 assert_eq!(replies.len(), 1);
17511 assert_eq!(
17512 replies[0].header.ty,
17513 FrameType::Response,
17514 "candidate catalog.update was refused: {:?}",
17515 serde_json::from_slice::<Value>(&replies[0].body).ok()
17516 );
17517 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
17518 assert_eq!(
17519 swap.registry
17520 .get_module("aft")
17521 .unwrap()
17522 .unwrap()
17523 .connection_id,
17524 INCUMBENT
17525 );
17526 }
17527 }
17528
17529 mod swap_admission {
17533 use super::*;
17534
17535 const INCUMBENT_NONCE: &str = "incumbent-nonce";
17536 const CANDIDATE_NONCE: &str = "candidate-nonce";
17537
17538 fn handler_with_incumbent(
17539 module_id: &str,
17540 reserved: bool,
17541 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
17542 let registry = Arc::new(Registry::default());
17543 let supervisor = SupervisorHandle::new();
17544 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
17545 if reserved {
17546 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
17547 }
17548 let handler =
17549 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
17550 let incumbent = handler
17551 .handle_control(
17552 ConnectionId::new(1),
17553 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
17554 )
17555 .unwrap();
17556 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
17557 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
17558 (registry, supervisor, handler)
17559 }
17560
17561 #[test]
17567 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
17568 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
17569
17570 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
17571 let replies = handler
17572 .handle_control(
17573 ConnectionId::new(connection),
17574 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
17575 )
17576 .unwrap();
17577 assert_eq!(replies[0].header.ty, FrameType::Error);
17578 assert_eq!(
17579 parse_error(&replies[0])["code"],
17580 "swap_token_invalid",
17581 "nonce {nonce:?}"
17582 );
17583 }
17584 assert!(registry.get_candidate("aft").unwrap().is_none());
17585 assert_eq!(
17586 registry.get_module("aft").unwrap().unwrap().connection_id,
17587 ConnectionId::new(1)
17588 );
17589
17590 let admitted = handler
17593 .handle_control(
17594 ConnectionId::new(4),
17595 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
17596 )
17597 .unwrap();
17598 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
17599 assert_eq!(
17600 registry
17601 .get_candidate("aft")
17602 .unwrap()
17603 .unwrap()
17604 .connection_id,
17605 ConnectionId::new(4)
17606 );
17607 assert_eq!(
17608 registry.get_module("aft").unwrap().unwrap().connection_id,
17609 ConnectionId::new(1),
17610 "the candidate must not take the active slot"
17611 );
17612 let replayed = handler
17613 .handle_control(
17614 ConnectionId::new(5),
17615 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
17616 )
17617 .unwrap();
17618 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
17619
17620 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
17624 let squatter = handler
17625 .handle_control(
17626 ConnectionId::new(6),
17627 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
17628 )
17629 .unwrap();
17630 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
17631 assert!(
17632 registry.get_module("aft").unwrap().is_none(),
17633 "a squatter took the active slot of an id being swapped"
17634 );
17635 }
17636
17637 #[test]
17642 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
17643 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
17644
17645 let replies = handler
17646 .handle_control(
17647 ConnectionId::new(2),
17648 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17649 )
17650 .unwrap();
17651
17652 assert_eq!(
17653 replies[0].header.ty,
17654 FrameType::HelloAck,
17655 "reserved candidate refused: {:?}",
17656 serde_json::from_slice::<Value>(&replies[0].body).ok()
17657 );
17658 assert_eq!(
17659 registry
17660 .get_candidate("vault")
17661 .unwrap()
17662 .unwrap()
17663 .connection_id,
17664 ConnectionId::new(2)
17665 );
17666 }
17667
17668 #[test]
17671 fn without_an_open_swap_the_ordinary_gates_decide() {
17672 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
17673 supervisor.close_swap("vault");
17674
17675 let candidate = handler
17676 .handle_control(
17677 ConnectionId::new(2),
17678 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17679 )
17680 .unwrap();
17681 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
17682 let duplicate = handler
17683 .handle_control(
17684 ConnectionId::new(3),
17685 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
17686 )
17687 .unwrap();
17688 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17689 assert!(registry.get_candidate("vault").unwrap().is_none());
17690 }
17691 }
17692
17693 mod scopes {
17697 mod idle_tests {
17698 use super::*;
17699 include!("scope_idle_tests.rs");
17700 }
17701 mod route_open_timings {
17702 use super::*;
17703 include!("route_open_timing_tests.rs");
17704 }
17705 mod route_open_refusal_log {
17706 use super::*;
17707 include!("route_open_refusal_log_tests.rs");
17708 }
17709 mod activity {
17710 include!("activity_scope_tests.rs");
17711 }
17712 mod delta {
17713 include!("scope_handler_delta_tests.rs");
17714 }
17715 mod call_modes {
17716 include!("scope_call_mode_tests.rs");
17717 }
17718 mod federated {
17719 include!("scope_federated_tests.rs");
17720 }
17721 #[tokio::test]
17722 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
17723 let rig = rig().await;
17724 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
17725 "s".into(),
17726 1,
17727 "tool".into(),
17728 ))
17729 .unwrap();
17730 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
17731 let (direct, _rx) = wide_ctx(999);
17732 for (ctx, expected) in [
17733 (&rig.owner, FrameType::Response),
17734 (&direct, FrameType::Error),
17735 ] {
17736 let frame =
17737 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
17738 .unwrap();
17739 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
17740 assert_eq!(replies[0].header.ty, expected);
17741 if expected == FrameType::Response {
17742 let reply: subc_control::ScopeServedByReply =
17743 serde_json::from_slice(&replies[0].body).unwrap();
17744 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
17745 } else {
17746 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
17747 }
17748 }
17749 let reply = call(
17750 &rig.handler,
17751 &rig.owner,
17752 &ModuleControlRequestFromModule::ScopeDescribe {
17753 owner: Principal::Reserved {
17754 module_id: OWNER.into(),
17755 },
17756 scope_ref: "s".into(),
17757 },
17758 )
17759 .await;
17760 assert!(matches!(
17761 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
17762 ModuleControlResponseToModule::ScopeDescribe { .. }
17763 ));
17764 }
17765
17766 use subc_protocol::scope::{
17767 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
17768 ScopeStatus,
17769 };
17770
17771 use super::*;
17772
17773 const OWNER: &str = "prefrontal-core";
17774
17775 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
17776 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
17777 }
17778
17779 async fn call(
17780 handler: &ControlHandler,
17781 ctx: &RouteCtx,
17782 request: &ModuleControlRequestFromModule,
17783 ) -> Frame {
17784 let body = serde_json::to_vec(request).unwrap();
17785 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
17786 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
17787 assert_eq!(replies.len(), 1, "{replies:?}");
17788 replies.pop().unwrap()
17789 }
17790
17791 async fn sync(
17792 handler: &ControlHandler,
17793 ctx: &RouteCtx,
17794 generation: u64,
17795 scopes: Vec<ScopeRecord>,
17796 ) -> Result<ModuleControlResponseToModule, String> {
17797 let reply = call(
17798 handler,
17799 ctx,
17800 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
17801 )
17802 .await;
17803 match reply.header.ty {
17804 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
17805 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
17806 }
17807 }
17808
17809 async fn describe(
17810 handler: &ControlHandler,
17811 ctx: &RouteCtx,
17812 owner: &str,
17813 scope_ref: &str,
17814 ) -> ModuleControlResponseToModule {
17815 let reply = call(
17816 handler,
17817 ctx,
17818 &ModuleControlRequestFromModule::ScopeDescribe {
17819 owner: Principal::Reserved {
17820 module_id: owner.to_string(),
17821 },
17822 scope_ref: scope_ref.to_string(),
17823 },
17824 )
17825 .await;
17826 assert_eq!(
17827 reply.header.ty,
17828 FrameType::Response,
17829 "{:?}",
17830 parse_error(&reply)
17831 );
17832 serde_json::from_slice(&reply.body).unwrap()
17833 }
17834
17835 #[tokio::test]
17836 async fn scope_describe_never_echoes_bind_proof_or_hash() {
17837 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17838 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17839 let mut rig = rig().await;
17840 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17841 .await;
17842 let _bound = rig
17843 .bound(
17844 Some(AFT),
17845 PLEXUS,
17846 Some(
17847 rig.selector("s", Some(1))
17848 .with_bind_proof(Some(PROOF.into())),
17849 ),
17850 )
17851 .await;
17852 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
17853 let output = serde_json::to_string(&reply).unwrap();
17854 assert!(output.contains("\"status\":\"live\""));
17855 assert!(!output.contains(PROOF));
17856 assert!(!output.contains(HASH));
17857 }
17858
17859 async fn module(
17861 handler: &ControlHandler,
17862 connection: u64,
17863 module_id: &str,
17864 nonce: Option<&str>,
17865 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17866 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
17867 hello_via_sink(
17868 handler,
17869 &ctx,
17870 &mut rx,
17871 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
17872 )
17873 .await;
17874 (ctx, rx)
17875 }
17876
17877 #[tokio::test]
17880 async fn a_client_connection_cannot_sync_or_describe() {
17881 let handler = ControlHandler::new(Arc::new(Registry::default()));
17882 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
17883 for request in [
17884 ModuleControlRequestFromModule::ScopeSync {
17885 generation: 1,
17886 scopes: vec![head("s", 1)],
17887 },
17888 ModuleControlRequestFromModule::ScopeDescribe {
17889 owner: Principal::Direct,
17890 scope_ref: "s".to_string(),
17891 },
17892 ] {
17893 let reply = call(&handler, &ctx, &request).await;
17894 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
17895 }
17896 assert!(
17897 !handler
17898 .scopes
17899 .read()
17900 .unwrap()
17901 .describe(
17902 &Principal::Reserved {
17903 module_id: OWNER.to_string()
17904 },
17905 "s"
17906 )
17907 .owner_synced
17908 );
17909 }
17910
17911 #[tokio::test]
17914 async fn a_module_without_a_supervised_launch_cannot_sync() {
17915 let handler = ControlHandler::new(Arc::new(Registry::default()));
17916 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
17917 assert_eq!(
17918 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
17919 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17920 );
17921 }
17922
17923 #[tokio::test]
17924 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
17925 let supervisor = SupervisorHandle::new();
17926 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17927 let handler = ControlHandler::new(Arc::new(Registry::default()))
17928 .with_supervisor(supervisor.clone());
17929 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17930 sync(&handler, &incumbent, 1, vec![head("s", 1)])
17931 .await
17932 .expect("the current launch syncs");
17933
17934 supervisor.open_swap(OWNER, "n2".to_string());
17937 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
17938 assert_eq!(
17939 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17940 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17941 );
17942 sync(&handler, &incumbent, 2, vec![head("s", 1)])
17943 .await
17944 .expect("the serving owner syncs during the swap");
17945
17946 supervisor.close_swap(OWNER);
17949 assert_eq!(
17950 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17951 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17952 );
17953 sync(&handler, &incumbent, 3, vec![head("s", 1)])
17954 .await
17955 .expect("the serving owner syncs after the rollback");
17956 handler.cleanup_connection(candidate.connection_id).unwrap();
17957
17958 supervisor.open_swap(OWNER, "n3".to_string());
17963 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
17964 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
17965 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
17966 .await
17967 .expect("the promoted launch takes authority");
17968 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17969 panic!("unexpected reply {reply:?}");
17970 };
17971 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
17972 assert_eq!(
17973 sync(&handler, &incumbent, 4, Vec::new()).await,
17974 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17975 );
17976 }
17977
17978 #[tokio::test]
17981 async fn closing_the_authority_connection_frees_sync_authority() {
17982 let supervisor = SupervisorHandle::new();
17983 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17984 let handler = ControlHandler::new(Arc::new(Registry::default()))
17985 .with_supervisor(supervisor.clone());
17986 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17987 sync(&handler, &first, 10, vec![head("s", 1)])
17988 .await
17989 .unwrap();
17990 handler.cleanup_connection(first.connection_id).unwrap();
17991
17992 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
17993 sync(&handler, &second, 1, vec![head("s", 1)])
17994 .await
17995 .expect("the next connection takes the released authority");
17996 }
17997
17998 #[tokio::test]
17999 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
18000 let supervisor = SupervisorHandle::new();
18001 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18002 let handler =
18003 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
18004 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
18005 sync(&handler, &first, 10, vec![head("s", 1)])
18006 .await
18007 .unwrap();
18008 handler
18009 .handle_control_frame(
18010 &first,
18011 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
18012 )
18013 .await
18014 .unwrap();
18015 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
18016 sync(&handler, &second, 1, vec![head("s", 1)])
18017 .await
18018 .expect("GOODBYE releases authority even if the old socket remains open");
18019 }
18020
18021 #[tokio::test]
18022 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
18023 let registry = Arc::new(Registry::default());
18024 let supervisor_handle = SupervisorHandle::new();
18025 let supervisor =
18026 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
18027 .with_handle(supervisor_handle.clone())
18028 .with_daemon_incarnation("incarnation-7".to_string());
18029 supervisor
18032 .supervise_configured(
18033 ModuleSpec {
18034 carry_containment: false,
18035 module_id: OWNER.to_string(),
18036 program: PathBuf::from("/nonexistent/prefrontal-core"),
18037 args: Vec::new(),
18038 env: Vec::new(),
18039 reserved: false,
18040 reserved_prefixes: Vec::new(),
18041 protocol: ModuleProtocol::Subc,
18042 overlap: Default::default(),
18043 },
18044 false,
18045 )
18046 .unwrap();
18047 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
18048 let handler =
18049 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
18050 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
18051
18052 let ModuleControlResponseToModule::ScopeDescribe {
18054 status,
18055 daemon_incarnation,
18056 owner_synced,
18057 owner_configured,
18058 scope,
18059 ..
18060 } = describe(&handler, &reader, OWNER, "s").await
18061 else {
18062 panic!("not a describe reply");
18063 };
18064 assert_eq!(status, ScopeStatus::NotLive);
18065 assert_eq!(daemon_incarnation, "incarnation-7");
18066 assert!(!owner_synced);
18067 assert!(owner_configured);
18068 assert!(scope.is_none());
18069
18070 let ModuleControlResponseToModule::ScopeDescribe {
18073 status,
18074 owner_configured,
18075 ..
18076 } = describe(&handler, &reader, "ghost", "s").await
18077 else {
18078 panic!("not a describe reply");
18079 };
18080 assert_eq!(status, ScopeStatus::NotLive);
18081 assert!(!owner_configured);
18082
18083 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
18085 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
18086 let ModuleControlResponseToModule::ScopeDescribe {
18087 status,
18088 scope_epoch,
18089 owner_synced,
18090 scope,
18091 ..
18092 } = describe(&handler, &reader, OWNER, "s").await
18093 else {
18094 panic!("not a describe reply");
18095 };
18096 assert_eq!(status, ScopeStatus::Live);
18097 assert_eq!(scope_epoch, Some(4));
18098 assert!(owner_synced);
18099 let stamp = scope.expect("a live scope carries its stamp");
18100 assert!(
18101 stamp.owner_authorized,
18102 "prefrontal-core is the default authority"
18103 );
18104 assert_eq!(stamp.kind, ScopeKind::Head);
18105 }
18106
18107 #[tokio::test]
18108 async fn scope_authority_owners_decides_owner_authorized() {
18109 let supervisor = SupervisorHandle::new();
18110 supervisor.set_spawn_nonce("broca", "b1".to_string());
18111 let handler = ControlHandler::new(Arc::new(Registry::default()))
18112 .with_supervisor(supervisor)
18113 .with_scope_authority_owners(vec!["broca".to_string()]);
18114 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
18115 let mut gated = head("s", 1);
18116 gated.attributes.agent_id = Some("agent".to_string());
18117 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
18118 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
18119 describe(&handler, &broca, "broca", "s").await
18120 else {
18121 panic!("not a describe reply");
18122 };
18123 assert!(scope.unwrap().owner_authorized);
18124 }
18125
18126 #[tokio::test]
18130 async fn scope_ops_and_the_scopes_capability_are_advertised() {
18131 let handler = ControlHandler::new(Arc::new(Registry::default()));
18132 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
18133 let ack = hello_via_sink(
18134 &handler,
18135 &ctx,
18136 &mut rx,
18137 hello_frame("m", PROTOCOL_VERSION, 1),
18138 )
18139 .await;
18140 let ack = parse_ack(&ack);
18141 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
18142 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
18143 }
18144 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
18145
18146 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
18147 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
18148 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
18149 let reply = handler
18150 .handle_control_frame(&client, frame)
18151 .await
18152 .unwrap()
18153 .pop()
18154 .unwrap();
18155 let ClientControlResponse::ServerDescribe { capabilities, .. } =
18156 serde_json::from_slice(&reply.body).unwrap()
18157 else {
18158 panic!("not a server.describe reply");
18159 };
18160 assert!(
18161 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
18162 "{capabilities:?}"
18163 );
18164 }
18165
18166 const PLEXUS: &str = "plexus";
18169 const OTHER: &str = "other";
18170 const AFT: &str = "aft";
18171 const BROCA: &str = "broca";
18172 const MAGIC: &str = "magic-context";
18173
18174 fn nonce(module_id: &str) -> String {
18175 format!("nonce-{module_id}")
18176 }
18177
18178 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
18179 let (tx, rx) = mpsc::channel(64);
18180 (
18181 RouteCtx {
18182 connection_id: ConnectionId::new(connection),
18183 egress: FrameSink::new(tx),
18184 },
18185 rx,
18186 )
18187 }
18188
18189 struct Rig {
18193 handler: ControlHandler,
18194 forwarding: Arc<ForwardingTable>,
18195 owner: RouteCtx,
18196 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18197 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
18198 generation: u64,
18199 next_connection: u64,
18200 _supervisor: Supervisor,
18201 }
18202
18203 async fn rig() -> Rig {
18204 rig_with_flow_support(true).await
18205 }
18206
18207 async fn rig_with_flow_support(flow_support: bool) -> Rig {
18208 rig_with_scope_capabilities(flow_support, false).await
18209 }
18210
18211 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
18212 let mut provides = Vec::new();
18213 if flow_support {
18214 provides.push("flow-scopes/v1");
18215 }
18216 if run_support {
18217 provides.push("agent-run-scopes/v1");
18218 }
18219 rig_with_target_capabilities(&provides).await
18220 }
18221
18222 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
18223 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
18224 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18225 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18228 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
18229 Frame::build(
18230 FrameType::Hello,
18231 control_flags(),
18232 0,
18233 0,
18234 connection,
18235 serde_json::to_vec(&body).unwrap(),
18236 )
18237 .unwrap()
18238 }
18239
18240 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
18241 let registry = Arc::new(Registry::default());
18242 let forwarding = Arc::new(ForwardingTable::default());
18243 let supervisor_handle = SupervisorHandle::new();
18244 let supervisor =
18245 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
18246 .with_handle(supervisor_handle.clone());
18247 supervisor
18248 .supervise_configured(
18249 ModuleSpec {
18250 carry_containment: false,
18251 module_id: OWNER.to_string(),
18252 program: PathBuf::from("/nonexistent/prefrontal-core"),
18253 args: Vec::new(),
18254 env: Vec::new(),
18255 reserved: false,
18256 reserved_prefixes: Vec::new(),
18257 protocol: ModuleProtocol::Subc,
18258 overlap: Default::default(),
18259 },
18260 false,
18261 )
18262 .unwrap();
18263 for module_id in [OWNER, AFT, BROCA, MAGIC] {
18264 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
18265 }
18266 let handler =
18267 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18268 .with_supervisor(supervisor_handle);
18269 let (owner, mut owner_rx) = wide_ctx(1);
18270 hello_via_sink(
18271 &handler,
18272 &owner,
18273 &mut owner_rx,
18274 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
18275 )
18276 .await;
18277 let mut modules = BTreeMap::new();
18278 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
18279 let (ctx, mut rx) = wide_ctx(connection);
18280 let hello = hello_with_capabilities(module_id, connection, provides);
18281 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
18282 modules.insert(module_id.to_string(), (ctx, rx));
18283 }
18284 Rig {
18285 handler,
18286 forwarding,
18287 owner,
18288 _owner_rx: owner_rx,
18289 modules,
18290 generation: 0,
18291 next_connection: 100,
18292 _supervisor: supervisor,
18293 }
18294 }
18295
18296 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
18297 ScopeCarrier::new(Principal::Reserved {
18298 module_id: module_id.to_string(),
18299 })
18300 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
18301 }
18302
18303 fn session(scope_epoch: u64) -> ScopeRecord {
18306 let mut record = head("s", scope_epoch);
18307 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
18308 record.attributes.agent_id = Some("agent-1".to_string());
18309 record.attributes.delegates = true;
18310 record
18311 }
18312
18313 impl Rig {
18314 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
18315 self.generation += 1;
18316 sync(&self.handler, &self.owner, self.generation, scopes)
18317 .await
18318 .expect("the owner's sync is accepted");
18319 }
18320
18321 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18322 ScopeSelector::new(
18323 Principal::Reserved {
18324 module_id: OWNER.to_string(),
18325 },
18326 scope_ref.to_string(),
18327 scope_epoch,
18328 )
18329 }
18330
18331 fn open_frame(
18332 &mut self,
18333 opener: Option<&str>,
18334 target: &str,
18335 scope: Option<ScopeSelector>,
18336 ) -> (
18337 RouteCtx,
18338 mpsc::Receiver<crate::router::OutboundFrame>,
18339 Frame,
18340 ) {
18341 self.next_connection += 1;
18342 let (ctx, rx) = wide_ctx(self.next_connection);
18343 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
18344 busy_refusals: None,
18345 target: RouteTarget::ToolProvider {
18346 module_id: target.to_string(),
18347 },
18348 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
18349 consumer_identity: opener.map(|module_id| ConsumerIdentity {
18350 module_id: module_id.to_string(),
18351 launch_nonce: nonce(module_id),
18352 }),
18353 consumer_capabilities: None,
18354 role_versions: None,
18355 admission_facts: None,
18356 scope,
18357 })
18358 .unwrap();
18359 let frame = Frame::build(
18360 FrameType::Request,
18361 control_flags(),
18362 0,
18363 0,
18364 self.next_connection,
18365 body,
18366 )
18367 .unwrap();
18368 (ctx, rx, frame)
18369 }
18370
18371 async fn refused(
18373 &mut self,
18374 opener: Option<&str>,
18375 target: &str,
18376 scope: Option<ScopeSelector>,
18377 ) -> String {
18378 self.refusal_body(opener, target, scope).await["code"]
18379 .as_str()
18380 .unwrap()
18381 .to_string()
18382 }
18383
18384 async fn refusal_body(
18385 &mut self,
18386 opener: Option<&str>,
18387 target: &str,
18388 scope: Option<ScopeSelector>,
18389 ) -> Value {
18390 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
18391 let replies = tokio::time::timeout(
18392 Duration::from_secs(2),
18393 self.handler.handle_control_frame(&ctx, frame),
18394 )
18395 .await
18396 .expect("the open must be refused before waiting for a bind ack")
18397 .unwrap();
18398 assert_eq!(replies.len(), 1, "{replies:?}");
18399 assert_eq!(replies[0].header.ty, FrameType::Error);
18400 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18401 assert!(
18402 module_rx.try_recv().is_err(),
18403 "a refused open relays nothing"
18404 );
18405 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
18406 parse_error(&replies[0])
18407 }
18408
18409 async fn relayed(
18411 &mut self,
18412 opener: Option<&str>,
18413 target: &str,
18414 scope: Option<ScopeSelector>,
18415 ) -> Relayed {
18416 self.relayed_with_legacy_root(opener, target, scope, None)
18417 .await
18418 }
18419
18420 async fn relayed_with_legacy_root(
18421 &mut self,
18422 opener: Option<&str>,
18423 target: &str,
18424 scope: Option<ScopeSelector>,
18425 legacy_root: Option<&std::path::Path>,
18426 ) -> Relayed {
18427 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18428 if let Some(root) = legacy_root {
18429 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18430 body["identity"]["project_root"] = serde_json::json!(root);
18431 frame.body = serde_json::to_vec(&body).unwrap();
18432 }
18433 let handler = self.handler.clone();
18434 let task_ctx = ctx.clone();
18435 let task = tokio::spawn(async move {
18436 handler
18437 .handle_control_frame(&task_ctx, frame)
18438 .await
18439 .unwrap()
18440 });
18441 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18442 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18443 .await
18444 .expect("the target receives the relayed route.bind")
18445 .unwrap()
18446 .frame;
18447 Relayed {
18448 target: target.to_string(),
18449 client: ctx,
18450 client_rx: rx,
18451 task,
18452 bind,
18453 }
18454 }
18455
18456 async fn ack(&self, relayed: &Relayed) {
18457 let (module, _) = &self.modules[&relayed.target];
18458 self.handler
18459 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
18460 .await
18461 .unwrap();
18462 }
18463
18464 async fn bound(
18466 &mut self,
18467 opener: Option<&str>,
18468 target: &str,
18469 scope: Option<ScopeSelector>,
18470 ) -> Bound {
18471 let relayed = self.relayed(opener, target, scope).await;
18472 self.commit_relayed(relayed).await
18473 }
18474
18475 async fn bound_with_legacy_root(
18476 &mut self,
18477 opener: Option<&str>,
18478 target: &str,
18479 scope: Option<ScopeSelector>,
18480 root: &std::path::Path,
18481 ) -> Bound {
18482 let relayed = self
18483 .relayed_with_legacy_root(opener, target, scope, Some(root))
18484 .await;
18485 self.commit_relayed(relayed).await
18486 }
18487
18488 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
18489 self.ack(&relayed).await;
18490 let Relayed {
18491 target,
18492 client,
18493 mut client_rx,
18494 task,
18495 bind,
18496 } = relayed;
18497 assert!(
18498 task.await.unwrap().is_empty(),
18499 "the open is answered by commit"
18500 );
18501 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
18502 Bound {
18503 target,
18504 client,
18505 client_rx,
18506 channel,
18507 epoch,
18508 bind,
18509 }
18510 }
18511
18512 fn live(&self, route: &Bound) -> bool {
18513 matches!(
18514 self.forwarding
18515 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18516 .unwrap(),
18517 DataRoute::Client(DataRouteState::Bound(_))
18518 )
18519 }
18520 }
18521
18522 struct Relayed {
18523 target: String,
18524 client: RouteCtx,
18525 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18526 task: tokio::task::JoinHandle<Vec<Frame>>,
18527 bind: Frame,
18528 }
18529
18530 struct Bound {
18531 target: String,
18532 client: RouteCtx,
18533 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18534 channel: u16,
18535 epoch: u32,
18536 bind: Frame,
18537 }
18538
18539 impl Bound {
18540 fn closed_reason(&mut self) -> RouteCloseReason {
18543 let mut reason = None;
18544 let mut goodbye = false;
18545 while let Ok(outbound) = self.client_rx.try_recv() {
18546 let frame = outbound.frame;
18547 match frame.header.ty {
18548 FrameType::Goodbye => {
18549 assert_eq!(
18550 (frame.header.channel, frame.header.epoch),
18551 (self.channel, self.epoch)
18552 );
18553 goodbye = true;
18554 }
18555 FrameType::Push => {
18556 let ClientControlPush::RouteClosed {
18557 reason: r,
18558 module_id,
18559 ..
18560 } = serde_json::from_slice(&frame.body).unwrap()
18561 else {
18562 panic!("unexpected push");
18563 };
18564 assert_eq!(module_id, self.target);
18565 reason = Some(r);
18566 }
18567 other => panic!("unexpected frame {other:?}"),
18568 }
18569 }
18570 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
18571 reason.expect("the client is told why the route closed")
18572 }
18573
18574 fn untouched(&mut self) -> bool {
18575 self.client_rx.try_recv().is_err()
18576 }
18577
18578 fn stamp(&self) -> Option<ScopeStamp> {
18579 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
18580 ModuleControlRequest::RouteBind { scope, .. } => scope,
18581 other => panic!("expected a route.bind, got {other:?}"),
18582 }
18583 }
18584 }
18585
18586 #[tokio::test]
18587 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
18588 ) {
18589 let mut rig = rig().await;
18590 let mut record = session(1);
18591 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
18592 record.child_owners = vec![Principal::Reserved {
18593 module_id: MAGIC.to_string(),
18594 }];
18595 rig.sync(vec![record]).await;
18596 let scope = || Some(rig_selector("s", Some(1)));
18597
18598 rig.bound(Some(OWNER), PLEXUS, scope()).await;
18601 rig.bound(Some(AFT), OTHER, scope()).await;
18602 rig.bound(Some(BROCA), PLEXUS, scope()).await;
18603
18604 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
18608 assert_eq!(
18609 rig.refused(opener, target, scope()).await,
18610 error_codes::SCOPE_NOT_CARRIER,
18611 "{opener:?} -> {target}"
18612 );
18613 }
18614 }
18615
18616 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18617 ScopeSelector::new(
18618 Principal::Reserved {
18619 module_id: OWNER.to_string(),
18620 },
18621 scope_ref.to_string(),
18622 scope_epoch,
18623 )
18624 }
18625
18626 #[tokio::test]
18627 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
18628 let mut rig = rig().await;
18629 rig.sync(vec![session(1)]).await;
18630 for opener in [OWNER, AFT] {
18631 assert_eq!(
18632 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
18633 .await,
18634 error_codes::SCOPE_EPOCH_REQUIRED,
18635 "{opener}"
18636 );
18637 }
18638 }
18639
18640 #[tokio::test]
18641 async fn admission_separates_not_synced_not_live_and_ended() {
18642 let mut rig = rig().await;
18643 let code = rig
18645 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18646 .await;
18647 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
18648 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
18649
18650 let ghost = ScopeSelector::new(
18652 Principal::Reserved {
18653 module_id: "ghost".to_string(),
18654 },
18655 "s".to_string(),
18656 Some(1),
18657 );
18658 assert_eq!(
18659 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
18660 error_codes::SCOPE_NOT_LIVE
18661 );
18662
18663 rig.sync(vec![session(2)]).await;
18664 assert_eq!(
18665 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
18666 .await,
18667 error_codes::SCOPE_NOT_LIVE
18668 );
18669 for epoch in [1, 3] {
18670 assert_eq!(
18671 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
18672 .await,
18673 error_codes::SCOPE_ENDED,
18674 "epoch {epoch}"
18675 );
18676 }
18677 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
18679 .await;
18680 }
18681
18682 #[tokio::test]
18683 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
18684 let mut rig = rig().await;
18685 rig.sync(vec![session(1)]).await;
18686 let route = rig
18687 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18688 .await;
18689 let stamp = route.stamp().expect("a scoped bind carries the stamp");
18690 assert_eq!(stamp.scope_ref, "s");
18691 assert_eq!(stamp.scope_epoch, 1);
18692 assert_eq!(stamp.kind, ScopeKind::Head);
18693 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
18694 assert!(stamp.attributes.delegates);
18695 assert!(stamp.owner_authorized);
18696 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18697 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
18698
18699 let (broca, mut broca_rx) = wide_ctx(50);
18702 hello_via_sink(
18703 &rig.handler,
18704 &broca,
18705 &mut broca_rx,
18706 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
18707 )
18708 .await;
18709 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
18710 .await
18711 .unwrap();
18712 let own = ScopeSelector::new(
18713 Principal::Reserved {
18714 module_id: BROCA.to_string(),
18715 },
18716 "b".to_string(),
18717 Some(1),
18718 );
18719 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
18720 assert!(!route.stamp().unwrap().owner_authorized);
18721 }
18722
18723 #[tokio::test]
18724 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
18725 let mut rig = rig().await;
18726 let mut record = head("s", 1);
18727 record.carriers = vec![carrier(AFT, None)];
18728 let flow_id = "Flow:run-7/step_2!~";
18729 record.attributes.flow_id = Some(flow_id.to_string());
18730 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
18731 .await
18732 .unwrap();
18733 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18734 panic!("not a sync reply");
18735 };
18736 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
18737 let route = rig
18738 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18739 .await;
18740 assert!(rig.live(&route), "the stamped bind committed");
18741 let stamp = route.stamp().expect("a flow scope carries a stamp");
18742 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
18743 assert_eq!(stamp.attributes.agent_id, None);
18744 assert!(!stamp.attributes.delegates);
18745 assert!(stamp.owner_authorized);
18746 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18747 assert_eq!(unscoped.stamp(), None);
18748 }
18749
18750 #[tokio::test]
18751 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
18752 let mut rig = rig_with_flow_support(false).await;
18753 let mut record = session(1);
18754 record.attributes.flow_id = Some("flow:7".to_string());
18755 rig.sync(vec![record]).await;
18756 for opener in [OWNER, AFT] {
18757 let body = rig
18758 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18759 .await;
18760 assert_eq!(body["code"], "target_flow_unsupported");
18761 let message = body["message"].as_str().unwrap();
18762 for required in [PLEXUS, "flow-scopes/v1"] {
18763 assert!(message.contains(required), "{message}");
18764 }
18765 }
18766 }
18767
18768 #[tokio::test]
18769 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
18770 let mut rig = rig_with_flow_support(true).await;
18771 let mut record = session(1);
18772 record.attributes.flow_id = Some("flow:7".to_string());
18773 rig.sync(vec![record]).await;
18774 for opener in [OWNER, AFT] {
18775 let route = rig
18776 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18777 .await;
18778 assert!(rig.live(&route));
18779 assert_eq!(
18780 route.stamp().unwrap().attributes.flow_id.as_deref(),
18781 Some("flow:7")
18782 );
18783 }
18784 }
18785
18786 #[tokio::test]
18787 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
18788 use std::future::Future;
18789
18790 let mut rig = rig_with_flow_support(true).await;
18791 let mut record = session(1);
18792 record.attributes.flow_id = Some("flow:7".to_string());
18793 rig.sync(vec![record]).await;
18794 let (client, mut client_rx, frame) =
18795 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
18796 for _ in 0..64 {
18799 client.egress.try_send(route_bind_ack(1)).unwrap();
18800 }
18801 let handler = rig.handler.clone();
18802 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18803 std::future::poll_fn(|cx| {
18804 assert!(open.as_mut().poll(cx).is_pending());
18805 std::task::Poll::Ready(())
18806 })
18807 .await;
18808 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18809
18810 let old_connection = rig.modules[PLEXUS].0.connection_id;
18811 rig.handler.cleanup_connection(old_connection).unwrap();
18812 let (replacement, mut replacement_rx) = wide_ctx(200);
18813 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
18814 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18815 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18816 let hello = Frame::build(
18817 FrameType::Hello,
18818 control_flags(),
18819 0,
18820 0,
18821 200,
18822 serde_json::to_vec(&body).unwrap(),
18823 )
18824 .unwrap();
18825 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
18826
18827 client_rx.try_recv().unwrap();
18828 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18829 .await
18830 .expect("the replacement is refused without waiting for a bind ack")
18831 .unwrap();
18832 assert_eq!(replies.len(), 1);
18833 let body = parse_error(&replies[0]);
18834 assert_eq!(body["code"], "target_flow_unsupported");
18835 for required in [PLEXUS, "flow-scopes/v1"] {
18836 assert!(body["message"].as_str().unwrap().contains(required));
18837 }
18838 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
18839 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18840 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18841 assert!(rig
18842 .handler
18843 .registry
18844 .get_module_by_connection(replacement.connection_id)
18845 .unwrap()
18846 .is_some());
18847 }
18848
18849 #[tokio::test]
18850 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
18851 use std::future::Future;
18852 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18853 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18854 const NEXT_HASH: &str =
18855 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18856 let mut rig = rig().await;
18857 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
18858 record.carriers.push(ScopeCarrier::new(Principal::Direct));
18859 rig.sync(vec![record.clone()]).await;
18860 let (client, mut rx, frame) = rig.open_frame(
18861 None,
18862 PLEXUS,
18863 Some(
18864 rig.selector("s", Some(1))
18865 .with_bind_proof(Some(PROOF.into())),
18866 ),
18867 );
18868 for _ in 0..64 {
18871 client.egress.try_send(route_bind_ack(1)).unwrap();
18872 }
18873 let handler = rig.handler.clone();
18874 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18875 std::future::poll_fn(|cx| {
18876 assert!(open.as_mut().poll(cx).is_pending());
18877 std::task::Poll::Ready(())
18878 })
18879 .await;
18880 record.bind_proof_sha256 = Some(NEXT_HASH.into());
18881 rig.sync(vec![record]).await;
18882 rx.try_recv().unwrap();
18883 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18884 .await
18885 .unwrap()
18886 .unwrap();
18887 assert_eq!(
18888 parse_error(&replies[0])["code"],
18889 error_codes::SCOPE_BIND_PROOF_MISMATCH
18890 );
18891 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
18892 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18893 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18894 }
18895
18896 #[tokio::test]
18897 async fn scope_bind_proof_reserved_openers_ignore_proof() {
18898 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18899 let mut rig = rig().await;
18900 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18901 .await;
18902 for opener in [OWNER, AFT] {
18903 for proof in [None, Some("malformed+proof=")] {
18904 let route = rig
18905 .bound(
18906 Some(opener),
18907 PLEXUS,
18908 Some(
18909 rig.selector("s", Some(1))
18910 .with_bind_proof(proof.map(str::to_string)),
18911 ),
18912 )
18913 .await;
18914 assert!(rig.live(&route));
18915 }
18916 }
18917 }
18918
18919 #[tokio::test]
18920 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
18921 use std::future::Future;
18922 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18923 const NEXT_HASH: &str =
18924 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18925 let mut rig = rig().await;
18926 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18927 .await;
18928 let (client, mut rx, frame) =
18929 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
18930 for _ in 0..64 {
18931 client.egress.try_send(route_bind_ack(1)).unwrap();
18932 }
18933 let handler = rig.handler.clone();
18934 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18935 std::future::poll_fn(|cx| {
18936 assert!(open.as_mut().poll(cx).is_pending());
18937 std::task::Poll::Ready(())
18938 })
18939 .await;
18940 rig.sync(vec![
18941 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
18942 ])
18943 .await;
18944 rx.try_recv().unwrap();
18945 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18946 let bind = std::future::poll_fn(|cx| {
18947 assert!(open.as_mut().poll(cx).is_pending());
18948 module_rx.poll_recv(cx)
18949 })
18950 .await
18951 .unwrap()
18952 .frame;
18953 handler
18954 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
18955 .await
18956 .unwrap();
18957 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18958 .await
18959 .unwrap()
18960 .unwrap();
18961 assert_eq!(replies.len(), 1);
18962 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
18965 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18966 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
18967 let route = rig
18968 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
18969 .await;
18970 assert!(rig.live(&route));
18971 }
18972
18973 #[tokio::test]
18974 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
18975 {
18976 let mut rig = rig_with_flow_support(false).await;
18977 rig.sync(vec![session(1)]).await;
18978 let route = rig
18979 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18980 .await;
18981 assert!(rig.live(&route));
18982 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
18983 let mut record = session(1);
18984 record.attributes.flow_id = Some("flow:7".to_string());
18985 rig.sync(vec![record]).await;
18986 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
18987 assert!(rig.live(&unscoped));
18988 assert_eq!(unscoped.stamp(), None);
18989 }
18990
18991 #[tokio::test]
18992 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
18993 let mut rig = rig().await;
18994 let mut record = session(1);
18995 record.attributes.flow_id = Some("flow:7".to_string());
18996 rig.sync(vec![record.clone()]).await;
18997 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18998 let mut owner_route = rig
18999 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19000 .await;
19001 let mut carrier_route = rig
19002 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
19003 .await;
19004 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19005 rig.sync(vec![record.clone()]).await;
19006 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19007 assert!(rig.live(&owner_route) && owner_route.untouched());
19008 assert!(rig.live(&carrier_route) && carrier_route.untouched());
19009
19010 record.attributes.flow_id = Some("flow:8".to_string());
19011 rig.sync(vec![record]).await;
19012 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
19013 let before = before.unwrap();
19014 assert_eq!(after.scope_epoch, before.scope_epoch);
19015 assert!(after.version > before.version);
19016 for route in [&mut owner_route, &mut carrier_route] {
19017 assert!(!rig.live(route));
19018 assert_eq!(
19019 route.closed_reason(),
19020 RouteCloseReason::ScopeDelegationChanged
19021 );
19022 }
19023 assert!(rig.live(&unscoped) && unscoped.untouched());
19024 for target in [PLEXUS, OTHER] {
19027 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
19028 let goodbye = module_rx
19029 .try_recv()
19030 .expect("the provider sees the drain")
19031 .frame;
19032 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19033 assert!(module_rx.try_recv().is_err());
19034 }
19035 let rebound = rig
19036 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19037 .await;
19038 assert_eq!(
19039 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
19040 Some("flow:8")
19041 );
19042 }
19043
19044 #[tokio::test]
19049 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
19050 let mut rig = rig().await;
19051 rig.sync(vec![session(1)]).await;
19052 let mut cotenant = rig
19053 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19054 .await;
19055
19056 let mut changed = session(1);
19057 changed.child_owners.push(Principal::Reserved {
19058 module_id: MAGIC.to_string(),
19059 });
19060 let mut ended = None;
19061 for (code, next) in [
19062 (error_codes::SCOPE_CHANGED, vec![changed]),
19063 (error_codes::SCOPE_ENDED, Vec::new()),
19064 ] {
19065 let relayed = rig
19066 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19067 .await;
19068 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
19069 ended = Some(next.is_empty());
19070 rig.sync(next).await;
19071 rig.ack(&relayed).await;
19072 let replies = relayed.task.await.unwrap();
19073 assert_eq!(replies.len(), 1, "{replies:?}");
19074 assert_eq!(parse_error(&replies[0])["code"], code);
19075 assert_eq!(
19076 subc_protocol::error_codes::is_retryable_route_open(code),
19077 code == error_codes::SCOPE_CHANGED
19078 );
19079 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19080 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19082 let mut goodbyes = Vec::new();
19085 while let Ok(outbound) = plexus_rx.try_recv() {
19086 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
19087 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
19088 }
19089 assert!(
19090 goodbyes.contains(&(bind_channel, bind_epoch)),
19091 "{goodbyes:?}"
19092 );
19093 assert!(rig
19094 .handler
19095 .registry
19096 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
19097 .unwrap()
19098 .is_some());
19099 }
19100 assert_eq!(ended, Some(true));
19101 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
19104 }
19105
19106 #[tokio::test]
19109 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
19110 struct Case {
19111 name: &'static str,
19112 change: fn(&mut ScopeRecord),
19113 closed: [Option<RouteCloseReason>; 4],
19116 }
19117 use RouteCloseReason::*;
19118 let cases = [
19119 Case {
19120 name: "a carrier entry removed",
19121 change: |r| {
19122 r.carriers.retain(|c| {
19123 c.principal
19124 != Principal::Reserved {
19125 module_id: AFT.to_string(),
19126 }
19127 })
19128 },
19129 closed: [None, Some(ScopeCarrierRemoved), None, None],
19130 },
19131 Case {
19132 name: "a target removed from a carrier",
19133 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
19134 closed: [None, None, None, Some(ScopeCarrierRemoved)],
19135 },
19136 Case {
19137 name: "a bare carrier narrowed to targets",
19138 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
19139 closed: [None, Some(ScopeCarrierRemoved), None, None],
19140 },
19141 Case {
19142 name: "delegates turned off",
19143 change: |r| r.attributes.delegates = false,
19144 closed: [Some(ScopeDelegationChanged); 4],
19145 },
19146 Case {
19147 name: "agent_id changed",
19148 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
19149 closed: [Some(ScopeDelegationChanged); 4],
19150 },
19151 Case {
19152 name: "a carrier added, child owners changed, the record re-sent",
19153 change: |r| {
19154 r.carriers.push(carrier(MAGIC, None));
19155 r.child_owners.push(Principal::Reserved {
19156 module_id: MAGIC.to_string(),
19157 });
19158 },
19159 closed: [None; 4],
19160 },
19161 Case {
19162 name: "a target added",
19163 change: |r| {
19164 r.carriers[1]
19165 .targets
19166 .as_mut()
19167 .unwrap()
19168 .push("third".to_string())
19169 },
19170 closed: [None; 4],
19171 },
19172 Case {
19173 name: "delegates turned on",
19174 change: |r| r.attributes.delegates = true,
19175 closed: [None; 4],
19176 },
19177 ];
19178 for case in cases {
19179 let mut rig = rig().await;
19180 rig.sync(vec![session(1)]).await;
19181 let scope = || Some(rig_selector("s", Some(1)));
19182 let mut routes = [
19183 rig.bound(Some(OWNER), PLEXUS, scope()).await,
19184 rig.bound(Some(AFT), PLEXUS, scope()).await,
19185 rig.bound(Some(BROCA), PLEXUS, scope()).await,
19186 rig.bound(Some(BROCA), OTHER, scope()).await,
19187 ];
19188 let mut record = session(1);
19189 (case.change)(&mut record);
19190 rig.sync(vec![record]).await;
19191 for (index, expected) in case.closed.iter().enumerate() {
19192 let route = &mut routes[index];
19193 match expected {
19194 Some(reason) => {
19195 assert!(!rig.live(route), "{}: route {index} still live", case.name);
19196 assert_eq!(
19197 route.closed_reason(),
19198 *reason,
19199 "{}: route {index}",
19200 case.name
19201 );
19202 }
19203 None => {
19204 assert!(rig.live(route), "{}: route {index} closed", case.name);
19205 assert!(
19206 route.untouched(),
19207 "{}: route {index} was told something",
19208 case.name
19209 );
19210 }
19211 }
19212 }
19213 }
19214 }
19215
19216 #[tokio::test]
19217 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
19218 for next in [Vec::new(), vec![session(2)]] {
19220 let mut rig = rig().await;
19221 rig.sync(vec![session(1)]).await;
19222 let mut route = rig
19223 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19224 .await;
19225 rig.sync(next).await;
19226 assert!(!rig.live(&route));
19227 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
19228 }
19229
19230 let mut rig = rig().await;
19233 let mut child = session(1);
19234 child.scope_ref = "child".to_string();
19235 child.kind = ScopeKind::Worker;
19236 child.parent = Some(ScopeParent::new(
19237 Principal::Reserved {
19238 module_id: OWNER.to_string(),
19239 },
19240 "s".to_string(),
19241 1,
19242 ));
19243 rig.sync(vec![session(1), child.clone()]).await;
19244 let mut child_route = rig
19245 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
19246 .await;
19247 assert_eq!(
19248 child_route.stamp().unwrap().parent_state,
19249 Some(ParentState::Linked)
19250 );
19251 rig.sync(vec![child]).await;
19252 assert!(!rig.live(&child_route));
19253 assert_eq!(
19254 child_route.closed_reason(),
19255 RouteCloseReason::ScopeParentEnded
19256 );
19257 }
19258
19259 #[tokio::test]
19260 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
19261 ) {
19262 let mut rig = rig().await;
19263 rig.sync(vec![session(1)]).await;
19264 let mut route = rig
19265 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19266 .await;
19267 let before = rig.forwarding.published_scope_tag(OWNER, "s");
19268 rig.sync(vec![session(1)]).await;
19269 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19270 assert!(rig.live(&route) && route.untouched());
19271
19272 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
19276 .forwarding
19277 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
19278 .unwrap()
19279 else {
19280 panic!("the route is bound");
19281 };
19282 binding.flow.acquire_tagged(9, false).await.unwrap();
19283 let mut widened = session(1);
19284 widened.carriers.push(carrier(MAGIC, None));
19285 rig.sync(vec![widened]).await;
19286 assert!(rig.live(&route) && route.untouched());
19287 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19288 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
19289 assert_eq!(binding.flow.in_flight(), 1);
19292 binding
19293 .flow
19294 .acquire_tagged(10, false)
19295 .await
19296 .expect("the flow is still open");
19297 }
19298
19299 #[tokio::test]
19302 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
19303 let mut rig = rig().await;
19304 rig.sync(vec![session(1)]).await;
19305 let mut on_incumbent = rig
19306 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19307 .await;
19308
19309 let (candidate, _candidate_rx) = wide_ctx(9);
19312 let registration = rig
19313 .handler
19314 .registry
19315 .register_candidate_with_control_ops(
19316 manifest(PLEXUS, PROTOCOL_VERSION),
19317 PROTOCOL_VERSION,
19318 candidate.connection_id,
19319 module_baseline_control_ops(),
19320 )
19321 .unwrap();
19322 rig.forwarding
19323 .register_candidate_module_connection(
19324 candidate.connection_id,
19325 PLEXUS.to_string(),
19326 PROTOCOL_VERSION,
19327 manifest_concurrency(®istration.manifest),
19328 candidate.egress.clone(),
19329 )
19330 .unwrap();
19331 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
19332 rig.handler
19333 .registry
19334 .promote_candidate(PLEXUS)
19335 .unwrap()
19336 .unwrap();
19337 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
19338
19339 rig.sync(Vec::new()).await;
19340 assert!(!rig.live(&on_incumbent));
19341 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
19342 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19343 let goodbye = incumbent_rx
19344 .try_recv()
19345 .expect("the superseded endpoint is told")
19346 .frame;
19347 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19348 }
19349 }
19350}
19351
19352#[cfg(test)]
19353mod concurrency_default_exposure_tests {
19354 use super::*;
19355
19356 fn hello_body(role_json: &str) -> Vec<u8> {
19357 format!(
19358 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
19359 )
19360 .into_bytes()
19361 }
19362
19363 fn manifest_from(body: &[u8]) -> ModuleManifest {
19364 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
19365 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
19366 .expect("manifest parses")
19367 }
19368
19369 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
19370
19371 #[test]
19372 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
19373 let body = hello_body(&format!(
19374 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
19375 ));
19376 let manifest = manifest_from(&body);
19377 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19380 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
19381 }
19382
19383 #[test]
19384 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
19385 let body = hello_body(&format!(
19386 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
19387 ));
19388 let manifest = manifest_from(&body);
19389 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19390 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19391 }
19392
19393 #[test]
19394 fn non_management_roles_are_never_reported() {
19395 let body = hello_body(
19396 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
19397 );
19398 let manifest = manifest_from(&body);
19399 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19400 }
19401}