1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6 fmt,
7 net::SocketAddr,
8 path::{Path, PathBuf},
9 sync::{Arc, Mutex, RwLock},
10 time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26 error_codes,
27 manifest::{
28 validate_hello_capability_grammar, validate_hello_confirm_operations,
29 validate_hello_display_name, validate_hello_event_declarations,
30 validate_hello_flow_operations, validate_hello_operation_deferral,
31 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32 validate_hello_self_signal_declarations, validate_role_declarations_value,
33 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
34 ProviderRole,
35 },
36 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
37 scope::{
38 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
39 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
40 SCOPE_SYNC_OP,
41 },
42 session::{
43 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
44 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
45 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
46 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS, ROLE_VERSIONS_FIELD,
47 },
48 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
49 Priority, RouteTarget, PROTOCOL_VERSION,
50};
51use tokio::time::{timeout_at, Instant};
52use tracing::{debug, info, instrument::WithSubscriber, warn};
53
54use crate::{
55 caller_identity::CallerIdentity,
56 capability_requirements::{
57 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
58 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
59 RequirementStatus, RuntimeModule,
60 },
61 child_roster::DaemonShutdownFlag,
62 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
63 forwarding::{
64 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
65 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
66 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
67 },
68 observability::{
69 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
70 },
71 privileged_prompt::{prompt_text, PrivilegedOperation},
72 provenance::{
73 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
74 },
75 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
76 router::{daemon_error_flags, RouteCtx, RouterError},
77 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
78 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
79 stderr_tail::{CaptureState, TailEntry},
80 supervise::{
81 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
82 SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
83 },
84 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
85};
86
87pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
93
94const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
95const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
96const CAP_PING_PONG: &str = "ping_pong_v1";
97const CAP_SESSION_ATTACH: &str = "session_attach_v1";
98const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
99
100const SUBC_CONTROL_OPS: &[&str] = &[
101 ops::DAEMON_SHUTDOWN_INTENT,
102 ops::ACTIVITY_RECORD,
103 ops::ACTIVITY_QUERY,
104 ops::ACTIVITY_SUBSCRIBE,
105 ops::SERVER_DESCRIBE,
106 ops::CATALOG_LIST,
107 ops::ROUTE_OPEN,
108 ops::ROUTE_POLL,
109 ops::ROUTE_CLOSING,
110 ops::ROUTE_CLOSED,
111 ops::SUPERVISOR_LIST,
112 ops::SUPERVISOR_RESTART,
113 ops::SUPERVISOR_SWAP,
114 ops::SUPERVISOR_RELOAD,
115 ops::SUPERVISOR_RESCAN,
116 ops::SUPERVISOR_RELEASE_RESERVED,
117 ops::SUPERVISOR_SET_ENABLED,
118 ops::SUPERVISOR_HEALTH_PROBE,
119 ops::SUPERVISOR_HEALTH,
120 ops::SUPERVISOR_STDERR_TAIL,
121 ops::SUPERVISOR_TERMINALS,
122 ops::SUPERVISOR_ROUTES,
123 ops::SUPERVISOR_PROVENANCE,
124 ops::SUPERVISOR_SPAWN_SNAPSHOT,
125 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
126];
127
128const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
135
136mod agent_commands;
137mod rescan_checked;
138use rescan_checked::{RescanMode, RescanPlan};
139
140const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
141 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
142 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
143 "supervisor.live_roots",
144 "supervisor.agent_commands",
145 SCOPE_SYNC_OP,
146 SCOPE_APPLY_OP,
147 SCOPE_DESCRIBE_OP,
148 subc_protocol::scope::ROUTE_AUTHORIZE_OP,
149 "operator.confirm",
150];
151
152const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
156
157const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
158
159pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
167
168pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
183
184pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
202
203const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
204const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
205
206fn reload_verdict(
207 configured: &Path,
208 spawned_from: Option<&Path>,
209 image: subc_control::RunningImageAgreement,
210) -> PendingReloadVerdict {
211 let path = match spawned_from {
212 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
213 ReloadPathAgreement::Match
214 }
215 Some(spawned_from) => ReloadPathAgreement::Mismatch {
216 configured: configured.to_path_buf(),
217 spawned_from: spawned_from.to_path_buf(),
218 },
219 None => ReloadPathAgreement::Unavailable {
220 reason: if matches!(
221 image,
222 subc_control::RunningImageAgreement::Unavailable {
223 reason: subc_control::RunningImageUnavailableReason::NotRunning
224 }
225 ) {
226 ReloadPathUnavailableReason::NotRunning
227 } else {
228 ReloadPathUnavailableReason::SpawnedPathUnavailable
229 },
230 },
231 };
232 PendingReloadVerdict::new(path, image)
233}
234
235fn launch_spec_agreement(
236 configured: &crate::supervise::ModuleSpec,
237 running: bool,
238 spawned: Option<&crate::supervise::LaunchSpecDigest>,
239) -> subc_control::LaunchSpecAgreement {
240 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
241 if !running {
242 return Agreement::Unavailable {
243 reason: Reason::NotRunning,
244 };
245 }
246 match spawned {
247 None => Agreement::Unavailable {
248 reason: Reason::SpawnedSpecUnavailable,
249 },
250 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
251 Agreement::Match
252 }
253 Some(_) => Agreement::Mismatch,
254 }
255}
256
257fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
258 if configured == spawned_from {
259 return true;
260 }
261 #[cfg(windows)]
262 {
263 match (
269 std::fs::canonicalize(configured),
270 std::fs::canonicalize(spawned_from),
271 ) {
272 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
273 _ => false,
274 }
275 }
276 #[cfg(not(windows))]
277 {
278 false
279 }
280}
281
282#[derive(Clone)]
283struct DaemonProvenanceFacts {
284 build: DaemonBuildProvenance,
285 pid: Option<u32>,
286 started_at_ms: Option<u64>,
287 start_clock: Option<crate::clock::StartClock>,
288 executable_path: Option<PathBuf>,
289 executable_identity: Option<SpawnedFileIdentity>,
290 process_start_time: Option<u64>,
291 probe: ExecutableIdentityProbe,
292}
293
294impl Default for DaemonProvenanceFacts {
295 fn default() -> Self {
296 Self {
297 build: DaemonBuildProvenance {
298 build_git_sha: None,
299 build_lock_digest: None,
300 },
301 pid: None,
302 started_at_ms: None,
303 start_clock: None,
304 executable_path: None,
305 executable_identity: None,
306 process_start_time: None,
307 probe: ExecutableIdentityProbe::default(),
308 }
309 }
310}
311
312#[derive(Debug, Clone)]
313struct SupervisorRescanContext {
314 supervisor: Supervisor,
315 config_path: PathBuf,
316 configured_port: Option<u16>,
317 storage_config: Option<crate::daemon_config::StorageConfig>,
318 admission_facts_carrier_module_id: Option<String>,
319 admission_facts_targets: Option<Vec<String>>,
320 scope_authority_owners: Vec<String>,
321 federated_scope_owner: String,
322 privileged_operator_prompt: PrivilegedOperatorPrompt,
323 generation: Arc<std::sync::atomic::AtomicU64>,
329}
330
331const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
352 "reloading",
353 "supervisor_not_live",
354 "registration_not_active",
355 "no_forwarding_connection",
356 "relay_send_failed",
357];
358
359#[derive(Clone)]
361pub struct ControlHandler {
362 registry: Arc<Registry>,
363 activity: Option<crate::activity::ActivityLedger>,
364 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
365 forwarding: Arc<ForwardingTable>,
366 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
367 supervisor: SupervisorHandle,
368 pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
369 agent_declarations: agent_commands::Queues,
370 list_admission: Arc<tokio::sync::Semaphore>,
371 subc_capabilities: Arc<[String]>,
372 route_bind_relay_timeout: Duration,
376 route_bind_relay_timeouts: BTreeMap<String, Duration>,
380 route_bind_breakers: RouteBindBreakers,
383 route_bind_concurrency: RouteBindConcurrency,
386 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
391 route_bind_breaker_threshold: u32,
393 route_bind_breaker_cooldown: Duration,
395 health_probe_timeout: Duration,
396 storage_config: Option<crate::daemon_config::StorageConfig>,
399 machine_id: Option<crate::machine_id::MachineId>,
403 admission_facts_carrier_module_id: Option<String>,
404 admission_facts_targets: Option<Vec<String>>,
405 scopes: Arc<RwLock<ScopeTable>>,
409 scope_deadline_added: Arc<tokio::sync::Notify>,
413 scope_authority_owners: Vec<String>,
416 federated_scope_owner: String,
417 privileged_operator_prompt: PrivilegedOperatorPrompt,
419 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
422 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
423 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
424 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
425 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
426 pub(crate) operator_apps:
427 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
428 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
433 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
439 #[cfg(test)]
441 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
442 #[cfg(test)]
443 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
444 #[allow(dead_code)]
448 #[cfg(windows)]
449 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
450 #[allow(dead_code)]
451 #[cfg(windows)]
452 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
453 #[allow(dead_code)]
457 #[cfg(windows)]
458 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
459 #[cfg(all(test, windows))]
462 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
463 #[cfg(test)]
464 caller_lookup: Option<TestCallerLookup>,
465 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
468 rescan: Option<SupervisorRescanContext>,
469 connected_clients: ConnectedClients,
470 counters: DaemonCounters,
471 local_endpoint_unavailable: Option<String>,
474 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
475 daemon_shutdown: DaemonShutdownFlag,
480 daemon_provenance: DaemonProvenanceFacts,
481 #[cfg(test)]
482 control_dispatch_delay: Option<Duration>,
483 #[cfg(test)]
484 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
485 #[cfg(test)]
489 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
490 #[cfg(test)]
496 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
497}
498
499impl fmt::Debug for ControlHandler {
500 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
501 f.debug_struct("ControlHandler")
502 .field("registry", &self.registry)
503 .field("forwarding", &self.forwarding)
504 .field("process_liveness", &self.process_liveness.is_some())
505 .field("supervisor", &self.supervisor)
506 .field("subc_capabilities", &self.subc_capabilities)
507 .finish()
508 }
509}
510
511struct RouteOpenRequest {
512 target: RouteTarget,
513 identity: BindIdentity,
514 consumer_identity: Option<ConsumerIdentity>,
515 consumer_capabilities: Option<Vec<String>>,
516 role_versions: Option<BTreeMap<String, String>>,
517 busy_refusals: Option<bool>,
518 admission_facts: Option<serde_json::Value>,
519 scope: Option<ScopeSelector>,
520}
521
522#[derive(Clone, Copy)]
523enum RouteOpenPhase {
524 Admission,
525 ProjectRoot,
526 EgressReservation,
527 BindSend,
528 BindAckWait,
529}
530
531struct RouteOpenTimings {
535 module_id: String,
536 principal: String,
537 admission: Duration,
538 project_root: Duration,
539 egress_reservation: Duration,
540 bind_send: Duration,
541 bind_ack_wait: Duration,
542 phase: Option<RouteOpenPhase>,
543 phase_started_at: Instant,
544}
545
546impl RouteOpenTimings {
547 fn new() -> Self {
548 Self {
549 module_id: String::new(),
550 principal: "unresolved".to_string(),
552 admission: Duration::ZERO,
553 project_root: Duration::ZERO,
554 egress_reservation: Duration::ZERO,
555 bind_send: Duration::ZERO,
556 bind_ack_wait: Duration::ZERO,
557 phase: Some(RouteOpenPhase::Admission),
558 phase_started_at: Instant::now(),
559 }
560 }
561
562 fn finish(&mut self) {
563 let elapsed = self.phase_started_at.elapsed();
564 match self.phase.take() {
565 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
566 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
567 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
568 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
569 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
570 None => {}
571 }
572 }
573
574 fn enter(&mut self, phase: RouteOpenPhase) {
575 self.finish();
576 self.phase = Some(phase);
577 self.phase_started_at = Instant::now();
578 }
579}
580
581struct RouteBindReservationGuard {
582 forwarding: Arc<ForwardingTable>,
583 endpoint: ModuleEndpointId,
584 relay_corr: u64,
585 armed: bool,
586}
587
588struct ModuleControlRpcGuard {
589 forwarding: Arc<ForwardingTable>,
590 endpoint: ModuleEndpointId,
591 corr: u64,
592 armed: bool,
593}
594
595impl ModuleControlRpcGuard {
596 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
597 Self {
598 forwarding,
599 endpoint,
600 corr,
601 armed: true,
602 }
603 }
604
605 fn disarm(&mut self) {
606 self.armed = false;
607 }
608}
609
610impl Drop for ModuleControlRpcGuard {
611 fn drop(&mut self) {
612 if self.armed {
613 let _ = self
614 .forwarding
615 .cancel_module_control_rpc(self.endpoint, self.corr);
616 }
617 }
618}
619
620impl RouteBindReservationGuard {
621 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
622 Self {
623 forwarding,
624 endpoint,
625 relay_corr,
626 armed: true,
627 }
628 }
629
630 fn release_and_disarm(&mut self) {
631 if !self.armed {
632 return;
633 }
634 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
635 self.endpoint,
636 self.relay_corr,
637 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
638 ) {
639 send_goodbye_target_best_effort(
640 &self.forwarding.counters(),
641 &target,
642 "canceled route.bind",
643 );
644 }
645 self.armed = false;
646 }
647
648 fn disarm(&mut self) {
649 self.armed = false;
650 }
651}
652
653impl Drop for RouteBindReservationGuard {
654 fn drop(&mut self) {
655 self.release_and_disarm();
656 }
657}
658
659#[derive(Debug, Clone, Default)]
689pub(crate) struct RouteBindBreakers {
690 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
691}
692
693#[derive(Debug, Clone, Default)]
694pub(crate) struct RouteBindConcurrency {
695 modules: Arc<Mutex<HashMap<String, usize>>>,
696}
697
698struct RouteBindConcurrencyGuard {
699 concurrency: RouteBindConcurrency,
700 module_id: String,
701}
702
703impl RouteBindConcurrency {
704 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
707 let mut modules = self
708 .modules
709 .lock()
710 .expect("route.bind concurrency mutex poisoned");
711 let in_flight = modules.entry(module_id.to_string()).or_default();
712 if *in_flight >= limit {
713 return Err(*in_flight);
714 }
715 *in_flight += 1;
716 Ok(RouteBindConcurrencyGuard {
717 concurrency: self.clone(),
718 module_id: module_id.to_string(),
719 })
720 }
721}
722
723impl Drop for RouteBindConcurrencyGuard {
724 fn drop(&mut self) {
725 let mut modules = self
726 .concurrency
727 .modules
728 .lock()
729 .expect("route.bind concurrency mutex poisoned");
730 let remove = {
731 let in_flight = modules
732 .get_mut(&self.module_id)
733 .expect("admitted route.bind has a concurrency entry");
734 *in_flight -= 1;
735 *in_flight == 0
736 };
737 if remove {
738 modules.remove(&self.module_id);
739 }
740 }
741}
742
743#[derive(Debug, Default)]
744struct ModuleBreakerState {
745 consecutive_timeouts: u32,
747 cooldown_until: Option<Instant>,
750 probe_in_flight: Option<Arc<()>>,
755}
756
757enum RouteBindAdmission<'a> {
759 Admitted {
760 guard: RouteBindBreakerGuard<'a>,
761 probe: bool,
764 },
765 Refused {
766 consecutive_timeouts: u32,
767 retry_in: Duration,
771 probe_in_flight: bool,
772 },
773}
774
775struct RouteBindBreakerGuard<'a> {
782 breakers: RouteBindBreakers,
783 module_id: &'a str,
784 probe_token: Option<Arc<()>>,
785 settled: bool,
786}
787
788impl RouteBindBreakerGuard<'_> {
789 fn record_accepted(&mut self) -> bool {
793 self.settled = true;
794 self.breakers.record_accepted(self.module_id)
795 }
796
797 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
800 self.settled = true;
801 self.breakers.record_timeout(
802 self.module_id,
803 self.probe_token.as_ref(),
804 threshold,
805 cooldown,
806 )
807 }
808
809 fn record_inconclusive(&mut self) {
818 self.settled = true;
819 self.breakers
820 .record_inconclusive(self.module_id, self.probe_token.as_ref());
821 }
822}
823
824impl Drop for RouteBindBreakerGuard<'_> {
825 fn drop(&mut self) {
826 if !self.settled {
827 self.breakers
828 .record_inconclusive(self.module_id, self.probe_token.as_ref());
829 }
830 }
831}
832
833struct BreakerOpened {
837 consecutive_timeouts: u32,
838 reopened_after_probe: bool,
841}
842
843impl RouteBindBreakers {
844 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
845 self.modules
846 .lock()
847 .expect("route.bind breaker mutex poisoned")
848 }
849
850 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
853 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
854 probe: probe_token.is_some(),
855 guard: RouteBindBreakerGuard {
856 breakers: self.clone(),
857 module_id,
858 probe_token,
859 settled: false,
860 },
861 };
862
863 let mut modules = self.lock();
864 let Some(state) = modules.get_mut(module_id) else {
865 return admitted(None);
866 };
867 let Some(cooldown_until) = state.cooldown_until else {
868 return admitted(None);
869 };
870 if state.probe_in_flight.is_some() {
871 return RouteBindAdmission::Refused {
872 consecutive_timeouts: state.consecutive_timeouts,
873 retry_in: Duration::ZERO,
874 probe_in_flight: true,
875 };
876 }
877 let now = Instant::now();
878 if now < cooldown_until {
879 return RouteBindAdmission::Refused {
880 consecutive_timeouts: state.consecutive_timeouts,
881 retry_in: cooldown_until - now,
882 probe_in_flight: false,
883 };
884 }
885 let token = Arc::new(());
886 state.probe_in_flight = Some(Arc::clone(&token));
887 admitted(Some(token))
888 }
889
890 fn record_accepted(&self, module_id: &str) -> bool {
891 self.lock()
892 .remove(module_id)
893 .is_some_and(|state| state.cooldown_until.is_some())
894 }
895
896 fn record_timeout(
897 &self,
898 module_id: &str,
899 probe_token: Option<&Arc<()>>,
900 threshold: u32,
901 cooldown: Duration,
902 ) -> Option<BreakerOpened> {
903 let mut modules = self.lock();
904 let state = modules.entry(module_id.to_string()).or_default();
905 let was_open = state.cooldown_until.is_some();
906 let was_probe = Self::owns_probe(state, probe_token);
907 if was_probe {
908 state.probe_in_flight = None;
909 }
910 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
911 if state.consecutive_timeouts < threshold {
912 return None;
913 }
914 state.cooldown_until = Some(Instant::now() + cooldown);
915 Some(BreakerOpened {
916 consecutive_timeouts: state.consecutive_timeouts,
917 reopened_after_probe: was_open && was_probe,
918 })
919 }
920
921 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
922 state
926 .probe_in_flight
927 .as_ref()
928 .zip(token)
929 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
930 }
931
932 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
933 if let Some(state) = self.lock().get_mut(module_id) {
934 if Self::owns_probe(state, probe_token) {
935 state.probe_in_flight = None;
936 }
937 }
938 }
939
940 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
955 self.lock()
956 .remove(module_id)
957 .map(|state| state.consecutive_timeouts)
958 .filter(|discarded| *discarded > 0)
959 }
960
961 fn open_snapshot(&self) -> Option<serde_json::Value> {
968 let now = Instant::now();
969 let modules = self.lock();
970 let open = modules
971 .iter()
972 .filter_map(|(module_id, state)| {
973 let cooldown_until = state.cooldown_until?;
974 Some((
975 module_id.clone(),
976 serde_json::json!({
977 "consecutive_timeouts": state.consecutive_timeouts,
978 "cooldown_remaining_ms":
979 cooldown_until.saturating_duration_since(now).as_millis() as u64,
980 "probe_in_flight": state.probe_in_flight.is_some(),
981 }),
982 ))
983 })
984 .collect::<serde_json::Map<String, serde_json::Value>>();
985 (!open.is_empty()).then_some(serde_json::Value::Object(open))
986 }
987}
988
989impl ControlHandler {
990 pub fn new(registry: Arc<Registry>) -> Self {
991 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
992 }
993
994 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
995 let counters = forwarding.counters();
996 let route_bind_breakers = forwarding.route_bind_breakers();
1000 let route_bind_concurrency = forwarding.route_bind_concurrency();
1001 let route_outages = forwarding.route_outages();
1002 Self {
1003 registry,
1004 activity: None,
1005 activity_streams: Arc::new(Mutex::new(HashMap::new())),
1006 forwarding,
1007 process_liveness: None,
1008 supervisor: SupervisorHandle::new(),
1009 resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1010 env!("CARGO_PKG_VERSION").to_owned(),
1011 )),
1012 agent_declarations: agent_commands::Queues::default(),
1013 list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1014 subc_capabilities: Arc::from([
1015 CAP_MANIFEST_REGISTRATION.to_string(),
1016 CAP_CHANNEL_LIFECYCLE.to_string(),
1017 CAP_PING_PONG.to_string(),
1018 CAP_SESSION_ATTACH.to_string(),
1019 CAP_ADMISSION_FACTS_RELAY.to_string(),
1020 CAP_SCOPES_V1.to_string(),
1021 subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1022 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1023 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1024 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1025 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1026 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1027 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1028 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1029 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1030 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1031 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1032 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1033 ]),
1034 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1035 route_bind_relay_timeouts: BTreeMap::new(),
1036 route_bind_breakers,
1037 route_bind_concurrency,
1038 route_outages,
1039 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1040 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1041 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1042 storage_config: None,
1043 machine_id: None,
1044 admission_facts_carrier_module_id: None,
1045 admission_facts_targets: None,
1046 scopes: Arc::new(RwLock::new(ScopeTable::new(
1047 crate::daemon_config::default_scope_authority_owners(),
1048 ))),
1049 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1050 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1051 federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1052 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1053 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1054 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1055 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1056 local_connections: Arc::new(Mutex::new(HashSet::new())),
1057 operator_app_policy: Default::default(),
1058 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1059 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1060 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1061 #[cfg(test)]
1062 app_open_test_platform: None,
1063 #[cfg(test)]
1064 app_open_test_launch: None,
1065 #[cfg(windows)]
1066 windows_verify: None,
1067 #[cfg(windows)]
1068 windows_flash: None,
1069 #[cfg(windows)]
1070 windows_track: None,
1071 #[cfg(all(test, windows))]
1072 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1073 #[cfg(test)]
1074 caller_lookup: None,
1075 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1076 rescan: None,
1077 connected_clients: ConnectedClients::new(),
1078 local_endpoint_unavailable: None,
1079 counters,
1080 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1081 daemon_shutdown: DaemonShutdownFlag::default(),
1082 daemon_provenance: DaemonProvenanceFacts::default(),
1083 #[cfg(test)]
1084 control_dispatch_delay: None,
1085 #[cfg(test)]
1086 provenance_probe_override: None,
1087 #[cfg(test)]
1088 route_bind_relay_sent_hold: None,
1089 #[cfg(test)]
1090 hello_registry_to_forwarding_hold: None,
1091 }
1092 }
1093
1094 pub fn with_storage_config(
1097 mut self,
1098 storage_config: Option<crate::daemon_config::StorageConfig>,
1099 ) -> Self {
1100 self.storage_config = storage_config;
1101 self
1102 }
1103
1104 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1107 self.activity = Some(crate::activity::ActivityLedger::open(
1108 run_dir.join("activity.jsonl"),
1109 incarnation,
1110 ));
1111 self.supervisor.set_activity(self.activity.clone());
1112 self
1113 }
1114
1115 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1118 self.machine_id = machine_id;
1119 self
1120 }
1121
1122 pub fn with_admission_facts_config(
1126 mut self,
1127 carrier_module_id: Option<String>,
1128 targets: Option<Vec<String>>,
1129 ) -> Self {
1130 self.admission_facts_carrier_module_id = carrier_module_id;
1131 self.admission_facts_targets = targets;
1132 self
1133 }
1134
1135 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1139 self.scopes = Arc::new(RwLock::new(
1140 ScopeTable::new(owners.iter().cloned())
1141 .with_federated_owner(self.federated_scope_owner.clone()),
1142 ));
1143 self.scope_authority_owners = owners;
1144 self
1145 }
1146
1147 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1149 self.scopes = Arc::new(RwLock::new(
1150 ScopeTable::new(self.scope_authority_owners.iter().cloned())
1151 .with_federated_owner(owner.clone()),
1152 ));
1153 self.federated_scope_owner = owner;
1154 self
1155 }
1156
1157 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1159 self.privileged_operator_prompt = mode;
1160 self
1161 }
1162
1163 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1166 self.route_bind_relay_timeout = timeout;
1167 self
1168 }
1169
1170 pub fn with_route_bind_relay_timeouts(
1176 mut self,
1177 timeouts: impl IntoIterator<Item = (String, Duration)>,
1178 ) -> Self {
1179 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1180 self
1181 }
1182
1183 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1189 self.route_bind_relay_timeouts
1190 .get(module_id)
1191 .copied()
1192 .unwrap_or(self.route_bind_relay_timeout)
1193 }
1194
1195 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1203 self.route_bind_breaker_threshold = threshold.max(1);
1204 self.route_bind_breaker_cooldown = cooldown;
1205 self
1206 }
1207
1208 #[cfg(test)]
1209 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1210 self.health_probe_timeout = timeout;
1211 self
1212 }
1213
1214 #[cfg(test)]
1215 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1216 self.control_dispatch_delay = Some(delay);
1217 self
1218 }
1219
1220 #[cfg(test)]
1224 pub(crate) fn with_route_bind_relay_sent_hold(
1225 mut self,
1226 hold: Arc<tokio::sync::Barrier>,
1227 ) -> Self {
1228 self.route_bind_relay_sent_hold = Some(hold);
1229 self
1230 }
1231
1232 #[cfg(test)]
1237 pub(crate) fn with_hello_registry_to_forwarding_hold(
1238 mut self,
1239 hold: Arc<std::sync::Barrier>,
1240 ) -> Self {
1241 self.hello_registry_to_forwarding_hold = Some(hold);
1242 self
1243 }
1244
1245 #[cfg(test)]
1246 fn hold_between_registry_and_forwarding(&self) {
1247 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1248 hold.wait();
1249 hold.wait();
1250 }
1251 }
1252
1253 pub fn with_process_liveness(
1254 mut self,
1255 process_liveness: Arc<dyn ModuleProcessLiveness>,
1256 ) -> Self {
1257 self.process_liveness = Some(process_liveness);
1258 self
1259 }
1260
1261 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1262 self.supervisor = supervisor;
1263 self.supervisor.set_activity(self.activity.clone());
1264 self
1265 }
1266
1267 pub fn with_daemon_provenance(
1268 mut self,
1269 pid: u32,
1270 started_at_ms: u64,
1271 executable_path: Option<PathBuf>,
1272 build_git_sha: Option<String>,
1273 build_lock_digest: Option<String>,
1274 ) -> Self {
1275 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1276 let process_start_time = process_start_time(pid);
1277 self.daemon_provenance = DaemonProvenanceFacts {
1278 build: DaemonBuildProvenance {
1279 build_git_sha,
1280 build_lock_digest,
1281 },
1282 pid: Some(pid),
1283 started_at_ms: Some(started_at_ms),
1284 start_clock: None,
1285 executable_path,
1286 executable_identity,
1287 process_start_time,
1288 probe: ExecutableIdentityProbe::default(),
1289 };
1290 self
1291 }
1292
1293 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1294 self.daemon_provenance.start_clock = Some(clock);
1295 self
1296 }
1297
1298 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1300 self.scopes
1301 .write()
1302 .unwrap_or_else(|poisoned| poisoned.into_inner())
1303 .set_wall_clock(Arc::new(clock));
1304 self
1305 }
1306
1307 #[cfg(test)]
1308 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1309 self.provenance_probe_override = Some(result);
1310 self
1311 }
1312
1313 pub fn with_capability_config(
1317 self,
1318 modules: impl IntoIterator<Item = (String, bool)>,
1319 reserved_capabilities: BTreeMap<String, String>,
1320 ) -> Self {
1321 self.capability_evaluator
1322 .configure(modules, reserved_capabilities);
1323 self
1324 }
1325
1326 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1330 self.daemon_shutdown = flag;
1331 self
1332 }
1333
1334 pub fn with_supervisor_rescan(
1335 mut self,
1336 supervisor: Supervisor,
1337 config_path: impl Into<PathBuf>,
1338 configured_port: Option<u16>,
1339 ) -> Self {
1340 self.rescan = Some(SupervisorRescanContext {
1341 supervisor,
1342 config_path: config_path.into(),
1343 configured_port,
1344 storage_config: self.storage_config.clone(),
1345 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1346 admission_facts_targets: self.admission_facts_targets.clone(),
1347 scope_authority_owners: self.scope_authority_owners.clone(),
1348 federated_scope_owner: self.federated_scope_owner.clone(),
1349 privileged_operator_prompt: self.privileged_operator_prompt,
1350 generation: Arc::default(),
1351 });
1352 self
1353 }
1354
1355 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1356 self.connected_clients = connected_clients;
1357 self
1358 }
1359
1360 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1363 self.local_endpoint_unavailable = reason;
1364 self
1365 }
1366
1367 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1368 Arc::clone(&self.forwarding)
1369 }
1370
1371 pub(crate) fn counters(&self) -> DaemonCounters {
1372 self.counters.clone()
1373 }
1374
1375 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1378 tokio::spawn(async move {
1379 loop {
1380 self.capability_evaluator
1381 .wait_for_change_or_deadline()
1382 .await;
1383 self.refresh_capability_requirements();
1384 }
1385 });
1386 }
1387
1388 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1394 tokio::spawn(async move {
1395 loop {
1396 let has_deadlines = self
1397 .scopes
1398 .read()
1399 .unwrap_or_else(|poisoned| poisoned.into_inner())
1400 .has_deadlines();
1401 if !has_deadlines {
1402 self.scope_deadline_added.notified().await;
1403 continue;
1404 }
1405 tokio::time::sleep(Duration::from_secs(1)).await;
1406 if let Err(error) = self.sweep_expired_scopes() {
1407 warn!(%error, "scope expiry sweep failed");
1408 }
1409 }
1410 });
1411 }
1412
1413 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1414 let mut table = self
1415 .scopes
1416 .write()
1417 .unwrap_or_else(|poisoned| poisoned.into_inner());
1418 let swept = table.sweep_expired();
1419 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1420 drop(table);
1421 self.close_scope_drained_routes(drained);
1422 Ok(())
1423 }
1424
1425 fn publish_scope_changes(
1426 &self,
1427 changes: &[crate::scopes::ScopeTagChange],
1428 expired: &[crate::scopes::ScopeExpired],
1429 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1430 if changes.is_empty() {
1431 return Ok(Vec::new());
1432 }
1433 let drained = self
1434 .forwarding
1435 .publish_scope_changes(changes)
1436 .map_err(RouterError::Forwarding)?;
1437 let mut counts = HashMap::new();
1438 for route in &drained {
1439 *counts
1440 .entry((
1441 route.scope.owner.as_str(),
1442 route.scope.scope_ref.as_str(),
1443 route.scope.tag.scope_epoch,
1444 ))
1445 .or_insert(0usize) += 1;
1446 }
1447 for scope in expired {
1448 let routes_closed = counts
1449 .get(&(
1450 scope.owner.as_str(),
1451 scope.scope_ref.as_str(),
1452 scope.scope_epoch,
1453 ))
1454 .copied()
1455 .unwrap_or(0);
1456 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1457 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1458 routes_closed, "scope expired");
1459 }
1460 Ok(drained)
1461 }
1462
1463 fn runtime_capability_snapshot(
1464 &self,
1465 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1466 let runtime = self
1467 .supervisor
1468 .list()
1469 .into_iter()
1470 .map(|module| {
1471 let status = module.status().map_err(|err| {
1472 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1473 })?;
1474 Ok(RuntimeModule {
1475 module_id: status.module_id,
1476 state: status.state,
1477 enabled: status.enabled,
1478 })
1479 })
1480 .collect::<Result<Vec<_>, RouterError>>()?;
1481 let registrations = self.registry.list_registered_modules().map_err(|err| {
1487 RouterError::backend(
1488 0,
1489 0,
1490 format!("failed to list capability registrations: {err}"),
1491 )
1492 })?;
1493 let registrations = registrations
1494 .into_iter()
1495 .map(|registration| RegisteredModule {
1496 module_id: registration.manifest.module_id,
1497 module_version: registration.manifest.module_version,
1498 capabilities: registration.manifest.capabilities,
1499 })
1500 .collect();
1501 Ok((runtime, registrations))
1502 }
1503
1504 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1511 let cached_registration = RegisteredModule {
1512 module_id: registration.manifest.module_id.clone(),
1513 module_version: registration.manifest.module_version.clone(),
1514 capabilities: registration.manifest.capabilities.clone(),
1515 };
1516 let change = self.capability_evaluator.record_hello(&cached_registration);
1517 log_manifest_change(&cached_registration.module_id, &change);
1518 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1519 self.enforce_capability_denies();
1520 }
1521 self.refresh_capability_requirements();
1522 }
1523
1524 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1528 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1529 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1530 self.supervisor.set_swap_promotion_observer(observer);
1531 }
1532
1533 pub fn refresh_capability_requirements(&self) {
1534 match self.runtime_capability_snapshot() {
1535 Ok((runtime, registrations)) => {
1536 log_requirement_events(
1537 self.capability_evaluator
1538 .evaluate_now(&runtime, ®istrations),
1539 &self.daemon_shutdown,
1540 );
1541 }
1542 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1543 }
1544 }
1545
1546 fn enforce_capability_denies(&self) {
1550 let registrations = match self.registry.list_registered_modules() {
1551 Ok(snapshot) => snapshot,
1552 Err(err) => {
1553 warn!(error = %err, "failed to read registrations for capability deny census");
1554 return;
1555 }
1556 };
1557 let manifests = registrations
1558 .into_iter()
1559 .map(|registration| {
1560 (
1561 registration.manifest.module_id.clone(),
1562 registration.manifest,
1563 )
1564 })
1565 .collect::<BTreeMap<_, _>>();
1566 let census = match self.forwarding.route_census(None) {
1567 Ok(census) => census,
1568 Err(err) => {
1569 warn!(error = %err, "failed to read route census for capability deny enforcement");
1570 return;
1571 }
1572 };
1573
1574 for (target_module_id, routes) in census {
1575 let Some(target_manifest) = manifests.get(&target_module_id) else {
1576 continue;
1577 };
1578 let mut closed_routes = Vec::new();
1579 let mut goodbyes = Vec::new();
1580 for route in routes {
1581 let Principal::Reserved {
1582 module_id: opening_module_id,
1583 } = &route.principal
1584 else {
1585 continue;
1586 };
1587 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1588 continue;
1589 };
1590 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1591 continue;
1592 };
1593
1594 match self.forwarding.release_client_route(
1595 route.goodbye_target.connection_id,
1596 route.goodbye_target.channel,
1597 route.goodbye_target.epoch,
1598 ) {
1599 Ok(RouteRelease::Removed(module_goodbye)) => {
1600 warn!(
1601 opening_module_id,
1602 target_module_id,
1603 capability,
1604 "force-closing route because an attested capability deny edge now matches"
1605 );
1606 goodbyes.push(module_goodbye);
1607 goodbyes.push(route.goodbye_target.clone());
1608 closed_routes.push(route);
1609 }
1610 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1611 Err(err) => warn!(
1612 opening_module_id,
1613 target_module_id,
1614 capability,
1615 error = %err,
1616 "failed to force-close capability-denied route"
1617 ),
1618 }
1619 }
1620
1621 if closed_routes.is_empty() {
1622 continue;
1623 }
1624 send_route_control_pushes(
1625 &self.forwarding,
1626 closed_routes,
1627 ClientControlPush::RouteClosed {
1628 module_id: target_module_id,
1629 channels: Vec::new(),
1630 reason: RouteCloseReason::CapabilityDenied,
1631 drained: false,
1632 abandoned: 0,
1633 excluded_subscriptions: 0,
1634 terminal: Some(false),
1635 },
1636 );
1637 self.emit_route_goodbyes(goodbyes);
1640 }
1641 }
1642
1643 fn not_ready_reason(
1649 &self,
1650 registration: &crate::registry::ModuleRegistration,
1651 ) -> Option<NotReadyReason> {
1652 if !registration.ready {
1653 return Some(NotReadyReason {
1654 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1655 capability: None,
1656 });
1657 }
1658 self.first_unprovided_required_capability(registration)
1659 .map(|capability| NotReadyReason {
1660 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1661 capability: Some(capability),
1662 })
1663 }
1664
1665 fn first_unprovided_required_capability(
1683 &self,
1684 registration: &crate::registry::ModuleRegistration,
1685 ) -> Option<String> {
1686 let required = registration
1687 .manifest
1688 .capabilities
1689 .iter()
1690 .flat_map(|declarations| declarations.requires.iter())
1691 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1692 .map(|requirement| requirement.capability.as_str())
1693 .collect::<BTreeSet<_>>();
1694 if required.is_empty() {
1695 return None;
1696 }
1697 let module_id = registration.manifest.module_id.as_str();
1698 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1699 if required
1700 .iter()
1701 .any(|capability| verdict(capability).is_none())
1702 {
1703 self.refresh_capability_requirements();
1704 }
1705 required
1706 .into_iter()
1707 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1708 .map(str::to_string)
1709 }
1710
1711 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1712 self.capability_evaluator
1713 .statuses()
1714 .into_iter()
1715 .map(capability_requirement_status)
1716 .collect()
1717 }
1718
1719 fn deregister_connection(
1726 &self,
1727 connection_id: ConnectionId,
1728 reason: RegistrationEndReason,
1729 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1730 self.registry
1731 .deregister_connection_with_reason(connection_id, reason)
1732 }
1733
1734 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1735 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1736 return None;
1737 }
1738 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1739 parse_client_control_request(&frame.body)
1740 else {
1741 return None;
1742 };
1743 Some(target_module_id(&target).to_string())
1744 }
1745
1746 pub(crate) fn route_open_capacity_refusal(
1747 &self,
1748 ctx: &RouteCtx,
1749 frame: &Frame,
1750 target_module_id: &str,
1751 in_flight: usize,
1752 limit: usize,
1753 ) -> Result<Frame, RouterError> {
1754 let caller = self.route_open_caller_from_frame(frame);
1757 self.route_open_admission_refusal_frame(
1758 ctx,
1759 &caller,
1760 frame,
1761 target_module_id,
1762 "open_admission_full",
1763 (in_flight, limit),
1764 format!(
1765 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1766 ),
1767 )
1768 }
1769
1770 fn route_open_target_capacity_refusal(
1771 &self,
1772 ctx: &RouteCtx,
1773 caller: &RouteOpenCaller,
1774 frame: &Frame,
1775 target_module_id: &str,
1776 in_flight: usize,
1777 ) -> Result<Frame, RouterError> {
1778 self.route_open_admission_refusal_frame(
1779 ctx,
1780 caller,
1781 frame,
1782 target_module_id,
1783 "target_binds_full",
1784 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1785 format!(
1786 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1787 ),
1788 )
1789 }
1790
1791 #[allow(clippy::too_many_arguments)]
1799 fn route_open_admission_refusal_frame(
1800 &self,
1801 ctx: &RouteCtx,
1802 caller: &RouteOpenCaller,
1803 frame: &Frame,
1804 target_module_id: &str,
1805 reason: &'static str,
1806 (in_flight, limit): (usize, usize),
1807 message: impl Into<String>,
1808 ) -> Result<Frame, RouterError> {
1809 let code = error_codes::TARGET_UNAVAILABLE;
1810 self.counters.increment_route_open_refused(code);
1811 info!(
1812 target: "control",
1813 code,
1814 reason,
1815 module_id = ?target_module_id,
1816 connection_id = ctx.connection_id.get(),
1817 principal = %caller.principal,
1818 scope = %caller.scope,
1819 harness = %caller.harness,
1820 in_flight,
1821 limit,
1822 "route.open refused"
1823 );
1824 control_error_frame(frame, code, message.into())
1825 }
1826
1827 #[cfg(test)]
1833 pub fn handle_control(
1834 &self,
1835 connection_id: ConnectionId,
1836 frame: Frame,
1837 ) -> Result<Vec<Frame>, RouterError> {
1838 match frame.header.ty {
1839 FrameType::Ping => Ok(vec![pong(&frame)?]),
1840 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1841 FrameType::Goodbye => self.handle_goodbye(connection_id),
1842 ty => Ok(vec![control_error_frame(
1843 &frame,
1844 "unsupported_control_frame",
1845 format!("unsupported channel-0 frame {ty:?}"),
1846 )?]),
1847 }
1848 }
1849
1850 pub async fn handle_control_frame(
1851 &self,
1852 ctx: &RouteCtx,
1853 frame: Frame,
1854 ) -> Result<Vec<Frame>, RouterError> {
1855 self.handle_control_frame_timed(ctx, frame, None).await
1856 }
1857
1858 #[cfg(any(test, feature = "test-support"))]
1859 pub fn with_test_operator_app_policy(
1860 mut self,
1861 policy: crate::operator_apps::TestOperatorAppPolicy,
1862 ) -> Self {
1863 self.operator_app_policy = policy.into();
1864 self
1865 }
1866
1867 #[cfg(all(windows, any(test, feature = "test-support")))]
1871 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1872 self.windows_verify = Some(verify);
1873 self
1874 }
1875
1876 #[cfg(all(windows, any(test, feature = "test-support")))]
1878 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1879 self.windows_flash = Some(flash);
1880 self
1881 }
1882
1883 #[cfg(all(windows, any(test, feature = "test-support")))]
1887 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1888 self.windows_track = Some(track);
1889 self
1890 }
1891
1892 pub(crate) fn record_captured_peer(
1893 &self,
1894 connection: ConnectionId,
1895 peer: subc_os::local_peer::CapturedPeer,
1896 ) {
1897 self.record_observed_peer(connection, peer.observation().clone());
1898 #[cfg(windows)]
1899 if let Some(stamp) = self.stamp_windows_launch(&peer) {
1900 self.operator_apps
1901 .lock()
1902 .unwrap_or_else(|p| p.into_inner())
1903 .insert(connection, stamp);
1904 }
1905 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1906 self.operator_apps
1907 .lock()
1908 .unwrap_or_else(|p| p.into_inner())
1909 .insert(connection, stamp);
1910 }
1911 }
1912
1913 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1914 self.local_connections
1915 .lock()
1916 .unwrap_or_else(|p| p.into_inner())
1917 .insert(connection);
1918 }
1919
1920 fn connection_operator_app(
1923 &self,
1924 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1925 ) -> Option<subc_protocol::peer::OperatorApp> {
1926 stamp
1927 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1928 .map(|stamp| stamp.app.clone())
1929 }
1930
1931 fn handle_connection_describe(
1932 &self,
1933 ctx: &RouteCtx,
1934 frame: Frame,
1935 ) -> Result<Vec<Frame>, RouterError> {
1936 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1937 return Ok(vec![control_error_frame(
1938 &frame,
1939 "invalid_control_body",
1940 "malformed connection.describe body",
1941 )?]);
1942 }
1943 let local = self
1944 .local_connections
1945 .lock()
1946 .unwrap_or_else(|p| p.into_inner())
1947 .contains(&ctx.connection_id);
1948 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1949 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1950 Ok(vec![control_response_body_frame(
1951 &frame,
1952 &subc_control::ConnectionDescribeReply::new(local, operator_app),
1953 ops::CONNECTION_DESCRIBE,
1954 )?])
1955 }
1956
1957 pub(crate) fn record_observed_peer(
1958 &self,
1959 connection: ConnectionId,
1960 peer: subc_os::local_peer::ObservedPeer,
1961 ) {
1962 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1963 let peer = match peer {
1964 OsPeer::Unobserved => ObservedPeer::unobserved(),
1965 OsPeer::Observed(peer) => {
1966 let observed = match peer.user {
1967 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1968 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1969 };
1970 observed
1971 .with_exe(
1972 peer.exe
1973 .and_then(|path| path.into_os_string().into_string().ok()),
1974 )
1975 .with_signing_id(peer.signing_id)
1976 }
1977 };
1978 self.observed_peers
1979 .lock()
1980 .unwrap_or_else(|p| p.into_inner())
1981 .insert(connection, peer);
1982 }
1983
1984 pub(crate) fn record_caller_endpoints(
1985 &self,
1986 connection: ConnectionId,
1987 endpoints: Option<(SocketAddr, SocketAddr)>,
1988 ) {
1989 let mut callers = self
1990 .caller_endpoints
1991 .lock()
1992 .unwrap_or_else(|p| p.into_inner());
1993 callers.remove(&connection);
1994 if let Some(endpoints) = endpoints {
1995 callers.insert(connection, endpoints);
1996 }
1997 }
1998
1999 #[cfg(test)]
2000 pub(crate) fn with_caller_lookup_for_test(
2001 mut self,
2002 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2003 ) -> Self {
2004 self.caller_lookup = Some(Arc::new(lookup));
2005 self
2006 }
2007
2008 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2009 let endpoints = self
2010 .caller_endpoints
2011 .lock()
2012 .unwrap_or_else(|p| p.into_inner())
2013 .get(&connection)
2014 .copied();
2015 #[cfg(test)]
2016 let lookup = self.caller_lookup.clone();
2017 tokio::task::spawn_blocking(move || {
2018 #[cfg(test)]
2019 if let Some(lookup) = lookup {
2020 return lookup(endpoints);
2021 }
2022 endpoints
2023 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2024 .unwrap_or_default()
2025 })
2026 .await
2027 .unwrap_or_default()
2028 }
2029
2030 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2031 frame.header.channel == 0
2032 && frame.header.ty == FrameType::Request
2033 && self
2034 .forwarding
2035 .module_endpoint_for_connection(ctx.connection_id)
2036 .is_ok_and(|endpoint| endpoint.is_none())
2037 && parse_client_control_request(&frame.body)
2038 .ok()
2039 .is_some_and(|request| privileged_operation(&request).is_some())
2040 }
2041
2042 pub(crate) async fn handle_control_frame_timed(
2043 &self,
2044 ctx: &RouteCtx,
2045 frame: Frame,
2046 dispatch_started_at: Option<StdInstant>,
2047 ) -> Result<Vec<Frame>, RouterError> {
2048 match frame.header.ty {
2049 FrameType::Ping => Ok(vec![pong(&frame)?]),
2050 FrameType::Hello => {
2051 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2052 }
2053 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2054 FrameType::Cancel => {
2055 if self
2058 .agent_declarations
2059 .cancel(ctx.connection_id, frame.header.corr)
2060 || self
2061 .forwarding
2062 .operator_confirms()
2063 .cancel(ctx.connection_id, frame.header.corr)
2064 || self
2065 .supervisor
2066 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2067 || self.cancel_activity_subscription(ctx, &frame)?
2068 {
2069 Ok(Vec::new())
2070 } else {
2071 Ok(vec![control_error_frame(
2072 &frame,
2073 "unknown_subscription",
2074 "no subscription has this correlation id",
2075 )?])
2076 }
2077 }
2078 FrameType::Request => {
2079 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2082 if op
2083 .as_ref()
2084 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2085 {
2086 return self.handle_activity_record(ctx, frame).await;
2087 }
2088 if op.as_ref().is_some_and(|probe| {
2089 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2090 }) {
2091 return self.handle_activity_read(ctx, frame).await;
2092 }
2093 if op
2094 .as_ref()
2095 .is_some_and(|probe| probe.op == "operator.confirm")
2096 {
2097 return self.handle_operator_confirm(ctx, frame);
2098 }
2099 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2100 return self
2101 .observe_control_wait(
2102 ctx,
2103 None,
2104 ops::APP_OPEN,
2105 self.handle_app_open(ctx, frame),
2106 )
2107 .await;
2108 }
2109 if op
2110 .as_ref()
2111 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2112 {
2113 return self.handle_connection_describe(ctx, frame);
2114 }
2115 if op
2116 .as_ref()
2117 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2118 {
2119 return self.handle_scope_served_by(ctx.connection_id, frame);
2120 }
2121 if self
2122 .forwarding
2123 .module_endpoint_for_connection(ctx.connection_id)
2124 .map_err(RouterError::Forwarding)?
2125 .is_some()
2126 {
2127 if !is_known_module_request_op(&frame.body) {
2128 return Ok(vec![control_error_frame(
2129 &frame,
2130 "unsupported_control_frame",
2131 "module-originated channel-0 REQUEST is not supported",
2132 )?]);
2133 }
2134 let request = match parse_module_control_request_from_module(&frame.body) {
2135 Ok(request) => request,
2136 Err((err, ControlRequestBodyError::UnknownOp)) => {
2137 return Ok(vec![control_error_frame(
2138 &frame,
2139 "unsupported_control_frame",
2140 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2141 )?])
2142 }
2143 Err((err, ControlRequestBodyError::InvalidBody)) => {
2144 return Ok(vec![control_error_frame(
2145 &frame,
2146 "invalid_control_body",
2147 format!("malformed module control body: {err}"),
2148 )?])
2149 }
2150 };
2151 let op = module_control_request_op(&request);
2152 let corr = frame.header.corr;
2153 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2154 if let ModuleControlRequestFromModule::AgentCommands {
2155 started,
2156 finished,
2157 known,
2158 } = request
2159 {
2160 return self
2161 .agent_declarations
2162 .enqueue(
2163 self,
2164 ctx,
2165 frame,
2166 crate::resource_totals::Declaration {
2167 started,
2168 finished,
2169 known,
2170 },
2171 )
2172 .await;
2173 }
2174 let result =
2175 self.handle_module_control_request(ctx.connection_id, frame, request);
2176 log_slow_control_dispatch(
2177 dispatch_started_at,
2178 op,
2179 ctx.connection_id,
2180 corr,
2181 None,
2182 );
2183 return result;
2184 }
2185
2186 if is_known_module_request_op(&frame.body) {
2187 let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2188 .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2189 let code = if is_agents {
2190 "not_supervised"
2191 } else if op
2192 .as_ref()
2193 .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2194 {
2195 error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2196 } else {
2197 "not_registered"
2198 };
2199 return Ok(vec![control_error_frame(
2200 &frame,
2201 code,
2202 "module-originated control requests require the module's own registered connection",
2203 )?]);
2204 }
2205
2206 let request = match parse_client_control_request(&frame.body) {
2207 Ok(request) => request,
2208 Err((err, ControlRequestBodyError::UnknownOp)) => {
2209 return Ok(vec![control_error_frame(
2210 &frame,
2211 "unknown_control_op",
2212 format!("unknown client control op: {err}"),
2213 )?])
2214 }
2215 Err((err, ControlRequestBodyError::InvalidBody)) => {
2216 return Ok(vec![control_error_frame(
2217 &frame,
2218 "invalid_control_body",
2219 format!("malformed client control body: {err}"),
2220 )?])
2221 }
2222 };
2223 let op = client_control_request_op(&request);
2224 let corr = frame.header.corr;
2225 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2226 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2227 #[cfg(test)]
2228 if let Some(delay) = self.control_dispatch_delay {
2229 tokio::time::sleep(delay).await;
2230 }
2231 let result = self
2232 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2233 .await;
2234 if let Some(timings) = route_open_timings.as_mut() {
2235 timings.finish();
2236 }
2237 log_slow_control_dispatch(
2238 dispatch_started_at,
2239 op,
2240 ctx.connection_id,
2241 corr,
2242 route_open_timings.as_ref(),
2243 );
2244 result
2245 }
2246 FrameType::Push => {
2247 let Some(endpoint) = self
2248 .forwarding
2249 .module_endpoint_for_connection(ctx.connection_id)
2250 .map_err(RouterError::Forwarding)?
2251 else {
2252 return Ok(vec![control_error_frame(
2253 &frame,
2254 "unsupported_control_frame",
2255 "client-originated channel-0 PUSH is not supported",
2256 )?]);
2257 };
2258 self.handle_status_update(endpoint, frame)
2259 }
2260 FrameType::Response | FrameType::Error
2261 if self
2262 .forwarding
2263 .module_endpoint_for_connection(ctx.connection_id)
2264 .map_err(RouterError::Forwarding)?
2265 .is_some() =>
2266 {
2267 self.handle_module_relay_response(ctx.connection_id, frame)
2268 }
2269 ty => Ok(vec![control_error_frame(
2270 &frame,
2271 "unsupported_control_frame",
2272 format!("unsupported channel-0 frame {ty:?}"),
2273 )?]),
2274 }
2275 }
2276
2277 pub fn cleanup_connection(
2278 &self,
2279 connection_id: ConnectionId,
2280 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2281 self.local_connections
2282 .lock()
2283 .unwrap_or_else(|p| p.into_inner())
2284 .remove(&connection_id);
2285 self.caller_endpoints
2286 .lock()
2287 .unwrap_or_else(|p| p.into_inner())
2288 .remove(&connection_id);
2289 self.build_class_warned
2290 .lock()
2291 .unwrap_or_else(|p| p.into_inner())
2292 .remove(&connection_id);
2293 self.observed_peers
2294 .lock()
2295 .unwrap_or_else(|p| p.into_inner())
2296 .remove(&connection_id);
2297 self.operator_apps
2298 .lock()
2299 .unwrap_or_else(|p| p.into_inner())
2300 .remove(&connection_id);
2301 self.forwarding
2302 .operator_confirms()
2303 .connection_closed(connection_id);
2304 self.cleanup_connection_with_end_reason(
2305 connection_id,
2306 RegistrationEndReason::ConnectionClosed,
2307 )
2308 }
2309
2310 fn cleanup_connection_with_end_reason(
2311 &self,
2312 connection_id: ConnectionId,
2313 requested_reason: RegistrationEndReason,
2314 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2315 self.agent_declarations.close(connection_id);
2316 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2317 self.registry
2318 .get_module_by_connection(connection_id)?
2319 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2320 .and_then(|module| module.registration_end_reason().ok().flatten())
2321 .unwrap_or(requested_reason)
2322 } else {
2323 requested_reason
2324 };
2325 let crash_closed = self
2326 .registry
2327 .get_module_by_connection(connection_id)?
2328 .and_then(|registration| {
2329 self.forwarding
2330 .module_endpoint_for_connection(connection_id)
2331 .ok()
2332 .flatten()
2333 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2334 .map(|routes| (registration.manifest.module_id, routes))
2335 });
2336 let crash_closed = crash_closed.map(|(module_id, routes)| {
2337 let terminal = match self.supervisor.get(&module_id) {
2338 None => false,
2339 Some(module) => match module.will_recover_after_connection_loss() {
2340 Ok(will_recover) => !will_recover,
2341 Err(err) => {
2342 warn!(
2343 %module_id,
2344 error = %err,
2345 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2346 );
2347 false
2348 }
2349 },
2350 };
2351 let (reason, drained) = match self
2355 .forwarding
2356 .daemon_drain_result_for_connection(connection_id)
2357 {
2358 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2359 Ok(None) => (RouteCloseReason::Crash, false),
2360 Err(err) => {
2361 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2362 (RouteCloseReason::Crash, false)
2363 }
2364 };
2365 (module_id, routes, reason, drained, terminal)
2366 });
2367 let registrations = self.deregister_connection(connection_id, end_reason);
2368 let cleanup = if crash_closed.is_some() {
2369 self.forwarding.cleanup_connection_counted(connection_id)
2370 } else {
2371 self.forwarding
2377 .cleanup_connection(connection_id)
2378 .map(|released| crate::forwarding::ConnectionCleanup {
2379 released,
2380 abandoned_relays: 0,
2381 })
2382 };
2383 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2388 let abandoned = cleanup
2389 .as_ref()
2390 .map_or(0, |cleanup| cleanup.abandoned_relays);
2391 send_route_control_pushes(
2392 &self.forwarding,
2393 routes,
2394 ClientControlPush::RouteClosed {
2395 module_id,
2396 channels: Vec::new(),
2397 reason,
2398 drained,
2399 abandoned,
2400 excluded_subscriptions: 0,
2401 terminal: Some(terminal),
2402 },
2403 );
2404 }
2405 if let Ok(cleanup) = cleanup {
2406 self.emit_route_goodbyes(cleanup.released);
2407 }
2408 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2412 crate::supervise::notify_registration_release();
2413 self.capability_evaluator.wake_deadline_loop();
2414 self.refresh_capability_requirements();
2415 }
2416 self.supervisor.remove_spawn_subscribers(connection_id);
2417 self.activity_streams
2418 .lock()
2419 .unwrap_or_else(|p| p.into_inner())
2420 .retain(|(id, _), task| {
2421 if *id == connection_id {
2422 task.abort();
2423 false
2424 } else {
2425 true
2426 }
2427 });
2428 self.hello_launch_nonces
2431 .lock()
2432 .unwrap_or_else(|poisoned| poisoned.into_inner())
2433 .forget(connection_id);
2434 self.scopes
2435 .write()
2436 .unwrap_or_else(|poisoned| poisoned.into_inner())
2437 .release_connection(connection_id);
2438 registrations
2439 }
2440
2441 pub(crate) fn handle_route_goodbye(
2442 &self,
2443 connection_id: ConnectionId,
2444 route_channel: u16,
2445 route_epoch: u32,
2446 ) -> Result<bool, RouterError> {
2447 debug!(
2448 connection_id = connection_id.get(),
2449 route_channel, route_epoch, "handling route GOODBYE"
2450 );
2451 let RouteRelease::Removed(released_route) = self
2452 .forwarding
2453 .release_client_route(connection_id, route_channel, route_epoch)
2454 .map_err(RouterError::Forwarding)?
2455 else {
2456 return Ok(false);
2457 };
2458 self.emit_route_goodbyes(vec![released_route]);
2459 Ok(true)
2460 }
2461
2462 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2463 for released in released_routes {
2464 let frame = match Frame::build_with_version(
2465 released.negotiated_ver,
2466 FrameType::Goodbye,
2467 control_flags(),
2468 released.channel,
2469 released.epoch,
2470 0,
2471 Vec::new(),
2472 ) {
2473 Ok(frame) => frame,
2474 Err(err) => {
2475 warn!(
2476 route_channel = released.channel,
2477 error = %err,
2478 "failed to build route GOODBYE frame"
2479 );
2480 continue;
2481 }
2482 };
2483 if !released.close_on_delivery_failure() {
2484 crate::forwarding::send_module_route_goodbye(
2485 &self.counters,
2486 &released.sink,
2487 frame,
2488 released.module_id.as_deref(),
2489 "client route released",
2490 );
2491 continue;
2492 }
2493 if let Err(err) = released.sink.try_send(frame) {
2494 warn!(
2495 target_connection_id = released.connection_id.get(),
2496 route_channel = released.channel,
2497 error = %err,
2498 "route GOODBYE was not delivered to client; closing target connection"
2499 );
2500 if self
2501 .forwarding
2502 .escalate_client_delivery_failure(
2503 released.connection_id,
2504 released.channel,
2505 released.epoch,
2506 CloseReason::new(
2507 "route_goodbye_delivery_failed",
2508 format!(
2509 "failed to enqueue route GOODBYE for channel {}: {err}",
2510 released.channel
2511 ),
2512 ),
2513 crate::forwarding::UndeliveredFrame {
2514 module_id: released.module_id.as_deref(),
2515 sink: &released.sink,
2516 },
2517 )
2518 .unwrap_or(false)
2519 {
2520 self.counters.increment_goodbye_relay_client_failed();
2521 }
2522 }
2523 }
2524 }
2525
2526 fn send_abandoned_route_bind_goodbye(
2538 &self,
2539 module_sink: &crate::FrameSink,
2540 negotiated_ver: u8,
2541 module_channel: u16,
2542 module_epoch: u32,
2543 ) {
2544 let frame = match Frame::build_with_version(
2545 negotiated_ver,
2546 FrameType::Goodbye,
2547 control_flags(),
2548 module_channel,
2549 module_epoch,
2550 0,
2551 Vec::new(),
2552 ) {
2553 Ok(frame) => frame,
2554 Err(err) => {
2555 warn!(
2556 route_channel = module_channel,
2557 error = %err,
2558 "failed to build GOODBYE for abandoned route.bind"
2559 );
2560 return;
2561 }
2562 };
2563 crate::forwarding::send_module_route_goodbye(
2564 &self.counters,
2565 module_sink,
2566 frame,
2567 None,
2568 "abandoned route.bind",
2569 );
2570 }
2571
2572 fn handle_hello(
2573 &self,
2574 connection_id: ConnectionId,
2575 sink: Option<crate::FrameSink>,
2576 frame: Frame,
2577 ) -> Result<Vec<Frame>, RouterError> {
2578 debug!(
2579 connection_id = connection_id.get(),
2580 corr = frame.header.corr,
2581 "handling HELLO"
2582 );
2583 if self
2587 .registry
2588 .get_module_by_connection(connection_id)
2589 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2590 .is_some()
2591 {
2592 return Ok(vec![control_error_frame(
2593 &frame,
2594 "invalid_hello",
2595 "connection is already registered as a module",
2596 )?]);
2597 }
2598 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2599 Ok(value) => value,
2600 Err(err) => {
2601 return Ok(vec![control_error_frame(
2602 &frame,
2603 "invalid_hello",
2604 format!("malformed HELLO body: {err}"),
2605 )?])
2606 }
2607 };
2608 if let Err(err) = validate_hello_display_name(&hello_value) {
2609 return Ok(vec![control_error_frame(
2610 &frame,
2611 "invalid_manifest",
2612 err.to_string(),
2613 )?]);
2614 }
2615 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2616 return Ok(vec![control_error_frame(
2617 &frame,
2618 "invalid_capability_grammar",
2619 err.to_string(),
2620 )?]);
2621 }
2622 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2623 return Ok(vec![control_error_frame(
2624 &frame,
2625 "invalid_manifest",
2626 err.to_string(),
2627 )?]);
2628 }
2629 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2630 return Ok(vec![control_error_frame(
2631 &frame,
2632 "invalid_event_declaration",
2633 err.to_string(),
2634 )?]);
2635 }
2636 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2637 return Ok(vec![control_error_frame(
2638 &frame,
2639 "invalid_manifest",
2640 err.to_string(),
2641 )?]);
2642 }
2643 if let Some(provenance) = hello_value
2644 .get("manifest")
2645 .and_then(|manifest| manifest.get("provenance"))
2646 {
2647 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2648 return Ok(vec![control_error_frame(
2649 &frame,
2650 "invalid_manifest",
2651 format!("malformed manifest provenance: {err}"),
2652 )?]);
2653 }
2654 }
2655 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2656 let module_id = hello_value["manifest"]["module_id"]
2657 .as_str()
2658 .unwrap_or("<unknown>");
2659 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2660 }
2661 if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
2662 let module_id = hello_value
2663 .get("manifest")
2664 .and_then(|m| m.get("module_id"))
2665 .and_then(serde_json::Value::as_str)
2666 .unwrap_or("unknown");
2667 warn!(module_id, %reason, "dropping operation_deferral from HELLO");
2668 }
2669 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2670 let module_id = hello_value["manifest"]["module_id"]
2671 .as_str()
2672 .unwrap_or("<unknown>");
2673 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2674 }
2675 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2676 let module_id = hello_value["manifest"]["module_id"]
2677 .as_str()
2678 .unwrap_or("<unknown>");
2679 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2680 }
2681 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2682 Ok(hello) => hello,
2683 Err(err) => {
2684 return Ok(vec![control_error_frame(
2685 &frame,
2686 "invalid_hello",
2687 format!("malformed HELLO body: {err}"),
2688 )?])
2689 }
2690 };
2691
2692 if hello.protocol_ver != hello.manifest.protocol_ver {
2693 return Ok(vec![control_error_frame(
2694 &frame,
2695 "invalid_manifest",
2696 format!(
2697 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2698 hello.protocol_ver, hello.manifest.protocol_ver
2699 ),
2700 )?]);
2701 }
2702
2703 if hello.manifest.module_id.trim().is_empty() {
2704 return Ok(vec![control_error_frame(
2705 &frame,
2706 "invalid_manifest",
2707 "manifest module_id must not be empty",
2708 )?]);
2709 }
2710
2711 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2712 Ok(negotiated_ver) => negotiated_ver,
2713 Err(message) => {
2714 return Ok(vec![control_error_frame(
2715 &frame,
2716 "version_unsupported",
2717 message,
2718 )?])
2719 }
2720 };
2721
2722 let swap_admission = self
2730 .supervisor
2731 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2732 if swap_admission == SwapHelloAdmission::Refused {
2733 warn!(
2734 module_id = %hello.manifest.module_id,
2735 connection_id = connection_id.get(),
2736 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2737 );
2738 return Ok(vec![control_error_frame(
2739 &frame,
2740 "swap_token_invalid",
2741 format!(
2742 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2743 hello.manifest.module_id
2744 ),
2745 )?]);
2746 }
2747 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2748
2749 let verification = if swap_candidate {
2754 ReservedHelloVerification {
2755 nonce_verified: true,
2756 prefix_owner: None,
2757 }
2758 } else {
2759 match self.supervisor.reserved_hello_verification(
2760 &hello.manifest.module_id,
2761 hello.launch_nonce.as_deref(),
2762 ) {
2763 Ok(verified) => verified,
2764 Err(rejection) => {
2765 let message = match rejection {
2766 ReservedHelloRejection::Exact { module_id } => format!(
2767 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2768 ),
2769 ReservedHelloRejection::Prefix {
2770 prefix,
2771 owner_module_id,
2772 } => format!(
2773 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2774 hello.manifest.module_id
2775 ),
2776 };
2777 return Ok(vec![control_error_frame(
2778 &frame,
2779 "reserved_module",
2780 message,
2781 )?]);
2782 }
2783 }
2784 };
2785
2786 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2787 &hello.manifest.module_id,
2788 hello.manifest.capabilities.as_ref(),
2789 );
2790 if let Some(refusal) = reserved_capability_refusals.first() {
2791 let capability = refusal.capability.clone();
2792 let bound_module = refusal.claimants[0].clone();
2793 log_duplicate_claim_events(reserved_capability_refusals);
2794 return Ok(vec![control_error_frame(
2795 &frame,
2796 "reserved_capability",
2797 format!(
2798 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2799 capability, bound_module, hello.manifest.module_id
2800 ),
2801 )?]);
2802 }
2803
2804 if self
2807 .forwarding
2808 .connection_has_client_routes(connection_id)
2809 .map_err(RouterError::Forwarding)?
2810 {
2811 return Ok(vec![control_error_frame(
2812 &frame,
2813 "invalid_hello",
2814 "connection has open client routes and cannot also register as a module",
2815 )?]);
2816 }
2817
2818 self.hello_launch_nonces
2823 .lock()
2824 .unwrap_or_else(|poisoned| poisoned.into_inner())
2825 .record(connection_id, hello.launch_nonce.as_deref());
2826 let control_ops = effective_module_control_ops(hello.control_ops);
2827 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2830 if swap_candidate {
2831 return self.register_swap_candidate(
2832 connection_id,
2833 sink,
2834 &frame,
2835 hello.manifest,
2836 negotiated_ver,
2837 control_ops,
2838 hello_ack,
2839 );
2840 }
2841 let registration = match self
2848 .registry
2849 .register_unpublished_verified_with_control_ops(
2850 hello.manifest,
2851 negotiated_ver,
2852 connection_id,
2853 control_ops,
2854 verification.nonce_verified,
2855 ) {
2856 Ok(registration) => registration,
2857 Err(RegistryError::DuplicateModuleId { module_id }) => {
2858 return Ok(vec![control_error_frame(
2859 &frame,
2860 "duplicate_module_id",
2861 format!(
2862 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2863 ),
2864 )?])
2865 }
2866 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2867 return Ok(vec![control_error_frame(
2868 &frame,
2869 "invalid_module_id",
2870 err.to_string(),
2871 )?])
2872 }
2873 Err(err) => {
2874 return Ok(vec![control_error_frame(
2875 &frame,
2876 "registry_error",
2877 err.to_string(),
2878 )?])
2879 }
2880 };
2881 #[cfg(test)]
2882 self.hold_between_registry_and_forwarding();
2883
2884 let reply = if let Some(sink) = sink {
2885 let concurrency = manifest_concurrency(®istration.manifest);
2901 if let Err(err) = self.forwarding.register_module_connection_acked(
2902 connection_id,
2903 registration.manifest.module_id.clone(),
2904 negotiated_ver,
2905 concurrency,
2906 sink,
2907 crate::forwarding::ModuleHelloAck {
2908 frame: hello_ack,
2909 verified_prefix_owner: verification.prefix_owner,
2910 },
2911 ) {
2912 if matches!(
2917 self.deregister_connection(
2918 connection_id,
2919 RegistrationEndReason::RegistrationFailed,
2920 ),
2921 Ok(r) if !r.is_empty()
2922 ) {
2923 crate::supervise::notify_registration_release();
2924 }
2925 return Ok(vec![control_error_frame(
2926 &frame,
2927 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2928 "invalid_hello"
2929 } else {
2930 forwarding_error_code(&err)
2931 },
2932 err.to_string(),
2933 )?]);
2934 }
2935 Vec::new()
2936 } else {
2937 vec![hello_ack]
2940 };
2941
2942 if !self
2947 .registry
2948 .publish_registration(connection_id)
2949 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2950 {
2951 debug!(
2952 module_id = %registration.manifest.module_id,
2953 connection_id = connection_id.get(),
2954 "registration was removed before it could be published"
2955 );
2956 }
2957
2958 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2967 info!(
2968 module_id = %registration.manifest.module_id,
2969 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2970 );
2971 }
2972
2973 self.apply_registration_capabilities(®istration);
2974
2975 self.supervisor
2976 .record_activity_registration(®istration.manifest.module_id);
2977
2978 info!(
2979 module_id = %registration.manifest.module_id,
2980 module_version = %registration.manifest.module_version,
2981 negotiated_ver,
2982 routable_provider = manifest_provides_routable_role(®istration.manifest),
2983 connection_id = connection_id.get(),
2984 "module registered"
2985 );
2986
2987 Ok(reply)
2988 }
2989
2990 #[allow(clippy::too_many_arguments)]
3007 fn register_swap_candidate(
3008 &self,
3009 connection_id: ConnectionId,
3010 sink: Option<crate::FrameSink>,
3011 frame: &Frame,
3012 manifest: ModuleManifest,
3013 negotiated_ver: u8,
3014 control_ops: Vec<String>,
3015 hello_ack: Frame,
3016 ) -> Result<Vec<Frame>, RouterError> {
3017 let module_id = manifest.module_id.clone();
3018 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
3019 manifest,
3020 negotiated_ver,
3021 connection_id,
3022 control_ops,
3023 ) {
3024 Ok(registration) => registration,
3025 Err(RegistryError::DuplicateModuleId { module_id }) => {
3026 return Ok(vec![control_error_frame(
3027 frame,
3028 "duplicate_module_id",
3029 format!(
3030 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
3031 ),
3032 )?])
3033 }
3034 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3035 return Ok(vec![control_error_frame(
3036 frame,
3037 "invalid_module_id",
3038 err.to_string(),
3039 )?])
3040 }
3041 Err(err) => {
3042 return Ok(vec![control_error_frame(
3043 frame,
3044 "registry_error",
3045 err.to_string(),
3046 )?])
3047 }
3048 };
3049 #[cfg(test)]
3050 self.hold_between_registry_and_forwarding();
3051 let reply = if let Some(sink) = sink {
3052 let concurrency = manifest_concurrency(®istration.manifest);
3056 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
3057 connection_id,
3058 module_id.clone(),
3059 negotiated_ver,
3060 concurrency,
3061 sink,
3062 hello_ack,
3063 ) {
3064 if matches!(
3065 self.deregister_connection(
3066 connection_id,
3067 RegistrationEndReason::RegistrationFailed,
3068 ),
3069 Ok(r) if !r.is_empty()
3070 ) {
3071 crate::supervise::notify_registration_release();
3072 }
3073 return Ok(vec![control_error_frame(
3074 frame,
3075 forwarding_error_code(&err),
3076 err.to_string(),
3077 )?]);
3078 }
3079 Vec::new()
3080 } else {
3081 vec![hello_ack]
3082 };
3083 if !self
3084 .registry
3085 .publish_registration(connection_id)
3086 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3087 {
3088 debug!(
3089 module_id = %module_id,
3090 connection_id = connection_id.get(),
3091 "swap candidate registration was removed before it could be published"
3092 );
3093 }
3094 self.supervisor.mark_swap_candidate_admitted(&module_id);
3095 info!(
3096 module_id = %module_id,
3097 module_version = %registration.manifest.module_version,
3098 negotiated_ver,
3099 ready = registration.ready,
3100 connection_id = connection_id.get(),
3101 "swap candidate registered; not routable until cutover"
3102 );
3103 Ok(reply)
3104 }
3105
3106 fn build_hello_ack(
3107 &self,
3108 frame: &Frame,
3109 negotiated_ver: u8,
3110 module_id: &str,
3111 ) -> Result<Frame, RouterError> {
3112 let ack = ModuleHelloAckBody {
3113 negotiated_ver,
3114 subc_ops: module_subc_ops(),
3115 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3116 storage: self
3117 .storage_config
3118 .as_ref()
3119 .map(|cfg| cfg.descriptor_for(module_id)),
3120 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3121 };
3122 let body = serde_json::to_vec(&ack).map_err(|err| {
3123 RouterError::backend(
3124 0,
3125 frame.header.corr,
3126 format!("failed to encode HELLO_ACK: {err}"),
3127 )
3128 })?;
3129
3130 Frame::build_with_version(
3131 negotiated_ver,
3132 FrameType::HelloAck,
3133 control_flags(),
3134 0,
3135 0,
3136 frame.header.corr,
3137 body,
3138 )
3139 .map_err(RouterError::FrameBuild)
3140 }
3141
3142 async fn handle_client_control_request(
3143 &self,
3144 ctx: &RouteCtx,
3145 frame: Frame,
3146 request: ClientControlRequest,
3147 route_open_timings: &mut Option<RouteOpenTimings>,
3148 ) -> Result<Vec<Frame>, RouterError> {
3149 if let Some(operation) = privileged_operation(&request) {
3150 let caller = self.resolve_caller(ctx.connection_id).await;
3151 let op = client_control_request_op(&request);
3152 let text = prompt_text(operation, &caller);
3153 match self.privileged_operator_prompt {
3154 PrivilegedOperatorPrompt::Log => {
3155 info!(target: "subc_daemon::privileged_prompt", operation = op,
3156 target = ?operation, caller = %caller.description(), summary = %text,
3157 "privileged request would prompt the operator");
3158 }
3159 PrivilegedOperatorPrompt::Enforce => {
3160 let key =
3161 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3162 let stamp = self
3163 .operator_apps
3164 .lock()
3165 .unwrap_or_else(|p| p.into_inner())
3166 .get(&ctx.connection_id)
3167 .cloned();
3168 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3169 &self.operator_app_policy,
3170 &self.launch_table,
3171 stamp.as_ref(),
3172 )
3173 .then(|| self.shortcut_context(stamp, None, None));
3174 let outcome = self
3175 .forwarding
3176 .operator_confirms()
3177 .approve_privileged_with_shortcut(
3178 ctx,
3179 &frame,
3180 op,
3181 text,
3182 caller.description(),
3183 key,
3184 shortcut,
3185 )
3186 .await;
3187 if outcome.code != "confirmed" {
3188 return Ok(vec![outcome.frame(&frame)]);
3189 }
3190 }
3191 }
3192 }
3193 match request {
3194 ClientControlRequest::DaemonShutdownIntent { intent } => {
3195 if intent.as_str() != Some("restart") {
3196 return Ok(vec![control_error_frame(
3197 &frame,
3198 "invalid_params",
3199 "intent must be 'restart'",
3200 )?]);
3201 }
3202 let response = ClientControlResponse::DaemonShutdownIntent {
3203 intent: "restart".to_owned(),
3204 expires_at_ms: self.supervisor.announce_daemon_restart(),
3205 };
3206 Ok(vec![control_response_body_frame(
3207 &frame,
3208 &response,
3209 "ClientControlResponse::DaemonShutdownIntent",
3210 )?])
3211 }
3212 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3213 ClientControlRequest::CatalogList { module_id } => {
3214 self.handle_catalog_list(frame, module_id)
3215 }
3216 ClientControlRequest::RouteOpen {
3217 target,
3218 identity,
3219 consumer_identity,
3220 consumer_capabilities,
3221 role_versions,
3222 busy_refusals,
3223 admission_facts,
3224 scope,
3225 ..
3226 } => {
3227 self.handle_route_open(
3228 ctx,
3229 frame,
3230 RouteOpenRequest {
3231 target,
3232 identity,
3233 consumer_identity,
3234 consumer_capabilities,
3235 role_versions,
3236 busy_refusals,
3237 admission_facts,
3238 scope,
3239 },
3240 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3241 )
3242 .await
3243 }
3244 ClientControlRequest::RoutePoll {
3245 route_channel,
3246 route_epoch,
3247 kind,
3248 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3249 ClientControlRequest::SupervisorList {} => {
3250 let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
3251 self.agent_declarations.refuse(
3252 self,
3253 ctx,
3254 control_error_frame(
3255 &frame,
3256 "busy",
3257 "supervisor list work is at capacity; retry",
3258 )?,
3259 );
3260 return Ok(Vec::new());
3261 };
3262 let handler = self.clone();
3264 let sink = ctx.egress.clone();
3265 tokio::spawn(
3266 async move {
3267 let _permit = permit;
3268 let result = handler.handle_supervisor_list(frame.clone()).await;
3269 let replies = match result {
3270 Ok(replies) => replies,
3271 Err(error) => {
3272 control_error_frame(&frame, "backend_error", error.to_string())
3273 .into_iter()
3274 .collect()
3275 }
3276 };
3277 for reply in replies {
3278 if !matches!(
3279 tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
3280 .await,
3281 Ok(Ok(()))
3282 ) {
3283 break;
3284 }
3285 }
3286 }
3287 .with_current_subscriber(),
3288 );
3289 Ok(Vec::new())
3290 }
3291 ClientControlRequest::SupervisorSpawnSnapshot {} => {
3292 self.handle_supervisor_spawn_snapshot(frame)
3293 }
3294 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3295 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3296 }
3297 ClientControlRequest::SupervisorRestart {
3298 module_id,
3299 drain_timeout_ms,
3300 } => {
3301 self.log_supervisor_request_received(
3302 ctx,
3303 frame.header.corr,
3304 ops::SUPERVISOR_RESTART,
3305 Some(&module_id),
3306 None,
3307 )?;
3308 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3309 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3310 .await
3311 }
3312 ClientControlRequest::SupervisorSwap {
3313 module_id,
3314 ready_timeout_ms,
3315 } => {
3316 self.log_supervisor_request_received(
3317 ctx,
3318 frame.header.corr,
3319 ops::SUPERVISOR_SWAP,
3320 Some(&module_id),
3321 None,
3322 )?;
3323 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3324 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3325 .await
3326 }
3327 ClientControlRequest::SupervisorReload { module_id } => {
3328 self.log_supervisor_request_received(
3329 ctx,
3330 frame.header.corr,
3331 ops::SUPERVISOR_RELOAD,
3332 Some(&module_id),
3333 None,
3334 )?;
3335 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3336 self.handle_supervisor_reload(frame, module_id, requester)
3337 .await
3338 }
3339 ClientControlRequest::SupervisorRescan { preview } => {
3340 if !preview {
3341 self.log_supervisor_request_received(
3342 ctx,
3343 frame.header.corr,
3344 ops::SUPERVISOR_RESCAN,
3345 None,
3346 None,
3347 )?;
3348 }
3349 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3350 let operation = self.handle_supervisor_rescan(
3351 frame,
3352 if preview {
3353 RescanMode::Preview
3354 } else {
3355 RescanMode::Execute
3356 },
3357 requester,
3358 );
3359 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3360 .await
3361 }
3362 ClientControlRequest::SupervisorRescanChecked {
3363 preview,
3364 candidate,
3365 expect,
3366 } => {
3367 self.observe_control_wait(
3368 ctx,
3369 None,
3370 ops::SUPERVISOR_RESCAN,
3371 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3372 )
3373 .await
3374 }
3375 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3376 self.log_supervisor_request_received(
3377 ctx,
3378 frame.header.corr,
3379 ops::SUPERVISOR_RELEASE_RESERVED,
3380 Some(&module_id),
3381 None,
3382 )?;
3383 self.handle_supervisor_release_reserved(frame, module_id)
3384 .await
3385 }
3386 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3387 self.log_supervisor_request_received(
3388 ctx,
3389 frame.header.corr,
3390 ops::SUPERVISOR_SET_ENABLED,
3391 Some(&module_id),
3392 Some(enabled),
3393 )?;
3394 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3395 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3396 .await
3397 }
3398 ClientControlRequest::SupervisorHealthProbe { module_id } => {
3399 self.handle_supervisor_health_probe(ctx, frame, module_id)
3400 .await
3401 }
3402 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3403 ClientControlRequest::SupervisorRoutes { module_id } => {
3404 self.handle_supervisor_routes(frame, module_id)
3405 }
3406 ClientControlRequest::SupervisorProvenance { module_id } => {
3407 self.handle_supervisor_provenance(ctx, frame, module_id)
3408 .await
3409 }
3410 ClientControlRequest::SupervisorStderrTail {
3411 module_id,
3412 max_lines,
3413 max_bytes,
3414 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3415 ClientControlRequest::SupervisorTerminals { module_id } => {
3416 self.handle_supervisor_terminals(ctx, frame, module_id)
3417 .await
3418 }
3419 }
3420 }
3421
3422 fn log_supervisor_request_received(
3423 &self,
3424 ctx: &RouteCtx,
3425 corr: u64,
3426 op: &'static str,
3427 module_id: Option<&str>,
3428 enabled: Option<bool>,
3429 ) -> Result<(), RouterError> {
3430 let caller = self.supervisor_requester(ctx, corr)?;
3431 let caller = principal_label(&caller);
3432 info!(
3435 target: "control",
3436 op,
3437 module_id,
3438 enabled,
3439 connection_id = ctx.connection_id.get(),
3440 caller = %caller,
3441 "supervisor request received"
3442 );
3443 Ok(())
3444 }
3445
3446 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3447 Ok(self
3448 .registry
3449 .get_module_by_connection(ctx.connection_id)
3450 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3451 .map(|registration| Principal::Reserved {
3452 module_id: registration.manifest.module_id,
3453 })
3454 .unwrap_or(Principal::Direct))
3455 }
3456
3457 fn handle_module_control_request(
3458 &self,
3459 connection_id: ConnectionId,
3460 frame: Frame,
3461 request: ModuleControlRequestFromModule,
3462 ) -> Result<Vec<Frame>, RouterError> {
3463 match request {
3464 ModuleControlRequestFromModule::RouteAuthorize {
3465 principal,
3466 scope,
3467 target,
3468 expected,
3469 } => self.handle_route_authorize(
3470 connection_id,
3471 frame,
3472 principal,
3473 scope,
3474 target,
3475 expected,
3476 ),
3477 ModuleControlRequestFromModule::CatalogUpdate {
3478 provides,
3479 capabilities,
3480 ready,
3481 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3482 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { role_declarations } => {
3483 self.handle_catalog_update_role_declarations(
3484 connection_id,
3485 frame,
3486 role_declarations,
3487 )
3488 }
3489 ModuleControlRequestFromModule::LiveRoots {} => {
3490 let registered = self
3491 .registry
3492 .get_module_by_connection(connection_id)
3493 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3494 let Some(registration) = registered else {
3495 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3496 };
3497 let response = self
3498 .forwarding
3499 .live_roots(®istration.manifest.module_id)
3500 .map_err(RouterError::Forwarding)?;
3501 Ok(vec![control_response_body_frame(
3502 &frame,
3503 &response,
3504 "ModuleControlResponseToModule::LiveRoots",
3505 )?])
3506 }
3507 ModuleControlRequestFromModule::AgentCommands {
3508 started,
3509 finished,
3510 known,
3511 } => self.handle_agent_commands(connection_id, frame, started, finished, known),
3512 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3513 self.handle_scope_sync(connection_id, frame, generation, scopes)
3514 }
3515 ModuleControlRequestFromModule::ScopeApply {
3516 generation,
3517 upsert,
3518 end,
3519 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3520 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3521 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3522 }
3523 _ => Ok(vec![control_error_frame(
3524 &frame,
3525 "unsupported_control_frame",
3526 "unsupported module control operation",
3527 )?]),
3528 }
3529 }
3530
3531 fn handle_scope_served_by(
3532 &self,
3533 connection_id: ConnectionId,
3534 frame: Frame,
3535 ) -> Result<Vec<Frame>, RouterError> {
3536 let registered = self
3537 .registry
3538 .get_module_by_connection(connection_id)
3539 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3540 let Some(registered) = registered else {
3541 return Ok(vec![control_error_frame(
3542 &frame,
3543 "not_registered",
3544 "scope.served_by requires an active module registration owned by this connection",
3545 )?]);
3546 };
3547 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3548 {
3549 Ok(request) => request,
3550 Err(err) => {
3551 return Ok(vec![control_error_frame(
3552 &frame,
3553 "invalid_control_body",
3554 format!("malformed scope.served_by body: {err}"),
3555 )?])
3556 }
3557 };
3558 let answer = self
3559 .scopes
3560 .read()
3561 .unwrap_or_else(|poisoned| poisoned.into_inner())
3562 .served_by(
3563 ®istered.manifest.module_id,
3564 &request.scope_ref,
3565 request.scope_epoch,
3566 &request.module_id,
3567 );
3568 let reply = subc_control::ScopeServedByReply::new(
3569 answer,
3570 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3571 );
3572 Ok(vec![control_response_body_frame(
3573 &frame,
3574 &reply,
3575 "ScopeServedByReply",
3576 )?])
3577 }
3578
3579 fn handle_operator_confirm(
3580 &self,
3581 ctx: &RouteCtx,
3582 frame: Frame,
3583 ) -> Result<Vec<Frame>, RouterError> {
3584 use crate::operator_confirm::{audit, Outcome};
3585 let registration = self
3586 .registry
3587 .get_module_by_connection(ctx.connection_id)
3588 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3589 let Some(registration) = registration else {
3590 let outcome = Outcome::refusal("not_registered");
3591 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3592 return Ok(vec![outcome.frame(&frame)]);
3593 };
3594 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3595 Ok(request) => request,
3596 Err(_) => {
3597 let outcome = Outcome::refusal("invalid_control_body");
3598 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3599 return Ok(vec![outcome.frame(&frame)]);
3600 }
3601 };
3602 let declared = matches!(
3603 request.kind,
3604 Some(subc_protocol::session::OperatorConfirmKind::Operation)
3605 ) && request.operation.as_ref().is_some_and(|operation| {
3606 registration
3607 .manifest
3608 .confirm_operations
3609 .as_ref()
3610 .is_some_and(|operations| operations.contains(operation))
3611 });
3612 let module_id = registration.manifest.module_id;
3613 let nonce = self
3617 .hello_launch_nonces
3618 .lock()
3619 .unwrap_or_else(|p| p.into_inner())
3620 .nonce(ctx.connection_id)
3621 .map(str::to_owned);
3622 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3623 self.supervisor
3624 .spawned_consumer_authorized(&module_id, nonce)
3625 });
3626 let confirms = self.forwarding.operator_confirms();
3627 self.forwarding
3628 .with_operator_route(
3629 ctx.connection_id,
3630 request.route_channel,
3631 request.route_epoch,
3632 |binding| {
3633 let shortcut = binding
3634 .filter(|binding| {
3635 nonce_proven
3636 && crate::operator_confirm::valid_summary(&request.summary)
3637 && declared
3638 && crate::operator_confirm::ShortcutContext::route_eligible(
3639 &self.operator_app_policy,
3640 &self.launch_table,
3641 binding,
3642 )
3643 })
3644 .map(|binding| {
3645 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3646 &self.operator_app_policy,
3647 binding,
3648 );
3649 let stamp = if phone.is_some() {
3650 None
3651 } else {
3652 binding.operator_app.clone()
3653 };
3654 self.shortcut_context(
3655 stamp,
3656 phone,
3657 Some((request.route_channel, request.route_epoch)),
3658 )
3659 });
3660 confirms.admit_with_shortcut(
3661 ctx,
3662 frame,
3663 module_id,
3664 nonce_proven,
3665 request,
3666 binding,
3667 shortcut,
3668 )
3669 },
3670 )
3671 .map_err(RouterError::Forwarding)
3672 }
3673
3674 fn shortcut_context(
3675 &self,
3676 stamp: Option<crate::operator_apps::OperatorAppStamp>,
3677 phone_fingerprint: Option<String>,
3678 route: Option<(u16, u32)>,
3679 ) -> crate::operator_confirm::ShortcutContext {
3680 crate::operator_confirm::ShortcutContext {
3681 ledger: self.activity.clone(),
3682 forwarding: self.forwarding.clone(),
3683 launches: self.launch_table.clone(),
3684 stamp,
3685 phone_fingerprint,
3686 route,
3687 connections: self.operator_apps.clone(),
3688 #[cfg(test)]
3689 hooks: Default::default(),
3690 }
3691 }
3692
3693 #[cfg(any(test, feature = "test-support"))]
3695 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3696 self.operator_app_policy.phone_enabled = enabled;
3697 self
3698 }
3699
3700 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3703 let mut scopes = ScopeRecords::from(scopes);
3704 for (index, record) in scopes.records.iter_mut().enumerate() {
3705 if let Some(refusal) = Self::canonicalize_workdir(record) {
3706 scopes.refusals.insert(index, refusal);
3707 }
3708 }
3709 scopes
3710 }
3711
3712 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3713 let workdir = record.workdir.as_mut()?;
3714 let path = std::path::Path::new(workdir);
3715 if !path.is_absolute() {
3716 return Some((
3717 error_codes::INVALID_WORKDIR,
3718 "workdir must be absolute".to_string(),
3719 ));
3720 }
3721 match ProjectRootId::from_path_allowing_missing(path) {
3722 Ok(root) => match root.as_path().to_str() {
3723 Some(canonical) => {
3724 *workdir = canonical.to_string();
3725 None
3726 }
3727 None => Some((
3728 error_codes::INVALID_WORKDIR,
3729 "canonical workdir is not UTF-8".to_string(),
3730 )),
3731 },
3732 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3733 }
3734 }
3735
3736 fn handle_scope_sync(
3740 &self,
3741 connection_id: ConnectionId,
3742 frame: Frame,
3743 generation: u64,
3744 scopes: Vec<ScopeRecord>,
3745 ) -> Result<Vec<Frame>, RouterError> {
3746 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3747 }
3748
3749 fn handle_scope_change(
3750 &self,
3751 connection_id: ConnectionId,
3752 frame: Frame,
3753 generation: u64,
3754 scopes: Vec<ScopeRecord>,
3755 end: Option<Vec<ScopeEnd>>,
3756 ) -> Result<Vec<Frame>, RouterError> {
3757 let is_apply = end.is_some();
3758 let Some(registration) = self
3759 .registry
3760 .get_module_by_connection(connection_id)
3761 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3762 else {
3763 return Ok(vec![control_error_frame(
3764 &frame,
3765 "not_registered",
3766 "scope.sync and scope.apply require an active module registration owned by this connection",
3767 )?]);
3768 };
3769 let owner = registration.manifest.module_id;
3770 let scopes = Self::prepare_scope_records(scopes);
3771 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3772 let is_current_launch = |connection: ConnectionId| {
3773 self.hello_launch_nonces
3774 .lock()
3775 .unwrap_or_else(|poisoned| poisoned.into_inner())
3776 .presented(connection, current_nonce.as_deref())
3777 };
3778 let mut table = self
3783 .scopes
3784 .write()
3785 .unwrap_or_else(|poisoned| poisoned.into_inner());
3786 let outcome = match end {
3787 Some(end) => table.apply(
3788 &owner,
3789 connection_id,
3790 is_current_launch,
3791 generation,
3792 scopes,
3793 end,
3794 ),
3795 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3796 };
3797 let drained = match &outcome {
3798 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3799 Err(_) => Vec::new(),
3800 };
3801 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3802 drop(table);
3803 if has_deadlines {
3804 self.scope_deadline_added.notify_one();
3805 }
3806 match outcome {
3807 Ok(applied) => {
3808 let counts = ScopeOutcomeCounts::of(&applied.results);
3809 info!(
3810 owner = %owner,
3811 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3812 generation,
3813 records = applied.results.len(),
3814 created = counts.created,
3815 replaced = counts.replaced,
3816 updated = counts.updated,
3817 unchanged = counts.unchanged,
3818 refused = counts.refused,
3819 ended = applied.ended.len(),
3820 tag_changes = applied.tag_changes.len(),
3821 routes_closed = drained.len(),
3822 "scope change accepted"
3823 );
3824 for refused in applied
3830 .results
3831 .iter()
3832 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3833 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3834 {
3835 warn!(
3836 owner = %owner,
3837 generation,
3838 scope_ref = %refused.scope_ref,
3839 scope_epoch = refused.scope_epoch,
3840 code = refused.code.as_deref().unwrap_or(""),
3841 "scope record refused"
3842 );
3843 }
3844 self.close_scope_drained_routes(drained);
3845 let response = if is_apply {
3846 ModuleControlResponseToModule::ScopeApply {
3847 generation,
3848 results: applied.results,
3849 end_results: applied.end_results,
3850 ended: applied.ended,
3851 }
3852 } else {
3853 ModuleControlResponseToModule::ScopeSync {
3854 generation,
3855 results: applied.results,
3856 ended: applied.ended,
3857 }
3858 };
3859 Ok(vec![control_response_body_frame(
3860 &frame,
3861 &response,
3862 "scope change response",
3863 )?])
3864 }
3865 Err(refusal) => {
3866 info!(
3867 owner = %owner,
3868 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3869 generation,
3870 code = refusal.code,
3871 "scope change refused"
3872 );
3873 Ok(vec![control_error_frame(
3874 &frame,
3875 refusal.code,
3876 refusal.message,
3877 )?])
3878 }
3879 }
3880 }
3881
3882 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3888 if drained.is_empty() {
3889 return;
3890 }
3891 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3892 BTreeMap::new();
3893 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3894 for route in drained {
3895 warn!(
3896 module_id = %route.module_id,
3897 reason = ?route.reason,
3898 client_connection_id = route.client.connection_id.get(),
3899 route_channel = route.client.channel,
3900 "closing route because its scope changed"
3901 );
3902 pushes
3903 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3904 .or_insert_with(|| (route.reason, Vec::new()))
3905 .1
3906 .push(EndpointRoute {
3907 goodbye_target: route.client.clone(),
3908 principal: Principal::Unverified,
3909 bound_at: Instant::now(),
3910 draining: false,
3911 drain_reason: None,
3912 });
3913 goodbyes.push(route.module);
3914 goodbyes.push(route.client);
3915 }
3916 for ((module_id, _), (reason, routes)) in pushes {
3917 send_route_control_pushes(
3918 &self.forwarding,
3919 routes,
3920 ClientControlPush::RouteClosed {
3921 module_id,
3922 channels: Vec::new(),
3923 reason,
3924 drained: false,
3925 abandoned: 0,
3926 excluded_subscriptions: 0,
3927 terminal: Some(false),
3928 },
3929 );
3930 }
3931 self.emit_route_goodbyes(goodbyes);
3932 }
3933
3934 fn handle_route_authorize(
3935 &self,
3936 connection_id: ConnectionId,
3937 frame: Frame,
3938 principal: Principal,
3939 scope: subc_protocol::scope::RouteAuthorizeScope,
3940 target: String,
3941 expected: subc_protocol::scope::RouteAuthorizeExpected,
3942 ) -> Result<Vec<Frame>, RouterError> {
3943 use crate::scopes::admission::{admit_route, AdmissionMode};
3944 use subc_protocol::scope::{
3945 RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
3946 };
3947 let registry = self
3948 .registry
3949 .route_admission_snapshot()
3950 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3951 let owner_configured = match &scope.owner {
3952 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3953 _ => false,
3954 };
3955 let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
3956 let answer = {
3957 let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
3958 let permitted = registry
3959 .iter()
3960 .find(|r| r.connection_id == connection_id)
3961 .is_some_and(|r| {
3962 scopes.is_federated_owner(&Principal::Reserved {
3963 module_id: r.manifest.module_id.clone(),
3964 }) && r.manifest.capabilities.as_ref().is_some_and(|caps| {
3965 caps.provides
3966 .iter()
3967 .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
3968 })
3969 });
3970 if !permitted {
3971 drop(scopes);
3972 return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
3973 "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
3974 }
3975 admit_route(
3976 &scopes,
3977 ®istry,
3978 &principal,
3979 &target,
3980 Some(&selector),
3981 owner_configured,
3982 AdmissionMode::Authorize(&expected),
3983 )
3984 };
3985 let response = match answer {
3986 Ok(admission) => {
3987 let mut response = RouteAuthorizeResponse::new(
3988 RouteAuthorizeOutcome::Authorized,
3989 "route admission authorized",
3990 );
3991 response.content_version = admission.content_version;
3992 response
3993 }
3994 Err(refusal) => refusal.response(),
3995 };
3996 Ok(vec![control_response_body_frame(
3997 &frame,
3998 &response,
3999 "RouteAuthorizeResponse",
4000 )?])
4001 }
4002
4003 fn handle_scope_describe(
4006 &self,
4007 connection_id: ConnectionId,
4008 frame: Frame,
4009 owner: Principal,
4010 scope_ref: String,
4011 ) -> Result<Vec<Frame>, RouterError> {
4012 let registered = self
4013 .registry
4014 .get_module_by_connection(connection_id)
4015 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4016 let Some(registered) = registered else {
4017 return Ok(vec![control_error_frame(
4018 &frame,
4019 "not_registered",
4020 "scope.describe requires an active module registration owned by this connection",
4021 )?]);
4022 };
4023 let scopes = self
4024 .scopes
4025 .read()
4026 .unwrap_or_else(|poisoned| poisoned.into_inner());
4027 if scopes.is_federated_owner(&owner)
4028 && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
4029 {
4030 return Ok(vec![control_error_frame(
4031 &frame,
4032 error_codes::SCOPE_FEDERATED_UNSUPPORTED,
4033 "scope.describe of the federated owner requires scope-federated/v1",
4034 )?]);
4035 }
4036 let mut description = scopes.describe(&owner, &scope_ref);
4037 drop(scopes);
4038 if owner
4039 != (Principal::Reserved {
4040 module_id: registered.manifest.module_id.clone(),
4041 })
4042 {
4043 if let Some(facts) = description
4044 .stamp
4045 .as_mut()
4046 .and_then(|stamp| stamp.attributes.federated.as_mut())
4047 {
4048 facts.stamp_required = false;
4049 }
4050 }
4051 if !registered
4054 .manifest
4055 .capabilities
4056 .as_ref()
4057 .is_some_and(|caps| {
4058 caps.provides
4059 .iter()
4060 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4061 })
4062 {
4063 if let Some(stamp) = description.stamp.as_mut() {
4064 stamp.attributes.on_behalf_of_agent = None;
4065 }
4066 }
4067 if !registered
4068 .manifest
4069 .capabilities
4070 .as_ref()
4071 .is_some_and(|caps| {
4072 caps.provides
4073 .iter()
4074 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
4075 })
4076 {
4077 if let Some(stamp) = description.stamp.as_mut() {
4078 stamp.attributes.flow_version = None;
4079 stamp.attributes.flow_code_hash = None;
4080 stamp.attributes.flow_install_id = None;
4081 }
4082 }
4083 let owner_configured = match &owner {
4084 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4089 _ => false,
4090 };
4091 let response = ModuleControlResponseToModule::ScopeDescribe {
4092 status: description.status,
4093 scope_epoch: description.scope_epoch,
4094 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4095 owner_synced: description.owner_synced,
4096 owner_configured,
4097 scope: description.stamp,
4098 };
4099 Ok(vec![control_response_body_frame(
4100 &frame,
4101 &response,
4102 "ModuleControlResponseToModule::ScopeDescribe",
4103 )?])
4104 }
4105
4106 fn handle_catalog_update_role_declarations(
4107 &self,
4108 connection_id: ConnectionId,
4109 frame: Frame,
4110 role_declarations: serde_json::Value,
4111 ) -> Result<Vec<Frame>, RouterError> {
4112 if let Err(err) = validate_role_declarations_value(Some(&role_declarations)) {
4113 return Ok(vec![control_error_frame(
4114 &frame,
4115 "invalid_role_declarations",
4116 err.to_string(),
4117 )?]);
4118 }
4119 let declarations = serde_json::from_value(role_declarations)
4120 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4121 if self
4122 .registry
4123 .replace_role_declarations_for_connection(connection_id, declarations)
4124 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4125 .is_none()
4126 {
4127 return Ok(vec![control_error_frame(
4128 &frame,
4129 "not_registered",
4130 "catalog.update_role_declarations requires an active module registration owned by this connection",
4131 )?]);
4132 }
4133 control_response_body_frame(
4134 &frame,
4135 &ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {},
4136 "ModuleControlResponseToModule::CatalogUpdateRoleDeclarations",
4137 )
4138 .map(|frame| vec![frame])
4139 }
4140
4141 fn handle_catalog_update(
4142 &self,
4143 connection_id: ConnectionId,
4144 frame: Frame,
4145 provides: Vec<ProviderRole>,
4146 capabilities: Option<CapabilityDeclarations>,
4147 ready: Option<bool>,
4148 ) -> Result<Vec<Frame>, RouterError> {
4149 self.refresh_capability_requirements();
4150 let Some(registration) = self
4151 .registry
4152 .get_module_by_connection(connection_id)
4153 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4154 else {
4155 return Ok(vec![control_error_frame(
4156 &frame,
4157 "not_registered",
4158 "catalog.update requires an active module registration owned by this connection",
4159 )?]);
4160 };
4161
4162 if let Some(message) =
4163 catalog_update_frozen_field_message(®istration.manifest, &provides)
4164 {
4165 return Ok(vec![control_error_frame(
4166 &frame,
4167 "catalog_update_frozen_field",
4168 message,
4169 )?]);
4170 }
4171
4172 let mut candidate = registration.manifest.clone();
4173 candidate.provides = provides.clone();
4174 candidate.capabilities = capabilities
4175 .clone()
4176 .or_else(|| registration.manifest.capabilities.clone());
4177 if let Some(rich_ui) = &candidate.rich_ui {
4178 if let Err(err) = rich_ui.validate() {
4179 return Ok(vec![control_error_frame(
4180 &frame,
4181 "invalid_manifest",
4182 err.to_string(),
4183 )?]);
4184 }
4185 }
4186 if let Err(err) = candidate.validate_capability_grammar() {
4187 return Ok(vec![control_error_frame(
4188 &frame,
4189 "invalid_capability_grammar",
4190 err.to_string(),
4191 )?]);
4192 }
4193
4194 let mut conflicts = self
4197 .capability_evaluator
4198 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
4199 if let Some(conflict) = conflicts.first() {
4200 let message = format!(
4201 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
4202 conflict.capability, conflict.claimants[0], candidate.module_id
4203 );
4204 for conflict in &mut conflicts {
4205 conflict.source = DuplicateClaimSource::CatalogUpdate;
4206 }
4207 log_duplicate_claim_events(conflicts);
4208 return Ok(vec![control_error_frame(
4209 &frame,
4210 "reserved_capability",
4211 message,
4212 )?]);
4213 }
4214
4215 let updated = match self.registry.replace_catalog_for_connection(connection_id, provides, capabilities, ready) {
4216 Err(crate::registry::RegistryError::FederatedCapabilityChanged) => return Ok(vec![control_error_frame(
4217 &frame, error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
4218 "scope-federated/v1 is fixed at HELLO; re-register on a new connection to change it",
4219 )?]),
4220 result => result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?,
4221 };
4222 if updated.is_none() {
4223 return Ok(vec![control_error_frame(
4224 &frame,
4225 "not_registered",
4226 "catalog.update requires an active module registration owned by this connection",
4227 )?]);
4228 }
4229 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
4230 log_duplicate_claim_events(
4231 self.capability_evaluator
4232 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
4233 );
4234 }
4235 if capability_census_trigger(
4236 registration.manifest.capabilities.as_ref(),
4237 updated
4238 .as_ref()
4239 .and_then(|entry| entry.manifest.capabilities.as_ref()),
4240 ) {
4241 self.enforce_capability_denies();
4242 }
4243 self.refresh_capability_requirements();
4244
4245 let response = ModuleControlResponseToModule::CatalogUpdate {};
4246 control_response_body_frame(
4247 &frame,
4248 &response,
4249 "ModuleControlResponseToModule::CatalogUpdate",
4250 )
4251 .map(|frame| vec![frame])
4252 }
4253
4254 pub(crate) fn with_activity_ledger_config(
4255 mut self,
4256 run_dir: &Path,
4257 incarnation: String,
4258 config: crate::activity::ActivityConfig,
4259 ) -> Self {
4260 self.activity = Some(crate::activity::ActivityLedger::open_config(
4261 run_dir.join("activity.jsonl"),
4262 incarnation,
4263 config,
4264 ));
4265 self.supervisor.set_activity(self.activity.clone());
4266 self
4267 }
4268
4269 pub(crate) async fn configure_activity(
4270 &self,
4271 config: crate::activity::ActivityConfig,
4272 ) -> Result<(), &'static str> {
4273 match &self.activity {
4274 Some(ledger) => ledger.configure(config).await,
4275 None => Ok(()),
4276 }
4277 }
4278
4279 fn cancel_activity_subscription(
4280 &self,
4281 ctx: &RouteCtx,
4282 frame: &Frame,
4283 ) -> Result<bool, RouterError> {
4284 let task = self
4285 .activity_streams
4286 .lock()
4287 .unwrap_or_else(|p| p.into_inner())
4288 .remove(&(ctx.connection_id, frame.header.corr));
4289 let Some(task) = task else {
4290 return Ok(false);
4291 };
4292 task.abort();
4293 let end = Frame::build_with_version(
4294 response_version(frame),
4295 FrameType::StreamEnd,
4296 control_flags(),
4297 0,
4298 0,
4299 frame.header.corr,
4300 Vec::new(),
4301 )
4302 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4303 let sink = ctx.egress.clone();
4304 tokio::spawn(async move {
4305 let _ = sink.send(end).await;
4306 });
4307 Ok(true)
4308 }
4309
4310 async fn handle_activity_read(
4311 &self,
4312 ctx: &RouteCtx,
4313 frame: Frame,
4314 ) -> Result<Vec<Frame>, RouterError> {
4315 use subc_control::{
4316 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4317 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4318 };
4319 #[derive(Deserialize)]
4320 struct Identity {
4321 #[serde(default)]
4322 consumer_identity: Option<ConsumerIdentity>,
4323 }
4324 let principal = match self
4328 .forwarding
4329 .module_id_for_connection(ctx.connection_id)
4330 .map_err(RouterError::Forwarding)?
4331 {
4332 Some(module_id) => Ok(Principal::Reserved { module_id }),
4333 None => {
4334 let identity: Identity = match serde_json::from_slice(&frame.body) {
4335 Ok(identity) => identity,
4336 Err(_) => {
4337 return Ok(vec![control_error_frame(
4338 &frame,
4339 ACTIVITY_MALFORMED,
4340 "malformed activity read",
4341 )?])
4342 }
4343 };
4344 self.route_open_principal(&frame, identity.consumer_identity)?
4345 }
4346 };
4347 let admitted = matches!(principal, Ok(Principal::Direct))
4348 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4349 if !admitted {
4350 return Ok(vec![control_error_frame(
4351 &frame,
4352 ACTIVITY_READ_FORBIDDEN,
4353 "activity reads require an operator or trusted mirror principal",
4354 )?]);
4355 }
4356 let Some(ledger) = &self.activity else {
4357 return Ok(vec![control_error_frame(
4358 &frame,
4359 ACTIVITY_UNAVAILABLE,
4360 "activity ledger unavailable",
4361 )?]);
4362 };
4363 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4364 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4365 {
4366 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4367 Ok(request) => ledger.query(request).await,
4368 Err(_) => Err(ACTIVITY_MALFORMED),
4369 };
4370 return match result {
4371 Ok(reply) => Ok(vec![control_response_body_frame(
4372 &frame,
4373 &reply,
4374 ops::ACTIVITY_QUERY,
4375 )?]),
4376 Err(code) => Ok(vec![control_error_frame(
4377 &frame,
4378 code,
4379 "activity query refused",
4380 )?]),
4381 };
4382 }
4383 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4384 Ok(request) => request,
4385 Err(_) => {
4386 return Ok(vec![control_error_frame(
4387 &frame,
4388 ACTIVITY_MALFORMED,
4389 "malformed activity subscription",
4390 )?])
4391 }
4392 };
4393 let mut subscription = match ledger.subscribe(request.since).await {
4394 Ok(subscription) => subscription,
4395 Err(code) => {
4396 return Ok(vec![control_error_frame(
4397 &frame,
4398 code,
4399 "activity subscription refused",
4400 )?])
4401 }
4402 };
4403 let key = (ctx.connection_id, frame.header.corr);
4404 let sink = ctx.egress.clone();
4405 let mut streams = self
4406 .activity_streams
4407 .lock()
4408 .unwrap_or_else(|p| p.into_inner());
4409 streams.retain(|_, task| !task.is_finished());
4410 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4411 return Ok(vec![control_error_frame(
4412 &frame,
4413 "duplicate_subscription",
4414 "activity subscription correlation id already in use",
4415 )?]);
4416 }
4417 let task = tokio::spawn(async move {
4418 loop {
4419 let entry = if let Some(entry) = subscription.replay.pop_front() {
4420 Some(entry)
4421 } else {
4422 subscription.receiver.recv().await
4423 };
4424 let Some(entry) = entry else {
4425 break;
4426 };
4427 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4428 return;
4429 };
4430 let Ok(next) = Frame::build_with_version(
4431 response_version(&frame),
4432 FrameType::StreamData,
4433 control_flags(),
4434 0,
4435 0,
4436 frame.header.corr,
4437 body,
4438 ) else {
4439 return;
4440 };
4441 if sink.send(next).await.is_err() {
4442 return;
4443 }
4444 }
4445 if let Some(code) = subscription.terminal_code() {
4448 if let Ok(end) =
4449 control_error_frame(&frame, code, "activity subscriber fell behind")
4450 {
4451 let _ = sink.send(end).await;
4452 }
4453 }
4454 });
4455 streams.insert(key, task.abort_handle());
4456 Ok(Vec::new())
4457 }
4458
4459 async fn handle_activity_record(
4460 &self,
4461 ctx: &RouteCtx,
4462 frame: Frame,
4463 ) -> Result<Vec<Frame>, RouterError> {
4464 use subc_control::{
4465 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4466 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4467 };
4468 let result = async {
4469 let module = self
4470 .forwarding
4471 .module_id_for_connection(ctx.connection_id)
4472 .ok()
4473 .flatten()
4474 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4475 let request: ActivityRecordRequest =
4476 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4477 crate::activity::validate(&module, &request)?;
4478 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4479 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4480 let at_ms = std::time::SystemTime::now()
4481 .duration_since(std::time::UNIX_EPOCH)
4482 .unwrap_or_default()
4483 .as_millis() as u64;
4484 ledger.record(module, request, actor, at_ms).await
4485 }
4486 .await;
4487 match result {
4488 Ok(entry) => Ok(vec![control_response_body_frame(
4489 &frame,
4490 &ActivityRecordResponse { seq: entry.seq },
4491 "activity.record",
4492 )?]),
4493 Err(code) => Ok(vec![control_error_frame(
4494 &frame,
4495 code,
4496 "activity recording refused",
4497 )?]),
4498 }
4499 }
4500
4501 fn activity_actor(
4502 &self,
4503 connection: ConnectionId,
4504 module: &str,
4505 on: Option<&subc_control::ActivityOn>,
4506 ) -> Result<subc_control::ActivityActor, &'static str> {
4507 use crate::forwarding::{DataRoute, DataRouteState};
4508 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4509 let principal = |id: &str| Principal::Reserved {
4510 module_id: id.to_owned(),
4511 };
4512 let mut actor = ActivityActor {
4513 principal: principal(module),
4514 scope_owner: None,
4515 agent_id: None,
4516 run_id: None,
4517 flow_id: None,
4518 project_root: None,
4519 workdir: None,
4520 project_id: None,
4521 };
4522 let scope = match on {
4523 None => return Ok(actor),
4524 Some(ActivityOn::Scope(scope)) => {
4525 let selector = subc_protocol::scope::ScopeSelector::new(
4526 principal(module),
4527 scope.scope_ref.clone(),
4528 Some(scope.epoch),
4529 );
4530 self.scopes
4531 .read()
4532 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4533 .attribute(&principal(module), module, &selector, true)
4534 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4535 }
4536 Some(ActivityOn::Route(route)) => {
4537 let DataRoute::Module(DataRouteState::Bound(binding)) = self
4538 .forwarding
4539 .lookup_data_route(connection, route.channel, route.epoch)
4540 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4541 else {
4542 return Err(ACTIVITY_ACTOR_UNKNOWN);
4543 };
4544 if binding.module_id != module {
4545 return Err(ACTIVITY_ACTOR_UNKNOWN);
4546 }
4547 actor.principal = binding.principal.clone();
4548 actor.project_root = binding
4549 .project_root
4550 .as_ref()
4551 .map(|root| root.as_path().to_path_buf());
4552 let Some(bound) = &binding.scope else {
4553 return Ok(actor);
4554 };
4555 let owner = principal(&bound.owner);
4556 let selector = subc_protocol::scope::ScopeSelector::new(
4557 owner.clone(),
4558 bound.scope_ref.clone(),
4559 Some(bound.tag.scope_epoch),
4560 );
4561 let admitted = self
4562 .scopes
4563 .read()
4564 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4565 .attribute(&owner, module, &selector, true)
4566 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4567 if admitted.tag != bound.tag {
4568 return Err(ACTIVITY_ACTOR_UNKNOWN);
4569 }
4570 admitted
4571 }
4572 };
4573 actor.scope_owner = Some(scope.stamp.owner);
4574 actor.agent_id = scope.stamp.attributes.agent_id;
4575 actor.run_id = scope.stamp.attributes.run_id;
4576 actor.flow_id = scope.stamp.attributes.flow_id;
4577 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4578 actor.project_id = scope.stamp.project_id;
4579 Ok(actor)
4580 }
4581
4582 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4583 self.refresh_capability_requirements();
4584 let mut counters = self.counters.snapshot();
4590 let losses = self.resource_tracker.loss_diagnostics();
4591 if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
4592 if let Some(obj) = counters.as_object_mut() {
4593 obj.insert("agent_command_declaration_loss_by_module".into(), losses);
4594 }
4595 }
4596 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4597 obj.insert(
4598 "activity_unparseable_lines".into(),
4599 ledger.read_errors().into(),
4600 );
4601 obj.insert(
4602 "activity_lifecycle_append_failures".into(),
4603 ledger.lifecycle_failures().into(),
4604 );
4605 obj.insert(
4606 "activity_started_digest_skips".into(),
4607 ledger.lifecycle_digest_skips().into(),
4608 );
4609 }
4610 if let (Ok((connections_with_routes, max)), Some(obj)) = (
4611 self.forwarding.client_route_concentration(),
4612 counters.as_object_mut(),
4613 ) {
4614 obj.insert(
4615 "client_connections_with_routes".into(),
4616 connections_with_routes.into(),
4617 );
4618 obj.insert("max_routes_on_one_connection".into(), max.into());
4619 }
4620 if let (Some(open_breakers), Some(obj)) = (
4626 self.route_bind_breakers.open_snapshot(),
4627 counters.as_object_mut(),
4628 ) {
4629 obj.insert("route_bind_breakers_open".into(), open_breakers);
4630 }
4631 if let (Some(reason), Some(obj)) = (
4636 self.local_endpoint_unavailable.clone(),
4637 counters.as_object_mut(),
4638 ) {
4639 obj.insert("local_endpoint_unavailable".into(), reason.into());
4640 }
4641 let response = ClientControlResponse::ServerDescribe {
4642 protocol_ver: PROTOCOL_VERSION,
4643 subc_ops: subc_ops(),
4644 capabilities: self.describe_capabilities(),
4645 connected_clients: self.connected_clients.count(),
4646 counters: Some(counters),
4647 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4648 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4649 capability_requirements: self.capability_requirement_statuses(),
4650 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4651 config_path: self
4652 .rescan
4653 .as_ref()
4654 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4655 };
4656 Ok(vec![control_response_body_frame(
4657 &frame,
4658 &response,
4659 "ClientControlResponse::ServerDescribe",
4660 )?])
4661 }
4662
4663 fn handle_catalog_list(
4664 &self,
4665 frame: Frame,
4666 module_id: Option<String>,
4667 ) -> Result<Vec<Frame>, RouterError> {
4668 let (generation, modules) = self
4669 .registry
4670 .list_modules_with_nonce_verification()
4671 .map_err(|err| {
4672 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4673 })?;
4674 let entries = modules
4675 .into_iter()
4676 .filter(|(registration, _)| {
4677 module_id
4678 .as_deref()
4679 .map(|wanted| registration.manifest.module_id == wanted)
4680 .unwrap_or(true)
4681 })
4682 .map(|(registration, nonce_verified)| {
4683 let not_ready = self.not_ready_reason(®istration);
4684 let roles = registration.manifest.provides;
4685 CatalogEntry::new(
4686 registration.manifest.module_id,
4687 roles,
4688 registration.control_ops,
4689 )
4690 .with_ready(not_ready.is_none())
4691 .with_display_name(registration.manifest.display_name)
4692 .with_not_ready(not_ready)
4693 .with_module_version(Some(registration.manifest.module_version))
4694 .with_provenance(registration.manifest.provenance)
4695 .with_capabilities(registration.manifest.capabilities)
4696 .with_self_signals(registration.manifest.self_signals)
4697 .with_events(registration.manifest.events)
4698 .with_operation_delivery(registration.manifest.operation_delivery)
4699 .with_operation_payload(registration.manifest.operation_payload)
4700 .with_flow_operations(registration.manifest.flow_operations)
4701 .with_operation_deferral(registration.manifest.operation_deferral)
4702 .with_confirm_operations(registration.manifest.confirm_operations)
4703 .with_rich_ui(registration.manifest.rich_ui)
4704 .with_role_declarations(registration.manifest.role_declarations)
4705 .with_nonce_verified(nonce_verified)
4706 })
4707 .collect();
4708 let response = ClientControlResponse::CatalogList {
4709 generation,
4710 modules: entries,
4711 subc_ops: subc_ops(),
4712 };
4713 Ok(vec![control_response_body_frame(
4714 &frame,
4715 &response,
4716 "ClientControlResponse::CatalogList",
4717 )?])
4718 }
4719
4720 fn attest_route_open_consumer(
4725 &self,
4726 consumer_identity: Option<&ConsumerIdentity>,
4727 ) -> Option<Principal> {
4728 let Some(consumer_identity) = consumer_identity else {
4729 return Some(Principal::Direct);
4730 };
4731 self.supervisor
4732 .spawned_consumer_authorized(
4733 &consumer_identity.module_id,
4734 &consumer_identity.launch_nonce,
4735 )
4736 .then(|| Principal::Reserved {
4737 module_id: consumer_identity.module_id.clone(),
4738 })
4739 }
4740
4741 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4745 let Ok(ClientControlRequest::RouteOpen {
4746 identity,
4747 consumer_identity,
4748 scope,
4749 ..
4750 }) = parse_client_control_request(&frame.body)
4751 else {
4752 return RouteOpenCaller::unknown();
4753 };
4754 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4755 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4756 }
4757
4758 fn route_open_principal(
4759 &self,
4760 frame: &Frame,
4761 consumer_identity: Option<ConsumerIdentity>,
4762 ) -> Result<Result<Principal, Frame>, RouterError> {
4763 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4764 self.route_open_principal_attested(frame, consumer_identity, attested)
4765 }
4766
4767 fn route_open_principal_attested(
4771 &self,
4772 frame: &Frame,
4773 consumer_identity: Option<ConsumerIdentity>,
4774 attested: Option<Principal>,
4775 ) -> Result<Result<Principal, Frame>, RouterError> {
4776 if let Some(principal) = attested {
4777 return Ok(Ok(principal));
4778 }
4779 let Some(consumer_identity) = consumer_identity else {
4780 return Ok(Ok(Principal::Direct));
4781 };
4782
4783 Ok(Err(control_error_frame(
4784 frame,
4785 "bad_consumer_identity",
4786 format!(
4787 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4788 consumer_identity.module_id
4789 ),
4790 )?))
4791 }
4792
4793 #[allow(clippy::too_many_arguments)]
4800 fn route_open_refusal_frame(
4801 &self,
4802 ctx: &RouteCtx,
4803 caller: &RouteOpenCaller,
4804 frame: &Frame,
4805 module_id: &str,
4806 reason: &'static str,
4807 code: &'static str,
4808 message: impl Into<String>,
4809 ) -> Result<Frame, RouterError> {
4810 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4811 control_error_frame(frame, code, message.into())
4812 }
4813
4814 #[allow(clippy::too_many_arguments)]
4823 fn route_bind_rejection(
4824 &self,
4825 ctx: &RouteCtx,
4826 caller: &RouteOpenCaller,
4827 frame: &Frame,
4828 target_module_id: &str,
4829 body: ErrorBody,
4830 daemon_authored: bool,
4831 ) -> Result<Vec<Frame>, RouterError> {
4832 if daemon_authored {
4838 let scope_code = match body.code.as_str() {
4839 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4840 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4841 _ => None,
4842 };
4843 if let Some(code) = scope_code {
4844 self.observe_route_open_refusal(
4845 ctx,
4846 caller,
4847 target_module_id,
4848 "scope_changed_before_commit",
4849 code,
4850 );
4851 return Ok(vec![control_error_body_frame(frame, body)?]);
4852 }
4853 }
4854 self.counters
4855 .increment_route_open_refused("module_rejected");
4856 info!(
4857 target: "control",
4858 code = "module_rejected",
4859 module_code = ?body.code,
4860 module_id = ?target_module_id,
4861 connection_id = ctx.connection_id.get(),
4862 principal = %caller.principal,
4863 scope = %caller.scope,
4864 harness = %caller.harness,
4865 "route.open refused"
4866 );
4867 let refusal = if daemon_authored {
4868 control_error_body_frame(frame, body)?
4869 } else {
4870 control_relayed_error_body_frame(frame, body)?
4871 };
4872 Ok(vec![refusal])
4873 }
4874
4875 #[allow(clippy::too_many_arguments)]
4893 fn route_open_breaker_refusal_frame(
4894 &self,
4895 ctx: &RouteCtx,
4896 caller: &RouteOpenCaller,
4897 frame: &Frame,
4898 module_id: &str,
4899 consecutive_timeouts: u32,
4900 retry_in: Duration,
4901 probe_in_flight: bool,
4902 ) -> Result<Frame, RouterError> {
4903 self.counters
4904 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4905 debug!(
4906 target: "control",
4907 code = "module_timeout",
4908 module_id = ?module_id,
4909 connection_id = ctx.connection_id.get(),
4910 principal = %caller.principal,
4911 scope = %caller.scope,
4912 harness = %caller.harness,
4913 consecutive_timeouts,
4914 retry_in_ms = retry_in.as_millis() as u64,
4915 probe_in_flight,
4916 "route.open refused by open bind-relay breaker"
4917 );
4918 let detail = if probe_in_flight {
4925 "one test route is already being tried; retry once it settles".to_string()
4926 } else {
4927 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4928 };
4929 control_error_frame(
4930 frame,
4931 "module_timeout",
4932 format!(
4933 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4934 timed out in a row); {detail}; its established routes are unaffected"
4935 ),
4936 )
4937 }
4938
4939 fn observe_route_open_refusal(
4952 &self,
4953 ctx: &RouteCtx,
4954 caller: &RouteOpenCaller,
4955 module_id: &str,
4956 reason: &'static str,
4957 code: &'static str,
4958 ) {
4959 self.counters.increment_route_open_refused(code);
4960 info!(
4961 target: "control",
4962 code,
4963 reason,
4964 module_id = ?module_id,
4965 connection_id = ctx.connection_id.get(),
4966 principal = %caller.principal,
4967 scope = %caller.scope,
4968 harness = %caller.harness,
4969 "route.open refused"
4970 );
4971 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4972 self.route_outages.record_not_serving(module_id, reason);
4973 }
4974 }
4975
4976 fn observe_route_open_accept(
5021 &self,
5022 ctx: &RouteCtx,
5023 module_id: &str,
5024 principal: &str,
5025 scope: &str,
5026 ) {
5027 self.route_outages.record_accepted(module_id);
5028 self.counters.increment_route_open_accepted(principal);
5029 info!(
5030 target: "control",
5031 principal,
5032 module_id,
5033 connection_id = ctx.connection_id.get(),
5034 scope = %scope,
5035 "route.open accepted"
5036 );
5037 }
5038
5039 fn supervised_absent_route_open_refusal_frame(
5040 &self,
5041 ctx: &RouteCtx,
5042 caller: &RouteOpenCaller,
5043 frame: &Frame,
5044 module_id: &str,
5045 code: &'static str,
5046 status: &crate::supervise::ModuleStatus,
5047 ) -> Result<Frame, RouterError> {
5048 self.counters.increment_route_open_refused(code);
5049 info!(
5050 target: "control",
5051 code,
5052 reason = "supervised_not_registered",
5053 module_id = ?module_id,
5054 connection_id = ctx.connection_id.get(),
5055 principal = %caller.principal,
5056 scope = %caller.scope,
5057 harness = %caller.harness,
5058 state = %status.state,
5059 enabled = status.enabled,
5060 live = status.live,
5061 "route.open refused"
5062 );
5063 self.route_outages
5067 .record_not_serving(module_id, "supervised_not_registered");
5068 let message = if let Some(pid) = status.waiting_for_previous_pid {
5069 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
5070 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
5071 })
5072 } else {
5073 format!(
5074 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
5075 status.state, status.enabled, status.live
5076 )
5077 };
5078 control_error_frame(frame, code, message)
5079 }
5080
5081 async fn handle_route_open(
5082 &self,
5083 ctx: &RouteCtx,
5084 frame: Frame,
5085 request: RouteOpenRequest,
5086 timings: &mut RouteOpenTimings,
5087 ) -> Result<Vec<Frame>, RouterError> {
5088 let RouteOpenRequest {
5089 target,
5090 mut identity,
5091 consumer_identity,
5092 consumer_capabilities,
5093 role_versions,
5094 busy_refusals,
5095 admission_facts,
5096 scope,
5097 } = request;
5098 let target_module_id = target_module_id(&target).to_string();
5099 timings.module_id.clone_from(&target_module_id);
5100 if self
5101 .registry
5102 .get_module_by_connection(ctx.connection_id)
5103 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5104 .is_some()
5105 {
5106 return Ok(vec![control_error_frame(
5107 &frame,
5108 "invalid_request",
5109 "module connections cannot open client routes",
5110 )?]);
5111 }
5112 debug!(
5113 connection_id = ctx.connection_id.get(),
5114 corr = frame.header.corr,
5115 module_id = %target_module_id,
5116 "handling route.open"
5117 );
5118
5119 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
5124 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
5125
5126 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
5132 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
5133 self.observe_route_open_refusal(
5134 ctx,
5135 &caller,
5136 &target_module_id,
5137 "invalid_role_versions",
5138 error_codes::INVALID_REQUEST,
5139 );
5140 return Ok(vec![control_error_body_frame(
5141 &frame,
5142 ErrorBody {
5143 code: error_codes::INVALID_REQUEST.to_string(),
5144 message: error.to_string(),
5145 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
5146 },
5147 )?]);
5148 }
5149
5150 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
5151 if status.waiting_for_previous_pid.is_some() {
5152 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
5153 ctx,
5154 &caller,
5155 &frame,
5156 &target_module_id,
5157 "module_warming",
5158 &status,
5159 )?]);
5160 }
5161 }
5162
5163 let admission_registry = self
5182 .registry
5183 .route_admission_snapshot()
5184 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5185 let Some(registration) =
5186 crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
5187 else {
5188 if let Some((status, warming)) =
5189 self.supervisor_status(&target_module_id, frame.header.corr)?
5190 {
5191 if status.protocol == ModuleProtocol::None {
5198 return Ok(vec![self.route_open_refusal_frame(
5199 ctx,
5200 &caller,
5201 &frame,
5202 &target_module_id,
5203 "protocol_none",
5204 error_codes::MODULE_NO_PROTOCOL,
5205 format!(
5206 "module_id '{target_module_id}' is declared protocol: none; \
5207 it speaks no subc wire and serves no routes"
5208 ),
5209 )?]);
5210 }
5211 let code = if warming {
5212 "module_warming"
5213 } else {
5214 "target_unavailable"
5215 };
5216 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
5217 ctx,
5218 &caller,
5219 &frame,
5220 &target_module_id,
5221 code,
5222 &status,
5223 )?]);
5224 }
5225 if let Some(removed_ago_ms) =
5226 self.supervisor.removal_tombstone_age_ms(&target_module_id)
5227 {
5228 return Ok(vec![self.route_open_refusal_frame(
5229 ctx,
5230 &caller,
5231 &frame,
5232 &target_module_id,
5233 "removed",
5234 error_codes::MODULE_REMOVED,
5235 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
5236 )?]);
5237 }
5238 return Ok(vec![self.route_open_refusal_frame(
5239 ctx,
5240 &caller,
5241 &frame,
5242 &target_module_id,
5243 "not_registered",
5244 error_codes::UNKNOWN_MODULE,
5245 format!("module_id '{target_module_id}' is not registered"),
5246 )?]);
5247 };
5248
5249 if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
5253 .is_err()
5254 {
5255 self.counters
5256 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
5257 info!(
5258 target: "control",
5259 code = error_codes::MODULE_WARMING,
5260 module_id = ?target_module_id,
5261 connection_id = ctx.connection_id.get(),
5262 principal = %caller.principal,
5263 scope = %caller.scope,
5264 harness = %caller.harness,
5265 reason = "declared_not_ready",
5266 "route.open refused"
5267 );
5268 self.route_outages
5271 .record_not_serving(&target_module_id, "declared_not_ready");
5272 return Ok(vec![control_error_body_frame(
5273 &frame,
5274 ErrorBody {
5275 code: error_codes::MODULE_WARMING.to_string(),
5276 message: format!(
5277 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5278 ),
5279 detail: Some(serde_json::json!({
5280 "reason": "declared_not_ready"
5281 })),
5282 },
5283 )?]);
5284 }
5285
5286 if let Some(capability) = self.first_unprovided_required_capability(registration) {
5305 self.counters
5306 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5307 info!(
5308 target: "control",
5309 code = error_codes::MODULE_WARMING,
5310 module_id = ?target_module_id,
5311 connection_id = ctx.connection_id.get(),
5312 principal = %caller.principal,
5313 scope = %caller.scope,
5314 harness = %caller.harness,
5315 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5316 capability = %capability,
5317 "route.open refused"
5318 );
5319 return Ok(vec![control_error_body_frame(
5320 &frame,
5321 ErrorBody {
5322 code: error_codes::MODULE_WARMING.to_string(),
5323 message: format!(
5324 "module_id '{target_module_id}' requires capability '{capability}', \
5325 which no registered module provides; retry"
5326 ),
5327 detail: Some(serde_json::json!({
5328 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5329 "capability": capability,
5330 })),
5331 },
5332 )?]);
5333 }
5334
5335 if !target_has_required_role(&target, ®istration.manifest.provides) {
5336 return Ok(vec![self.route_open_refusal_frame(
5337 ctx,
5338 &caller,
5339 &frame,
5340 &target_module_id,
5341 "role_not_provided",
5342 "target_unavailable",
5343 format!("module_id '{target_module_id}' does not provide the requested target"),
5344 )?]);
5345 }
5346
5347 if registration.state != ChannelState::Active {
5348 return Ok(vec![self.route_open_refusal_frame(
5349 ctx,
5350 &caller,
5351 &frame,
5352 &target_module_id,
5353 "registration_not_active",
5354 "target_unavailable",
5355 format!("module_id '{target_module_id}' is not active"),
5356 )?]);
5357 }
5358
5359 if self
5360 .forwarding
5361 .module_is_draining(&target_module_id)
5362 .map_err(RouterError::Forwarding)?
5363 {
5364 return Ok(vec![self.route_open_refusal_frame(
5365 ctx,
5366 &caller,
5367 &frame,
5368 &target_module_id,
5369 "reloading",
5370 "module_reloading",
5371 format!("module_id '{target_module_id}' is reloading"),
5372 )?]);
5373 }
5374
5375 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5376 process_liveness.process_live(&target_module_id) == Some(false)
5377 }) {
5378 if process_liveness.process_replacing(&target_module_id) {
5385 return Ok(vec![self.route_open_refusal_frame(
5386 ctx,
5387 &caller,
5388 &frame,
5389 &target_module_id,
5390 "reloading",
5391 "module_reloading",
5392 format!("module_id '{target_module_id}' is reloading"),
5393 )?]);
5394 }
5395 return Ok(vec![self.route_open_refusal_frame(
5396 ctx,
5397 &caller,
5398 &frame,
5399 &target_module_id,
5400 "supervisor_not_live",
5401 "target_unavailable",
5402 format!("module_id '{target_module_id}' is not live"),
5403 )?]);
5404 }
5405
5406 if !self
5407 .forwarding
5408 .has_live_module_connection(&target_module_id)
5409 .map_err(RouterError::Forwarding)?
5410 {
5411 return Ok(vec![self.route_open_refusal_frame(
5412 ctx,
5413 &caller,
5414 &frame,
5415 &target_module_id,
5416 "no_forwarding_connection",
5417 "target_unavailable",
5418 format!("module_id '{target_module_id}' has no live forwarding connection"),
5419 )?]);
5420 }
5421
5422 if let Some(error) =
5423 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5424 {
5425 self.observe_route_open_refusal(
5426 ctx,
5427 &caller,
5428 &target_module_id,
5429 "op_not_allowed",
5430 "op_not_allowed",
5431 );
5432 return Ok(vec![error]);
5433 }
5434
5435 let principal =
5436 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5437 Ok(principal) => principal,
5438 Err(error) => {
5439 self.observe_route_open_refusal(
5440 ctx,
5441 &caller,
5442 &target_module_id,
5443 "bad_consumer_identity",
5444 "bad_consumer_identity",
5445 );
5446 return Ok(vec![error]);
5447 }
5448 };
5449 timings.principal = principal_label(&principal);
5450
5451 let build_class = identity.build_class.take().filter(|build_class| {
5454 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5455 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5456 } else {
5457 if self
5458 .build_class_warned
5459 .lock()
5460 .unwrap_or_else(|p| p.into_inner())
5461 .insert(ctx.connection_id)
5462 {
5463 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5464 "ignoring build_class from a non-Callosum opener");
5465 }
5466 false
5467 }
5468 });
5469
5470 if let Principal::Reserved {
5474 module_id: opening_module_id,
5475 } = &principal
5476 {
5477 if let Some(opening_registration) = admission_registry
5478 .iter()
5479 .find(|r| r.manifest.module_id == *opening_module_id)
5480 {
5481 if let Some(capability) =
5482 denied_capability(&opening_registration.manifest, ®istration.manifest)
5483 {
5484 warn!(
5485 opening_module_id,
5486 target_module_id,
5487 capability,
5488 "refusing route.open because an attested capability deny edge matches"
5489 );
5490 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5491 &frame,
5492 &target_module_id,
5493 "capability_deny_edge",
5494 "capability_forbidden",
5495 format!(
5496 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5497 ),
5498 )?]);
5499 }
5500 }
5501 }
5502
5503 if admission_facts.is_some() {
5504 let carrier_matches = matches!(
5505 &principal,
5506 Principal::Reserved { module_id }
5507 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5508 );
5509 if !carrier_matches {
5510 return Ok(vec![self.route_open_refusal_frame(
5511 ctx,
5512 &caller,
5513 &frame,
5514 &target_module_id,
5515 "admission_facts_carrier_not_permitted",
5516 "admission_facts_not_permitted",
5517 "admission facts may only be carried by the configured reserved module",
5518 )?]);
5519 }
5520
5521 let target_allowed = self
5522 .admission_facts_targets
5523 .as_ref()
5524 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5525 if !target_allowed {
5526 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5527 &frame,
5528 &target_module_id,
5529 "admission_facts_target_not_listed",
5530 "admission_facts_target_not_allowed",
5531 format!(
5532 "admission facts are not permitted for target module_id '{target_module_id}'"
5533 ),
5534 )?]);
5535 }
5536
5537 }
5541
5542 let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
5548 None => (None, None, false),
5549 Some(selector) => {
5550 let owner_configured = match &selector.owner {
5551 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5557 _ => false,
5558 };
5559 let mut scopes = self
5560 .scopes
5561 .write()
5562 .unwrap_or_else(|poisoned| poisoned.into_inner());
5563 let admitted = crate::scopes::admission::admit_route(
5564 &scopes,
5565 &admission_registry,
5566 &principal,
5567 &target_module_id,
5568 Some(selector),
5569 owner_configured,
5570 crate::scopes::admission::AdmissionMode::Open,
5571 )
5572 .map(|route| {
5573 let admission = route.scope.expect("scoped admission");
5574 let bound = scopes.reserve_bind(
5575 &admission,
5576 &principal,
5577 ctx.connection_id,
5578 selector.hold_presence,
5579 );
5580 (admission, bound)
5581 });
5582 match admitted {
5583 Ok((admission, bound)) => {
5584 (Some(bound), Some(admission.stamp), admission.stamp_required)
5585 }
5586 Err(refusal) => {
5587 return Ok(vec![self.route_open_refusal_frame(
5588 ctx,
5589 &caller,
5590 &frame,
5591 &target_module_id,
5592 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
5593 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
5594 refusal.message,
5595 )?]);
5596 }
5597 }
5598 }
5599 };
5600 if let Some(stamp) = &scope_stamp {
5603 caller.scope = route_open_scope_label(stamp);
5604 }
5605
5606 timings.enter(RouteOpenPhase::ProjectRoot);
5614 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5615 Ok(project) => project,
5616 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5617 };
5618 timings.enter(RouteOpenPhase::Admission);
5619 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5620 Ok(root) => root,
5621 Err(err) => {
5622 return Ok(vec![control_error_frame(
5623 &frame,
5624 "invalid_project_root",
5625 err.to_string(),
5626 )?])
5627 }
5628 };
5629 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5635
5636 let _concurrency_guard = match self
5647 .route_bind_concurrency
5648 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5649 {
5650 Ok(guard) => guard,
5651 Err(in_flight) => {
5652 return Ok(vec![self.route_open_target_capacity_refusal(
5653 ctx,
5654 &caller,
5655 &frame,
5656 &target_module_id,
5657 in_flight,
5658 )?]);
5659 }
5660 };
5661
5662 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5665 RouteBindAdmission::Admitted { guard, probe } => {
5666 if probe {
5667 info!(
5668 module_id = %target_module_id,
5669 connection_id = ctx.connection_id.get(),
5670 "route.bind breaker half-open: admitting one probe"
5671 );
5672 }
5673 guard
5674 }
5675 RouteBindAdmission::Refused {
5676 consecutive_timeouts,
5677 retry_in,
5678 probe_in_flight,
5679 } => {
5680 return Ok(vec![self.route_open_breaker_refusal_frame(
5681 ctx,
5682 &caller,
5683 &frame,
5684 &target_module_id,
5685 consecutive_timeouts,
5686 retry_in,
5687 probe_in_flight,
5688 )?]);
5689 }
5690 };
5691
5692 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5697 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5698 let operator_app = self
5699 .operator_apps
5700 .lock()
5701 .unwrap_or_else(|p| p.into_inner())
5702 .get(&ctx.connection_id)
5703 .cloned();
5704 timings.enter(RouteOpenPhase::EgressReservation);
5705 let pending = match self
5706 .forwarding
5707 .begin_route_bind_relay_for(
5708 ctx.connection_id,
5709 ctx.egress.clone(),
5710 response_version(&frame),
5711 frame.header.corr,
5712 &target_module_id,
5713 principal.clone(),
5714 bound_scope,
5715 project_root,
5716 identity.harness.clone(),
5717 build_class,
5718 operator_app.clone(),
5719 relay_deadline,
5720 busy_refusals == Some(true),
5721 )
5722 .await
5723 {
5724 Ok(pending) => pending,
5725 Err(err) => {
5726 return Ok(vec![self.route_open_refusal_frame(
5727 ctx,
5728 &caller,
5729 &frame,
5730 &target_module_id,
5731 "relay_reservation_failed",
5732 forwarding_error_code(&err),
5733 err.to_string(),
5734 )?])
5735 }
5736 };
5737 timings.enter(RouteOpenPhase::Admission);
5738 let crate::forwarding::PendingRouteBindRelay {
5739 endpoint,
5740 module_sink,
5741 negotiated_ver,
5742 client_channel,
5743 client_epoch,
5744 module_channel,
5745 module_epoch,
5746 corr: relay_corr,
5747 receiver,
5748 } = pending;
5749 let mut reservation =
5750 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5751
5752 let relay_registration = self
5757 .registry
5758 .get_module_by_connection(endpoint.connection_id)
5759 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5760 let relay_capabilities = relay_registration
5761 .as_ref()
5762 .and_then(|registration| registration.manifest.capabilities.as_ref());
5763 if let Some(selector) = &scope {
5764 if let Err(refusal) = self
5767 .scopes
5768 .read()
5769 .unwrap_or_else(|poisoned| poisoned.into_inner())
5770 .check_bind_proof(&principal, selector)
5771 {
5772 reservation.release_and_disarm();
5773 return Ok(vec![self.route_open_refusal_frame(
5774 ctx,
5775 &caller,
5776 &frame,
5777 &target_module_id,
5778 refusal.code,
5779 refusal.code,
5780 refusal.message,
5781 )?]);
5782 }
5783 }
5784 if let Some(stamp) = scope_stamp.as_ref() {
5785 if let Err(refusal) = crate::scopes::check_target_required_stamp(
5786 stamp_required,
5787 &target_module_id,
5788 relay_capabilities,
5789 )
5790 .and_then(|()| {
5791 crate::scopes::check_target_flow_support(
5792 stamp,
5793 &target_module_id,
5794 relay_capabilities,
5795 )
5796 })
5797 .and_then(|()| {
5798 crate::scopes::check_target_agent_run_support(
5799 stamp,
5800 &target_module_id,
5801 relay_capabilities,
5802 )
5803 }) {
5804 reservation.release_and_disarm();
5805 return Ok(vec![self.route_open_refusal_frame(
5806 ctx,
5807 &caller,
5808 &frame,
5809 &target_module_id,
5810 refusal.code,
5811 refusal.code,
5812 refusal.message,
5813 )?]);
5814 }
5815 }
5816
5817 if scope_stamp
5820 .as_ref()
5821 .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
5822 && !crate::scopes::target_supports_federated(relay_capabilities)
5823 {
5824 scope_stamp = None;
5826 }
5827 if !relay_capabilities.is_some_and(|caps| {
5828 caps.provides
5829 .iter()
5830 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5831 }) {
5832 if let Some(stamp) = scope_stamp.as_mut() {
5833 stamp.workdir = None;
5834 stamp.project_id = None;
5835 }
5836 }
5837 let worker_run = scope_stamp
5842 .as_ref()
5843 .is_some_and(crate::scopes::is_worker_run)
5844 && relay_capabilities.is_some_and(|caps| {
5845 caps.provides
5846 .iter()
5847 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5848 });
5849 if !worker_run
5850 && !relay_capabilities.is_some_and(|caps| {
5851 caps.provides
5852 .iter()
5853 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5854 })
5855 {
5856 if let Some(stamp) = scope_stamp.as_mut() {
5857 stamp.attributes.on_behalf_of_agent = None;
5858 }
5859 }
5860 if !relay_capabilities.is_some_and(|caps| {
5861 caps.provides
5862 .iter()
5863 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5864 }) {
5865 if let Some(stamp) = scope_stamp.as_mut() {
5866 stamp.attributes.flow_version = None;
5867 stamp.attributes.flow_code_hash = None;
5868 stamp.attributes.flow_install_id = None;
5869 }
5870 }
5871 let call_mode = if worker_run
5872 || relay_capabilities
5873 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5874 {
5875 Some(derive_call_mode(scope_stamp.as_ref()))
5876 } else {
5877 if let Some(stamp) = scope_stamp.as_mut() {
5878 stamp.expires_at_ms = None;
5879 }
5880 None
5881 };
5882
5883 debug!(
5884 connection_id = ctx.connection_id.get(),
5885 client_channel,
5886 client_epoch,
5887 module_channel,
5888 module_epoch,
5889 "reserved route handle pair"
5890 );
5891 let route_principal_label = principal_label(&principal);
5894 let relay = ModuleControlRequest::RouteBind {
5895 route_channel: module_channel,
5896 epoch: module_epoch,
5897 target,
5898 identity,
5899 principal: Some(principal),
5900 consumer_capabilities,
5901 role_versions,
5902 admission_facts,
5903 scope: scope_stamp,
5904 call_mode,
5905 };
5906 let peer = RouteBindPeer::new()
5907 .with_observed_peer(
5908 relay_capabilities
5909 .is_some_and(|caps| {
5910 caps.provides
5911 .iter()
5912 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5913 })
5914 .then(|| {
5915 self.observed_peers
5916 .lock()
5917 .unwrap_or_else(|p| p.into_inner())
5918 .get(&ctx.connection_id)
5919 .cloned()
5920 .unwrap_or_else(ObservedPeer::unobserved)
5921 }),
5922 )
5923 .with_operator_app(
5924 self.connection_operator_app(operator_app.as_ref())
5925 .filter(|_| {
5926 relay_capabilities.is_some_and(|caps| {
5927 caps.provides
5928 .iter()
5929 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5930 })
5931 }),
5932 );
5933 let relay_body = crate::bind_compat::encode_bind(
5934 &relay,
5935 &peer,
5936 &legacy_bind_project,
5937 relay_capabilities,
5938 )
5939 .map_err(|err| {
5940 RouterError::backend(
5941 0,
5942 frame.header.corr,
5943 format!("failed to encode route.bind request: {err}"),
5944 )
5945 })?;
5946 let relay_frame = Frame::build_with_version(
5947 negotiated_ver,
5948 FrameType::Request,
5949 control_flags(),
5950 0,
5951 0,
5952 relay_corr,
5953 relay_body,
5954 )
5955 .map_err(RouterError::FrameBuild)?;
5956
5957 timings.enter(RouteOpenPhase::BindSend);
5958 let sent = module_sink.send(relay_frame).await;
5959 timings.enter(RouteOpenPhase::Admission);
5960 if let Err(err) = sent {
5961 reservation.release_and_disarm();
5962 return Ok(vec![self.route_open_refusal_frame(
5963 ctx,
5964 &caller,
5965 &frame,
5966 &target_module_id,
5967 "relay_send_failed",
5968 "target_unavailable",
5969 err.to_string(),
5970 )?]);
5971 }
5972
5973 #[cfg(test)]
5974 if let Some(hold) = &self.route_bind_relay_sent_hold {
5975 hold.wait().await;
5979 hold.wait().await;
5980 }
5981
5982 if self
5987 .forwarding
5988 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5989 .map_err(RouterError::Forwarding)?
5990 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5991 {
5992 self.send_abandoned_route_bind_goodbye(
5993 &module_sink,
5994 negotiated_ver,
5995 module_channel,
5996 module_epoch,
5997 );
5998 }
5999
6000 timings.enter(RouteOpenPhase::BindAckWait);
6001 let outcome = timeout_at(relay_deadline, receiver).await;
6002 timings.enter(RouteOpenPhase::Admission);
6003 match outcome {
6004 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
6005 reservation.disarm();
6006 if breaker.record_accepted() {
6007 info!(
6008 module_id = %target_module_id,
6009 "route.bind breaker closed: the probe was accepted"
6010 );
6011 }
6012 self.observe_route_open_accept(
6013 ctx,
6014 &target_module_id,
6015 &route_principal_label,
6016 &caller.scope,
6017 );
6018 Ok(Vec::new())
6019 }
6020 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
6021 reservation.release_and_disarm();
6022 breaker.record_inconclusive();
6026 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
6027 }
6028 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
6029 reservation.release_and_disarm();
6030 breaker.record_inconclusive();
6031 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
6032 }
6033 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
6034 reservation.release_and_disarm();
6035 breaker.record_inconclusive();
6036 tracing::warn!(
6043 module_id = %target_module_id,
6044 "route.bind relay abandoned: {message}"
6045 );
6046 Ok(vec![self.route_open_refusal_frame(
6047 ctx,
6048 &caller,
6049 &frame,
6050 &target_module_id,
6051 "relay_abandoned",
6052 "target_unavailable",
6053 message,
6054 )?])
6055 }
6056 Ok(Err(_)) => {
6057 reservation.release_and_disarm();
6058 breaker.record_inconclusive();
6059 Ok(vec![self.route_open_refusal_frame(
6060 ctx,
6061 &caller,
6062 &frame,
6063 &target_module_id,
6064 "relay_waiter_canceled",
6065 "target_unavailable",
6066 "route.bind relay waiter was canceled before the module responded",
6067 )?])
6068 }
6069 Err(_) => {
6070 reservation.release_and_disarm();
6071 if let Some(opened) = breaker.record_timeout(
6075 self.route_bind_breaker_threshold,
6076 self.route_bind_breaker_cooldown,
6077 ) {
6078 warn!(
6079 module_id = %target_module_id,
6080 consecutive_timeouts = opened.consecutive_timeouts,
6081 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
6082 reopened_after_probe = opened.reopened_after_probe,
6083 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
6084 );
6085 }
6086 tracing::warn!(
6094 module_id = %target_module_id,
6095 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
6096 "route.bind relay timed out: module did not ack within budget"
6097 );
6098 Ok(vec![self.route_open_refusal_frame(
6099 ctx,
6100 &caller,
6101 &frame,
6102 &target_module_id,
6103 "relay_timed_out",
6104 "module_timeout",
6105 format!(
6106 "module_id '{target_module_id}' did not answer route.bind within {:?}",
6107 route_bind_relay_timeout
6108 ),
6109 )?])
6110 }
6111 }
6112 }
6113
6114 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6115 let response = ClientControlResponse::SupervisorSpawnSnapshot {
6116 snapshot: self.supervisor.spawn_snapshot(),
6117 };
6118 Ok(vec![control_response_body_frame(
6119 &frame,
6120 &response,
6121 "ClientControlResponse::SupervisorSpawnSnapshot",
6122 )?])
6123 }
6124
6125 fn handle_supervisor_spawn_subscribe(
6126 &self,
6127 ctx: &RouteCtx,
6128 frame: Frame,
6129 since: Option<SpawnCursor>,
6130 ) -> Result<Vec<Frame>, RouterError> {
6131 match self.supervisor.subscribe_spawns(
6132 ctx.connection_id,
6133 frame.header.corr,
6134 response_version(&frame),
6135 since,
6136 ctx.egress.clone(),
6137 ) {
6138 Ok(()) => Ok(Vec::new()),
6139 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
6140 Ok(vec![control_error_body_frame(
6141 &frame,
6142 ErrorBody {
6143 code: "spawn_cursor_incarnation_mismatch".to_string(),
6144 message: "spawn cursor belongs to a different daemon incarnation"
6145 .to_string(),
6146 detail: Some(serde_json::json!({
6147 "current_daemon_incarnation": current
6148 })),
6149 },
6150 )?])
6151 }
6152 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
6153 &frame,
6154 ErrorBody {
6155 code: "spawn_cursor_too_old".to_string(),
6156 message: "spawn cursor predates the retained event ring".to_string(),
6157 detail: Some(serde_json::json!({
6158 "oldest_retained_cursor": oldest
6159 })),
6160 },
6161 )?]),
6162 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
6163 0,
6164 frame.header.corr,
6165 format!("failed to open supervisor spawn subscription: {error}"),
6166 )),
6167 }
6168 }
6169
6170 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6171 let handler = self.clone();
6174 let runtime = tokio::runtime::Handle::current();
6175 let subscriber = tracing::dispatcher::get_default(Clone::clone);
6178 let corr = frame.header.corr;
6179 tokio::task::spawn_blocking(move || {
6180 runtime.block_on(
6181 list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
6182 .with_subscriber(subscriber),
6183 )
6184 })
6185 .await
6186 .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
6187 }
6188
6189 async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6190 let generation = self
6191 .registry
6192 .generation()
6193 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6194 let supervised = self.supervisor.list();
6195 let mut statuses = BTreeMap::new();
6196 let mut not_running = BTreeSet::new();
6197 for module in &supervised {
6198 let status = module.status_for_control("list").map_err(|err| {
6199 RouterError::backend(
6200 0,
6201 frame.header.corr,
6202 format!("failed to read supervisor status: {err}"),
6203 )
6204 })?;
6205 if !status.enabled || !status.process_alive {
6206 not_running.insert(module.module_id().to_owned());
6207 }
6208 statuses.insert(module.module_id().to_owned(), status);
6209 }
6210 let mut process_resources = BTreeMap::new();
6213 for module in &supervised {
6214 process_resources.insert(
6215 module.module_id().to_owned(),
6216 crate::subtree_resources::read_process(module.clone()).await,
6217 );
6218 }
6219 let mut image_readings = BTreeMap::new();
6222 for module in &supervised {
6223 image_readings.insert(
6224 module.module_id().to_owned(),
6225 module.running_image_agreement().await,
6226 );
6227 }
6228 let readings = self.resource_tracker.list(supervised.clone());
6229 let mut modules = Vec::new();
6230 for module in supervised {
6231 let status = statuses
6232 .remove(module.module_id())
6233 .expect("each supervised module has a status snapshot");
6234 let (configured, _) = module.configuration().map_err(|err| {
6235 RouterError::backend(
6236 0,
6237 frame.header.corr,
6238 format!("failed to read module configuration: {err}"),
6239 )
6240 })?;
6241 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
6242 RouterError::backend(
6243 0,
6244 frame.header.corr,
6245 format!("failed to read retry time: {err}"),
6246 )
6247 })?;
6248 let image = image_readings
6249 .remove(module.module_id())
6250 .expect("each supervised module has an image snapshot");
6251 let resources = process_resources.remove(module.module_id());
6252 let mut verdict =
6253 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
6254 verdict.spec = Some(launch_spec_agreement(
6255 &configured,
6256 status.process_alive,
6257 status.launch_spec_digest.as_ref(),
6258 ));
6259 let pending_reload = Some(verdict);
6260 let configured_enabled = self
6261 .capability_evaluator
6262 .configured_enabled(&status.module_id);
6263 modules.push(
6266 SupervisorEntry::new(
6267 status.module_id,
6268 status.state.to_string(),
6269 status.enabled,
6270 status.live,
6271 status.health.status,
6272 )
6273 .with_configured_enabled(configured_enabled)
6274 .with_next_retry_at_ms(next_retry_at_ms)
6275 .with_launch_nonce_env(Some(!cfg!(unix)))
6276 .with_launch_nonce_source(status.launch_nonce_source)
6277 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
6278 .with_protocol(status.protocol)
6279 .with_pending_reload(pending_reload)
6280 .with_last_probe_ms(status.health.last_probe_ms)
6281 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
6282 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
6283 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
6284 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
6285 .with_restart_count(Some(status.restart_count))
6286 .with_max_restarts(Some(status.max_restarts))
6287 .with_lifetime_restarts(Some(status.lifetime_restarts))
6288 .with_spawn_generation(Some(status.spawn_generation))
6289 .with_restart_window_secs(Some(status.restart_window.as_secs()))
6290 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
6291 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
6292 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
6293 .with_resources(resources),
6294 );
6295 }
6296 for module in &mut modules {
6297 if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
6298 module.tree = tree.clone();
6299 module.cpu_last_hour = hour.clone();
6300 }
6301 module.tree_unavailable = readings
6302 .reasons
6303 .get(&module.module_id)
6304 .map(|reason| (*reason).to_owned());
6305 if module.tree.is_none() {
6306 module.tree_unavailable = Some(
6307 if not_running.contains(&module.module_id) {
6308 "not_running"
6309 } else if let Some(reason) = readings.reasons.get(&module.module_id) {
6310 reason
6311 } else {
6312 "not_yet_measured"
6313 }
6314 .to_owned(),
6315 );
6316 }
6317 }
6318 let response = ClientControlResponse::SupervisorList {
6319 generation,
6320 modules,
6321 daemon: readings.daemon,
6322 agent_commands: readings.agent_commands,
6323 };
6324 Ok(vec![control_response_body_frame(
6325 &frame,
6326 &response,
6327 "ClientControlResponse::SupervisorList",
6328 )?])
6329 }
6330
6331 fn handle_supervisor_stderr_tail(
6332 &self,
6333 frame: Frame,
6334 module_id: String,
6335 max_lines: Option<u32>,
6336 max_bytes: Option<u32>,
6337 ) -> Result<Vec<Frame>, RouterError> {
6338 let Some(module) = self.supervisor.get(&module_id) else {
6339 return Ok(vec![control_error_frame(
6340 &frame,
6341 "unknown_module",
6342 format!("module_id '{module_id}' is not supervised"),
6343 )?]);
6344 };
6345
6346 let snapshot = module.stderr_tail(
6347 max_lines.map(|value| value as usize),
6348 max_bytes.map(|value| value as usize),
6349 );
6350
6351 let response = ClientControlResponse::SupervisorStderrTail {
6352 module_id,
6353 tail: StderrTail {
6354 capture: match snapshot.capture {
6355 CaptureState::Captured => StderrCaptureState::Captured,
6356 CaptureState::Incomplete { reason } => {
6357 StderrCaptureState::Incomplete { reason }
6358 }
6359 CaptureState::NotCaptured { reason } => {
6360 StderrCaptureState::NotCaptured { reason }
6361 }
6362 },
6363 entries: snapshot
6364 .entries
6365 .into_iter()
6366 .map(|entry| match entry {
6367 TailEntry::Line {
6368 text,
6369 truncated,
6370 at_ms,
6371 } => StderrTailEntry::Line {
6372 text,
6373 truncated,
6374 at_ms,
6375 },
6376 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6377 })
6378 .collect(),
6379 dropped_lines: snapshot.dropped_lines,
6380 },
6381 };
6382 Ok(vec![control_response_body_frame(
6383 &frame,
6384 &response,
6385 "ClientControlResponse::SupervisorStderrTail",
6386 )?])
6387 }
6388
6389 async fn observe_control_wait<F: std::future::Future>(
6390 &self,
6391 ctx: &RouteCtx,
6392 target: Option<&str>,
6393 kind: &'static str,
6394 future: F,
6395 ) -> F::Output {
6396 self.counters
6397 .observe_reader_wait(
6398 || {
6399 let module = self
6400 .forwarding
6401 .module_id_for_connection(ctx.connection_id)
6402 .ok()
6403 .flatten();
6404 let principal = if module.is_some() { "module" } else { "direct" };
6405 (
6406 self.resolve_reader_wait_target(kind, target)
6407 .or(module)
6408 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6409 principal.into(),
6410 )
6411 },
6412 kind,
6413 future,
6414 )
6415 .await
6416 }
6417
6418 pub(crate) fn resolve_reader_wait_target(
6421 &self,
6422 op: &str,
6423 selector: Option<&str>,
6424 ) -> Option<String> {
6425 let selector = selector?;
6426 match op {
6427 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6428 .supervisor
6429 .get(selector)
6430 .map(|module| module.module_id().to_owned()),
6431 ops::SUPERVISOR_HEALTH_PROBE => self
6432 .registry
6433 .get_module(selector)
6434 .ok()
6435 .flatten()
6436 .map(|registration| registration.manifest.module_id),
6437 _ => None,
6438 }
6439 }
6440
6441 async fn handle_supervisor_terminals(
6442 &self,
6443 ctx: &RouteCtx,
6444 frame: Frame,
6445 module_id: String,
6446 ) -> Result<Vec<Frame>, RouterError> {
6447 let Some(module) = self.supervisor.get(&module_id) else {
6448 return Ok(vec![control_error_frame(
6449 &frame,
6450 "unknown_module",
6451 format!("module_id '{module_id}' is not supervised"),
6452 )?]);
6453 };
6454
6455 let terminals = self
6458 .observe_control_wait(
6459 ctx,
6460 Some(module.module_id()),
6461 ops::SUPERVISOR_TERMINALS,
6462 module.read_durable_terminal_history(),
6463 )
6464 .await
6465 .map_err(|error| {
6466 RouterError::backend(
6467 0,
6468 frame.header.corr,
6469 format!("failed to read terminal history: {error}"),
6470 )
6471 })?;
6472 let response = ClientControlResponse::SupervisorTerminals {
6473 module_id,
6474 terminals,
6475 };
6476 Ok(vec![control_response_body_frame(
6477 &frame,
6478 &response,
6479 "ClientControlResponse::SupervisorTerminals",
6480 )?])
6481 }
6482
6483 fn handle_supervisor_routes(
6484 &self,
6485 frame: Frame,
6486 module_id: Option<String>,
6487 ) -> Result<Vec<Frame>, RouterError> {
6488 let modules = self
6489 .forwarding
6490 .route_census(module_id.as_deref())
6491 .map_err(RouterError::Forwarding)?
6492 .into_iter()
6493 .map(|(module_id, routes)| SupervisorRouteModule {
6494 module_id,
6495 routes: routes
6496 .into_iter()
6497 .map(|route| SupervisorRoute {
6498 consumer: match route.principal {
6499 Principal::Reserved { module_id } => {
6500 SupervisorRouteConsumer::Reserved { module_id }
6501 }
6502 Principal::Direct | Principal::Unverified => {
6503 SupervisorRouteConsumer::Direct {
6504 connection_id: route.goodbye_target.connection_id.get(),
6505 }
6506 }
6507 },
6508 age_ms: Instant::now()
6509 .saturating_duration_since(route.bound_at)
6510 .as_millis()
6511 .try_into()
6512 .unwrap_or(u64::MAX),
6513 draining: route.draining,
6514 drain_reason: route.drain_reason,
6515 })
6516 .collect(),
6517 })
6518 .collect();
6519 let response = ClientControlResponse::SupervisorRoutes { modules };
6520 Ok(vec![control_response_body_frame(
6521 &frame,
6522 &response,
6523 "ClientControlResponse::SupervisorRoutes",
6524 )?])
6525 }
6526
6527 async fn handle_supervisor_provenance(
6528 &self,
6529 ctx: &RouteCtx,
6530 frame: Frame,
6531 module_id: Option<String>,
6532 ) -> Result<Vec<Frame>, RouterError> {
6533 let targeted = module_id.is_some();
6534 let selected = if let Some(module_id) = module_id {
6535 let Some(module) = self.supervisor.get(&module_id) else {
6536 return Ok(vec![control_error_frame(
6537 &frame,
6538 "unknown_module",
6539 format!("module_id '{module_id}' is not supervised"),
6540 )?]);
6541 };
6542 vec![module]
6543 } else {
6544 self.supervisor.list()
6545 };
6546
6547 let target = targeted.then(|| selected[0].module_id());
6548 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6549 let mut modules = Vec::with_capacity(selected.len());
6550 for module in &selected {
6551 let (status, observed_image) = module
6552 .status_and_running_image_agreement()
6553 .await
6554 .map_err(|err| {
6555 RouterError::backend(
6556 0,
6557 frame.header.corr,
6558 format!("failed to read supervisor status: {err}"),
6559 )
6560 })?;
6561 let module_declared = self
6562 .registry
6563 .get_module(&status.module_id)
6564 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6565 .and_then(|registration| registration.manifest.provenance)
6566 .map(|build| ModuleDeclaredProvenance::Reported { build })
6567 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6568 #[cfg(test)]
6569 let running_image = match &self.provenance_probe_override {
6570 Some(result) => result.clone(),
6571 None => observed_image,
6572 };
6573 #[cfg(not(test))]
6574 let running_image = observed_image;
6575 modules.push(SupervisorModuleProvenance {
6576 module_id: status.module_id,
6577 module_declared,
6578 daemon_observed: SupervisorObservedProcess {
6579 launch_nonce_source: status.launch_nonce_source,
6580 pid: status.pid,
6581 spawned_at_ms: status.spawned_at_ms,
6582 spawned_from: status.spawned_from,
6583 running_image,
6584 },
6585 });
6586 }
6587 let daemon = SupervisorDaemonProvenance {
6588 daemon_build: self.daemon_provenance.build.clone(),
6589 daemon_observed: DaemonObservedProcess {
6590 pid: self.daemon_provenance.pid,
6591 started_at_ms: self
6592 .daemon_provenance
6593 .start_clock
6594 .map(|clock| clock.started_at_ms())
6595 .or(self.daemon_provenance.started_at_ms),
6596 running_image: self
6597 .daemon_provenance
6598 .probe
6599 .observe(
6600 self.daemon_provenance.pid,
6601 self.daemon_provenance.executable_path.as_deref(),
6602 self.daemon_provenance.executable_identity.clone(),
6603 self.daemon_provenance.process_start_time,
6604 )
6605 .await,
6606 },
6607 };
6608 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6609 Ok(vec![control_response_body_frame(
6610 &frame,
6611 &response,
6612 "ClientControlResponse::SupervisorProvenance",
6613 )?])
6614 })
6615 .await
6616 }
6617
6618 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6619 self.refresh_capability_requirements();
6620 let generation = self
6621 .registry
6622 .generation()
6623 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6624 let modules = self
6625 .supervisor
6626 .list()
6627 .into_iter()
6628 .map(|module| {
6629 let status = module.status_for_control("health").map_err(|err| {
6630 RouterError::backend(
6631 0,
6632 frame.header.corr,
6633 format!("failed to read supervisor health: {err}"),
6634 )
6635 })?;
6636 let module_id = status.module_id;
6637 let capability_detail = self
6638 .capability_evaluator
6639 .required_problem_detail(&module_id);
6640 Ok(SupervisorHealthEntry {
6641 module_id,
6642 status: status.health.status,
6643 detail: append_capability_problem_detail(
6644 status.health.detail,
6645 capability_detail,
6646 ),
6647 metrics: status.health.metrics,
6648 consecutive_failures: status.health.consecutive_failures,
6649 late_answer_count: status.health.late_answer_count,
6650 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6651 last_action: status.health.last_action,
6652 last_action_ms: status.health.last_action_ms,
6653 last_probe_ms: status.health.last_probe_ms,
6654 })
6655 })
6656 .collect::<Result<Vec<_>, RouterError>>()?;
6657 let response = ClientControlResponse::SupervisorHealth {
6658 generation,
6659 modules,
6660 };
6661 Ok(vec![control_response_body_frame(
6662 &frame,
6663 &response,
6664 "ClientControlResponse::SupervisorHealth",
6665 )?])
6666 }
6667
6668 async fn handle_supervisor_restart(
6669 &self,
6670 frame: Frame,
6671 module_id: String,
6672 drain_timeout_ms: Option<u64>,
6673 requester: Principal,
6674 ) -> Result<Vec<Frame>, RouterError> {
6675 let operation_lock = self.supervisor.operation_lock();
6676 let _operation_guard = operation_lock.lock().await;
6677 let Some(module) = self.supervisor.get(&module_id) else {
6678 return Ok(vec![control_error_frame(
6679 &frame,
6680 "unknown_module",
6681 format!("module_id '{module_id}' is not supervised"),
6682 )?]);
6683 };
6684
6685 self.route_outages.mark_operator_action(&module_id);
6686 if let Err(err) = module
6687 .restart_requested(drain_timeout_ms, Some(requester))
6688 .await
6689 {
6690 self.route_outages
6691 .operator_action_ended_unrefused(&module_id);
6692 let (code, message) = match err {
6693 crate::supervise::SuperviseError::Disabled { .. } => {
6694 ("module_disabled", err.to_string())
6695 }
6696 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6697 ("swap_in_progress", err.to_string())
6698 }
6699 _ => (
6700 "target_unavailable",
6701 format!("failed to restart module_id '{module_id}': {err}"),
6702 ),
6703 };
6704 return Ok(vec![control_error_frame(&frame, code, message)?]);
6705 }
6706
6707 let response = ClientControlResponse::SupervisorAck {
6708 module_id,
6709 applied: true,
6710 };
6711 Ok(vec![control_response_body_frame(
6712 &frame,
6713 &response,
6714 "ClientControlResponse::SupervisorAck",
6715 )?])
6716 }
6717
6718 async fn handle_supervisor_swap(
6722 &self,
6723 frame: Frame,
6724 module_id: String,
6725 ready_timeout_ms: Option<u64>,
6726 requester: Principal,
6727 ) -> Result<Vec<Frame>, RouterError> {
6728 let module = {
6735 let operation_lock = self.supervisor.operation_lock();
6736 let _operation_guard = operation_lock.lock().await;
6737 self.supervisor.get(&module_id)
6738 };
6739 let Some(module) = module else {
6740 return Ok(vec![control_error_frame(
6741 &frame,
6742 "unknown_module",
6743 format!("module_id '{module_id}' is not supervised"),
6744 )?]);
6745 };
6746
6747 self.route_outages.mark_operator_action(&module_id);
6748 if let Err(err) = module
6749 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6750 .await
6751 {
6752 self.route_outages
6753 .operator_action_ended_unrefused(&module_id);
6754 use crate::supervise::SuperviseError;
6755 let message = err.to_string();
6756 let error = match err {
6757 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6758 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6759 code: "swap_refused".to_string(),
6760 message,
6761 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6762 },
6763 SuperviseError::SwapFailed {
6764 arm,
6765 candidate_exit,
6766 ..
6767 } => ErrorBody {
6768 code: "swap_failed".to_string(),
6769 message,
6770 detail: Some(serde_json::json!({
6771 "arm": arm.as_str(),
6772 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6773 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6774 })),
6775 },
6776 _ => ErrorBody::new(
6777 "target_unavailable",
6778 format!("failed to swap module_id '{module_id}': {message}"),
6779 ),
6780 };
6781 return Ok(vec![control_error_body_frame(&frame, error)?]);
6782 }
6783 self.route_outages
6787 .operator_action_ended_unrefused(&module_id);
6788
6789 let response = ClientControlResponse::SupervisorAck {
6790 module_id,
6791 applied: true,
6792 };
6793 Ok(vec![control_response_body_frame(
6794 &frame,
6795 &response,
6796 "ClientControlResponse::SupervisorAck",
6797 )?])
6798 }
6799
6800 async fn handle_supervisor_reload(
6801 &self,
6802 frame: Frame,
6803 module_id: String,
6804 requester: Principal,
6805 ) -> Result<Vec<Frame>, RouterError> {
6806 let operation_lock = self.supervisor.operation_lock();
6807 let _operation_guard = operation_lock.lock().await;
6808 let Some(module) = self.supervisor.get(&module_id) else {
6809 return Ok(vec![control_error_frame(
6810 &frame,
6811 "unknown_module",
6812 format!("module_id '{module_id}' is not supervised"),
6813 )?]);
6814 };
6815
6816 self.route_outages.mark_operator_action(&module_id);
6817 if let Err(err) = module.reload_requested(Some(requester)).await {
6818 self.route_outages
6819 .operator_action_ended_unrefused(&module_id);
6820 let (code, message) = match err {
6821 crate::supervise::SuperviseError::Disabled { .. } => {
6822 ("module_disabled", err.to_string())
6823 }
6824 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6825 ("swap_in_progress", err.to_string())
6826 }
6827 _ => (
6828 "reload_failed",
6829 format!("failed to reload module_id '{module_id}': {err}"),
6830 ),
6831 };
6832 return Ok(vec![control_error_frame(&frame, code, message)?]);
6833 }
6834
6835 let response = ClientControlResponse::SupervisorAck {
6836 module_id,
6837 applied: true,
6838 };
6839 Ok(vec![control_response_body_frame(
6840 &frame,
6841 &response,
6842 "ClientControlResponse::SupervisorAck",
6843 )?])
6844 }
6845
6846 async fn handle_supervisor_rescan(
6850 &self,
6851 frame: Frame,
6852 mode: RescanMode,
6853 requester: Principal,
6854 ) -> Result<Vec<Frame>, RouterError> {
6855 let preview = mode.previews();
6856 let Some(context) = self.rescan.clone() else {
6857 return Ok(vec![control_error_frame(
6858 &frame,
6859 "rescan_unavailable",
6860 "the daemon was not started with a reloadable config path".to_string(),
6861 )?]);
6862 };
6863
6864 let operation_lock = self.supervisor.operation_lock();
6865 let _operation_guard = operation_lock.lock().await;
6866 let loaded = match self.load_rescan_config(&mode, &context) {
6870 Ok(config) => config,
6871 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6872 };
6873 let Some((config, activity_config)) = loaded else {
6881 return Ok(vec![control_error_frame(
6882 &frame,
6883 "invalid_daemon_config",
6884 format!(
6885 "daemon config not found at {}; refusing to rescan (an absent config would \
6886 retire every supervised module)",
6887 context.config_path.display()
6888 ),
6889 )?]);
6890 };
6891 let (
6892 configured_port,
6893 storage_config,
6894 admission_facts_carrier_module_id,
6895 admission_facts_targets,
6896 scope_authority_owners,
6897 federated_scope_owner,
6898 privileged_operator_prompt,
6899 modules,
6900 reserved_capabilities,
6901 ) = (
6902 config.port,
6903 config.storage,
6904 config.admission_facts_carrier_module_id,
6905 config.admission_facts_targets,
6906 config.scope_authority_owners,
6907 config.federated_scope_owner,
6908 config.privileged_operator_prompt,
6909 config.modules,
6910 config.reserved_capabilities,
6911 );
6912
6913 let mut restart_required = Vec::new();
6922 for section in RestartRequiredSection::ALL {
6923 let changed = match section {
6924 RestartRequiredSection::Port => configured_port != context.configured_port,
6925 RestartRequiredSection::Storage => storage_config != context.storage_config,
6926 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6927 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6928 }
6929 RestartRequiredSection::AdmissionFactsTargets => {
6930 admission_facts_targets != context.admission_facts_targets
6931 }
6932 RestartRequiredSection::ScopeAuthorityOwners => {
6933 scope_authority_owners != context.scope_authority_owners
6934 }
6935 RestartRequiredSection::FederatedScopeOwner => {
6936 federated_scope_owner != context.federated_scope_owner
6937 }
6938 RestartRequiredSection::PrivilegedOperatorPrompt => {
6939 privileged_operator_prompt != context.privileged_operator_prompt
6940 }
6941 };
6942 if changed {
6943 restart_required.push(section.label().to_string());
6944 }
6945 }
6946 if !restart_required.is_empty() {
6947 warn!(
6948 config_path = %context.config_path.display(),
6949 sections = %restart_required.join(", "),
6950 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6951 );
6952 }
6953
6954 for configured in &modules {
6955 if let Err(err) = validate_spec(&configured.module_spec()) {
6956 return Ok(vec![control_error_frame(
6957 &frame,
6958 "invalid_daemon_config",
6959 format!("supervisor rescan rejected daemon config: {err}"),
6960 )?]);
6961 }
6962 }
6963
6964 let configured_capabilities = modules
6965 .iter()
6966 .map(|module| (module.module_id.clone(), module.enabled))
6967 .collect::<Vec<_>>();
6968 let preview_capability_warnings = if preview {
6969 let (_, registrations) = self.runtime_capability_snapshot()?;
6970 let current_modules = self
6971 .supervisor
6972 .list()
6973 .into_iter()
6974 .map(|module| module.module_id().to_string())
6975 .collect::<BTreeSet<_>>();
6976 let resulting_modules = configured_capabilities.clone();
6977 let removed = current_modules
6978 .into_iter()
6979 .filter(|module_id| {
6980 !resulting_modules
6981 .iter()
6982 .any(|(configured_id, _)| configured_id == module_id)
6983 })
6984 .collect::<Vec<_>>();
6985 let disabled = match &mode {
6988 RescanMode::Candidate(_) => resulting_modules
6989 .iter()
6990 .filter(|(_, enabled)| !enabled)
6991 .map(|(module_id, _)| module_id.clone())
6992 .collect::<Vec<_>>(),
6993 _ => Vec::new(),
6994 };
6995 self.capability_evaluator.preview_warnings(
6996 resulting_modules,
6997 &removed,
6998 &disabled,
6999 ®istrations,
7000 )
7001 } else {
7002 Vec::new()
7003 };
7004 let reconciled = match &mode {
7005 RescanMode::Candidate(candidate) => self
7006 .preview_candidate(&context, modules, candidate)
7007 .map(|result| (result, Vec::new())),
7008 _ => {
7009 self.reconcile_supervised_modules(
7010 &context.supervisor,
7011 modules,
7012 preview,
7013 requester.clone(),
7014 )
7015 .await
7016 }
7017 };
7018 let (result, changed) = match reconciled {
7019 Ok(result) => result,
7020 Err(message) => {
7021 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
7022 }
7023 };
7024 if !preview {
7025 if let Some(ledger) = &self.activity {
7026 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
7027 ledger.config_changed(&result.added, &result.removed, &changed, requester);
7028 }
7029 if let Err(code) = self.configure_activity(activity_config).await {
7030 warn!(
7031 code,
7032 "activity retention could not be applied during rescan"
7033 );
7034 }
7035 self.capability_evaluator
7036 .configure(configured_capabilities, reserved_capabilities);
7037 self.capability_evaluator.wake_deadline_loop();
7038 self.refresh_capability_requirements();
7039 }
7040 let mut result = result;
7041 result.restart_required = restart_required;
7042 result.capability_warnings = preview_capability_warnings;
7043 let response = ClientControlResponse::SupervisorRescan { result };
7044 Ok(vec![control_response_body_frame(
7045 &frame,
7046 &response,
7047 "ClientControlResponse::SupervisorRescan",
7048 )?])
7049 }
7050
7051 async fn handle_supervisor_release_reserved(
7052 &self,
7053 frame: Frame,
7054 module_id: String,
7055 ) -> Result<Vec<Frame>, RouterError> {
7056 let Some(context) = self.rescan.clone() else {
7057 return Ok(vec![control_error_frame(
7058 &frame,
7059 "release_unavailable",
7060 "reserved-id release requires a daemon started with a reloadable config path",
7061 )?]);
7062 };
7063 let operation_lock = self.supervisor.operation_lock();
7064 let _operation_guard = operation_lock.lock().await;
7065 let loaded = match crate::daemon_config::load(&context.config_path) {
7066 Ok(Some(config)) => config,
7067 Ok(None) => {
7068 return Ok(vec![control_error_frame(
7069 &frame,
7070 "invalid_daemon_config",
7071 format!(
7072 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
7073 context.config_path.display()
7074 ),
7075 )?])
7076 }
7077 Err(err) => {
7078 return Ok(vec![control_error_frame(
7079 &frame,
7080 "invalid_daemon_config",
7081 format!("unable to verify reserved-id release against daemon config: {err}"),
7082 )?])
7083 }
7084 };
7085 if loaded
7086 .modules
7087 .iter()
7088 .any(|configured| configured.module_id == module_id)
7089 {
7090 return Ok(vec![control_error_frame(
7091 &frame,
7092 "reserved_module_configured",
7093 format!(
7094 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
7095 ),
7096 )?]);
7097 }
7098 if !self.supervisor.release_retained_reserved_gate(&module_id) {
7099 return Ok(vec![control_error_frame(
7100 &frame,
7101 "reserved_gate_not_retained",
7102 format!(
7103 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
7104 ),
7105 )?]);
7106 }
7107
7108 let response = ClientControlResponse::SupervisorAck {
7109 module_id,
7110 applied: true,
7111 };
7112 Ok(vec![control_response_body_frame(
7113 &frame,
7114 &response,
7115 "ClientControlResponse::SupervisorAck",
7116 )?])
7117 }
7118
7119 fn plan_rescan(
7123 &self,
7124 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
7125 ) -> Result<RescanPlan, String> {
7126 let mut current = BTreeMap::new();
7127 for module in self.supervisor.list() {
7128 let (spec, health) = module.configuration().map_err(|err| {
7129 format!(
7130 "failed to read configuration for module_id '{}': {err}",
7131 module.module_id()
7132 )
7133 })?;
7134 let enabled = module
7135 .status()
7136 .map_err(|err| {
7137 format!(
7138 "failed to read status for module_id '{}': {err}",
7139 module.module_id()
7140 )
7141 })?
7142 .enabled;
7143 current.insert(
7144 module.module_id().to_string(),
7145 (module, spec, health, enabled),
7146 );
7147 }
7148 let configured = configured_modules
7149 .into_iter()
7150 .map(|module| (module.module_id.clone(), module))
7151 .collect::<BTreeMap<_, _>>();
7152
7153 let added = configured
7154 .keys()
7155 .filter(|module_id| !current.contains_key(*module_id))
7156 .cloned()
7157 .collect::<Vec<_>>();
7158 let removed = current
7159 .keys()
7160 .filter(|module_id| !configured.contains_key(*module_id))
7161 .cloned()
7162 .collect::<Vec<_>>();
7163 let mut changed_pending_reload = Vec::new();
7164 let mut configuration_changes = BTreeSet::new();
7165 let mut enabled_changes = BTreeSet::new();
7166 let mut unchanged = 0_u32;
7167
7168 for (module_id, configured_module) in &configured {
7169 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
7170 else {
7171 continue;
7172 };
7173 let launch_changed = *current_spec != configured_module.module_spec();
7177 let configuration_changed =
7178 launch_changed || *current_health != configured_module.health;
7179 let enabled_changed = *current_enabled != configured_module.enabled
7182 || self
7183 .capability_evaluator
7184 .configured_enabled(module_id)
7185 .is_some_and(|enabled| enabled != configured_module.enabled);
7186 if configuration_changed {
7187 configuration_changes.insert(module_id.clone());
7188 }
7189 if launch_changed {
7190 changed_pending_reload.push(module_id.clone());
7191 }
7192 if enabled_changed {
7193 enabled_changes.insert(module_id.clone());
7194 }
7195 if !configuration_changed && !enabled_changed {
7196 unchanged = unchanged.saturating_add(1);
7197 }
7198 }
7199 Ok(RescanPlan {
7200 current,
7201 configured,
7202 added,
7203 removed,
7204 changed_pending_reload,
7205 configuration_changes,
7206 enabled_changes,
7207 unchanged,
7208 })
7209 }
7210
7211 async fn reconcile_supervised_modules(
7220 &self,
7221 supervisor: &Supervisor,
7222 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
7223 preview: bool,
7224 requester: Principal,
7225 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
7226 let plan = self.plan_rescan(configured_modules)?;
7227
7228 if preview {
7234 return Ok((plan.preview_result(), Vec::new()));
7235 }
7236 let changed = plan
7237 .configuration_changes
7238 .union(&plan.enabled_changes)
7239 .cloned()
7240 .collect::<Vec<_>>();
7241 let RescanPlan {
7242 current,
7243 configured,
7244 added,
7245 removed,
7246 changed_pending_reload,
7247 configuration_changes,
7248 enabled_changes,
7249 unchanged,
7250 } = plan;
7251 if !(added.is_empty()
7256 && removed.is_empty()
7257 && configuration_changes.is_empty()
7258 && enabled_changes.is_empty())
7259 {
7260 if let Some(context) = &self.rescan {
7261 context
7262 .generation
7263 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7264 }
7265 }
7266
7267 for module_id in &removed {
7268 let module = ¤t
7269 .get(module_id)
7270 .expect("removed module came from current supervisor state")
7271 .0;
7272 module
7273 .retire_requested(Some(requester.clone()))
7274 .await
7275 .map_err(|err| {
7276 format!("failed to retire module_id '{module_id}' during rescan: {err}")
7277 })?;
7278 self.supervisor.record_rescan_removal(module_id);
7312 self.supervisor.retire(module_id);
7313 self.route_outages.forget(module_id);
7314 }
7315
7316 for module_id in configured.keys() {
7317 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
7318 continue;
7319 };
7320 let configured_module = configured
7321 .get(module_id)
7322 .expect("configured module id came from configured map");
7323 if configuration_changes.contains(module_id) {
7324 module
7325 .update_configuration(
7326 configured_module.module_spec(),
7327 configured_module.health.clone(),
7328 configured_module.drain_timeout_ms,
7329 )
7330 .await
7331 .map_err(|err| {
7332 format!(
7333 "failed to update module_id '{module_id}' configuration during rescan: {err}"
7334 )
7335 })?;
7336 }
7337 if *current_enabled != configured_module.enabled {
7340 self.route_outages.mark_operator_action(module_id);
7343 module
7344 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
7345 .await
7346 .map_err(|err| {
7347 self.route_outages.operator_action_ended_unrefused(module_id);
7348 format!(
7349 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
7350 configured_module.enabled
7351 )
7352 })?;
7353 }
7354 }
7355
7356 for module_id in &added {
7357 let configured_module = configured
7358 .get(module_id)
7359 .expect("added module id came from configured map");
7360 supervisor
7361 .supervise_configured_requested(
7362 configured_module.module_spec(),
7363 configured_module.enabled,
7364 configured_module.health.clone(),
7365 configured_module.drain_timeout_ms,
7366 configured_module.restart,
7367 configured_module.register_timeout,
7368 Some(requester.clone()),
7369 )
7370 .map_err(|err| {
7371 format!("failed to add module_id '{module_id}' during rescan: {err}")
7372 })?;
7373 }
7374
7375 Ok((
7376 SupervisorRescanResult {
7377 added,
7378 removed,
7379 changed_pending_reload,
7380 enabled_changes: enabled_changes.iter().cloned().collect(),
7381 unchanged,
7382 preview: false,
7383 restart_required: Vec::new(),
7386 capability_warnings: Vec::new(),
7387 module_actions: Vec::new(),
7388 earlier_pending: Vec::new(),
7389 expect: None,
7390 },
7391 changed,
7392 ))
7393 }
7394
7395 async fn handle_supervisor_set_enabled(
7396 &self,
7397 frame: Frame,
7398 module_id: String,
7399 enabled: bool,
7400 requester: Principal,
7401 ) -> Result<Vec<Frame>, RouterError> {
7402 let operation_lock = self.supervisor.operation_lock();
7403 let _operation_guard = operation_lock.lock().await;
7404 let Some(module) = self.supervisor.get(&module_id) else {
7405 return Ok(vec![control_error_frame(
7406 &frame,
7407 "unknown_module",
7408 format!("module_id '{module_id}' is not supervised"),
7409 )?]);
7410 };
7411
7412 self.route_outages.mark_operator_action(&module_id);
7415 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7416 Ok(applied) => applied,
7417 Err(err) => {
7418 self.route_outages
7419 .operator_action_ended_unrefused(&module_id);
7420 return Ok(vec![control_error_frame(
7421 &frame,
7422 "target_unavailable",
7423 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7424 )?]);
7425 }
7426 };
7427 if !applied {
7428 self.route_outages
7431 .operator_action_ended_unrefused(&module_id);
7432 }
7433
7434 self.capability_evaluator.wake_deadline_loop();
7435 self.refresh_capability_requirements();
7436 let response = ClientControlResponse::SupervisorAck { module_id, applied };
7437 Ok(vec![control_response_body_frame(
7438 &frame,
7439 &response,
7440 "ClientControlResponse::SupervisorAck",
7441 )?])
7442 }
7443
7444 async fn handle_supervisor_health_probe(
7445 &self,
7446 ctx: &RouteCtx,
7447 frame: Frame,
7448 module_id: String,
7449 ) -> Result<Vec<Frame>, RouterError> {
7450 self.refresh_capability_requirements();
7451 let Some(registration) = self
7452 .registry
7453 .get_module(&module_id)
7454 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7455 else {
7456 return Ok(vec![control_error_frame(
7457 &frame,
7458 "unknown_module",
7459 format!("module_id '{module_id}' is not registered"),
7460 )?]);
7461 };
7462
7463 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7476 {
7477 return Ok(vec![control_error_frame(
7478 &frame,
7479 "health_not_advertised",
7480 format!("module_id '{module_id}' did not advertise health.check"),
7481 )?]);
7482 }
7483
7484 self.observe_control_wait(
7485 ctx,
7486 Some(®istration.manifest.module_id),
7487 ops::SUPERVISOR_HEALTH_PROBE,
7488 async {
7489 let deadline = Instant::now() + self.health_probe_timeout;
7490 let pending = match self.forwarding.begin_module_control_rpc_for(
7491 &module_id,
7492 MODULE_CONTROL_OP_HEALTH_CHECK,
7493 deadline,
7494 ) {
7495 Ok(pending) => pending,
7496 Err(err) => {
7497 return Ok(vec![control_error_frame(
7498 &frame,
7499 forwarding_error_code(&err),
7500 err.to_string(),
7501 )?])
7502 }
7503 };
7504
7505 let PendingModuleControlRpc {
7506 endpoint,
7507 module_sink,
7508 negotiated_ver,
7509 corr: probe_corr,
7510 receiver,
7511 } = pending;
7512 let mut guard =
7513 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7514 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7515 .map_err(|err| {
7516 RouterError::backend(
7517 0,
7518 frame.header.corr,
7519 format!("failed to encode health.check request: {err}"),
7520 )
7521 })?;
7522 let probe_frame = Frame::build_with_version(
7523 negotiated_ver,
7524 FrameType::Request,
7525 control_flags(),
7526 0,
7527 0,
7528 probe_corr,
7529 probe_body,
7530 )
7531 .map_err(RouterError::FrameBuild)?;
7532
7533 match timeout_at(deadline, module_sink.send(probe_frame)).await {
7536 Ok(Ok(())) => {}
7537 Ok(Err(err)) => {
7538 return Ok(vec![control_error_frame(
7539 &frame,
7540 "target_unavailable",
7541 err.to_string(),
7542 )?]);
7543 }
7544 Err(_) => {
7545 return Ok(vec![control_error_frame(
7546 &frame,
7547 "module_timeout",
7548 format!(
7549 "module_id '{module_id}' did not answer health.check within {:?}",
7550 self.health_probe_timeout
7551 ),
7552 )?]);
7553 }
7554 }
7555
7556 match timeout_at(deadline, receiver).await {
7557 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7558 guard.disarm();
7559 let Some(report) = response.health_report() else {
7560 return Ok(vec![control_error_frame(
7561 &frame,
7562 "invalid_control_body",
7563 "health.check RPC returned a non-health response",
7564 )?]);
7565 };
7566 let HealthReport {
7571 status,
7572 detail,
7573 metrics,
7574 } = report;
7575 let capability_detail = self
7576 .capability_evaluator
7577 .required_problem_detail(&module_id);
7578 let response = ClientControlResponse::SupervisorHealthProbe {
7579 module_id,
7580 status,
7581 detail: append_capability_problem_detail(detail, capability_detail),
7582 metrics,
7583 };
7584 Ok(vec![control_response_body_frame(
7585 &frame,
7586 &response,
7587 "ClientControlResponse::SupervisorHealthProbe",
7588 )?])
7589 }
7590 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7591 guard.disarm();
7592 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7597 }
7598 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7599 guard.disarm();
7600 Ok(vec![control_error_frame(
7601 &frame,
7602 "target_unavailable",
7603 message,
7604 )?])
7605 }
7606 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7607 guard.disarm();
7608 Ok(vec![control_error_frame(
7609 &frame,
7610 "invalid_control_body",
7611 message,
7612 )?])
7613 }
7614 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7615 guard.disarm();
7616 Ok(vec![control_error_frame(
7617 &frame,
7618 "invalid_control_body",
7619 format!("expected module-control op '{expected}', got '{actual}'"),
7620 )?])
7621 }
7622 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7623 guard.disarm();
7624 Ok(vec![control_error_frame(
7625 &frame,
7626 "module_timeout",
7627 format!(
7628 "module_id '{module_id}' answered health.check after {:?}",
7629 self.health_probe_timeout
7630 ),
7631 )?])
7632 }
7633 Ok(Err(_)) => Ok(vec![control_error_frame(
7634 &frame,
7635 "target_unavailable",
7636 "health.check waiter was canceled before the module responded",
7637 )?]),
7638 Err(_) => Ok(vec![control_error_frame(
7639 &frame,
7640 "module_timeout",
7641 format!(
7642 "module_id '{module_id}' did not answer health.check within {:?}",
7643 self.health_probe_timeout
7644 ),
7645 )?]),
7646 }
7647 },
7648 )
7649 .await
7650 }
7651
7652 fn supervisor_status(
7653 &self,
7654 module_id: &str,
7655 corr: u64,
7656 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7657 self.supervisor
7658 .get(module_id)
7659 .map(|module| {
7660 let warming = module.is_warming_for_control("status").map_err(|err| {
7661 RouterError::backend(
7662 0,
7663 corr,
7664 format!(
7665 "failed to read supervisor warming state for module_id '{module_id}': {err}"
7666 ),
7667 )
7668 })?;
7669 module.status_for_control("status").map_err(|err| {
7670 RouterError::backend(
7671 0,
7672 corr,
7673 format!(
7674 "failed to read supervisor status for module_id '{module_id}': {err}"
7675 ),
7676 )
7677 }).map(|status| (status, warming))
7678 })
7679 .transpose()
7680 }
7681
7682 fn guard_module_control_op(
7683 &self,
7684 frame: &Frame,
7685 module_id: &str,
7686 op: &str,
7687 ) -> Result<Option<Frame>, RouterError> {
7688 if self.module_grants_op(module_id, op, frame.header.corr)? {
7689 return Ok(None);
7690 }
7691
7692 Ok(Some(control_error_frame(
7693 frame,
7694 "op_not_allowed",
7695 format!("module_id '{module_id}' did not grant control op '{op}'"),
7696 )?))
7697 }
7698
7699 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7700 let Some(registration) = self
7701 .registry
7702 .get_module(module_id)
7703 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7704 else {
7705 return Ok(false);
7706 };
7707 Ok(module_registration_grants_op(®istration.control_ops, op))
7708 }
7709
7710 fn handle_status_update(
7711 &self,
7712 endpoint: ModuleEndpointId,
7713 frame: Frame,
7714 ) -> Result<Vec<Frame>, RouterError> {
7715 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7716 Ok(update) => update,
7717 Err(err) => {
7718 if is_known_module_push_op(&frame.body) {
7723 return Ok(vec![control_error_frame(
7724 &frame,
7725 "invalid_control_body",
7726 format!("malformed module control push body: {err}"),
7727 )?]);
7728 }
7729 return Ok(Vec::new());
7730 }
7731 };
7732
7733 match update {
7734 ModuleControlPush::RouteStatus {
7735 route_channel,
7736 route_epoch,
7737 status,
7738 } => {
7739 self.forwarding
7740 .cache_status(endpoint, route_channel, route_epoch, status)
7741 .map_err(RouterError::Forwarding)?;
7742 }
7743 }
7744 Ok(Vec::new())
7745 }
7746
7747 fn handle_route_poll(
7748 &self,
7749 ctx: &RouteCtx,
7750 frame: Frame,
7751 route_channel: u16,
7752 route_epoch: u32,
7753 kind: PollKind,
7754 ) -> Result<Vec<Frame>, RouterError> {
7755 let snapshot = self
7756 .forwarding
7757 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7758 .map_err(RouterError::Forwarding)?;
7759 let response = match (kind, snapshot) {
7760 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7761 ClientControlResponse::RoutePoll {
7762 route_channel,
7763 route_epoch,
7764 status,
7765 live: None,
7766 }
7767 }
7768 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7769 route_channel,
7770 route_epoch,
7771 status: None,
7772 live: None,
7773 },
7774 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7775 let live = self
7801 .process_liveness
7802 .as_ref()
7803 .and_then(|source| source.process_live(&module_id))
7804 .unwrap_or(true);
7805 ClientControlResponse::RoutePoll {
7806 route_channel,
7807 route_epoch,
7808 status: None,
7809 live: Some(live),
7810 }
7811 }
7812 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7813 route_channel,
7814 route_epoch,
7815 status: None,
7816 live: Some(false),
7817 },
7818 };
7819
7820 Ok(vec![control_response_body_frame(
7821 &frame,
7822 &response,
7823 "ClientControlResponse::RoutePoll",
7824 )?])
7825 }
7826
7827 pub(crate) fn observe_module_control_completion(
7828 &self,
7829 completion: ModuleControlRpcCompletion,
7830 ) -> bool {
7831 match completion {
7832 ModuleControlRpcCompletion::Unknown => false,
7833 ModuleControlRpcCompletion::Settled => true,
7834 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7835 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7836 info!(
7837 module_id = %module_id,
7838 latency_ms,
7839 "late health.check answer proves the module is alive"
7840 );
7841 match self
7842 .supervisor
7843 .record_late_health_answer(&module_id, latency_ms)
7844 {
7845 Ok(true) => {}
7846 Ok(false) => debug!(
7847 module_id = %module_id,
7848 latency_ms,
7849 "late health.check answer has no active supervisor snapshot"
7850 ),
7851 Err(err) => warn!(
7852 module_id = %module_id,
7853 latency_ms,
7854 error = %err,
7855 "failed to record late health.check answer"
7856 ),
7857 }
7858 true
7859 }
7860 }
7861 }
7862
7863 fn refuse_to_end_module_connection_for_a_client(
7884 &self,
7885 module_connection_id: ConnectionId,
7886 corr: u64,
7887 err: ForwardingError,
7888 ) -> Result<(), RouterError> {
7889 if let ForwardingError::ConnectionClosing { connection_id } = err {
7890 if connection_id != module_connection_id {
7891 warn!(
7892 module_connection_id = module_connection_id.get(),
7893 client_connection_id = connection_id.get(),
7894 corr,
7895 "dropping a route.bind response for a closing client; the module connection keeps serving"
7896 );
7897 return Ok(());
7898 }
7899 }
7900 Err(RouterError::Forwarding(err))
7901 }
7902
7903 fn handle_module_relay_response(
7904 &self,
7905 connection_id: ConnectionId,
7906 frame: Frame,
7907 ) -> Result<Vec<Frame>, RouterError> {
7908 let mut secondary_error = None;
7909 let outcome = match frame.header.ty {
7910 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7911 Ok(probe) if probe.op == "route.bind" => {
7912 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7913 Ok(ModuleControlResponse::RouteBindAck {}) => {
7914 RouteBindRelayOutcome::Accepted
7915 }
7916 Ok(other) => {
7917 let message =
7918 format!("route.bind response carried unexpected body: {other:?}");
7919 secondary_error = Some(control_error_frame(
7920 &frame,
7921 "invalid_control_body",
7922 message.clone(),
7923 )?);
7924 RouteBindRelayOutcome::ModuleGone(message)
7925 }
7926 Err(err) => {
7927 let message = format!("malformed route.bind response body: {err}");
7928 secondary_error = Some(control_error_frame(
7929 &frame,
7930 "invalid_control_body",
7931 message.clone(),
7932 )?);
7933 RouteBindRelayOutcome::ModuleGone(message)
7934 }
7935 }
7936 }
7937 Ok(probe) => {
7938 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7939 {
7940 Ok(response) => ModuleControlRpcOutcome::Response(response),
7941 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7942 "malformed {} response body: {err}",
7943 probe.op
7944 )),
7945 };
7946 let completion = self
7947 .forwarding
7948 .complete_module_control_rpc(
7949 connection_id,
7950 frame.header.corr,
7951 Some(&probe.op),
7952 outcome,
7953 )
7954 .map_err(RouterError::Forwarding)?;
7955 if !self.observe_module_control_completion(completion) {
7956 debug!(
7957 connection_id = connection_id.get(),
7958 corr = frame.header.corr,
7959 op = %probe.op,
7960 "dropping late or unknown module-control RPC response"
7961 );
7962 }
7963 return Ok(Vec::new());
7964 }
7965 Err(err) => {
7966 if let Some(expected_op) = self
7967 .forwarding
7968 .pending_module_control_op(connection_id, frame.header.corr)
7969 .map_err(RouterError::Forwarding)?
7970 {
7971 let completion = self
7972 .forwarding
7973 .complete_module_control_rpc(
7974 connection_id,
7975 frame.header.corr,
7976 None,
7977 ModuleControlRpcOutcome::MalformedResponse(format!(
7978 "malformed {expected_op} response body: {err}"
7979 )),
7980 )
7981 .map_err(RouterError::Forwarding)?;
7982 if !self.observe_module_control_completion(completion) {
7983 debug!(
7984 connection_id = connection_id.get(),
7985 corr = frame.header.corr,
7986 "dropping late malformed module-control RPC response"
7987 );
7988 }
7989 return Ok(Vec::new());
7990 }
7991 let message = format!("malformed route.bind response body: {err}");
7992 secondary_error = Some(control_error_frame(
7993 &frame,
7994 "invalid_control_body",
7995 message.clone(),
7996 )?);
7997 RouteBindRelayOutcome::ModuleGone(message)
7998 }
7999 },
8000 FrameType::Error => {
8001 if self
8002 .forwarding
8003 .pending_module_control_op(connection_id, frame.header.corr)
8004 .map_err(RouterError::Forwarding)?
8005 .is_some()
8006 {
8007 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
8008 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
8009 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
8010 "malformed module-control ERROR body: {err}"
8011 )),
8012 };
8013 let completion = self
8014 .forwarding
8015 .complete_module_control_rpc(
8016 connection_id,
8017 frame.header.corr,
8018 None,
8019 outcome,
8020 )
8021 .map_err(RouterError::Forwarding)?;
8022 if !self.observe_module_control_completion(completion) {
8023 debug!(
8024 connection_id = connection_id.get(),
8025 corr = frame.header.corr,
8026 "dropping late or unknown module-control RPC error"
8027 );
8028 }
8029 return Ok(Vec::new());
8030 }
8031 match serde_json::from_slice::<ErrorBody>(&frame.body) {
8032 Ok(body) => RouteBindRelayOutcome::Rejected(body),
8033 Err(err) => {
8034 let message = format!("malformed route.bind ERROR body: {err}");
8035 secondary_error = Some(control_error_frame(
8036 &frame,
8037 "invalid_control_body",
8038 message.clone(),
8039 )?);
8040 RouteBindRelayOutcome::ModuleGone(message)
8041 }
8042 }
8043 }
8044 ty => {
8045 return Ok(vec![control_error_frame(
8046 &frame,
8047 "unsupported_control_frame",
8048 format!("unsupported module channel-0 frame {ty:?}"),
8049 )?])
8050 }
8051 };
8052
8053 let settled =
8054 self.forwarding
8055 .complete_pending_relay(connection_id, frame.header.corr, outcome);
8056 let completion = match settled {
8057 Ok(completion) => completion,
8058 Err(err) => {
8059 self.refuse_to_end_module_connection_for_a_client(
8060 connection_id,
8061 frame.header.corr,
8062 err,
8063 )?;
8064 return Ok(secondary_error.into_iter().collect());
8065 }
8066 };
8067 if let Some(target) = completion.abandoned.as_ref() {
8068 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
8069 }
8070 if !completion.settled {
8071 debug!(
8072 connection_id = connection_id.get(),
8073 corr = frame.header.corr,
8074 frame_type = ?frame.header.ty,
8075 "dropping late or unknown route.bind relay response"
8076 );
8077 }
8078 Ok(secondary_error.into_iter().collect())
8079 }
8080
8081 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
8082 debug!(connection_id = connection_id.get(), "handling GOODBYE");
8083 self.cleanup_connection_with_end_reason(
8087 connection_id,
8088 RegistrationEndReason::ExplicitGoodbye,
8089 )
8090 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
8091 Ok(Vec::new())
8092 }
8093}
8094
8095impl Default for ControlHandler {
8096 fn default() -> Self {
8097 Self::new(Arc::new(Registry::default()))
8098 }
8099}
8100
8101impl crate::supervise::SwapPromotionObserver for ControlHandler {
8102 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
8103 self.apply_registration_capabilities(registration);
8104 }
8105}
8106
8107fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
8108 CapabilityRequirementStatus {
8109 consumer: status.consumer,
8110 capability: status.capability,
8111 need: match status.need {
8112 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
8113 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
8114 },
8115 verdict: status.verdict.as_str().to_string(),
8116 episode_seq: status.episode_seq,
8117 config_satisfiable: status.config_satisfiable,
8118 runtime_available: status.runtime_available,
8119 detail: status.detail,
8120 }
8121}
8122
8123fn append_capability_problem_detail(
8124 detail: Option<String>,
8125 capability_detail: Option<String>,
8126) -> Option<String> {
8127 match (detail, capability_detail) {
8128 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
8129 (Some(detail), None) => Some(detail),
8130 (None, Some(capability_detail)) => Some(capability_detail),
8131 (None, None) => None,
8132 }
8133}
8134
8135fn subc_ops() -> Vec<String> {
8136 SUBC_CONTROL_OPS
8137 .iter()
8138 .map(|op| (*op).to_string())
8139 .collect()
8140}
8141
8142fn module_subc_ops() -> Vec<String> {
8143 SUBC_CONTROL_OPS
8144 .iter()
8145 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
8146 .map(|op| (*op).to_string())
8147 .collect()
8148}
8149
8150#[cfg(test)]
8151fn module_baseline_control_ops() -> Vec<String> {
8152 MODULE_BASELINE_CONTROL_OPS
8153 .iter()
8154 .map(|op| (*op).to_string())
8155 .collect()
8156}
8157
8158fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
8159 let mut seen = HashSet::new();
8160 let mut effective = Vec::new();
8161 for op in MODULE_BASELINE_CONTROL_OPS {
8162 if seen.insert((*op).to_string()) {
8163 effective.push((*op).to_string());
8164 }
8165 }
8166 for op in declared.unwrap_or_default() {
8167 if seen.insert(op.clone()) {
8168 effective.push(op);
8169 }
8170 }
8171 effective
8172}
8173
8174fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
8175 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
8176}
8177
8178fn target_module_id(target: &RouteTarget) -> &str {
8179 match target {
8180 RouteTarget::ToolProvider { module_id }
8181 | RouteTarget::ManagementSurface { module_id }
8182 | RouteTarget::InternalService { module_id, .. } => module_id,
8183 }
8184}
8185
8186fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
8187 roles.iter().any(|role| match (target, role) {
8188 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
8189 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
8190 (
8191 RouteTarget::InternalService { service_id, .. },
8192 ProviderRole::InternalService {
8193 service_id: provided,
8194 ..
8195 },
8196 ) => service_id == provided,
8197 _ => false,
8198 })
8199}
8200
8201fn is_routable_role(role: &ProviderRole) -> bool {
8202 matches!(
8203 role,
8204 ProviderRole::ToolProvider { .. }
8205 | ProviderRole::ManagementSurface { .. }
8206 | ProviderRole::InternalService { .. }
8207 )
8208}
8209
8210#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8211enum ControlRequestBodyError {
8212 UnknownOp,
8213 InvalidBody,
8214}
8215
8216#[derive(Debug, Deserialize)]
8217struct ControlOpProbe {
8218 op: String,
8219}
8220
8221const MODULE_PUSH_OPS: &[&str] = &["route.status"];
8224
8225fn is_known_module_push_op(body: &[u8]) -> bool {
8226 serde_json::from_slice::<ControlOpProbe>(body)
8227 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
8228 .unwrap_or(false)
8229}
8230
8231fn is_known_module_request_op(body: &[u8]) -> bool {
8232 serde_json::from_slice::<ControlOpProbe>(body)
8233 .map(|probe| is_module_to_subc_op(&probe.op))
8234 .unwrap_or(false)
8235}
8236
8237fn is_module_to_subc_op(op: &str) -> bool {
8238 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
8239}
8240
8241fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
8242 debug!(
8243 op = %op,
8244 connection_id = connection_id.get(),
8245 corr,
8246 "control dispatch"
8247 );
8248}
8249
8250fn log_slow_control_dispatch(
8251 dispatch_started_at: Option<StdInstant>,
8252 op: &'static str,
8253 connection_id: ConnectionId,
8254 corr: u64,
8255 route_open_timings: Option<&RouteOpenTimings>,
8256) {
8257 let Some(dispatch_started_at) = dispatch_started_at else {
8258 return;
8259 };
8260 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
8262 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
8263 if let Some(timings) = route_open_timings {
8264 warn!(
8265 op = %op,
8266 connection_id = connection_id.get(),
8267 corr,
8268 elapsed_ms = elapsed.as_millis() as u64,
8269 module_id = ?timings.module_id,
8270 principal = %timings.principal,
8271 admission_ms = timings.admission.as_millis() as u64,
8272 project_root_ms = timings.project_root.as_millis() as u64,
8273 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
8274 bind_send_ms = timings.bind_send.as_millis() as u64,
8275 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
8276 "slow control dispatch"
8277 );
8278 return;
8279 }
8280 warn!(
8281 op = %op,
8282 connection_id = connection_id.get(),
8283 corr,
8284 elapsed_ms = elapsed.as_millis() as u64,
8285 "slow control dispatch"
8286 );
8287 }
8288}
8289
8290#[cfg(test)]
8292type TestCallerLookup =
8293 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
8294
8295fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
8297 match request {
8298 ClientControlRequest::SupervisorRestart { module_id, .. } => {
8299 Some(PrivilegedOperation::Restart { module_id })
8300 }
8301 ClientControlRequest::SupervisorSwap { module_id, .. } => {
8302 Some(PrivilegedOperation::Swap { module_id })
8303 }
8304 ClientControlRequest::SupervisorReload { module_id } => {
8305 Some(PrivilegedOperation::Reload { module_id })
8306 }
8307 ClientControlRequest::SupervisorRescan { preview } => {
8308 if *preview {
8309 None
8310 } else {
8311 Some(PrivilegedOperation::Rescan)
8312 }
8313 }
8314 ClientControlRequest::SupervisorRescanChecked {
8319 preview,
8320 candidate,
8321 expect,
8322 } => (!*preview && candidate.is_none() && expect.is_some())
8323 .then_some(PrivilegedOperation::Rescan),
8324 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
8325 Some(PrivilegedOperation::SetEnabled {
8326 module_id,
8327 enabled: *enabled,
8328 })
8329 }
8330 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
8331 Some(PrivilegedOperation::ReleaseReserved { module_id })
8332 }
8333 ClientControlRequest::ServerDescribe {}
8334 | ClientControlRequest::DaemonShutdownIntent { .. }
8337 | ClientControlRequest::CatalogList { .. }
8338 | ClientControlRequest::RouteOpen { .. }
8339 | ClientControlRequest::RoutePoll { .. }
8340 | ClientControlRequest::SupervisorList {}
8341 | ClientControlRequest::SupervisorHealth {}
8342 | ClientControlRequest::SupervisorHealthProbe { .. }
8343 | ClientControlRequest::SupervisorRoutes { .. }
8344 | ClientControlRequest::SupervisorProvenance { .. }
8345 | ClientControlRequest::SupervisorStderrTail { .. }
8346 | ClientControlRequest::SupervisorTerminals { .. }
8347 | ClientControlRequest::SupervisorSpawnSnapshot {}
8348 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
8349 }
8350}
8351
8352fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
8353 match request {
8354 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
8355 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
8356 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
8357 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
8358 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
8359 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8360 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8361 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8362 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8363 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8364 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8365 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8366 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8367 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8368 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8369 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8370 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8371 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8372 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8373 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8374 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8375 }
8376}
8377
8378struct RouteOpenCaller {
8386 principal: String,
8390 scope: String,
8393 harness: String,
8396}
8397
8398impl RouteOpenCaller {
8399 fn new(
8400 principal: Option<&Principal>,
8401 scope: Option<&ScopeSelector>,
8402 identity: &BindIdentity,
8403 ) -> Self {
8404 Self {
8405 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8406 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8407 harness: bounded_log_text(&identity.harness),
8408 }
8409 }
8410
8411 fn unknown() -> Self {
8414 Self {
8415 principal: "unknown".to_string(),
8416 scope: "unknown".to_string(),
8417 harness: "unknown".to_string(),
8418 }
8419 }
8420}
8421
8422const MAX_LOGGED_TEXT_CHARS: usize = 64;
8425
8426fn bounded_log_text(value: &str) -> String {
8430 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8431 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8432 bounded.push('…');
8433 }
8434 format!("{bounded:?}")
8435}
8436
8437fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8443 use subc_protocol::scope::ScopeKind;
8444 let kind = match stamp.kind {
8445 ScopeKind::Head => "head",
8446 ScopeKind::Worker => "worker",
8447 ScopeKind::Ephemeral => "ephemeral",
8448 ScopeKind::Federated => "federated",
8449 _ => "unknown",
8450 };
8451 format!(
8452 "{kind}/{}/{}@{}",
8453 principal_label(&stamp.owner),
8454 bounded_log_text(&stamp.scope_ref),
8455 stamp.scope_epoch
8456 )
8457}
8458
8459fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8466 let epoch = selector
8467 .scope_epoch
8468 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8469 format!(
8470 "requested/{}/{}@{epoch}",
8471 bounded_log_text(&principal_label(&selector.owner)),
8472 bounded_log_text(&selector.scope_ref),
8473 )
8474}
8475
8476fn principal_label(principal: &Principal) -> String {
8477 match principal {
8478 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8479 Principal::Direct => "direct".to_string(),
8480 other => format!("{other:?}"),
8481 }
8482}
8483
8484fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8485 match request {
8486 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8487 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { .. } => {
8488 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS
8489 }
8490 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8491 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8492 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8493 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8494 ModuleControlRequestFromModule::RouteAuthorize { .. } => {
8495 subc_protocol::scope::ROUTE_AUTHORIZE_OP
8496 }
8497 ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
8498 _ => "unsupported",
8499 }
8500}
8501
8502fn parse_client_control_request(
8503 body: &[u8],
8504) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8505 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8506 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8507 Ok(probe)
8508 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8509 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8510 {
8511 ControlRequestBodyError::InvalidBody
8512 }
8513 Ok(_) => ControlRequestBodyError::UnknownOp,
8514 Err(_) => ControlRequestBodyError::InvalidBody,
8515 };
8516 (err, classification)
8517 })
8518}
8519
8520fn parse_module_control_request_from_module(
8521 body: &[u8],
8522) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8523 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8524 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8525 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8526 Ok(_) => ControlRequestBodyError::UnknownOp,
8527 Err(_) => ControlRequestBodyError::InvalidBody,
8528 };
8529 (err, classification)
8530 })
8531}
8532
8533#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8534enum ProviderRoleKind {
8535 ToolProvider,
8536 PipelineStage,
8537 ManagementSurface,
8538 InternalService,
8539}
8540
8541fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8542 match role {
8543 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8544 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8545 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8546 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8547 }
8548}
8549
8550fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8551 roles.iter().map(provider_role_kind).collect()
8552}
8553
8554const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8557
8558#[derive(Debug, Default, PartialEq, Eq)]
8560struct ScopeOutcomeCounts {
8561 created: usize,
8562 replaced: usize,
8563 updated: usize,
8564 unchanged: usize,
8565 refused: usize,
8566}
8567
8568impl ScopeOutcomeCounts {
8569 fn of(results: &[ScopeRecordResult]) -> Self {
8570 let mut counts = Self::default();
8571 for result in results {
8572 let slot = match result.outcome {
8573 ScopeRecordOutcome::Created => &mut counts.created,
8574 ScopeRecordOutcome::Replaced => &mut counts.replaced,
8575 ScopeRecordOutcome::Updated => &mut counts.updated,
8576 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8577 ScopeRecordOutcome::Refused => &mut counts.refused,
8578 };
8579 *slot += 1;
8580 }
8581 counts
8582 }
8583}
8584
8585#[cfg(test)]
8586mod scope_outcome_count_tests {
8587 use super::*;
8588
8589 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8590 ScopeRecordResult {
8591 scope_ref: "r".to_string(),
8592 scope_epoch: 1,
8593 outcome,
8594 code: None,
8595 message: None,
8596 version: None,
8597 parent_state: None,
8598 }
8599 }
8600
8601 #[test]
8604 fn every_outcome_is_counted_in_its_own_field() {
8605 let results = [
8606 result(ScopeRecordOutcome::Created),
8607 result(ScopeRecordOutcome::Created),
8608 result(ScopeRecordOutcome::Replaced),
8609 result(ScopeRecordOutcome::Updated),
8610 result(ScopeRecordOutcome::Unchanged),
8611 result(ScopeRecordOutcome::Refused),
8612 result(ScopeRecordOutcome::Refused),
8613 result(ScopeRecordOutcome::Refused),
8614 ];
8615 assert_eq!(
8616 ScopeOutcomeCounts::of(&results),
8617 ScopeOutcomeCounts {
8618 created: 2,
8619 replaced: 1,
8620 updated: 1,
8621 unchanged: 1,
8622 refused: 3,
8623 }
8624 );
8625 }
8626}
8627
8628fn capability_census_trigger(
8632 old: Option<&CapabilityDeclarations>,
8633 new: Option<&CapabilityDeclarations>,
8634) -> bool {
8635 let old_provides = old
8636 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8637 .unwrap_or_default();
8638 let old_denies = old
8639 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8640 .unwrap_or_default();
8641 let new = new.cloned().unwrap_or(CapabilityDeclarations {
8642 provides: Vec::new(),
8643 requires: Vec::new(),
8644 must_never_reach: Vec::new(),
8645 });
8646
8647 new.provides
8648 .iter()
8649 .any(|capability| !old_provides.contains(capability))
8650 || new
8651 .must_never_reach
8652 .iter()
8653 .any(|capability| !old_denies.contains(capability))
8654}
8655
8656fn catalog_update_frozen_field_message(
8657 registered: &ModuleManifest,
8658 provides: &[ProviderRole],
8659) -> Option<String> {
8660 let old_has_provides = !registered.provides.is_empty();
8661 let new_has_provides = !provides.is_empty();
8662 if old_has_provides != new_has_provides {
8663 return Some(format!(
8664 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8665 registered.module_id
8666 ));
8667 }
8668
8669 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
8670 return Some(format!(
8671 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8672 registered.module_id
8673 ));
8674 }
8675
8676 let registered_concurrency = manifest_concurrency(registered);
8677 let mut candidate = registered.clone();
8678 candidate.provides = provides.to_vec();
8679 let candidate_concurrency = manifest_concurrency(&candidate);
8680 if candidate_concurrency != registered_concurrency {
8681 return Some(format!(
8682 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8683 registered.module_id, registered_concurrency, candidate_concurrency
8684 ));
8685 }
8686
8687 None
8690}
8691
8692fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8693 manifest.provides.iter().any(is_routable_role)
8694}
8695
8696fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8702 manifest
8703 .provides
8704 .iter()
8705 .find_map(|provider| match provider {
8706 ProviderRole::ToolProvider { concurrency, .. }
8707 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8708 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8709 })
8710 .unwrap_or(Concurrency::ModuleManaged)
8711}
8712
8713fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8720 let has_management_surface = manifest
8721 .provides
8722 .iter()
8723 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8724 if !has_management_surface {
8725 return false;
8726 }
8727 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8728 return false;
8729 };
8730 let Some(provides) = raw
8731 .get("manifest")
8732 .and_then(|manifest| manifest.get("provides"))
8733 .and_then(serde_json::Value::as_array)
8734 else {
8735 return false;
8736 };
8737 provides.iter().any(|role| {
8742 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8743 && role.get("concurrency").is_none()
8744 })
8745}
8746
8747fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8748 if peer_version != PROTOCOL_VERSION {
8749 return Err(format!(
8750 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8751 ));
8752 }
8753 Ok(PROTOCOL_VERSION)
8754}
8755
8756fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8757 Frame::build_with_version(
8758 response_version(frame),
8759 FrameType::Pong,
8760 frame.header.flags,
8761 0,
8762 0,
8763 frame.header.corr,
8764 Vec::new(),
8765 )
8766 .map_err(RouterError::FrameBuild)
8767}
8768
8769pub(crate) fn control_error_frame(
8770 frame: &Frame,
8771 code: &'static str,
8772 message: impl Into<String>,
8773) -> Result<Frame, RouterError> {
8774 control_error_body_frame(
8775 frame,
8776 ErrorBody {
8777 code: code.to_string(),
8778 message: message.into(),
8779 detail: None,
8780 },
8781 )
8782}
8783
8784fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8792 control_error_frame_with_flags(frame, error, daemon_error_flags())
8793}
8794
8795fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8801 control_error_frame_with_flags(frame, error, control_flags())
8802}
8803
8804fn control_error_frame_with_flags(
8805 frame: &Frame,
8806 error: ErrorBody,
8807 flags: Flags,
8808) -> Result<Frame, RouterError> {
8809 let body = serde_json::to_vec(&error).map_err(|err| {
8810 RouterError::backend(
8811 0,
8812 frame.header.corr,
8813 format!("failed to encode control ERROR: {err}"),
8814 )
8815 })?;
8816
8817 Frame::build_with_version(
8818 response_version(frame),
8819 FrameType::Error,
8820 flags,
8821 0,
8822 0,
8823 frame.header.corr,
8824 body,
8825 )
8826 .map_err(RouterError::FrameBuild)
8827}
8828
8829pub(crate) fn control_response_body_frame<T: Serialize>(
8830 frame: &Frame,
8831 reply: &T,
8832 label: &'static str,
8833) -> Result<Frame, RouterError> {
8834 let body = serde_json::to_vec(reply).map_err(|err| {
8835 RouterError::backend(
8836 0,
8837 frame.header.corr,
8838 format!("failed to encode {label}: {err}"),
8839 )
8840 })?;
8841
8842 Frame::build_with_version(
8843 response_version(frame),
8844 FrameType::Response,
8845 control_flags(),
8846 0,
8847 0,
8848 frame.header.corr,
8849 body,
8850 )
8851 .map_err(RouterError::FrameBuild)
8852}
8853
8854fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8884 match err {
8885 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8886 ForwardingError::NoModuleConnection => "target_unavailable",
8887 ForwardingError::ModuleReloading { .. } => "module_reloading",
8888 ForwardingError::ClientRouteChannelExhausted { .. }
8889 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8890 ForwardingError::StaleModuleEndpoint
8891 | ForwardingError::UnknownReservation { .. }
8892 | ForwardingError::ConnectionClosing { .. }
8893 | ForwardingError::ClientEgressClosed { .. }
8894 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8895 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8898 ForwardingError::RelayCorrelationExhausted
8899 | ForwardingError::RouteOpenBuild(_)
8900 | ForwardingError::Poisoned => "forwarding_error",
8901 }
8902}
8903
8904fn response_version(frame: &Frame) -> u8 {
8905 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8906 frame.header.ver
8907 } else {
8908 PROTOCOL_VERSION
8909 }
8910}
8911
8912fn control_flags() -> Flags {
8913 Flags::new(false, Priority::Passive, false)
8914}
8915
8916fn send_goodbye_target_best_effort(
8921 counters: &DaemonCounters,
8922 target: &GoodbyeTarget,
8923 context: &'static str,
8924) {
8925 let Ok(frame) = Frame::build_with_version(
8926 target.negotiated_ver,
8927 FrameType::Goodbye,
8928 control_flags(),
8929 target.channel,
8930 target.epoch,
8931 0,
8932 Vec::new(),
8933 ) else {
8934 return;
8935 };
8936 crate::forwarding::send_module_route_goodbye(
8937 counters,
8938 &target.sink,
8939 frame,
8940 target.module_id.as_deref(),
8941 context,
8942 );
8943}
8944
8945pub(crate) fn send_route_control_pushes(
8946 forwarding: &ForwardingTable,
8947 routes: Vec<EndpointRoute>,
8948 push: ClientControlPush,
8949) {
8950 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8951 for route in routes {
8952 let target = route.goodbye_target;
8953 if let Some((existing, channels)) = targets
8954 .iter_mut()
8955 .find(|(existing, _)| existing.connection_id == target.connection_id)
8956 {
8957 debug_assert_eq!(
8958 existing.negotiated_ver, target.negotiated_ver,
8959 "one connection cannot negotiate multiple frame versions"
8960 );
8961 if !channels.contains(&target.channel) {
8962 channels.push(target.channel);
8963 }
8964 continue;
8965 }
8966 let channel = target.channel;
8967 targets.push((target, vec![channel]));
8968 }
8969 for (target, mut channels) in targets {
8970 channels.sort_unstable();
8971 let mut push = push.clone();
8972 match &mut push {
8973 ClientControlPush::RouteClosing {
8974 channels: covered, ..
8975 }
8976 | ClientControlPush::RouteClosed {
8977 channels: covered, ..
8978 } => *covered = channels,
8979 }
8980 let body = match serde_json::to_vec(&push) {
8981 Ok(body) => body,
8982 Err(err) => {
8983 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8984 continue;
8985 }
8986 };
8987 let frame = match Frame::build_with_version(
8988 target.negotiated_ver,
8989 FrameType::Push,
8990 control_flags(),
8991 0,
8992 0,
8993 0,
8994 body.clone(),
8995 ) {
8996 Ok(frame) => frame,
8997 Err(err) => {
8998 warn!(
8999 route_channel = target.channel,
9000 error = %err,
9001 "failed to build route lifecycle control PUSH frame"
9002 );
9003 continue;
9004 }
9005 };
9006 if let Err(err) = target.sink.try_send(frame) {
9007 if target.close_on_delivery_failure() {
9008 warn!(
9009 target_connection_id = target.connection_id.get(),
9010 route_channel = target.channel,
9011 error = %err,
9012 "route lifecycle control PUSH was not delivered to client; closing target connection"
9013 );
9014 let _ = forwarding.escalate_client_delivery_failure(
9015 target.connection_id,
9016 target.channel,
9017 target.epoch,
9018 CloseReason::new(
9019 "route_lifecycle_push_delivery_failed",
9020 format!(
9021 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
9022 target.channel
9023 ),
9024 ),
9025 crate::forwarding::UndeliveredFrame {
9026 module_id: target.module_id.as_deref(),
9027 sink: &target.sink,
9028 },
9029 );
9030 }
9031 }
9032 }
9033}
9034
9035#[cfg(test)]
9036mod tests {
9037 include!("control/agent_commands_tests.rs");
9038 include!("control/resource_revision_tests.rs");
9039 #[tokio::test]
9040 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
9041 let registry = Arc::new(Registry::default());
9042 let handler = ControlHandler::new(registry);
9043 assert_eq!(
9044 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
9045 None
9046 );
9047 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
9048 hello_via_sink(
9049 &handler,
9050 &ctx,
9051 &mut rx,
9052 hello_frame("daemon", PROTOCOL_VERSION, 1),
9053 )
9054 .await;
9055 assert_eq!(
9056 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
9057 Some("daemon".into())
9058 );
9059 assert_eq!(
9060 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
9061 None
9062 );
9063 assert_eq!(
9064 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
9065 None,
9066 "a registered module is not necessarily supervised"
9067 );
9068 assert_eq!(
9069 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
9070 None,
9071 "aggregate provenance has no single target"
9072 );
9073 assert_eq!(
9074 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
9075 None
9076 );
9077 }
9078
9079 mod peer_manifest {
9080 include!("peer_manifest_tests.rs");
9081 }
9082 mod activity_read {
9083 include!("activity_read_tests.rs");
9084 }
9085
9086 mod lifecycle_activity {
9087 include!("control_lifecycle_tests.rs");
9088 }
9089 #[cfg(unix)]
9090 #[tokio::test]
9091 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
9092 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
9093 let path = dir.join("subc.jsonc");
9094 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
9095 "program":"/bin/sleep","args":["60"],"protocol":"none",
9096 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
9097 }}}).to_string()).unwrap();
9098 let mut configured = crate::daemon_config::load(&path)
9099 .unwrap()
9100 .unwrap()
9101 .modules
9102 .pop()
9103 .unwrap();
9104 let registry = std::sync::Arc::new(crate::Registry::default());
9105 let handle = crate::SupervisorHandle::new();
9106 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
9107 .with_handle(handle.clone());
9108 let module = supervisor
9109 .supervise_configured_with_health(
9110 configured.module_spec(),
9111 true,
9112 configured.health.clone(),
9113 None,
9114 configured.restart,
9115 )
9116 .unwrap();
9117 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
9118 let before = module.status().unwrap().pid;
9119 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
9120 configured.health.cadence = std::time::Duration::from_secs(3600);
9121 let health_only = handler
9122 .reconcile_supervised_modules(
9123 &supervisor,
9124 vec![configured.clone()],
9125 false,
9126 Principal::Direct,
9127 )
9128 .await
9129 .unwrap();
9130 assert!(
9131 health_only.0.changed_pending_reload.is_empty(),
9132 "health policy is already applied live"
9133 );
9134 assert_eq!(module.status().unwrap().pid, before);
9135 assert_eq!(
9136 module.configuration().unwrap().1.http,
9137 configured.health.http
9138 );
9139 configured.args = vec!["61".into()];
9140 let launch = handler
9141 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
9142 .await
9143 .unwrap();
9144 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
9145 assert_eq!(
9146 module.status().unwrap().pid,
9147 before,
9148 "a launch edit is stored until reload"
9149 );
9150 module.drain().await.unwrap();
9151 }
9152 use cortexkit_test_support::ScratchDir;
9153 use std::{
9154 collections::BTreeMap,
9155 fmt,
9156 path::PathBuf,
9157 sync::{Arc, Mutex},
9158 time::Duration,
9159 };
9160
9161 use serde_json::{json, Value};
9162 use subc_protocol::{
9163 manifest::{
9164 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
9165 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
9166 },
9167 session::HealthStatus,
9168 FrameType,
9169 };
9170
9171 use super::*;
9172 use crate::{
9173 forwarding::{DataRoute, DataRouteState},
9174 registry::ChannelState,
9175 router::FrameSink,
9176 stderr_tail::DEFAULT_MAX_LINE_BYTES,
9177 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
9178 RouteCtx, Router,
9179 };
9180 use tokio::{
9181 sync::mpsc,
9182 time::{sleep, Instant},
9183 };
9184 use tracing::{
9185 field::{Field, Visit},
9186 Event, Subscriber,
9187 };
9188 use tracing_subscriber::{layer::Context, prelude::*, Layer};
9189
9190 fn fake_aft_stub_path() -> PathBuf {
9209 let mut path = std::env::current_exe().expect("current_exe available in tests");
9210 path.pop(); path.pop(); path.push(if cfg!(windows) {
9213 "fake-aft-stub.exe"
9214 } else {
9215 "fake-aft-stub"
9216 });
9217 assert!(
9218 path.exists(),
9219 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
9220 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
9221 path.display()
9222 );
9223 path
9224 }
9225
9226 fn client_retries(code: &str) -> bool {
9231 subc_protocol::error_codes::is_retryable_route_open(code)
9232 }
9233
9234 #[test]
9245 fn retryability_of_forwarding_codes_matches_the_failure() {
9246 let transient = [
9249 ForwardingError::NoModuleConnection,
9250 ForwardingError::ModuleReloading {
9251 module_id: "m".into(),
9252 },
9253 ForwardingError::StaleModuleEndpoint,
9254 ForwardingError::UnknownReservation {
9255 client_channel: 1,
9256 module_channel: 1,
9257 },
9258 ForwardingError::ConnectionClosing {
9259 connection_id: ConnectionId::new(1),
9260 },
9261 ForwardingError::ClientEgressClosed {
9262 connection_id: ConnectionId::new(1),
9263 },
9264 ForwardingError::ModuleEgressUnavailable {
9265 connection_id: ConnectionId::new(1),
9266 },
9267 ];
9268 for err in transient {
9269 let code = forwarding_error_code(&err);
9270 assert!(
9271 client_retries(code),
9272 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
9273 );
9274 }
9275
9276 let permanent = [
9281 ForwardingError::ConnectionRoleConflict {
9282 connection_id: ConnectionId::new(1),
9283 },
9284 ForwardingError::ClientRouteChannelExhausted {
9285 connection_id: ConnectionId::new(1),
9286 },
9287 ForwardingError::ModuleRouteChannelExhausted {
9288 endpoint: ModuleEndpointId {
9289 connection_id: ConnectionId::new(1),
9290 generation: 1,
9291 },
9292 },
9293 ForwardingError::RelayCorrelationExhausted,
9294 ForwardingError::RouteOpenBuild("x".into()),
9295 ForwardingError::Poisoned,
9296 ];
9297 for err in permanent {
9298 let code = forwarding_error_code(&err);
9299 assert!(
9300 !client_retries(code),
9301 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
9302 );
9303 }
9304 }
9305
9306 #[tokio::test]
9334 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
9335 let handler = ControlHandler::default();
9336 let frame =
9337 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
9338
9339 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
9343 assert_eq!(
9344 stamped,
9345 Principal::Direct,
9346 "a caller that proved nothing must not be stamped as a supervised module"
9347 );
9348
9349 let forged = handler
9354 .route_open_principal(
9355 &frame,
9356 Some(ConsumerIdentity {
9357 module_id: "aft".to_string(),
9358 launch_nonce: "not-a-real-nonce".to_string(),
9359 }),
9360 )
9361 .unwrap();
9362 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9363 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9364 }
9365
9366 #[test]
9383 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9384 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9385 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9386 let ClientControlRequest::RouteOpen {
9387 consumer_identity, ..
9388 } = parsed
9389 else {
9390 panic!("route.open body must parse as RouteOpen");
9391 };
9392 assert_eq!(
9393 consumer_identity,
9394 Some(ConsumerIdentity {
9395 module_id: "aft".to_string(),
9396 launch_nonce: "n".to_string(),
9397 }),
9398 "the wire field name must reach the value route_open_principal reads"
9399 );
9400 }
9401
9402 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9403 ModuleManifest::builder(module_id, "0.1.0")
9404 .protocol_ver(protocol_ver)
9405 .provides(vec![ProviderRole::ToolProvider {
9406 tools: vec![Tool {
9407 name: "read".to_string(),
9408 description: None,
9409 execution_mode: ExecutionMode::Pure,
9410 schema: json!({"type": "object"}),
9411 }],
9412 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9413 concurrency: Concurrency::ModuleManaged,
9414 emits_push: true,
9415 sub_supervises: true,
9416 }])
9417 .build()
9418 }
9419
9420 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9421 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9422 }
9423
9424 fn hello_frame_with_control_ops(
9425 module_id: &str,
9426 protocol_ver: u8,
9427 corr: u64,
9428 control_ops: Option<Vec<String>>,
9429 ) -> Frame {
9430 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9431 }
9432
9433 fn hello_frame_with_nonce(
9434 module_id: &str,
9435 protocol_ver: u8,
9436 corr: u64,
9437 launch_nonce: Option<&str>,
9438 ) -> Frame {
9439 hello_frame_full(
9440 module_id,
9441 protocol_ver,
9442 corr,
9443 None,
9444 launch_nonce.map(ToOwned::to_owned),
9445 )
9446 }
9447
9448 fn hello_frame_full(
9449 module_id: &str,
9450 protocol_ver: u8,
9451 corr: u64,
9452 control_ops: Option<Vec<String>>,
9453 launch_nonce: Option<String>,
9454 ) -> Frame {
9455 let body = serde_json::to_vec(&ModuleHelloBody {
9456 manifest: manifest(module_id, protocol_ver),
9457 protocol_ver,
9458 control_ops,
9459 launch_nonce,
9460 })
9461 .unwrap();
9462 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9463 }
9464
9465 fn hello_frame_with_rich_ui(
9466 module_id: &str,
9467 protocol_ver: u8,
9468 corr: u64,
9469 launch_nonce: Option<&str>,
9470 rich_ui: Value,
9471 ) -> Frame {
9472 let mut body = serde_json::to_value(ModuleHelloBody {
9473 manifest: manifest(module_id, protocol_ver),
9474 protocol_ver,
9475 control_ops: None,
9476 launch_nonce: launch_nonce.map(str::to_string),
9477 })
9478 .unwrap();
9479 body["manifest"]["rich_ui"] = rich_ui;
9480 Frame::build(
9481 FrameType::Hello,
9482 control_flags(),
9483 0,
9484 0,
9485 corr,
9486 serde_json::to_vec(&body).unwrap(),
9487 )
9488 .unwrap()
9489 }
9490
9491 fn non_routable_hello_frame_with_control_ops(
9492 module_id: &str,
9493 corr: u64,
9494 control_ops: Option<Vec<String>>,
9495 ) -> Frame {
9496 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9497 manifest.provides.clear();
9498 let body = serde_json::to_vec(&ModuleHelloBody {
9499 manifest,
9500 protocol_ver: PROTOCOL_VERSION,
9501 control_ops,
9502 launch_nonce: None,
9503 })
9504 .unwrap();
9505 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9506 }
9507
9508 fn capability_grammar_hello_frame(
9509 capabilities: Value,
9510 runtime_computed: Option<Value>,
9511 corr: u64,
9512 ) -> Frame {
9513 let mut body = serde_json::to_value(ModuleHelloBody {
9514 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9515 protocol_ver: PROTOCOL_VERSION,
9516 control_ops: None,
9517 launch_nonce: None,
9518 })
9519 .expect("HELLO body serializes");
9520 body["manifest"]["capabilities"] = capabilities;
9521 if let Some(runtime_computed) = runtime_computed {
9522 body["runtime_computed"] = runtime_computed;
9523 }
9524 Frame::build(
9525 FrameType::Hello,
9526 control_flags(),
9527 0,
9528 0,
9529 corr,
9530 serde_json::to_vec(&body).expect("HELLO body reserializes"),
9531 )
9532 .expect("HELLO frame builds")
9533 }
9534
9535 fn channel_request(channel: u16, corr: u64) -> Frame {
9536 Frame::build(
9537 FrameType::Request,
9538 Flags::new(true, Priority::Interactive, false),
9539 channel,
9540 0,
9541 corr,
9542 b"opaque".to_vec(),
9543 )
9544 .unwrap()
9545 }
9546
9547 fn route_ctx(
9548 connection_id: ConnectionId,
9549 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9550 let (tx, rx) = mpsc::channel(8);
9551 (
9552 RouteCtx {
9553 connection_id,
9554 egress: FrameSink::new(tx),
9555 },
9556 rx,
9557 )
9558 }
9559
9560 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9561 serde_json::from_slice(&frame.body).unwrap()
9562 }
9563
9564 async fn hello_via_sink(
9569 handler: &ControlHandler,
9570 ctx: &RouteCtx,
9571 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9572 hello: Frame,
9573 ) -> Frame {
9574 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9575 assert!(
9576 replies.is_empty(),
9577 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9578 );
9579 let ack = rx
9580 .try_recv()
9581 .expect("HELLO_ACK is queued on the module sink")
9582 .frame;
9583 assert_eq!(ack.header.ty, FrameType::HelloAck);
9584 ack
9585 }
9586
9587 fn parse_error(frame: &Frame) -> Value {
9588 serde_json::from_slice(&frame.body).unwrap()
9589 }
9590
9591 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9592 serde_json::from_slice(&frame.body).unwrap()
9593 }
9594
9595 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9596 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9597 route_channel,
9598 route_epoch: 0,
9599 kind,
9600 })
9601 .unwrap();
9602 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9603 }
9604
9605 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9606 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9607 module_id: module_id.to_string(),
9608 })
9609 .unwrap();
9610 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9611 }
9612
9613 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9614 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9615 }
9616
9617 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9619 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9620 let mut frame = route_open_frame(corr, module_id, root);
9621 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9622 body["identity"]["project_root"] = serde_json::json!(path.as_path());
9623 frame.body = serde_json::to_vec(&body).unwrap();
9624 frame
9625 }
9626
9627 fn route_open_frame_with_consumer_capabilities(
9628 corr: u64,
9629 module_id: &str,
9630 _project_root: ScratchDir,
9631 consumer_capabilities: Option<Vec<String>>,
9632 ) -> Frame {
9633 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9634 busy_refusals: None,
9635 target: RouteTarget::ToolProvider {
9636 module_id: module_id.to_string(),
9637 },
9638 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9639 consumer_identity: None,
9640 consumer_capabilities,
9641 role_versions: None,
9642 admission_facts: None,
9643 scope: None,
9644 })
9645 .unwrap();
9646 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9647 }
9648
9649 fn route_open_frame_with_role_versions(
9650 corr: u64,
9651 module_id: &str,
9652 _project_root: ScratchDir,
9653 role_versions: Option<BTreeMap<String, String>>,
9654 ) -> Frame {
9655 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9656 busy_refusals: None,
9657 target: RouteTarget::ToolProvider {
9658 module_id: module_id.to_string(),
9659 },
9660 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9661 consumer_identity: None,
9662 consumer_capabilities: None,
9663 role_versions,
9664 admission_facts: None,
9665 scope: None,
9666 })
9667 .unwrap();
9668 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9669 }
9670
9671 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9672 entries
9673 .iter()
9674 .map(|(role, version)| (role.to_string(), version.to_string()))
9675 .collect()
9676 }
9677
9678 fn route_open_frame_with_admission_facts(
9679 corr: u64,
9680 module_id: &str,
9681 _project_root: ScratchDir,
9682 consumer_identity: Option<subc_control::ConsumerIdentity>,
9683 facts: Option<Value>,
9684 ) -> Frame {
9685 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9686 busy_refusals: None,
9687 target: RouteTarget::ToolProvider {
9688 module_id: module_id.to_string(),
9689 },
9690 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9691 consumer_identity,
9692 consumer_capabilities: None,
9693 role_versions: None,
9694 admission_facts: facts,
9695 scope: None,
9696 })
9697 .unwrap();
9698 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9699 }
9700
9701 #[derive(Clone, Default)]
9702 struct EventCapture {
9703 events: Arc<Mutex<Vec<CapturedEvent>>>,
9704 }
9705
9706 #[derive(Clone, Debug)]
9707 struct CapturedEvent {
9708 target: String,
9709 level: tracing::Level,
9710 fields: BTreeMap<String, String>,
9711 }
9712
9713 impl EventCapture {
9714 fn events(&self) -> Vec<CapturedEvent> {
9715 self.events.lock().unwrap().clone()
9716 }
9717 }
9718
9719 impl<S> Layer<S> for EventCapture
9720 where
9721 S: Subscriber,
9722 {
9723 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9724 let mut visitor = EventFieldVisitor::default();
9725 event.record(&mut visitor);
9726 self.events.lock().unwrap().push(CapturedEvent {
9727 target: event.metadata().target().to_string(),
9728 level: *event.metadata().level(),
9729 fields: visitor.fields,
9730 });
9731 }
9732 }
9733
9734 #[derive(Default)]
9735 struct EventFieldVisitor {
9736 fields: BTreeMap<String, String>,
9737 }
9738
9739 impl Visit for EventFieldVisitor {
9740 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9741 self.fields
9742 .insert(field.name().to_string(), format!("{value:?}"));
9743 }
9744 }
9745
9746 fn health_response(corr: u64, status: HealthStatus) -> Frame {
9747 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9748 status,
9749 detail: Some("warming".to_string()),
9750 metrics: Some(json!({"queue_depth": 3})),
9751 })
9752 .unwrap();
9753 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9754 }
9755
9756 fn route_bind_ack(corr: u64) -> Frame {
9757 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9758 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9759 }
9760
9761 fn unique_project_root(label: &str) -> ScratchDir {
9762 ScratchDir::new(label)
9763 }
9764
9765 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9766 match parse_route_poll(frame) {
9767 ClientControlResponse::RoutePoll {
9768 status: None,
9769 live: Some(live),
9770 ..
9771 } => assert_eq!(live, expected_live),
9772 other => panic!("unexpected route.poll response: {other:?}"),
9773 }
9774 }
9775
9776 fn bind_liveness_route(
9777 registry: &Registry,
9778 forwarding: &ForwardingTable,
9779 module_id: &str,
9780 ) -> (RouteCtx, u16, u32) {
9781 let module_connection = ConnectionId::new(101);
9782 let client_connection = ConnectionId::new(202);
9783 let registration = registry
9784 .register_with_control_ops(
9785 manifest(module_id, PROTOCOL_VERSION),
9786 PROTOCOL_VERSION,
9787 module_connection,
9788 module_baseline_control_ops(),
9789 )
9790 .unwrap();
9791 let (module_tx, _module_rx) = mpsc::channel(8);
9792 let endpoint = forwarding
9793 .register_module_connection(
9794 module_connection,
9795 module_id.to_string(),
9796 PROTOCOL_VERSION,
9797 manifest_concurrency(®istration.manifest),
9798 FrameSink::new(module_tx),
9799 )
9800 .unwrap();
9801 let (client_ctx, _client_rx) = route_ctx(client_connection);
9802 let pending = forwarding
9803 .begin_route_bind_relay_for_test(
9804 client_connection,
9805 client_ctx.egress.clone(),
9806 1,
9807 module_id,
9808 )
9809 .unwrap();
9810 assert_eq!(pending.endpoint, endpoint);
9811 let route_channel = pending.client_channel;
9812 let route_epoch = pending.client_epoch;
9813 forwarding
9814 .complete_pending_relay(
9815 module_connection,
9816 pending.corr,
9817 RouteBindRelayOutcome::Accepted,
9818 )
9819 .unwrap();
9820 (client_ctx, route_channel, route_epoch)
9821 }
9822
9823 struct FakeProcessLiveness {
9824 live: Option<bool>,
9825 }
9826
9827 impl ModuleProcessLiveness for FakeProcessLiveness {
9828 fn process_live(&self, _module_id: &str) -> Option<bool> {
9829 self.live
9830 }
9831 }
9832
9833 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9834 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9835 {
9836 let registry = Arc::new(Registry::default());
9837 let supervisor_handle = SupervisorHandle::new();
9838 let supervisor = Supervisor::new_for_test(
9839 Arc::clone(®istry),
9840 RestartPolicy::new(1, Duration::from_millis(10)),
9841 )
9842 .with_handle(supervisor_handle.clone());
9843 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9844 let module = supervisor
9845 .spawn(ModuleSpec {
9846 carry_containment: false,
9847 module_id: "stderr-tail-wire".to_string(),
9848 program: fake_aft_stub_path(),
9849 args: Vec::new(),
9850 env: vec![
9851 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9852 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9853 ],
9854 reserved: false,
9855 reserved_prefixes: Vec::new(),
9856 protocol: ModuleProtocol::Subc,
9857 overlap: Default::default(),
9858 })
9859 .unwrap();
9860
9861 let deadline = Instant::now() + Duration::from_secs(5);
9862 loop {
9863 let tail = module.stderr_tail(None, None);
9864 if tail
9865 .entries
9866 .iter()
9867 .any(|entry| matches!(entry, TailEntry::ProcessStart))
9868 && tail.entries.iter().any(|entry| {
9869 matches!(
9870 entry,
9871 TailEntry::Line {
9872 truncated: true,
9873 ..
9874 }
9875 )
9876 })
9877 {
9878 break;
9879 }
9880 assert!(
9881 Instant::now() < deadline,
9882 "module did not produce a truncated line and restart boundary: {tail:?}"
9883 );
9884 sleep(Duration::from_millis(10)).await;
9885 }
9886
9887 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9888 let request = ClientControlRequest::SupervisorStderrTail {
9889 module_id: "stderr-tail-wire".to_string(),
9890 max_lines: None,
9891 max_bytes: None,
9892 };
9893 let frame = Frame::build(
9894 FrameType::Request,
9895 control_flags(),
9896 0,
9897 0,
9898 1,
9899 serde_json::to_vec(&request).unwrap(),
9900 )
9901 .unwrap();
9902 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9903 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9904 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9905 serde_json::from_slice(&responses[0].body).unwrap()
9906 else {
9907 panic!("expected supervisor.stderr_tail response");
9908 };
9909
9910 assert!(
9911 tail.entries
9912 .iter()
9913 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9914 "the control response lost the restart boundary"
9915 );
9916 let Some(StderrTailEntry::Line {
9917 text,
9918 truncated,
9919 at_ms,
9920 }) = tail.entries.iter().find(|entry| {
9921 matches!(
9922 entry,
9923 StderrTailEntry::Line {
9924 truncated: true,
9925 ..
9926 }
9927 )
9928 })
9929 else {
9930 panic!("the control response lost the truncated line");
9931 };
9932 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9933 assert!(*truncated);
9934 assert!(
9935 at_ms.is_some(),
9936 "the control response lost the line's capture time"
9937 );
9938 }
9939
9940 #[tokio::test(flavor = "current_thread")]
9945 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9946 let dir = ScratchDir::new("terminals-off-worker");
9947 let journal_path = dir.join("terminals.jsonl");
9948 let registry = Arc::new(Registry::default());
9949 let supervisor_handle = SupervisorHandle::new();
9950 let supervisor =
9951 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9952 .with_handle(supervisor_handle.clone())
9953 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9954 let module = supervisor
9955 .spawn(ModuleSpec {
9956 carry_containment: false,
9957 module_id: "terminal-off-worker".to_string(),
9958 program: fake_aft_stub_path(),
9959 args: Vec::new(),
9960 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9961 reserved: false,
9962 reserved_prefixes: Vec::new(),
9963 protocol: ModuleProtocol::Subc,
9964 overlap: Default::default(),
9965 })
9966 .unwrap();
9967 let deadline = Instant::now() + Duration::from_secs(5);
9968 while module.terminal_history().entries.len() != 2 {
9969 assert!(Instant::now() < deadline, "module did not record two exits");
9970 sleep(Duration::from_millis(10)).await;
9971 }
9972
9973 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9974 let handler =
9975 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
9976 let frame = Frame::build(
9977 FrameType::Request,
9978 control_flags(),
9979 0,
9980 0,
9981 1,
9982 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9983 module_id: "terminal-off-worker".to_string(),
9984 })
9985 .unwrap(),
9986 )
9987 .unwrap();
9988 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9989 let spawned_at = std::time::Instant::now();
9990 let read = tokio::spawn({
9991 let handler = Arc::clone(&handler);
9992 async move { handler.handle_control_frame(&ctx, frame).await }
9993 });
9994 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9997 .await
9998 .unwrap()
9999 .expect("the history read reached its pause");
10000 let elapsed = spawned_at.elapsed();
10001 assert!(
10002 elapsed < Duration::from_secs(2) && !read.is_finished(),
10003 "this task could not run while the history read was paused \
10004 (resumed after {elapsed:?}, read finished: {})",
10005 read.is_finished()
10006 );
10007
10008 drop(release);
10009 let responses = read.await.unwrap().unwrap();
10010 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
10011 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
10012 panic!("expected supervisor.terminals response");
10013 };
10014 assert_eq!(terminals.entries.len(), 2);
10015 assert_eq!(terminals.journal_skipped_lines, 0);
10016 assert_eq!(terminals.journal_read_errors, 0);
10017 }
10018
10019 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10020 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
10021 let registry = Arc::new(Registry::default());
10022 let supervisor_handle = SupervisorHandle::new();
10023 let supervisor =
10024 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
10025 .with_handle(supervisor_handle.clone());
10026 let module = supervisor
10027 .spawn(ModuleSpec {
10028 carry_containment: false,
10029 module_id: "terminal-golden".to_string(),
10030 program: fake_aft_stub_path(),
10031 args: Vec::new(),
10032 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
10033 reserved: false,
10034 reserved_prefixes: Vec::new(),
10035 protocol: ModuleProtocol::Subc,
10036 overlap: Default::default(),
10037 })
10038 .unwrap();
10039
10040 let deadline = Instant::now() + Duration::from_secs(5);
10041 while module.terminal_history().entries.len() != 2 {
10042 assert!(
10043 Instant::now() < deadline,
10044 "module did not retain two terminal exits: {:?}",
10045 module.terminal_history()
10046 );
10047 sleep(Duration::from_millis(10)).await;
10048 }
10049
10050 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10051 let request = ClientControlRequest::SupervisorTerminals {
10052 module_id: "terminal-golden".to_string(),
10053 };
10054 let frame = Frame::build(
10055 FrameType::Request,
10056 control_flags(),
10057 0,
10058 0,
10059 1,
10060 serde_json::to_vec(&request).unwrap(),
10061 )
10062 .unwrap();
10063 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
10064 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
10065 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
10066 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
10067 panic!("expected supervisor.terminals response");
10068 };
10069 assert_eq!(terminals.entries.len(), 2);
10070 assert_eq!(terminals.dropped, 0);
10071
10072 let mut rendered = serde_json::to_value(response).unwrap();
10073 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
10076 for (index, entry) in rendered["entries"]
10077 .as_array_mut()
10078 .expect("terminal response entries array")
10079 .iter_mut()
10080 .enumerate()
10081 {
10082 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
10083 }
10084
10085 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10086 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
10087 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
10088 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10089 std::fs::write(&golden_path, &serialized).unwrap();
10090 }
10091 let expected: Value =
10092 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
10093 assert_eq!(rendered, expected);
10094 }
10095
10096 #[test]
10097 fn hello_registers_manifest_and_returns_ack() {
10098 let registry = Arc::new(Registry::default());
10099 let handler = ControlHandler::new(Arc::clone(®istry));
10100 let conn = ConnectionId::new(1);
10101
10102 let responses = handler
10103 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
10104 .unwrap();
10105
10106 assert_eq!(responses.len(), 1);
10107 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10108 assert_eq!(responses[0].header.channel, 0);
10109 assert_eq!(responses[0].header.corr, 7);
10110 let ack = parse_ack(&responses[0]);
10111 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10112 assert!(ack
10113 .subc_capabilities
10114 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
10115 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
10116 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
10117 assert!(ack
10118 .subc_ops
10119 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
10120 assert!(ack
10121 .subc_ops
10122 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
10123 assert!(ack
10124 .subc_ops
10125 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS.to_string()));
10126
10127 let registration = registry.get_module("aft").unwrap().unwrap();
10128 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
10129 assert_eq!(registration.state, ChannelState::Active);
10130 assert_eq!(registration.connection_id, conn);
10131 assert_eq!(registration.control_ops, module_baseline_control_ops());
10132 assert!(registration.manifest.display_name.is_none());
10133 }
10134
10135 #[test]
10136 fn hello_display_name_refuses_invalid_manifest_by_field() {
10137 let registry = Arc::new(Registry::default());
10138 let handler = ControlHandler::new(Arc::clone(®istry));
10139 for (index, invalid) in [
10140 json!(""),
10141 json!("a".repeat(41)),
10142 json!(" Routing"),
10143 json!("a\nb"),
10144 json!("a\u{202e}b"),
10145 json!(7),
10146 ]
10147 .into_iter()
10148 .enumerate()
10149 {
10150 let module_id = format!("bad-display-name-{index}");
10151 let mut body = serde_json::to_value(ModuleHelloBody {
10152 manifest: manifest(&module_id, PROTOCOL_VERSION),
10153 protocol_ver: PROTOCOL_VERSION,
10154 control_ops: None,
10155 launch_nonce: None,
10156 })
10157 .unwrap();
10158 body["manifest"]["display_name"] = invalid;
10159 let hello = Frame::build(
10160 FrameType::Hello,
10161 control_flags(),
10162 0,
10163 0,
10164 7,
10165 serde_json::to_vec(&body).unwrap(),
10166 )
10167 .unwrap();
10168 let response = handler
10169 .handle_control(ConnectionId::new(100 + index as u64), hello)
10170 .unwrap();
10171 assert_eq!(response[0].header.ty, FrameType::Error);
10172 let error = parse_error(&response[0]);
10173 assert_eq!(error["code"], "invalid_manifest");
10174 assert!(error["message"].as_str().unwrap().contains("display_name"));
10175 assert!(registry.get_module(&module_id).unwrap().is_none());
10176 }
10177 }
10178
10179 #[test]
10180 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
10181 let invalid_identifiers = [
10182 ("case_change", "credentials-Provider/v1"),
10183 ("leading_zero", "credentials-provider/v01"),
10184 ("trailing_hyphen", "credentials-provider-/v1"),
10185 ("consecutive_hyphens", "credentials--provider/v1"),
10186 ("uppercase", "Credentials-provider/v1"),
10187 ("missing_v", "credentials-provider/1"),
10188 ("whitespace", "credentials provider/v1"),
10189 ("zero_version", "credentials-provider/v0"),
10190 ("out_of_range_version", "credentials-provider/v4294967296"),
10191 (
10192 "overlength_name",
10193 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
10194 ),
10195 ];
10196 let mut cases = invalid_identifiers
10197 .into_iter()
10198 .map(|(name, identifier)| {
10199 (
10200 format!("identifier_{name}"),
10201 "capabilities.provides[0]".to_string(),
10202 identifier.to_string(),
10203 json!({ "provides": [identifier] }),
10204 None,
10205 )
10206 })
10207 .collect::<Vec<_>>();
10208 cases.extend([
10209 (
10210 "unknown_need".to_string(),
10211 "capabilities.requires[0].need".to_string(),
10212 "deferred".to_string(),
10213 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
10214 None,
10215 ),
10216 (
10217 "duplicate_provides".to_string(),
10218 "capabilities.provides[1]".to_string(),
10219 "credentials-provider/v1".to_string(),
10220 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
10221 None,
10222 ),
10223 (
10224 "duplicate_must_never_reach".to_string(),
10225 "capabilities.must_never_reach[1]".to_string(),
10226 "credentials-provider/v1".to_string(),
10227 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
10228 None,
10229 ),
10230 (
10231 "duplicate_requires_same_need".to_string(),
10232 "capabilities.requires[1]".to_string(),
10233 "credentials-provider/v1".to_string(),
10234 json!({ "requires": [
10235 { "capability": "credentials-provider/v1", "need": "required" },
10236 { "capability": "credentials-provider/v1", "need": "required" }
10237 ] }),
10238 None,
10239 ),
10240 (
10241 "duplicate_requires_conflicting_need".to_string(),
10242 "capabilities.requires[1]".to_string(),
10243 "credentials-provider/v1".to_string(),
10244 json!({ "requires": [
10245 { "capability": "credentials-provider/v1", "need": "required" },
10246 { "capability": "credentials-provider/v1", "need": "optional" }
10247 ] }),
10248 None,
10249 ),
10250 (
10251 "capabilities_root_pointer".to_string(),
10252 "runtime_computed[0]".to_string(),
10253 "/capabilities".to_string(),
10254 json!({}),
10255 Some(json!(["/capabilities"])),
10256 ),
10257 (
10258 "capabilities_descendant_pointer".to_string(),
10259 "runtime_computed[0]".to_string(),
10260 "/capabilities/provides".to_string(),
10261 json!({}),
10262 Some(json!(["/capabilities/provides"])),
10263 ),
10264 (
10265 "malformed_pointer_without_leading_slash".to_string(),
10266 "runtime_computed[0]".to_string(),
10267 "capabilities".to_string(),
10268 json!({}),
10269 Some(json!(["capabilities"])),
10270 ),
10271 (
10272 "malformed_pointer_escape".to_string(),
10273 "runtime_computed[0]".to_string(),
10274 "/roles/~2/tools".to_string(),
10275 json!({}),
10276 Some(json!(["/roles/~2/tools"])),
10277 ),
10278 (
10279 "unknown_capabilities_field".to_string(),
10280 "capabilities.future".to_string(),
10281 "<array>".to_string(),
10282 json!({ "future": [] }),
10283 None,
10284 ),
10285 ]);
10286
10287 for (index, (name, field, value, capabilities, runtime_computed)) in
10288 cases.into_iter().enumerate()
10289 {
10290 let registry = Arc::new(Registry::default());
10291 let handler = ControlHandler::new(Arc::clone(®istry));
10292 let response = handler
10293 .handle_control(
10294 ConnectionId::new((index + 1) as u64),
10295 capability_grammar_hello_frame(
10296 capabilities,
10297 runtime_computed,
10298 index as u64 + 1,
10299 ),
10300 )
10301 .expect("invalid HELLO returns a refusal");
10302
10303 assert_eq!(response.len(), 1, "{name} must emit one refusal");
10304 let error = parse_error(&response[0]);
10305 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
10306 let message = error["message"]
10307 .as_str()
10308 .expect("error message is a string");
10309 assert!(
10310 message.contains(&field),
10311 "{name}: field missing from {message}"
10312 );
10313 assert!(
10314 message.contains(&value),
10315 "{name}: value missing from {message}"
10316 );
10317 assert_eq!(
10318 registry
10319 .active_registration_count()
10320 .expect("registry reads"),
10321 0,
10322 "{name}: refused HELLO must not create a catalog entry"
10323 );
10324 }
10325 }
10326
10327 #[test]
10328 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
10329 let registry = Arc::new(Registry::default());
10330 let handler = ControlHandler::new(Arc::clone(®istry));
10331 let capabilities = json!({
10332 "provides": ["credentials-provider/v1"],
10333 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
10334 "must_never_reach": ["federation-transport/v1"]
10335 });
10336 let response = handler
10337 .handle_control(
10338 ConnectionId::new(99),
10339 capability_grammar_hello_frame(
10340 capabilities.clone(),
10341 Some(json!(["/roles/0/tools"])),
10342 99,
10343 ),
10344 )
10345 .expect("valid HELLO registers");
10346 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10347
10348 let request = Frame::build(
10349 FrameType::Request,
10350 control_flags(),
10351 0,
10352 0,
10353 100,
10354 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10355 .expect("catalog request serializes"),
10356 )
10357 .expect("catalog request frame builds");
10358 let response = handler
10359 .handle_catalog_list(request, None)
10360 .expect("catalog list succeeds");
10361 let ClientControlResponse::CatalogList { modules, .. } =
10362 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
10363 else {
10364 panic!("catalog request must return catalog.list");
10365 };
10366 assert_eq!(modules.len(), 1);
10367 assert_eq!(
10368 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
10369 capabilities
10370 );
10371 }
10372
10373 #[test]
10374 fn catalog_list_mirrors_management_operation_description() {
10375 let registry = Arc::new(Registry::default());
10376 let handler = ControlHandler::new(Arc::clone(®istry));
10377 let description = "List managed records and return their identifiers and metadata.";
10378 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
10379 manifest.provides = vec![ProviderRole::ManagementSurface {
10380 operations: vec![ManagementOperation {
10381 name: "records.list".to_string(),
10382 kind: ManagementOperationKind::Query,
10383 description: Some(description.to_string()),
10384 }],
10385 config_schema: json!({"type": "object"}),
10386 observability: vec![ObservabilitySurface {
10387 name: "records.stats".to_string(),
10388 kind: ObservabilityKind::Snapshot,
10389 }],
10390 identity_scope: vec![IdentityScope::Project],
10391 concurrency: Concurrency::ModuleManaged,
10392 }];
10393 registry
10394 .register_with_control_ops(
10395 manifest,
10396 PROTOCOL_VERSION,
10397 ConnectionId::new(99),
10398 Vec::new(),
10399 )
10400 .expect("described management manifest registers");
10401
10402 let request = Frame::build(
10403 FrameType::Request,
10404 control_flags(),
10405 0,
10406 0,
10407 100,
10408 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10409 .expect("catalog request serializes"),
10410 )
10411 .expect("catalog request frame builds");
10412 let response = handler
10413 .handle_catalog_list(request, None)
10414 .expect("catalog list succeeds");
10415 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10416 assert_eq!(
10417 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10418 "catalog.list must preserve the declared operation description verbatim"
10419 );
10420 }
10421
10422 #[test]
10423 fn catalog_list_mirrors_registered_operation_delivery() {
10424 let registry = Arc::new(Registry::default());
10425 let handler = ControlHandler::new(Arc::clone(®istry));
10426 let delivery =
10427 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10428 let mut body = serde_json::to_value(ModuleHelloBody {
10429 manifest: manifest("watch-management", PROTOCOL_VERSION),
10430 protocol_ver: PROTOCOL_VERSION,
10431 control_ops: None,
10432 launch_nonce: None,
10433 })
10434 .unwrap();
10435 body["manifest"]["provides"] = json!([{
10436 "role": "management_surface",
10437 "operations": [
10438 {"name": "status", "kind": "query"},
10439 {"name": "future", "kind": "query"},
10440 {"name": "reset", "kind": "mutate"}
10441 ],
10442 "config_schema": {}, "observability": [], "identity_scope": []
10443 }]);
10444 body["manifest"]["operation_delivery"] = delivery.clone();
10445 let hello = Frame::build(
10446 FrameType::Hello,
10447 control_flags(),
10448 0,
10449 0,
10450 99,
10451 serde_json::to_vec(&body).unwrap(),
10452 )
10453 .unwrap();
10454 let response = handler
10455 .handle_control(ConnectionId::new(99), hello)
10456 .unwrap();
10457 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10458
10459 let (_, registrations) = registry.list_modules().unwrap();
10460 let registered = ®istrations[0].manifest;
10461 assert!(registered.is_watch_operation("status"));
10462 assert!(!registered.is_watch_operation("future"));
10463 assert!(!registered.is_watch_operation("reset"));
10464 assert!(!registered.is_watch_operation("missing"));
10465
10466 let request = Frame::build(
10467 FrameType::Request,
10468 control_flags(),
10469 0,
10470 0,
10471 100,
10472 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10473 )
10474 .unwrap();
10475 let response = handler.handle_catalog_list(request, None).unwrap();
10476 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10477 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10478 let ClientControlResponse::CatalogList { modules, .. } =
10479 serde_json::from_slice(&response[0].body).unwrap()
10480 else {
10481 panic!("expected catalog.list");
10482 };
10483 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10484 }
10485
10486 #[test]
10487 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10488 let registry = Arc::new(Registry::default());
10489 let supervisor = SupervisorHandle::new();
10490 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10491 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10492 let descriptor = json!({
10493 "schema": "ckios.richui.manifest/1",
10494 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10495 "bytes": 4096,
10496 "chunks": 4
10497 });
10498 let connection = ConnectionId::new(201);
10499 let hello = handler
10500 .handle_control(
10501 connection,
10502 hello_frame_with_rich_ui(
10503 "vault",
10504 PROTOCOL_VERSION,
10505 201,
10506 Some("the-real-nonce"),
10507 descriptor.clone(),
10508 ),
10509 )
10510 .unwrap();
10511 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10512
10513 let request =
10514 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10515 let provides = registry
10516 .get_module_by_connection(connection)
10517 .unwrap()
10518 .unwrap()
10519 .manifest
10520 .provides;
10521 handler
10522 .handle_catalog_update(connection, request, provides, None, Some(false))
10523 .unwrap();
10524
10525 let request = Frame::build(
10526 FrameType::Request,
10527 control_flags(),
10528 0,
10529 0,
10530 203,
10531 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10532 )
10533 .unwrap();
10534 let response = handler.handle_catalog_list(request, None).unwrap();
10535 let ClientControlResponse::CatalogList { modules, .. } =
10536 serde_json::from_slice(&response[0].body).unwrap()
10537 else {
10538 panic!("expected catalog.list");
10539 };
10540 assert_eq!(
10541 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10542 descriptor
10543 );
10544 assert!(modules[0].nonce_verified);
10545 }
10546
10547 fn hello_with_role_declarations(declarations: Value) -> Frame {
10548 let mut body = serde_json::to_value(ModuleHelloBody {
10549 manifest: manifest("role-test", PROTOCOL_VERSION),
10550 protocol_ver: PROTOCOL_VERSION,
10551 control_ops: None,
10552 launch_nonce: None,
10553 })
10554 .unwrap();
10555 body["manifest"]["role_declarations"] = declarations;
10556 Frame::build(
10557 FrameType::Hello,
10558 control_flags(),
10559 0,
10560 0,
10561 301,
10562 serde_json::to_vec(&body).unwrap(),
10563 )
10564 .unwrap()
10565 }
10566
10567 fn update_role_declarations_frame(declarations: Value) -> Frame {
10568 Frame::build(
10569 FrameType::Request,
10570 control_flags(),
10571 0,
10572 0,
10573 303,
10574 serde_json::to_vec(&json!({
10575 "op": "catalog.update_role_declarations",
10576 "role_declarations": declarations,
10577 }))
10578 .unwrap(),
10579 )
10580 .unwrap()
10581 }
10582
10583 #[tokio::test]
10584 async fn role_declarations_update_replaces_then_clears_catalog() {
10585 let registry = Arc::new(Registry::default());
10586 let handler = ControlHandler::new(Arc::clone(®istry));
10587 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
10588 let initial = json!({"media-provider/v1": {"engines": ["old"]}, "old/v1": {}});
10589 hello_via_sink(
10590 &handler,
10591 &ctx,
10592 &mut rx,
10593 hello_with_role_declarations(initial.clone()),
10594 )
10595 .await;
10596 assert_eq!(
10597 role_declarations_catalog(&handler).0["role_declarations"],
10598 initial
10599 );
10600 let before = registry.get_module("role-test").unwrap().unwrap();
10601 for declarations in [
10602 json!({"media-provider/v1": {"engines": ["new"]}}),
10603 json!({}),
10604 ] {
10605 let response = handler
10606 .handle_control_frame(&ctx, update_role_declarations_frame(declarations.clone()))
10607 .await
10608 .unwrap();
10609 assert_eq!(response[0].header.ty, FrameType::Response);
10610 assert_eq!(
10611 serde_json::from_slice::<Value>(&response[0].body).unwrap(),
10612 json!({"op": "catalog.update_role_declarations"})
10613 );
10614 let (raw, entry) = role_declarations_catalog(&handler);
10615 assert_eq!(
10616 serde_json::to_value(entry.role_declarations).unwrap(),
10617 declarations
10618 );
10619 if declarations.as_object().unwrap().is_empty() {
10620 assert!(raw.get("role_declarations").is_none());
10621 } else {
10622 assert_eq!(raw["role_declarations"], declarations);
10623 }
10624 let after = registry.get_module("role-test").unwrap().unwrap();
10625 let mut expected = before.manifest.clone();
10626 expected.role_declarations = serde_json::from_value(declarations).unwrap();
10627 assert_eq!(after.manifest, expected);
10628 assert_eq!(after.ready, before.ready);
10629 }
10630 }
10631
10632 #[tokio::test]
10633 async fn role_declarations_update_refuses_invalid_maps_without_changing_catalog() {
10634 let registry = Arc::new(Registry::default());
10635 let handler = ControlHandler::new(registry);
10636 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
10637 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
10638 hello_via_sink(
10639 &handler,
10640 &ctx,
10641 &mut rx,
10642 hello_with_role_declarations(initial.clone()),
10643 )
10644 .await;
10645 for (invalid, reason) in [
10646 (json!(null), "role_declarations must be an object"),
10647 (json!([]), "role_declarations must be an object"),
10648 (
10649 json!({"bad key": {}}),
10650 "role_declarations key must use capability-name grammar",
10651 ),
10652 (
10653 json!({"media-provider/v1": []}),
10654 "role_declarations values must be objects",
10655 ),
10656 (
10657 json!({"media-provider/v1": {"padding": "x".repeat(16384)}}),
10658 "role_declarations exceeds 16384 compact JSON bytes",
10659 ),
10660 ] {
10661 let response = handler
10662 .handle_control_frame(&ctx, update_role_declarations_frame(invalid))
10663 .await
10664 .unwrap();
10665 let error = parse_error(&response[0]);
10666 assert_eq!(error["code"], "invalid_role_declarations");
10667 assert_eq!(error["message"], reason);
10668 assert_eq!(
10669 role_declarations_catalog(&handler).0["role_declarations"],
10670 initial
10671 );
10672 }
10673 }
10674
10675 #[tokio::test]
10676 async fn role_declarations_update_is_owned_by_the_registered_connection() {
10677 let registry = Arc::new(Registry::default());
10678 let handler = ControlHandler::new(Arc::clone(®istry));
10679 let (owner, mut owner_rx) = route_ctx(ConnectionId::new(301));
10680 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
10681 hello_via_sink(
10682 &handler,
10683 &owner,
10684 &mut owner_rx,
10685 hello_with_role_declarations(initial.clone()),
10686 )
10687 .await;
10688 let (other, mut other_rx) = route_ctx(ConnectionId::new(302));
10689 let replacement = json!({"media-provider/v1": {"engines": ["other"]}});
10690 let response = handler
10691 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
10692 .await
10693 .unwrap();
10694 assert_eq!(parse_error(&response[0])["code"], "not_registered");
10695 hello_via_sink(
10696 &handler,
10697 &other,
10698 &mut other_rx,
10699 hello_frame("other", PROTOCOL_VERSION, 304),
10700 )
10701 .await;
10702 let response = handler
10703 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
10704 .await
10705 .unwrap();
10706 assert_eq!(response[0].header.ty, FrameType::Response);
10707 assert_eq!(
10708 serde_json::to_value(
10709 registry
10710 .get_module("role-test")
10711 .unwrap()
10712 .unwrap()
10713 .manifest
10714 .role_declarations
10715 )
10716 .unwrap(),
10717 initial
10718 );
10719 assert_eq!(
10720 serde_json::to_value(
10721 registry
10722 .get_module("other")
10723 .unwrap()
10724 .unwrap()
10725 .manifest
10726 .role_declarations
10727 )
10728 .unwrap(),
10729 replacement
10730 );
10731 }
10732
10733 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10734 let request = Frame::build(
10735 FrameType::Request,
10736 control_flags(),
10737 0,
10738 0,
10739 302,
10740 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10741 )
10742 .unwrap();
10743 let response = handler.handle_catalog_list(request, None).unwrap();
10744 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10745 let ClientControlResponse::CatalogList { mut modules, .. } =
10746 serde_json::from_slice(&response[0].body).unwrap()
10747 else {
10748 panic!("expected catalog.list");
10749 };
10750 (raw["modules"][0].clone(), modules.remove(0))
10751 }
10752
10753 #[test]
10754 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10755 let registry = Arc::new(Registry::default());
10756 let handler = ControlHandler::new(Arc::clone(®istry));
10757 let declarations = json!({
10758 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10759 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10760 });
10761 let connection = ConnectionId::new(301);
10762 let response = handler
10763 .handle_control(
10764 connection,
10765 hello_with_role_declarations(declarations.clone()),
10766 )
10767 .unwrap();
10768 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10769 let registered = registry
10770 .get_module_by_connection(connection)
10771 .unwrap()
10772 .unwrap()
10773 .manifest;
10774 assert_eq!(
10778 serde_json::to_value(®istered.role_declarations).unwrap(),
10779 declarations
10780 );
10781 let (raw, entry) = role_declarations_catalog(&handler);
10782 assert_eq!(raw["role_declarations"], declarations);
10783 assert_eq!(entry.role_declarations, registered.role_declarations);
10784
10785 let request =
10786 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10787 handler
10788 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10789 .unwrap();
10790 let (raw, entry) = role_declarations_catalog(&handler);
10791 assert_eq!(raw["role_declarations"], declarations);
10792 assert_eq!(entry.role_declarations, registered.role_declarations);
10793 }
10794
10795 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10796 let registry = Arc::new(Registry::default());
10797 let handler = ControlHandler::new(Arc::clone(®istry));
10798 let capture = EventCapture::default();
10799 let _subscriber =
10800 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10801 let response = handler
10802 .handle_control(
10803 ConnectionId::new(301),
10804 hello_with_role_declarations(declarations),
10805 )
10806 .unwrap();
10807 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10808 assert!(registry
10809 .get_module_by_connection(ConnectionId::new(301))
10810 .unwrap()
10811 .unwrap()
10812 .manifest
10813 .role_declarations
10814 .is_empty());
10815 let (raw, entry) = role_declarations_catalog(&handler);
10816 assert!(raw.get("role_declarations").is_none());
10817 assert!(entry.role_declarations.is_empty());
10818 let warnings: Vec<_> = capture
10819 .events()
10820 .into_iter()
10821 .filter(|event| {
10822 event.fields.get("message").map(String::as_str)
10823 == Some("dropping whole role_declarations map from HELLO")
10824 })
10825 .collect();
10826 assert_eq!(warnings.len(), 1);
10827 assert_eq!(warnings[0].level, tracing::Level::WARN);
10828 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10829 assert_eq!(warnings[0].fields["reason"], reason);
10830 }
10831
10832 #[test]
10833 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10834 for value in [
10835 json!(null),
10836 json!(false),
10837 json!(0),
10838 json!("opaque"),
10839 json!([]),
10840 ] {
10841 assert_hello_drops_role_declarations_with_warning(
10842 json!({"routing/v1": {}, "broken/v1": value}),
10843 "role_declarations values must be objects",
10844 );
10845 }
10846 assert_hello_drops_role_declarations_with_warning(
10847 json!([]),
10848 "role_declarations must be an object",
10849 );
10850 }
10851
10852 #[test]
10853 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10854 assert_hello_drops_role_declarations_with_warning(
10855 json!({"routing/v1": {}, "bad key": {}}),
10856 "role_declarations key must use capability-name grammar",
10857 );
10858 }
10859
10860 #[test]
10861 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10862 assert_hello_drops_role_declarations_with_warning(
10863 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10864 "role_declarations exceeds 16384 compact JSON bytes",
10865 );
10866 }
10867
10868 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10869 let mut body = serde_json::to_value(ModuleHelloBody {
10870 manifest: manifest("flow-ops", PROTOCOL_VERSION),
10871 protocol_ver: PROTOCOL_VERSION,
10872 control_ops: None,
10873 launch_nonce: None,
10874 })
10875 .unwrap();
10876 body["manifest"]["provides"] = json!([{
10877 "role": "management_surface",
10878 "operations": [
10879 {"name": "diagnostics.findings", "kind": "query"},
10880 {"name": "reset", "kind": "mutate"}
10881 ],
10882 "config_schema": {}, "observability": [], "identity_scope": []
10883 }]);
10884 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10885 if let Some(flow_operations) = flow_operations {
10886 body["manifest"]["flow_operations"] = flow_operations;
10887 }
10888 Frame::build(
10889 FrameType::Hello,
10890 control_flags(),
10891 0,
10892 0,
10893 401,
10894 serde_json::to_vec(&body).unwrap(),
10895 )
10896 .unwrap()
10897 }
10898
10899 #[test]
10900 fn catalog_list_mirrors_flow_operations_unchanged() {
10901 for (capability, flow_operations) in [
10904 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10905 ("call-modes/v1", json!(["diagnostics.findings"])),
10906 ("flow-scopes/v1", json!([])),
10907 ] {
10908 let registry = Arc::new(Registry::default());
10909 let handler = ControlHandler::new(Arc::clone(®istry));
10910 let connection = ConnectionId::new(401);
10911 let response = handler
10912 .handle_control(
10913 connection,
10914 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10915 )
10916 .unwrap();
10917 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10918 let registered = registry
10919 .get_module_by_connection(connection)
10920 .unwrap()
10921 .unwrap()
10922 .manifest;
10923 assert_eq!(
10924 serde_json::to_value(®istered.flow_operations).unwrap(),
10925 flow_operations
10926 );
10927 let (raw, entry) = role_declarations_catalog(&handler);
10928 assert_eq!(raw["flow_operations"], flow_operations);
10929 assert_eq!(entry.flow_operations, registered.flow_operations);
10930 }
10931
10932 let registry = Arc::new(Registry::default());
10933 let handler = ControlHandler::new(Arc::clone(®istry));
10934 let response = handler
10935 .handle_control(
10936 ConnectionId::new(401),
10937 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10938 )
10939 .unwrap();
10940 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10941 let (raw, entry) = role_declarations_catalog(&handler);
10942 assert!(raw.get("flow_operations").is_none());
10943 assert_eq!(entry.flow_operations, None);
10944 }
10945
10946 #[test]
10947 fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
10948 let oversized: serde_json::Map<String, Value> = (0..257)
10949 .map(|n| (format!("tool:t{n}"), json!(true)))
10950 .collect();
10951 for (capabilities, map, valid) in [
10952 (
10953 json!(["scope-federated/v1"]),
10954 json!({"tool:read":true,"management:reset":true}),
10955 true,
10956 ),
10957 (json!(["scope-federated/v1"]), json!({}), true),
10958 (json!([]), json!({"tool:read":true}), false),
10959 (
10960 json!(["scope-federated/v1"]),
10961 json!({"tool:valid":true,"tool:bad":false}),
10962 false,
10963 ),
10964 (
10965 json!(["scope-federated/v1"]),
10966 json!({"tool:valid":true,"management:missing":true}),
10967 false,
10968 ),
10969 (
10970 json!(["scope-federated/v1"]),
10971 Value::Object(oversized),
10972 false,
10973 ),
10974 ] {
10975 let capture = EventCapture::default();
10976 let _subscriber = tracing::subscriber::set_default(
10977 tracing_subscriber::registry().with(capture.clone()),
10978 );
10979 let registry = Arc::new(Registry::default());
10980 let handler = ControlHandler::new(Arc::clone(®istry));
10981 let hello = hello_with_flow_operations(capabilities, None);
10982 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
10983 body["manifest"]["operation_deferral"] = map.clone();
10984 let hello = Frame::build(
10985 FrameType::Hello,
10986 control_flags(),
10987 0,
10988 0,
10989 401,
10990 serde_json::to_vec(&body).unwrap(),
10991 )
10992 .unwrap();
10993 let response = handler
10994 .handle_control(ConnectionId::new(401), hello)
10995 .unwrap();
10996 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10997 let (raw, entry) = role_declarations_catalog(&handler);
10998 if valid {
10999 assert_eq!(raw["operation_deferral"], map);
11000 assert_eq!(
11001 serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
11002 map
11003 );
11004 } else {
11005 assert!(entry.operation_deferral.is_none());
11006 assert!(raw.get("operation_deferral").is_none());
11007 assert_eq!(
11008 capture
11009 .events()
11010 .iter()
11011 .filter(|event| event.fields.get("message").map(String::as_str)
11012 == Some("dropping operation_deferral from HELLO"))
11013 .count(),
11014 1
11015 );
11016 }
11017 }
11018 }
11019
11020 fn assert_hello_drops_flow_operations_with_warning(
11021 capabilities: Value,
11022 flow_operations: Value,
11023 reason: &str,
11024 ) {
11025 let registry = Arc::new(Registry::default());
11026 let handler = ControlHandler::new(Arc::clone(®istry));
11027 let capture = EventCapture::default();
11028 let _subscriber =
11029 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11030 let response = handler
11031 .handle_control(
11032 ConnectionId::new(401),
11033 hello_with_flow_operations(capabilities, Some(flow_operations)),
11034 )
11035 .unwrap();
11036 assert_eq!(response[0].header.ty, FrameType::HelloAck);
11037 let registered = registry
11038 .get_module_by_connection(ConnectionId::new(401))
11039 .unwrap()
11040 .expect("the module still registers")
11041 .manifest;
11042 assert_eq!(registered.module_id, "flow-ops");
11043 assert_eq!(registered.flow_operations, None);
11044 let (raw, entry) = role_declarations_catalog(&handler);
11045 assert_eq!(raw["module_id"], "flow-ops");
11046 assert!(raw.get("flow_operations").is_none());
11047 assert_eq!(entry.flow_operations, None);
11048 let warnings: Vec<_> = capture
11049 .events()
11050 .into_iter()
11051 .filter(|event| {
11052 event.fields.get("message").map(String::as_str)
11053 == Some("dropping flow_operations from HELLO")
11054 })
11055 .collect();
11056 assert_eq!(warnings.len(), 1);
11057 assert_eq!(warnings[0].level, tracing::Level::WARN);
11058 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
11059 assert_eq!(warnings[0].fields["reason"], reason);
11060 }
11061
11062 #[test]
11063 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
11064 assert_hello_drops_flow_operations_with_warning(
11065 json!(["flow-scopes/v1"]),
11066 json!(["diagnostics.findings", "missing"]),
11067 "flow_operations names 'missing', which is not a declared management operation",
11068 );
11069 }
11070
11071 #[test]
11072 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
11073 assert_hello_drops_flow_operations_with_warning(
11074 json!([]),
11075 json!(["diagnostics.findings"]),
11076 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
11077 );
11078 assert_hello_drops_flow_operations_with_warning(
11080 json!(["agent-run-scopes/v1"]),
11081 json!([]),
11082 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
11083 );
11084 }
11085
11086 #[test]
11087 fn hello_malformed_flow_operations_is_dropped_with_warning() {
11088 assert_hello_drops_flow_operations_with_warning(
11089 json!(["flow-scopes/v1"]),
11090 json!(["reset", "reset"]),
11091 "flow_operations names 'reset' more than once",
11092 );
11093 assert_hello_drops_flow_operations_with_warning(
11094 json!(["flow-scopes/v1"]),
11095 json!(vec!["reset"; 257]),
11096 "flow_operations lists 257 names, more than 256",
11097 );
11098 assert_hello_drops_flow_operations_with_warning(
11099 json!(["flow-scopes/v1"]),
11100 json!("reset"),
11101 "flow_operations must be an array of operation names",
11102 );
11103 }
11104
11105 #[test]
11106 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
11107 let registry = Arc::new(Registry::default());
11108 let handler = ControlHandler::new(registry.clone());
11109 let descriptor = json!({
11110 "schema": "ckios.richui.manifest/1",
11111 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
11112 "bytes": 128,
11113 "chunks": 2
11114 });
11115 let malformed = [
11116 ("schema", json!("")),
11117 ("schema", json!("a".repeat(65))),
11118 ("schema", json!("Bad schema")),
11119 (
11120 "manifest_digest",
11121 json!(format!("sha256:{}", "A".repeat(64))),
11122 ),
11123 ("manifest_digest", json!("sha256:abc")),
11124 ("bytes", json!(0)),
11125 ("bytes", json!(16_777_217)),
11126 ("bytes", json!("128")),
11127 ("chunks", json!(0)),
11128 ("chunks", json!(1025)),
11129 ("chunks", json!("2")),
11130 ];
11131 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
11132 let mut invalid = descriptor.clone();
11133 invalid[field] = invalid_value;
11134 let response = handler
11135 .handle_control(
11136 ConnectionId::new(210 + index as u64),
11137 hello_frame_with_rich_ui(
11138 &format!("bad-rich-ui-{index}"),
11139 PROTOCOL_VERSION,
11140 210 + index as u64,
11141 None,
11142 invalid,
11143 ),
11144 )
11145 .unwrap();
11146 assert_eq!(response[0].header.ty, FrameType::Error);
11147 let error = parse_error(&response[0]);
11148 assert_eq!(error["code"], "invalid_manifest");
11149 assert!(error["message"]
11150 .as_str()
11151 .unwrap()
11152 .contains(&format!("rich_ui.{field}")));
11153 assert!(registry
11154 .get_module(&format!("bad-rich-ui-{index}"))
11155 .unwrap()
11156 .is_none());
11157 }
11158 }
11159
11160 #[test]
11161 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
11162 let registry = Arc::new(Registry::default());
11163 let handler = ControlHandler::new(Arc::clone(®istry));
11164 let connection = ConnectionId::new(219);
11165 registry
11166 .register_with_control_ops(
11167 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
11168 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
11169 "bad schema",
11170 format!("sha256:{}", "a".repeat(64)),
11171 128,
11172 2,
11173 )))
11174 .build(),
11175 PROTOCOL_VERSION,
11176 connection,
11177 Vec::new(),
11178 )
11179 .unwrap();
11180 let request =
11181 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
11182 let response = handler
11183 .handle_catalog_update(connection, request, Vec::new(), None, None)
11184 .unwrap();
11185 assert_eq!(response[0].header.ty, FrameType::Error);
11186 let error = parse_error(&response[0]);
11187 assert_eq!(error["code"], "invalid_manifest");
11188 assert!(error["message"]
11189 .as_str()
11190 .unwrap()
11191 .contains("rich_ui.schema"));
11192 }
11193
11194 #[test]
11195 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
11196 let registry = Arc::new(Registry::default());
11197 let supervisor = SupervisorHandle::new();
11198 supervisor.open_swap("provider", "candidate-nonce".to_string());
11199 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11200 let connection = ConnectionId::new(218);
11201 let hello = handler
11202 .handle_control(
11203 connection,
11204 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
11205 )
11206 .unwrap();
11207 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11208 assert!(registry.list_modules().unwrap().1.is_empty());
11209
11210 registry.promote_candidate("provider").unwrap().unwrap();
11211 let request = Frame::build(
11212 FrameType::Request,
11213 control_flags(),
11214 0,
11215 0,
11216 219,
11217 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
11218 )
11219 .unwrap();
11220 let response = handler.handle_catalog_list(request, None).unwrap();
11221 let ClientControlResponse::CatalogList { modules, .. } =
11222 serde_json::from_slice(&response[0].body).unwrap()
11223 else {
11224 panic!("expected catalog.list");
11225 };
11226 assert_eq!(modules.len(), 1);
11227 assert!(modules[0].nonce_verified);
11228 }
11229
11230 #[test]
11231 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
11232 for provenance_json in [
11233 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
11234 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
11235 ] {
11236 let registry = Arc::new(Registry::default());
11237 let handler = ControlHandler::new(registry);
11238 let hello = Frame::build(
11239 FrameType::Hello,
11240 control_flags(),
11241 0,
11242 0,
11243 220,
11244 serde_json::to_vec(&ModuleHelloBody {
11245 manifest: ModuleManifest::builder("built", "0.1.0")
11246 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
11247 .build(),
11248 protocol_ver: PROTOCOL_VERSION,
11249 control_ops: None,
11250 launch_nonce: None,
11251 })
11252 .unwrap(),
11253 )
11254 .unwrap();
11255 assert_eq!(
11256 handler
11257 .handle_control(ConnectionId::new(220), hello)
11258 .unwrap()[0]
11259 .header
11260 .ty,
11261 FrameType::HelloAck
11262 );
11263 let request = Frame::build(
11264 FrameType::Request,
11265 control_flags(),
11266 0,
11267 0,
11268 221,
11269 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
11270 )
11271 .unwrap();
11272 let response = handler.handle_catalog_list(request, None).unwrap();
11273 let ClientControlResponse::CatalogList { modules, .. } =
11274 serde_json::from_slice(&response[0].body).unwrap()
11275 else {
11276 panic!("expected catalog.list");
11277 };
11278 assert_eq!(modules.len(), 1);
11279 assert_eq!(
11280 serde_json::to_vec(&modules[0]).unwrap(),
11281 format!(
11282 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
11283 )
11284 .as_bytes()
11285 );
11286 }
11287 }
11288
11289 #[test]
11290 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
11291 let registry = Arc::new(Registry::default());
11292 let handler = ControlHandler::new(Arc::clone(®istry));
11293 let hello = Frame::build(
11294 FrameType::Hello,
11295 control_flags(),
11296 0,
11297 0,
11298 220,
11299 serde_json::to_vec(&ModuleHelloBody {
11300 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
11301 protocol_ver: PROTOCOL_VERSION,
11302 control_ops: None,
11303 launch_nonce: Some("unverified".to_string()),
11304 })
11305 .unwrap(),
11306 )
11307 .unwrap();
11308 assert_eq!(
11309 handler
11310 .handle_control(ConnectionId::new(220), hello)
11311 .unwrap()[0]
11312 .header
11313 .ty,
11314 FrameType::HelloAck
11315 );
11316 let request = Frame::build(
11317 FrameType::Request,
11318 control_flags(),
11319 0,
11320 0,
11321 221,
11322 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
11323 )
11324 .unwrap();
11325 let response = handler.handle_catalog_list(request, None).unwrap();
11326 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
11327 assert!(wire["modules"][0].get("provenance").is_none());
11328 let ClientControlResponse::CatalogList { modules, .. } =
11329 serde_json::from_slice(&response[0].body).unwrap()
11330 else {
11331 panic!("expected catalog.list");
11332 };
11333 assert_eq!(
11334 serde_json::to_vec(&modules[0]).unwrap(),
11335 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
11336 );
11337 assert!(!modules[0].nonce_verified);
11338 assert!(modules[0].provenance.is_none());
11339 }
11340
11341 #[test]
11342 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
11343 let registry = Arc::new(Registry::default());
11344 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11345 [("vault".to_string(), true), ("squatter".to_string(), true)],
11346 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
11347 );
11348 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
11349 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
11350 provides: vec!["credentials-provider/v1".to_string()],
11351 requires: Vec::new(),
11352 must_never_reach: Vec::new(),
11353 });
11354 let frame = Frame::build(
11355 FrameType::Hello,
11356 control_flags(),
11357 0,
11358 0,
11359 77,
11360 serde_json::to_vec(&ModuleHelloBody {
11361 manifest: squatter,
11362 protocol_ver: PROTOCOL_VERSION,
11363 control_ops: None,
11364 launch_nonce: None,
11365 })
11366 .expect("HELLO serializes"),
11367 )
11368 .expect("HELLO frame builds");
11369 let response = handler
11370 .handle_control(ConnectionId::new(77), frame)
11371 .expect("reserved claim receives a typed refusal");
11372 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
11373 assert_eq!(
11374 registry
11375 .active_registration_count()
11376 .expect("registry reads"),
11377 0,
11378 "a reserved capability refusal must not leave a catalog entry"
11379 );
11380 }
11381
11382 #[test]
11383 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
11384 for settlement in ["timeout", "inconclusive", "drop"] {
11385 let breakers = RouteBindBreakers::default();
11386 let RouteBindAdmission::Admitted {
11387 guard: mut old,
11388 probe: false,
11389 } = breakers.admit("prov")
11390 else {
11391 panic!("ordinary relay admitted")
11392 };
11393 let RouteBindAdmission::Admitted {
11394 guard: mut opener, ..
11395 } = breakers.admit("prov")
11396 else {
11397 panic!("second relay admitted")
11398 };
11399 assert!(
11400 !opener
11401 .record_timeout(1, Duration::ZERO)
11402 .unwrap()
11403 .reopened_after_probe
11404 );
11405 let RouteBindAdmission::Admitted {
11406 guard: mut probe,
11407 probe: true,
11408 } = breakers.admit("prov")
11409 else {
11410 panic!("one cooldown probe admitted")
11411 };
11412 match settlement {
11413 "timeout" => assert!(
11414 !old.record_timeout(1, Duration::ZERO)
11415 .unwrap()
11416 .reopened_after_probe
11417 ),
11418 "inconclusive" => old.record_inconclusive(),
11419 "drop" => drop(old),
11420 _ => unreachable!(),
11421 }
11422 assert!(
11423 matches!(
11424 breakers.admit("prov"),
11425 RouteBindAdmission::Refused {
11426 probe_in_flight: true,
11427 ..
11428 }
11429 ),
11430 "{settlement} of a pre-open relay cannot release the real probe"
11431 );
11432 assert!(
11433 probe
11434 .record_timeout(1, Duration::ZERO)
11435 .unwrap()
11436 .reopened_after_probe
11437 );
11438 assert!(matches!(
11439 breakers.admit("prov"),
11440 RouteBindAdmission::Admitted { probe: true, .. }
11441 ));
11442 }
11443 let breakers = RouteBindBreakers::default();
11444 let admit = || match breakers.admit("prov") {
11445 RouteBindAdmission::Admitted { guard, .. } => guard,
11446 _ => panic!("relay admitted"),
11447 };
11448 admit().record_timeout(1, Duration::ZERO);
11449 let mut old_probe = admit();
11450 breakers.reset_for_new_module_connection("prov");
11451 admit().record_timeout(1, Duration::ZERO);
11452 let _new_probe = admit();
11453 old_probe.record_inconclusive();
11454 assert!(matches!(
11455 breakers.admit("prov"),
11456 RouteBindAdmission::Refused {
11457 probe_in_flight: true,
11458 ..
11459 }
11460 ));
11461 }
11462
11463 #[tokio::test]
11464 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
11465 for candidate in [false, true] {
11466 let registry = Arc::new(Registry::default());
11467 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11468 [("vault".to_string(), true), ("squatter".to_string(), true)],
11469 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
11470 );
11471 let conn = ConnectionId::new(77);
11472 let (ctx, mut rx) = route_ctx(conn);
11473 let initial = capability_manifest("squatter", &[], &[]);
11474 if candidate {
11475 registry
11476 .register_candidate_with_control_ops(
11477 initial.clone(),
11478 PROTOCOL_VERSION,
11479 conn,
11480 module_baseline_control_ops(),
11481 )
11482 .unwrap();
11483 handler
11484 .forwarding
11485 .register_candidate_module_connection(
11486 conn,
11487 "squatter".to_string(),
11488 PROTOCOL_VERSION,
11489 manifest_concurrency(&initial),
11490 ctx.egress.clone(),
11491 )
11492 .unwrap();
11493 } else {
11494 hello_via_sink(
11495 &handler,
11496 &ctx,
11497 &mut rx,
11498 hello_frame_with_manifest(initial.clone(), 1),
11499 )
11500 .await;
11501 }
11502 let response = handler
11503 .handle_control_frame(
11504 &ctx,
11505 catalog_update_with_capabilities_frame(
11506 2,
11507 capability_manifest("squatter", &["credentials-provider/v1"], &[])
11508 .capabilities
11509 .unwrap(),
11510 ),
11511 )
11512 .await
11513 .unwrap();
11514 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
11515 assert_eq!(
11516 registry
11517 .get_module_by_connection(conn)
11518 .unwrap()
11519 .unwrap()
11520 .manifest,
11521 initial
11522 );
11523 }
11524 }
11525
11526 #[test]
11527 fn server_describe_surfaces_required_capability_verdict_fields() {
11528 let registry = Arc::new(Registry::default());
11529 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11530 [
11531 ("consumer".to_string(), true),
11532 ("provider".to_string(), false),
11533 ],
11534 BTreeMap::new(),
11535 );
11536 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11537 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
11538 provides: Vec::new(),
11539 requires: vec![subc_protocol::manifest::CapabilityRequirement {
11540 capability: "credentials-provider/v1".to_string(),
11541 need: subc_protocol::manifest::CapabilityNeed::Required,
11542 }],
11543 must_never_reach: Vec::new(),
11544 });
11545 let hello = Frame::build(
11546 FrameType::Hello,
11547 control_flags(),
11548 0,
11549 0,
11550 78,
11551 serde_json::to_vec(&ModuleHelloBody {
11552 manifest: consumer,
11553 protocol_ver: PROTOCOL_VERSION,
11554 control_ops: None,
11555 launch_nonce: None,
11556 })
11557 .expect("HELLO serializes"),
11558 )
11559 .expect("HELLO frame builds");
11560 handler
11561 .handle_control(ConnectionId::new(78), hello)
11562 .expect("consumer registers");
11563 let describe = Frame::build(
11564 FrameType::Request,
11565 control_flags(),
11566 0,
11567 0,
11568 79,
11569 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
11570 .expect("request serializes"),
11571 )
11572 .expect("describe frame builds");
11573 let response = handler
11574 .handle_server_describe(describe)
11575 .expect("server.describe succeeds");
11576 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
11577 let requirement = &rendered["capability_requirements"][0];
11578 assert_eq!(requirement["consumer"], "consumer");
11579 assert_eq!(requirement["verdict"], "never_provided");
11580 assert_eq!(requirement["episode_seq"], 1);
11581 assert_eq!(requirement["config_satisfiable"], false);
11582 assert_eq!(requirement["runtime_available"], false);
11583 assert!(requirement["detail"]
11584 .as_str()
11585 .expect("detail string")
11586 .contains("credentials-provider/v1"));
11587 }
11588
11589 fn deregister_required_provider(
11595 daemon_shutdown_begun: bool,
11596 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
11597 const CAPABILITY: &str = "project-identity/v1";
11598 let registry = Arc::new(Registry::default());
11599 let daemon_shutdown = DaemonShutdownFlag::default();
11600 let handler = ControlHandler::new(Arc::clone(®istry))
11601 .with_capability_config(
11602 [
11603 ("consumer".to_string(), true),
11604 ("provider".to_string(), true),
11605 ],
11606 BTreeMap::new(),
11607 )
11608 .with_daemon_shutdown_flag(daemon_shutdown.clone());
11609 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11610 consumer.capabilities = Some(CapabilityDeclarations {
11611 provides: Vec::new(),
11612 requires: vec![subc_protocol::manifest::CapabilityRequirement {
11613 capability: CAPABILITY.to_string(),
11614 need: subc_protocol::manifest::CapabilityNeed::Required,
11615 }],
11616 must_never_reach: Vec::new(),
11617 });
11618 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
11619 let provider_conn = ConnectionId::new(182);
11620 for (conn, manifest, corr) in [
11621 (ConnectionId::new(181), consumer, 181),
11622 (provider_conn, provider, 182),
11623 ] {
11624 let hello = handler
11625 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
11626 .expect("module registers");
11627 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11628 }
11629 assert_eq!(
11630 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11631 Some(CapabilityVerdict::Provided),
11632 "the registered provider must satisfy the requirement first"
11633 );
11634 if daemon_shutdown_begun {
11635 daemon_shutdown.set_for_test();
11636 }
11637
11638 let capture = EventCapture::default();
11639 let _subscriber =
11640 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11641 handler
11642 .cleanup_connection(provider_conn)
11643 .expect("provider deregisters");
11644 let events = capture
11645 .events()
11646 .into_iter()
11647 .filter(|event| {
11648 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11649 })
11650 .collect();
11651 (
11652 events,
11653 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11654 )
11655 }
11656
11657 #[test]
11658 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11659 let (events, verdict) = deregister_required_provider(true);
11660 assert_eq!(
11661 verdict,
11662 Some(CapabilityVerdict::NeverProvided),
11663 "shutdown must not change the verdict route.open refusals read"
11664 );
11665 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11666 let event = &events[0];
11667 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11668 assert_eq!(
11669 event.fields.get("cause").map(String::as_str),
11670 Some("\"daemon_shutdown\""),
11671 "{event:?}"
11672 );
11673 assert_eq!(
11674 event.fields.get("verdict").map(String::as_str),
11675 Some("\"never_provided\""),
11676 "{event:?}"
11677 );
11678 }
11679
11680 #[test]
11681 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11682 let (events, verdict) = deregister_required_provider(false);
11683 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11684 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11685 let event = &events[0];
11686 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11687 assert_eq!(event.fields.get("cause"), None, "{event:?}");
11688 assert_eq!(
11689 event.fields.get("verdict").map(String::as_str),
11690 Some("\"never_provided\""),
11691 "{event:?}"
11692 );
11693 }
11694
11695 #[test]
11696 fn catalog_list_omits_capabilities_for_legacy_manifest() {
11697 let registry = Arc::new(Registry::default());
11698 let handler = ControlHandler::new(Arc::clone(®istry));
11699 let hello = handler
11700 .handle_control(
11701 ConnectionId::new(101),
11702 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11703 )
11704 .expect("legacy HELLO registers");
11705 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11706
11707 let request = Frame::build(
11708 FrameType::Request,
11709 control_flags(),
11710 0,
11711 0,
11712 102,
11713 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11714 .expect("catalog request serializes"),
11715 )
11716 .expect("catalog request frame builds");
11717 let response = handler
11718 .handle_catalog_list(request, None)
11719 .expect("catalog list succeeds");
11720 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11721 assert!(
11722 body["modules"][0].get("capabilities").is_none(),
11723 "legacy manifest must retain an absent capabilities field on catalog.list"
11724 );
11725 }
11726
11727 #[test]
11728 fn hello_ack_omits_storage_when_no_storage_config() {
11729 let registry = Arc::new(Registry::default());
11730 let handler = ControlHandler::new(Arc::clone(®istry));
11731 let responses = handler
11732 .handle_control(
11733 ConnectionId::new(1),
11734 hello_frame("aft", PROTOCOL_VERSION, 7),
11735 )
11736 .unwrap();
11737 let ack = parse_ack(&responses[0]);
11738 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11739 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11740 }
11741
11742 #[tokio::test]
11743 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11744 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11745 let registry = Arc::new(Registry::default());
11746 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
11747 let responses = handler
11748 .handle_control(
11749 ConnectionId::new(1),
11750 hello_frame("aft", PROTOCOL_VERSION, 7),
11751 )
11752 .unwrap();
11753 let ack = parse_ack(&responses[0]);
11754 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11755
11756 let described = handler
11757 .handle_control_frame(
11758 &route_ctx(ConnectionId::new(2)).0,
11759 Frame::build(
11760 FrameType::Request,
11761 control_flags(),
11762 0,
11763 0,
11764 9,
11765 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11766 )
11767 .unwrap(),
11768 )
11769 .await
11770 .unwrap();
11771 let ClientControlResponse::ServerDescribe { machine_id, .. } =
11772 serde_json::from_slice(&described[0].body).unwrap()
11773 else {
11774 panic!("server.describe answered with another shape");
11775 };
11776 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11777 }
11778
11779 #[test]
11780 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11781 let registry = Arc::new(Registry::default());
11784 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
11785 crate::daemon_config::StorageConfig::Sqlite {
11786 data_home: std::path::PathBuf::from("/data"),
11787 },
11788 ));
11789
11790 let responses = handler
11791 .handle_control(
11792 ConnectionId::new(1),
11793 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11794 )
11795 .unwrap();
11796 let ack = parse_ack(&responses[0]);
11797 assert_eq!(
11798 ack.storage,
11799 Some(serde_json::json!({
11800 "module_id": "alfonso-routing",
11801 "storage_namespace": "default",
11802 "isolation": { "kind": "module" },
11803 "backend": {
11804 "backend": "sqlite",
11805 "path": "/data/cortexkit/alfonso-routing/store.db"
11806 }
11807 })),
11808 "the delivered descriptor is the module's own sqlite store path"
11809 );
11810 }
11811
11812 #[test]
11813 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11814 let registry = Arc::new(Registry::default());
11815 let handler = ControlHandler::new(Arc::clone(®istry));
11816 let conn = ConnectionId::new(1);
11817 let responses = handler
11818 .handle_control(
11819 conn,
11820 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11821 )
11822 .unwrap();
11823 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11824 let registration = registry.get_module("aft").unwrap().unwrap();
11825 assert_eq!(registration.control_ops, module_baseline_control_ops());
11826
11827 let frame =
11828 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11829 assert!(handler
11830 .guard_module_control_op(&frame, "aft", "route.bind")
11831 .unwrap()
11832 .is_none());
11833 let error = handler
11834 .guard_module_control_op(&frame, "aft", "test.synthetic")
11835 .unwrap()
11836 .expect("synthetic ungranted op should be rejected");
11837 assert_eq!(error.header.ty, FrameType::Error);
11838 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11839 }
11840
11841 #[test]
11842 fn hello_control_ops_some_adds_optional_grants() {
11843 let registry = Arc::new(Registry::default());
11844 let handler = ControlHandler::new(Arc::clone(®istry));
11845 handler
11846 .handle_control(
11847 ConnectionId::new(1),
11848 hello_frame_with_control_ops(
11849 "aft",
11850 PROTOCOL_VERSION,
11851 7,
11852 Some(vec![
11853 "future.synthetic".to_string(),
11854 "route.bind".to_string(),
11855 ]),
11856 ),
11857 )
11858 .unwrap();
11859 let registration = registry.get_module("aft").unwrap().unwrap();
11860 assert_eq!(
11861 registration.control_ops,
11862 vec![
11863 "route.bind".to_string(),
11864 "route.status".to_string(),
11865 "future.synthetic".to_string(),
11866 ]
11867 );
11868 let frame =
11869 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11870 assert!(handler
11871 .guard_module_control_op(&frame, "aft", "future.synthetic")
11872 .unwrap()
11873 .is_none());
11874 }
11875
11876 #[tokio::test]
11877 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11878 let registry = Arc::new(Registry::default());
11879 let forwarding = Arc::new(ForwardingTable::default());
11880 let handler =
11881 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11882 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11883 hello_via_sink(
11884 &handler,
11885 &module_ctx,
11886 &mut module_rx,
11887 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11888 )
11889 .await;
11890
11891 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11892 let responses = handler
11893 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11894 .await
11895 .unwrap();
11896 assert_eq!(responses.len(), 1);
11897 assert_eq!(responses[0].header.ty, FrameType::Error);
11898 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11899 assert!(module_rx.try_recv().is_err());
11900 }
11901
11902 #[tokio::test]
11903 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11904 let registry = Arc::new(Registry::default());
11905 let forwarding = Arc::new(ForwardingTable::default());
11906 let handler =
11907 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11908 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11909 hello_via_sink(
11910 &handler,
11911 &module_ctx,
11912 &mut module_rx,
11913 hello_frame_with_control_ops(
11914 "aft",
11915 PROTOCOL_VERSION,
11916 7,
11917 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11918 ),
11919 )
11920 .await;
11921
11922 let project_root = unique_project_root("demux");
11923 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11924 let route_handler = handler.clone();
11925 let route_task = tokio::spawn(async move {
11926 route_handler
11927 .handle_control_frame(
11928 &route_client_ctx,
11929 route_open_frame(100, "aft", project_root),
11930 )
11931 .await
11932 .unwrap()
11933 });
11934 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11935 .await
11936 .unwrap()
11937 .unwrap();
11938 assert!(matches!(
11939 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11940 ModuleControlRequest::RouteBind { .. }
11941 ));
11942
11943 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11944 let health_handler = handler.clone();
11945 let health_task = tokio::spawn(async move {
11946 health_handler
11947 .handle_control_frame(
11948 &health_client_ctx,
11949 supervisor_health_probe_frame(101, "aft"),
11950 )
11951 .await
11952 .unwrap()
11953 });
11954 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11955 .await
11956 .unwrap()
11957 .unwrap();
11958 assert_eq!(
11959 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11960 ModuleControlRequest::HealthCheck {}
11961 );
11962
11963 handler
11964 .handle_control_frame(
11965 &module_ctx,
11966 health_response(health_frame.header.corr, HealthStatus::Degraded),
11967 )
11968 .await
11969 .unwrap();
11970 let health_response = health_task.await.unwrap();
11971 assert_eq!(health_response.len(), 1);
11972 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11973 ClientControlResponse::SupervisorHealthProbe {
11974 module_id,
11975 status,
11976 detail,
11977 metrics,
11978 } => {
11979 assert_eq!(module_id, "aft");
11980 assert_eq!(status, HealthStatus::Degraded);
11981 assert_eq!(detail.as_deref(), Some("warming"));
11982 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11983 }
11984 other => panic!("unexpected health response: {other:?}"),
11985 }
11986
11987 handler
11988 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11989 .await
11990 .unwrap();
11991 let route_response = route_task.await.unwrap();
11992 assert!(route_response.is_empty());
11993 let published = route_client_rx.recv().await.unwrap();
11994 assert!(matches!(
11995 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11996 ClientControlResponse::RouteOpen { .. }
11997 ));
11998 }
11999
12000 async fn relay_route_open(
12005 handler: &ControlHandler,
12006 client_connection: ConnectionId,
12007 client_egress: &FrameSink,
12008 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12009 corr: u64,
12010 module_id: &str,
12011 project_root_label: &str,
12012 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
12013 let ctx = RouteCtx {
12014 connection_id: client_connection,
12015 egress: client_egress.clone(),
12016 };
12017 let handler = handler.clone();
12018 let project_root = unique_project_root(project_root_label);
12019 let module_id = module_id.to_string();
12020 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
12021 let task = tokio::spawn(async move {
12022 let _guard = tracing::dispatcher::set_default(&dispatch);
12023 handler
12024 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
12025 .await
12026 .unwrap()
12027 });
12028 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
12029 .await
12030 .expect("module receives the relayed route.bind")
12031 .expect("module egress is open");
12032 (task, bind.frame)
12033 }
12034
12035 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
12036 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
12037 ModuleControlRequest::RouteBind {
12038 route_channel,
12039 epoch,
12040 ..
12041 } => (route_channel, epoch),
12042 other => panic!("expected a route.bind request, got {other:?}"),
12043 }
12044 }
12045
12046 fn published_route(frame: &Frame) -> (u16, u32) {
12047 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
12048 ClientControlResponse::RouteOpen {
12049 route_channel,
12050 route_epoch,
12051 ..
12052 } => (route_channel, route_epoch),
12053 other => panic!("expected a route.open response, got {other:?}"),
12054 }
12055 }
12056
12057 #[tokio::test]
12077 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
12078 let registry = Arc::new(Registry::default());
12079 let forwarding = Arc::new(ForwardingTable::default());
12080 let handler =
12081 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12082
12083 let module_connection = ConnectionId::new(30);
12084 let (module_ctx, mut module_rx) = route_ctx(module_connection);
12085 hello_via_sink(
12086 &handler,
12087 &module_ctx,
12088 &mut module_rx,
12089 hello_frame("aft", PROTOCOL_VERSION, 7),
12090 )
12091 .await;
12092
12093 let dying_client = ConnectionId::new(31);
12094 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
12095
12096 let (first_task, first_bind) = relay_route_open(
12099 &handler,
12100 dying_client,
12101 &dying_ctx.egress,
12102 &mut module_rx,
12103 100,
12104 "aft",
12105 "closing-first",
12106 )
12107 .await;
12108 handler
12109 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
12110 .await
12111 .unwrap();
12112 assert!(first_task.await.unwrap().is_empty());
12113 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
12114
12115 let (second_task, second_bind) = relay_route_open(
12117 &handler,
12118 dying_client,
12119 &dying_ctx.egress,
12120 &mut module_rx,
12121 101,
12122 "aft",
12123 "closing-second",
12124 )
12125 .await;
12126 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
12127
12128 assert!(forwarding
12131 .escalate_client_delivery_failure(
12132 dying_client,
12133 first_channel,
12134 first_epoch,
12135 CloseReason::new(
12136 "module_to_client_delivery_failed",
12137 "client egress refused a module frame",
12138 ),
12139 crate::forwarding::UndeliveredFrame {
12140 module_id: None,
12141 sink: &dying_ctx.egress,
12142 },
12143 )
12144 .unwrap());
12145 assert!(!dying_ctx.egress.is_closed());
12146
12147 let ack = handler
12149 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
12150 .await;
12151 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
12152 if module_loop_error.is_some() {
12153 handler.cleanup_connection(module_connection).unwrap();
12157 }
12158 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12162 .await
12163 .ok()
12164 .flatten();
12165
12166 assert!(
12168 registry
12169 .get_module_by_connection(module_connection)
12170 .unwrap()
12171 .is_some(),
12172 "one client's closing connection ended the shared module connection: \
12173 {module_loop_error:?}"
12174 );
12175 let cotenant = ConnectionId::new(32);
12177 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
12178 let (cotenant_task, cotenant_bind) = relay_route_open(
12179 &handler,
12180 cotenant,
12181 &cotenant_ctx.egress,
12182 &mut module_rx,
12183 102,
12184 "aft",
12185 "closing-cotenant",
12186 )
12187 .await;
12188 handler
12189 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
12190 .await
12191 .unwrap();
12192 assert!(cotenant_task.await.unwrap().is_empty());
12193 let (cotenant_channel, cotenant_epoch) =
12194 published_route(&cotenant_rx.recv().await.unwrap());
12195 assert!(matches!(
12196 forwarding
12197 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
12198 .unwrap(),
12199 DataRoute::Client(DataRouteState::Bound(_))
12200 ));
12201
12202 let goodbye = post_ack_module_frame
12205 .expect("module receives a GOODBYE for the abandoned route channel");
12206 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12207 assert_eq!(goodbye.header.channel, abandoned_channel);
12208 assert_eq!(goodbye.header.epoch, abandoned_epoch);
12209
12210 assert!(dying_rx.try_recv().is_err());
12214 let second_response = second_task.await.unwrap();
12215 assert_eq!(second_response.len(), 1);
12216 assert_eq!(
12217 parse_error(&second_response[0])["code"],
12218 "target_unavailable"
12219 );
12220 }
12221
12222 async fn catalog_snapshot(
12225 handler: &ControlHandler,
12226 client_ctx: &RouteCtx,
12227 corr: u64,
12228 ) -> (u64, Vec<(String, bool)>) {
12229 let body =
12230 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
12231 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
12232 let reply = handler
12233 .handle_control_frame(client_ctx, request)
12234 .await
12235 .unwrap()
12236 .pop()
12237 .unwrap();
12238 let ClientControlResponse::CatalogList {
12239 generation,
12240 modules,
12241 ..
12242 } = serde_json::from_slice(&reply.body).unwrap()
12243 else {
12244 panic!("not a catalog.list reply");
12245 };
12246 (
12247 generation,
12248 modules
12249 .into_iter()
12250 .map(|entry| (entry.module_id, entry.ready))
12251 .collect(),
12252 )
12253 }
12254
12255 fn hello_held_before_forwarding() -> (
12261 ControlHandler,
12262 Arc<std::sync::Barrier>,
12263 std::thread::JoinHandle<Vec<Frame>>,
12264 RouteCtx,
12265 mpsc::Receiver<crate::router::OutboundFrame>,
12266 ) {
12267 let hold = Arc::new(std::sync::Barrier::new(2));
12268 let handler = ControlHandler::with_forwarding(
12269 Arc::new(Registry::default()),
12270 Arc::new(ForwardingTable::default()),
12271 )
12272 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
12273 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
12274 let hello = {
12275 let handler = handler.clone();
12276 let module_ctx = module_ctx.clone();
12277 std::thread::spawn(move || {
12278 handler
12279 .handle_hello(
12280 module_ctx.connection_id,
12281 Some(module_ctx.egress.clone()),
12282 hello_frame("aft", PROTOCOL_VERSION, 7),
12283 )
12284 .unwrap()
12285 })
12286 };
12287 (handler, hold, hello, module_ctx, module_rx)
12288 }
12289
12290 #[tokio::test]
12299 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
12300 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
12301 let client = ConnectionId::new(41);
12302 let (client_ctx, mut client_rx) = route_ctx(client);
12303
12304 hold.wait();
12305 assert!(handler.registry.get_module("aft").unwrap().is_some());
12307 assert!(!handler
12308 .forwarding
12309 .has_live_module_connection("aft")
12310 .unwrap());
12311 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
12312 assert!(
12313 listed.is_empty(),
12314 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
12315 );
12316 let refused = handler
12317 .handle_control_frame(
12318 &client_ctx,
12319 route_open_frame(301, "aft", unique_project_root("listed-window")),
12320 )
12321 .await
12322 .unwrap();
12323 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
12324 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
12325 let duplicate = handler
12326 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
12327 .await
12328 .unwrap();
12329 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12330
12331 hold.wait();
12332 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
12333 let ack = module_rx.try_recv().unwrap().frame;
12334 assert_eq!(ack.header.ty, FrameType::HelloAck);
12335 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
12336 assert_eq!(listed, vec![("aft".to_string(), true)]);
12337
12338 let (open_task, bind) = relay_route_open(
12339 &handler,
12340 client,
12341 &client_ctx.egress,
12342 &mut module_rx,
12343 303,
12344 "aft",
12345 "listed-after",
12346 )
12347 .await;
12348 assert!(handler
12349 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12350 .await
12351 .unwrap()
12352 .is_empty());
12353 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
12354 published_route(&client_rx.try_recv().unwrap().frame);
12355 }
12356
12357 #[tokio::test]
12364 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
12365 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
12366 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12367 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
12368
12369 hold.wait();
12370 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
12371 assert!(listed.is_empty());
12372 assert_eq!(
12373 during, before,
12374 "the generation moved while the module was not listed"
12375 );
12376
12377 hold.wait();
12378 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
12379 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
12380 assert_eq!(listed, vec![("aft".to_string(), true)]);
12381 assert_eq!(
12382 after,
12383 before + 1,
12384 "the module appeared without exactly one generation bump"
12385 );
12386 }
12387
12388 #[tokio::test]
12393 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
12394 let registry = Arc::new(Registry::default());
12395 let handler = ControlHandler::with_forwarding(
12396 Arc::clone(®istry),
12397 Arc::new(ForwardingTable::default()),
12398 );
12399 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12400 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
12401 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
12402 drop(module_rx);
12403
12404 let replies = handler
12405 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
12406 .await
12407 .unwrap();
12408 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
12409 assert!(registry.get_module("aft").unwrap().is_none());
12410 assert!(registry
12411 .get_module_by_connection(module_ctx.connection_id)
12412 .unwrap()
12413 .is_none());
12414 assert_eq!(
12415 catalog_snapshot(&handler, &client_ctx, 301).await,
12416 (before, Vec::new())
12417 );
12418 }
12419
12420 async fn route_open_held_after_bind_send() -> (
12424 ControlHandler,
12425 Arc<tokio::sync::Barrier>,
12426 RouteCtx,
12427 mpsc::Receiver<crate::router::OutboundFrame>,
12428 ) {
12429 let hold = Arc::new(tokio::sync::Barrier::new(2));
12430 let handler = ControlHandler::with_forwarding(
12431 Arc::new(Registry::default()),
12432 Arc::new(ForwardingTable::default()),
12433 )
12434 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
12435 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12436 hello_via_sink(
12437 &handler,
12438 &module_ctx,
12439 &mut module_rx,
12440 hello_frame("aft", PROTOCOL_VERSION, 7),
12441 )
12442 .await;
12443 (handler, hold, module_ctx, module_rx)
12444 }
12445
12446 #[tokio::test]
12453 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
12454 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12455 let client = ConnectionId::new(41);
12456 let (client_ctx, mut client_rx) = route_ctx(client);
12457
12458 let (open_task, bind) = relay_route_open(
12459 &handler,
12460 client,
12461 &client_ctx.egress,
12462 &mut module_rx,
12463 200,
12464 "aft",
12465 "acked-before-mark",
12466 )
12467 .await;
12468 let (module_channel, module_epoch) = route_bind_channel(&bind);
12469 hold.wait().await;
12471
12472 assert!(handler
12474 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12475 .await
12476 .unwrap()
12477 .is_empty());
12478 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
12479
12480 hold.wait().await;
12481 assert!(
12482 open_task.await.unwrap().is_empty(),
12483 "route.open answers accepted"
12484 );
12485
12486 match module_rx.try_recv() {
12489 Err(mpsc::error::TryRecvError::Empty) => {}
12490 Ok(outbound) => panic!(
12491 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
12492 outbound.frame.header.ty,
12493 outbound.frame.header.channel,
12494 outbound.frame.header.epoch
12495 ),
12496 Err(err) => panic!("module egress closed: {err}"),
12497 }
12498
12499 let router = Router::with_control_handler(Arc::new(handler.clone()));
12502 let request = Frame::build(
12503 FrameType::Request,
12504 Flags::new(true, Priority::Interactive, false),
12505 client_channel,
12506 client_epoch,
12507 201,
12508 b"first request".to_vec(),
12509 )
12510 .unwrap();
12511 router
12512 .route_for_connection(&client_ctx, request)
12513 .await
12514 .unwrap();
12515 let delivered = module_rx.try_recv().unwrap().frame;
12516 assert_eq!(delivered.header.ty, FrameType::Request);
12517 assert_eq!(
12518 (delivered.header.channel, delivered.header.epoch),
12519 (module_channel, module_epoch)
12520 );
12521 assert_eq!(delivered.body, b"first request");
12522 }
12523
12524 #[tokio::test]
12530 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
12531 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12532 let client = ConnectionId::new(42);
12533 let (client_ctx, _client_rx) = route_ctx(client);
12534
12535 let (open_task, bind) = relay_route_open(
12536 &handler,
12537 client,
12538 &client_ctx.egress,
12539 &mut module_rx,
12540 300,
12541 "aft",
12542 "released-before-mark",
12543 )
12544 .await;
12545 let (module_channel, module_epoch) = route_bind_channel(&bind);
12546 hold.wait().await;
12547
12548 handler.cleanup_connection(client).unwrap();
12549 assert!(
12550 module_rx.try_recv().is_err(),
12551 "client teardown must not GOODBYE a bind route.open is still queueing"
12552 );
12553
12554 hold.wait().await;
12555 let response = open_task.await.unwrap();
12556 assert_eq!(response.len(), 1);
12557 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12558
12559 let goodbye = module_rx
12560 .try_recv()
12561 .expect("module is told to drop the binding the daemon abandoned")
12562 .frame;
12563 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12564 assert_eq!(
12565 (goodbye.header.channel, goodbye.header.epoch),
12566 (module_channel, module_epoch)
12567 );
12568 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12569 }
12570
12571 #[tokio::test]
12576 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
12577 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12578 let client = ConnectionId::new(43);
12579 let (client_ctx, client_rx) = route_ctx(client);
12580
12581 let (open_task, bind) = relay_route_open(
12582 &handler,
12583 client,
12584 &client_ctx.egress,
12585 &mut module_rx,
12586 400,
12587 "aft",
12588 "gone-client-before-mark",
12589 )
12590 .await;
12591 let (module_channel, module_epoch) = route_bind_channel(&bind);
12592 hold.wait().await;
12593
12594 drop(client_rx);
12595 assert!(client_ctx.egress.is_closed());
12596 handler
12597 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12598 .await
12599 .unwrap();
12600
12601 hold.wait().await;
12602 let response = open_task.await.unwrap();
12603 assert_eq!(response.len(), 1);
12604 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12605
12606 let goodbye = module_rx
12607 .try_recv()
12608 .expect("module is told to drop the binding it acked")
12609 .frame;
12610 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12611 assert_eq!(
12612 (goodbye.header.channel, goodbye.header.epoch),
12613 (module_channel, module_epoch)
12614 );
12615 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12616 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12617 }
12618
12619 #[test]
12626 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
12627 let handler = ControlHandler::default();
12628 let module_connection = ConnectionId::new(30);
12629 let client_connection = ConnectionId::new(31);
12630
12631 handler
12632 .refuse_to_end_module_connection_for_a_client(
12633 module_connection,
12634 77,
12635 ForwardingError::ConnectionClosing {
12636 connection_id: client_connection,
12637 },
12638 )
12639 .expect("a closing client must never end the module connection");
12640
12641 assert!(matches!(
12642 handler.refuse_to_end_module_connection_for_a_client(
12643 module_connection,
12644 78,
12645 ForwardingError::ConnectionClosing {
12646 connection_id: module_connection,
12647 },
12648 ),
12649 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12650 connection_id
12651 })) if connection_id == module_connection
12652 ));
12653 assert!(matches!(
12654 handler.refuse_to_end_module_connection_for_a_client(
12655 module_connection,
12656 79,
12657 ForwardingError::Poisoned,
12658 ),
12659 Err(RouterError::Forwarding(ForwardingError::Poisoned))
12660 ));
12661 assert!(matches!(
12662 handler.refuse_to_end_module_connection_for_a_client(
12663 module_connection,
12664 80,
12665 ForwardingError::StaleModuleEndpoint,
12666 ),
12667 Err(RouterError::Forwarding(
12668 ForwardingError::StaleModuleEndpoint
12669 ))
12670 ));
12671 }
12672
12673 #[tokio::test]
12686 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12687 let registry = Arc::new(Registry::default());
12688 let forwarding = Arc::new(ForwardingTable::default());
12689 let supervisor = SupervisorHandle::new();
12690 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12691 let handler =
12692 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12693 .with_supervisor(supervisor);
12694
12695 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12696 hello_via_sink(
12697 &handler,
12698 &target_ctx,
12699 &mut target_rx,
12700 hello_frame("target", PROTOCOL_VERSION, 1),
12701 )
12702 .await;
12703
12704 let wrong_nonce = handler
12708 .handle_control_frame(
12709 &route_ctx(ConnectionId::new(91)).0,
12710 route_open_frame_with_admission_facts(
12711 20,
12712 "target",
12713 unique_project_root("admission-facts"),
12714 Some(subc_control::ConsumerIdentity {
12715 module_id: "fed".to_string(),
12716 launch_nonce: "not-the-real-nonce".to_string(),
12717 }),
12718 None,
12719 ),
12720 )
12721 .await
12722 .unwrap();
12723 assert_eq!(
12724 parse_error(&wrong_nonce[0])["code"],
12725 "bad_consumer_identity",
12726 "a mismatched launch nonce must be refused, not stamped Reserved"
12727 );
12728
12729 let never_spawned = handler
12733 .handle_control_frame(
12734 &route_ctx(ConnectionId::new(92)).0,
12735 route_open_frame_with_admission_facts(
12736 21,
12737 "target",
12738 unique_project_root("admission-facts"),
12739 Some(subc_control::ConsumerIdentity {
12740 module_id: "never-spawned".to_string(),
12741 launch_nonce: "any-nonce".to_string(),
12742 }),
12743 None,
12744 ),
12745 )
12746 .await
12747 .unwrap();
12748 assert_eq!(
12749 parse_error(&never_spawned[0])["code"],
12750 "bad_consumer_identity",
12751 "an unspawned module_id must be refused rather than accepted for lack of a record"
12752 );
12753 }
12754
12755 #[tokio::test]
12778 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12779 let registry = Arc::new(Registry::default());
12780 let forwarding = Arc::new(ForwardingTable::default());
12781 let supervisor = SupervisorHandle::new();
12782 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12783 let handler =
12784 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12785 .with_supervisor(supervisor);
12786
12787 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12788 hello_via_sink(
12789 &handler,
12790 &target_ctx,
12791 &mut target_rx,
12792 hello_frame("target", PROTOCOL_VERSION, 1),
12793 )
12794 .await;
12795
12796 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12797 let route_handler = handler.clone();
12798 let route_task = tokio::spawn(async move {
12799 route_handler
12800 .handle_control_frame(
12801 &client_ctx,
12802 route_open_frame_with_admission_facts(
12803 30,
12804 "target",
12805 unique_project_root("admission-facts"),
12806 Some(subc_control::ConsumerIdentity {
12807 module_id: "fed".to_string(),
12808 launch_nonce: "fed-nonce".to_string(),
12809 }),
12810 None,
12811 ),
12812 )
12813 .await
12814 .unwrap()
12815 });
12816
12817 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12824 .await
12825 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12826 .expect("module control channel closed before route.bind");
12827 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12828 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12829 panic!("expected route.bind")
12830 };
12831 assert_eq!(
12832 principal,
12833 Some(Principal::Reserved {
12834 module_id: "fed".to_string()
12835 }),
12836 "a correctly attested consumer must be stamped Reserved for its own id"
12837 );
12838
12839 handler
12840 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12841 .await
12842 .unwrap();
12843 assert!(route_task.await.unwrap().is_empty());
12844 assert!(
12845 matches!(
12846 serde_json::from_slice::<ClientControlResponse>(
12847 &client_rx.recv().await.unwrap().body
12848 )
12849 .unwrap(),
12850 ClientControlResponse::RouteOpen { .. }
12851 ),
12852 "the route must actually open, not merely avoid an error"
12853 );
12854 }
12855
12856 #[tokio::test(start_paused = true)]
12857 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12858 let registry = Arc::new(Registry::default());
12859 let forwarding = Arc::new(ForwardingTable::default());
12860 let supervisor = SupervisorHandle::new();
12861 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12862 let handler =
12863 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12864 .with_supervisor(supervisor);
12865
12866 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12867 hello_via_sink(
12868 &handler,
12869 &target_ctx,
12870 &mut target_rx,
12871 hello_frame("target", PROTOCOL_VERSION, 1),
12872 )
12873 .await;
12874
12875 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12876 let direct_handler = handler.clone();
12877 let direct_open = tokio::spawn(async move {
12878 direct_handler
12879 .handle_control_frame(
12880 &direct_ctx,
12881 route_open_frame(2, "target", unique_project_root("route-census-direct")),
12882 )
12883 .await
12884 .unwrap()
12885 });
12886 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12887 .await
12888 .expect("no direct route.bind within 5s")
12889 .expect("target control channel closed before direct route.bind");
12890 handler
12891 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12892 .await
12893 .unwrap();
12894 assert!(direct_open.await.unwrap().is_empty());
12895 let _ = direct_rx.recv().await.unwrap();
12896
12897 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12898 let reserved_handler = handler.clone();
12899 let reserved_open = tokio::spawn(async move {
12900 reserved_handler
12901 .handle_control_frame(
12902 &reserved_ctx,
12903 route_open_frame_with_admission_facts(
12904 3,
12905 "target",
12906 unique_project_root("admission-facts"),
12907 Some(ConsumerIdentity {
12908 module_id: "fed".to_string(),
12909 launch_nonce: "fed-nonce".to_string(),
12910 }),
12911 None,
12912 ),
12913 )
12914 .await
12915 .unwrap()
12916 });
12917 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12918 .await
12919 .expect("no reserved route.bind within 5s")
12920 .expect("target control channel closed before reserved route.bind");
12921 handler
12922 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12923 .await
12924 .unwrap();
12925 assert!(reserved_open.await.unwrap().is_empty());
12926 let _ = reserved_rx.recv().await.unwrap();
12927
12928 forwarding
12929 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12930 .unwrap();
12931 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12932 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12933 module_id: Some("target".to_string()),
12934 })
12935 .unwrap();
12936 let census_frame =
12937 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12938 let response = handler
12939 .handle_control_frame(&census_ctx, census_frame)
12940 .await
12941 .unwrap()
12942 .pop()
12943 .unwrap();
12944 let actual: Value = serde_json::from_slice(&response.body).unwrap();
12945 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12946 assert!(matches!(
12947 decoded,
12948 ClientControlResponse::SupervisorRoutes { .. }
12949 ));
12950 let routes = actual["modules"][0]["routes"].as_array().unwrap();
12951 assert_eq!(routes.len(), 2);
12952 assert!(routes.iter().all(|route| route["draining"] == true));
12953 assert!(
12956 routes.iter().all(|route| route["drain_reason"] == "reload"),
12957 "draining routes must name the drain's reason: {routes:?}"
12958 );
12959 assert!(routes.iter().any(|route| {
12960 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12961 }));
12962 assert!(routes.iter().any(|route| {
12963 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12964 }));
12965
12966 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12967 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12968 if std::env::var_os("UPDATE_GOLDEN").is_some() {
12969 std::fs::write(
12970 &golden_path,
12971 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12972 )
12973 .unwrap();
12974 }
12975 let expected: Value =
12976 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12977 assert_eq!(actual, expected);
12978 }
12979
12980 async fn query_live_roots(
12981 handler: &ControlHandler,
12982 module_ctx: &RouteCtx,
12983 ) -> ModuleControlResponseToModule {
12984 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12985 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12986 let response = handler
12987 .handle_control_frame(module_ctx, frame)
12988 .await
12989 .unwrap()
12990 .pop()
12991 .unwrap();
12992 serde_json::from_slice(&response.body).unwrap()
12993 }
12994
12995 #[tokio::test(start_paused = true)]
12996 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12997 let registry = Arc::new(Registry::default());
12998 let forwarding = Arc::new(ForwardingTable::default());
12999 let handler = ControlHandler::with_forwarding(registry, forwarding);
13000 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
13001 hello_via_sink(
13002 &handler,
13003 &target_ctx,
13004 &mut target_rx,
13005 hello_frame("target", PROTOCOL_VERSION, 1),
13006 )
13007 .await;
13008 let root = unique_project_root("live-roots-known");
13009 let path = ProjectRootId::from_path_allowing_missing(root.path())
13010 .unwrap()
13011 .as_path()
13012 .to_path_buf();
13013 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
13014 let open_handler = handler.clone();
13015 let opened = tokio::spawn(async move {
13016 open_handler
13017 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
13018 .await
13019 .unwrap()
13020 });
13021 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
13022 .await
13023 .unwrap()
13024 .unwrap();
13025 handler
13026 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
13027 .await
13028 .unwrap();
13029 assert!(opened.await.unwrap().is_empty());
13030 let _ = client_rx.recv().await.unwrap();
13031
13032 let root = unique_project_root("live-roots-pending");
13033 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
13034 .unwrap()
13035 .as_path()
13036 .to_path_buf();
13037 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
13038 let open_handler = handler.clone();
13039 let pending = tokio::spawn(async move {
13040 open_handler
13041 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
13042 .await
13043 .unwrap()
13044 });
13045 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
13046 .await
13047 .unwrap()
13048 .unwrap();
13049 let actual = query_live_roots(&handler, &target_ctx).await;
13050 let ModuleControlResponseToModule::LiveRoots {
13051 roots,
13052 unknown_root_bindings,
13053 total_bindings,
13054 } = actual
13055 else {
13056 panic!("expected live roots")
13057 };
13058 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
13059 assert_eq!(unknown_root_bindings, 0);
13060 assert_eq!(
13061 roots.len(),
13062 2,
13063 "root-known arm must retain each canonical root"
13064 );
13065 assert_eq!(
13066 total_bindings,
13067 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
13068 );
13069 let counts = roots
13070 .iter()
13071 .map(|root| (root.project_root.clone(), root.bound, root.pending))
13072 .collect::<Vec<_>>();
13073 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
13074 expected.sort_by(|a, b| a.0.cmp(&b.0));
13075 assert_eq!(
13076 counts, expected,
13077 "roots must sort by path and count pending separately"
13078 );
13079 handler
13080 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
13081 .await
13082 .unwrap();
13083 assert!(pending.await.unwrap().is_empty());
13084 }
13085
13086 #[tokio::test(start_paused = true)]
13087 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
13088 let forwarding = Arc::new(ForwardingTable::default());
13089 let handler =
13090 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
13091 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
13092 hello_via_sink(
13093 &handler,
13094 &target_ctx,
13095 &mut target_rx,
13096 hello_frame("target", PROTOCOL_VERSION, 1),
13097 )
13098 .await;
13099 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
13100 let pending = forwarding
13101 .begin_route_bind_relay_for_test(
13102 client_ctx.connection_id,
13103 client_ctx.egress.clone(),
13104 2,
13105 "target",
13106 )
13107 .unwrap();
13108 forwarding
13109 .complete_pending_relay(
13110 target_ctx.connection_id,
13111 pending.corr,
13112 RouteBindRelayOutcome::Accepted,
13113 )
13114 .unwrap();
13115 let actual = query_live_roots(&handler, &target_ctx).await;
13116 let ModuleControlResponseToModule::LiveRoots {
13117 roots,
13118 unknown_root_bindings,
13119 total_bindings,
13120 } = actual
13121 else {
13122 panic!("expected live roots")
13123 };
13124 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
13125 assert_eq!(
13126 unknown_root_bindings, 1,
13127 "unknown-root arm must not read as no bindings"
13128 );
13129 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
13130 assert_eq!(
13131 total_bindings,
13132 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
13133 );
13134 }
13135
13136 #[tokio::test(start_paused = true)]
13144 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
13145 let forwarding = Arc::new(ForwardingTable::default());
13146 let handler =
13147 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
13148 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
13149 let replies = handler
13150 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
13151 .await
13152 .unwrap();
13153 let queued_by_hello = module_rx.len();
13154
13155 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
13156 let open_handler = handler.clone();
13157 let open = tokio::spawn(async move {
13158 open_handler
13159 .handle_control_frame(
13160 &client_ctx,
13161 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
13162 )
13163 .await
13164 .unwrap()
13165 });
13166 let mut spins = 0;
13168 while module_rx.len() == queued_by_hello {
13169 spins += 1;
13170 assert!(spins < 10_000, "route.open never queued a route.bind");
13171 tokio::task::yield_now().await;
13172 }
13173
13174 for reply in replies {
13176 module_ctx.egress.send(reply).await.unwrap();
13177 }
13178
13179 let first = module_rx.recv().await.unwrap().frame;
13180 assert_eq!(
13181 first.header.ty,
13182 FrameType::HelloAck,
13183 "the first frame a registering module reads must be its HELLO_ACK"
13184 );
13185 assert_eq!(first.header.corr, 7);
13186 let second = module_rx.recv().await.unwrap().frame;
13187 assert_eq!(second.header.ty, FrameType::Request);
13188 assert!(
13189 matches!(
13190 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
13191 ModuleControlRequest::RouteBind { .. }
13192 ),
13193 "the route.bind follows the ack"
13194 );
13195 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
13196
13197 handler
13198 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
13199 .await
13200 .unwrap();
13201 assert!(open.await.unwrap().is_empty());
13202 let _ = client_rx.recv().await.unwrap();
13203 }
13204
13205 #[tokio::test(start_paused = true)]
13206 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
13207 let handler = ControlHandler::with_forwarding(
13208 Arc::new(Registry::default()),
13209 Arc::new(ForwardingTable::default()),
13210 );
13211 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
13212 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
13213 hello_via_sink(
13214 &handler,
13215 &first_ctx,
13216 &mut first_rx,
13217 hello_frame("first", PROTOCOL_VERSION, 1),
13218 )
13219 .await;
13220 hello_via_sink(
13221 &handler,
13222 &second_ctx,
13223 &mut second_rx,
13224 hello_frame("second", PROTOCOL_VERSION, 2),
13225 )
13226 .await;
13227 let root = unique_project_root("second-only");
13228 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
13229 let cloned = handler.clone();
13230 let open = tokio::spawn(async move {
13231 cloned
13232 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
13233 .await
13234 .unwrap()
13235 });
13236 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
13237 .await
13238 .unwrap()
13239 .unwrap();
13240 let first = query_live_roots(&handler, &first_ctx).await;
13241 let second = query_live_roots(&handler, &second_ctx).await;
13242 assert!(
13243 matches!(
13244 first,
13245 ModuleControlResponseToModule::LiveRoots {
13246 total_bindings: 0,
13247 ..
13248 }
13249 ),
13250 "cross-module scope must not expose another module's roots"
13251 );
13252 assert!(
13253 matches!(
13254 second,
13255 ModuleControlResponseToModule::LiveRoots {
13256 total_bindings: 1,
13257 ..
13258 }
13259 ),
13260 "second module must see its pending route"
13261 );
13262 handler
13263 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
13264 .await
13265 .unwrap();
13266 assert!(open.await.unwrap().is_empty());
13267 }
13268
13269 #[tokio::test(start_paused = true)]
13270 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
13271 let handler = ControlHandler::with_forwarding(
13272 Arc::new(Registry::default()),
13273 Arc::new(ForwardingTable::default()),
13274 );
13275 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
13276 hello_via_sink(
13277 &handler,
13278 &target_ctx,
13279 &mut target_rx,
13280 hello_frame("target", PROTOCOL_VERSION, 1),
13281 )
13282 .await;
13283 let actual = query_live_roots(&handler, &target_ctx).await;
13284 let ModuleControlResponseToModule::LiveRoots {
13285 roots,
13286 unknown_root_bindings,
13287 total_bindings,
13288 } = actual
13289 else {
13290 panic!("expected live roots")
13291 };
13292 assert!(roots.is_empty());
13293 assert_eq!(unknown_root_bindings, 0);
13294 assert_eq!(total_bindings, 0);
13295 assert_eq!(
13296 total_bindings,
13297 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
13298 );
13299 }
13300
13301 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
13308 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
13309 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
13310 let text = std::fs::read_to_string(&path)
13311 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
13312 let vectors: Vec<(String, Value)> = text
13313 .lines()
13314 .filter(|line| !line.trim().is_empty())
13315 .map(|line| {
13316 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
13317 let id = entry["corpus_id"]
13318 .as_str()
13319 .expect("every vector carries a corpus_id")
13320 .to_string();
13321 (id, entry["package"].clone())
13322 })
13323 .collect();
13324 assert_eq!(
13328 vectors.len(),
13329 3,
13330 "vendored fed corpus changed size; re-sync from subc-federation"
13331 );
13332
13333 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
13348 let richest = vectors
13349 .iter()
13350 .filter_map(|(_, package)| package.as_object())
13351 .map(|object| {
13352 object
13353 .keys()
13354 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
13355 .count()
13356 })
13357 .max()
13358 .unwrap_or(0);
13359 assert!(
13360 richest >= 2,
13361 "vendored corpus no longer carries a package with unmodelled fields, \
13362 so the relay test can no longer distinguish a verbatim relay from a lossy one"
13363 );
13364
13365 vectors
13366 }
13367
13368 #[tokio::test]
13376 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
13377 for (corpus_id, package) in fed_admission_facts_vectors() {
13378 let registry = Arc::new(Registry::default());
13379 let forwarding = Arc::new(ForwardingTable::default());
13380 let supervisor = SupervisorHandle::new();
13381 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
13382 let handler =
13383 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13384 .with_supervisor(supervisor)
13385 .with_admission_facts_config(
13386 Some("fed".to_string()),
13387 Some(vec!["target".to_string()]),
13388 );
13389
13390 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
13391 hello_via_sink(
13392 &handler,
13393 &target_ctx,
13394 &mut target_rx,
13395 hello_frame("target", PROTOCOL_VERSION, 1),
13396 )
13397 .await;
13398
13399 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
13400 let route_handler = handler.clone();
13401 let expected = package.clone();
13402 let route_task = tokio::spawn(async move {
13403 route_handler
13404 .handle_control_frame(
13405 &client_ctx,
13406 route_open_frame_with_admission_facts(
13407 20,
13408 "target",
13409 unique_project_root("admission-facts"),
13410 Some(subc_control::ConsumerIdentity {
13411 module_id: "fed".to_string(),
13412 launch_nonce: "fed-nonce".to_string(),
13413 }),
13414 Some(package),
13415 ),
13416 )
13417 .await
13418 .unwrap()
13419 });
13420
13421 let bind_frame = target_rx.recv().await.unwrap();
13422 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13423 let ModuleControlRequest::RouteBind {
13424 admission_facts, ..
13425 } = bind
13426 else {
13427 panic!("{corpus_id}: expected route.bind")
13428 };
13429 assert_eq!(
13430 admission_facts,
13431 Some(expected),
13432 "{corpus_id}: relay must not add, drop or reshape any field"
13433 );
13434
13435 handler
13436 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13437 .await
13438 .unwrap();
13439 route_task.await.unwrap();
13440 }
13441 }
13442
13443 #[tokio::test]
13444 async fn route_bind_never_relays_the_openers_build_class() {
13445 let registry = Arc::new(Registry::default());
13446 let forwarding = Arc::new(ForwardingTable::default());
13447 let handler =
13448 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13449 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
13450 hello_via_sink(
13451 &handler,
13452 &target_ctx,
13453 &mut target_rx,
13454 hello_frame("target", PROTOCOL_VERSION, 1),
13455 )
13456 .await;
13457
13458 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
13459 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
13460 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13461 busy_refusals: None,
13462 target: RouteTarget::ToolProvider {
13463 module_id: "target".to_string(),
13464 },
13465 identity,
13466 consumer_identity: None,
13467 consumer_capabilities: None,
13468 role_versions: None,
13469 admission_facts: None,
13470 scope: None,
13471 })
13472 .unwrap();
13473 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
13474
13475 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
13476 let route_handler = handler.clone();
13477 let route_task = tokio::spawn(async move {
13478 route_handler
13479 .handle_control_frame(&client_ctx, open)
13480 .await
13481 .unwrap()
13482 });
13483 let bind_frame = target_rx.recv().await.unwrap();
13484 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13485 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
13486 panic!("expected route.bind")
13487 };
13488 assert_eq!(
13489 identity.build_class, None,
13490 "route.bind carried the build class"
13491 );
13492 assert!(
13493 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
13494 "the route.bind bytes still name build_class"
13495 );
13496 handler
13497 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13498 .await
13499 .unwrap();
13500 route_task.await.unwrap();
13501 }
13502
13503 #[tokio::test]
13504 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
13505 let registry = Arc::new(Registry::default());
13506 let forwarding = Arc::new(ForwardingTable::default());
13507 let supervisor = SupervisorHandle::new();
13508 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
13509 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
13510 let handler =
13511 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13512 .with_supervisor(supervisor)
13513 .with_admission_facts_config(
13514 Some("fed".to_string()),
13515 Some(vec!["target".to_string()]),
13516 );
13517
13518 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
13519 hello_via_sink(
13520 &handler,
13521 &target_ctx,
13522 &mut target_rx,
13523 hello_frame("target", PROTOCOL_VERSION, 1),
13524 )
13525 .await;
13526 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
13527 hello_via_sink(
13528 &handler,
13529 &other_ctx,
13530 &mut other_rx,
13531 hello_frame("other", PROTOCOL_VERSION, 2),
13532 )
13533 .await;
13534
13535 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
13536 let expected_facts = facts.clone();
13537 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
13538 let route_handler = handler.clone();
13539 let route_task = tokio::spawn(async move {
13540 route_handler
13541 .handle_control_frame(
13542 &client_ctx,
13543 route_open_frame_with_admission_facts(
13544 10,
13545 "target",
13546 unique_project_root("admission-facts"),
13547 Some(subc_control::ConsumerIdentity {
13548 module_id: "fed".to_string(),
13549 launch_nonce: "fed-nonce".to_string(),
13550 }),
13551 Some(facts.clone()),
13552 ),
13553 )
13554 .await
13555 .unwrap()
13556 });
13557 let bind_frame = target_rx.recv().await.unwrap();
13558 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13559 let ModuleControlRequest::RouteBind {
13560 admission_facts, ..
13561 } = bind
13562 else {
13563 panic!("expected route.bind")
13564 };
13565 assert_eq!(admission_facts, Some(expected_facts));
13566 handler
13567 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13568 .await
13569 .unwrap();
13570 assert!(route_task.await.unwrap().is_empty());
13571 assert!(matches!(
13572 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
13573 .unwrap(),
13574 ClientControlResponse::RouteOpen { .. }
13575 ));
13576
13577 let direct = handler
13578 .handle_control_frame(
13579 &route_ctx(ConnectionId::new(73)).0,
13580 route_open_frame_with_admission_facts(
13581 11,
13582 "target",
13583 unique_project_root("admission-facts"),
13584 None,
13585 Some(json!({"x": 1})),
13586 ),
13587 )
13588 .await
13589 .unwrap();
13590 assert_eq!(
13591 parse_error(&direct[0])["code"],
13592 "admission_facts_not_permitted"
13593 );
13594
13595 let different_reserved = handler
13596 .handle_control_frame(
13597 &route_ctx(ConnectionId::new(77)).0,
13598 route_open_frame_with_admission_facts(
13599 15,
13600 "target",
13601 unique_project_root("admission-facts"),
13602 Some(subc_control::ConsumerIdentity {
13603 module_id: "other".to_string(),
13604 launch_nonce: "other-nonce".to_string(),
13605 }),
13606 Some(json!({"x": 1})),
13607 ),
13608 )
13609 .await
13610 .unwrap();
13611 assert_eq!(
13612 parse_error(&different_reserved[0])["code"],
13613 "admission_facts_not_permitted"
13614 );
13615
13616 let other_target = handler
13617 .handle_control_frame(
13618 &route_ctx(ConnectionId::new(74)).0,
13619 route_open_frame_with_admission_facts(
13620 12,
13621 "other",
13622 unique_project_root("admission-facts"),
13623 Some(subc_control::ConsumerIdentity {
13624 module_id: "fed".to_string(),
13625 launch_nonce: "fed-nonce".to_string(),
13626 }),
13627 Some(json!({"x": 1})),
13628 ),
13629 )
13630 .await
13631 .unwrap();
13632 assert_eq!(
13633 parse_error(&other_target[0])["code"],
13634 "admission_facts_target_not_allowed"
13635 );
13636
13637 let nonexistent = handler
13638 .handle_control_frame(
13639 &route_ctx(ConnectionId::new(75)).0,
13640 route_open_frame_with_admission_facts(
13641 13,
13642 "missing",
13643 unique_project_root("admission-facts"),
13644 None,
13645 Some(json!({"x": 1})),
13646 ),
13647 )
13648 .await
13649 .unwrap();
13650 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13651
13652 let described = handler
13653 .handle_control_frame(
13654 &route_ctx(ConnectionId::new(76)).0,
13655 Frame::build(
13656 FrameType::Request,
13657 control_flags(),
13658 0,
13659 0,
13660 14,
13661 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13662 )
13663 .unwrap(),
13664 )
13665 .await
13666 .unwrap();
13667 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13668 serde_json::from_slice(&described[0].body).unwrap()
13669 else {
13670 panic!("expected server.describe response")
13671 };
13672 assert!(capabilities
13673 .iter()
13674 .any(|cap| cap == "admission_facts_relay_v1"));
13675 }
13676
13677 #[tokio::test]
13678 async fn admission_facts_without_configured_carrier_are_rejected() {
13679 let registry = Arc::new(Registry::default());
13680 let forwarding = Arc::new(ForwardingTable::default());
13681 let handler = ControlHandler::with_forwarding(registry, forwarding);
13682 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13683 hello_via_sink(
13684 &handler,
13685 &target_ctx,
13686 &mut target_rx,
13687 hello_frame("target", PROTOCOL_VERSION, 1),
13688 )
13689 .await;
13690
13691 let responses = handler
13692 .handle_control_frame(
13693 &route_ctx(ConnectionId::new(79)).0,
13694 route_open_frame_with_admission_facts(
13695 16,
13696 "target",
13697 unique_project_root("admission-facts"),
13698 None,
13699 Some(json!({"x": 1})),
13700 ),
13701 )
13702 .await
13703 .unwrap();
13704 assert_eq!(
13705 parse_error(&responses[0])["code"],
13706 "admission_facts_not_permitted"
13707 );
13708 }
13709
13710 #[tokio::test]
13711 async fn route_open_relays_consumer_capabilities_verbatim() {
13712 let registry = Arc::new(Registry::default());
13713 let forwarding = Arc::new(ForwardingTable::default());
13714 let handler =
13715 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13716 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13717 hello_via_sink(
13718 &handler,
13719 &module_ctx,
13720 &mut module_rx,
13721 hello_frame("aft", PROTOCOL_VERSION, 7),
13722 )
13723 .await;
13724
13725 let expected = vec!["elicitation".to_string(), "roots".to_string()];
13726 let expected_for_request = expected.clone();
13727 let project_root = unique_project_root("consumer-capabilities-present");
13728 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13729 let route_handler = handler.clone();
13730 let route_task = tokio::spawn(async move {
13731 route_handler
13732 .handle_control_frame(
13733 &client_ctx,
13734 route_open_frame_with_consumer_capabilities(
13735 401,
13736 "aft",
13737 project_root,
13738 Some(expected_for_request),
13739 ),
13740 )
13741 .await
13742 .unwrap()
13743 });
13744 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13745 .await
13746 .unwrap()
13747 .unwrap();
13748 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13749 let ModuleControlRequest::RouteBind {
13750 consumer_capabilities,
13751 ..
13752 } = bind
13753 else {
13754 panic!("expected route.bind request, got {bind:?}");
13755 };
13756 assert_eq!(consumer_capabilities, Some(expected.clone()));
13757
13758 handler
13759 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13760 .await
13761 .unwrap();
13762 let route_response = route_task.await.unwrap();
13763 assert!(route_response.is_empty());
13764 let published = client_rx.recv().await.unwrap();
13765 assert!(matches!(
13766 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13767 ClientControlResponse::RouteOpen { .. }
13768 ));
13769 }
13770
13771 #[tokio::test]
13772 async fn route_open_without_consumer_capabilities_relays_none() {
13773 let registry = Arc::new(Registry::default());
13774 let forwarding = Arc::new(ForwardingTable::default());
13775 let handler =
13776 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13777 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13778 hello_via_sink(
13779 &handler,
13780 &module_ctx,
13781 &mut module_rx,
13782 hello_frame("aft", PROTOCOL_VERSION, 7),
13783 )
13784 .await;
13785
13786 let project_root = unique_project_root("consumer-capabilities-absent");
13787 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13788 let route_handler = handler.clone();
13789 let route_task = tokio::spawn(async move {
13790 route_handler
13791 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13792 .await
13793 .unwrap()
13794 });
13795 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13796 .await
13797 .unwrap()
13798 .unwrap();
13799 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13800 let ModuleControlRequest::RouteBind {
13801 consumer_capabilities,
13802 ..
13803 } = bind
13804 else {
13805 panic!("expected route.bind request, got {bind:?}");
13806 };
13807 assert_eq!(consumer_capabilities, None);
13808
13809 handler
13810 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13811 .await
13812 .unwrap();
13813 let route_response = route_task.await.unwrap();
13814 assert!(route_response.is_empty());
13815 let published = client_rx.recv().await.unwrap();
13816 assert!(matches!(
13817 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13818 ClientControlResponse::RouteOpen { .. }
13819 ));
13820 }
13821
13822 async fn bind_role_versions_for(
13826 sent: Option<BTreeMap<String, String>>,
13827 connection: u64,
13828 ) -> Option<BTreeMap<String, String>> {
13829 let registry = Arc::new(Registry::default());
13830 let forwarding = Arc::new(ForwardingTable::default());
13831 let handler =
13832 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13833 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13834 hello_via_sink(
13835 &handler,
13836 &module_ctx,
13837 &mut module_rx,
13838 hello_frame("aft", PROTOCOL_VERSION, 7),
13839 )
13840 .await;
13841 let project_root = unique_project_root("role-versions");
13842 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13843 let route_handler = handler.clone();
13844 let route_task = tokio::spawn(async move {
13845 route_handler
13846 .handle_control_frame(
13847 &client_ctx,
13848 route_open_frame_with_role_versions(403, "aft", project_root, sent),
13849 )
13850 .await
13851 .unwrap()
13852 });
13853 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13854 .await
13855 .expect("a well-formed route.open reaches the module as a bind")
13856 .unwrap();
13857 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13858 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13859 panic!("expected route.bind request, got {bind:?}");
13860 };
13861 handler
13862 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13863 .await
13864 .unwrap();
13865 assert!(route_task.await.unwrap().is_empty());
13866 let published = client_rx.recv().await.unwrap();
13867 assert!(matches!(
13868 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13869 ClientControlResponse::RouteOpen { .. }
13870 ));
13871 role_versions
13872 }
13873
13874 #[tokio::test]
13875 async fn route_open_relays_role_versions_verbatim() {
13876 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13877 assert_eq!(
13878 bind_role_versions_for(Some(sent.clone()), 141).await,
13879 Some(sent)
13880 );
13881 }
13882
13883 #[tokio::test]
13886 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13887 assert_eq!(bind_role_versions_for(None, 143).await, None);
13888 assert_eq!(
13889 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13890 None
13891 );
13892 }
13893
13894 #[tokio::test]
13895 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13896 let registry = Arc::new(Registry::default());
13897 let forwarding = Arc::new(ForwardingTable::default());
13898 let handler =
13899 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13900 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13901 hello_via_sink(
13902 &handler,
13903 &module_ctx,
13904 &mut module_rx,
13905 hello_frame("aft", PROTOCOL_VERSION, 7),
13906 )
13907 .await;
13908
13909 let nine: BTreeMap<String, String> = (0..9)
13910 .map(|index| (format!("role-{index}"), "v1".to_string()))
13911 .collect();
13912 for (label, malformed) in [
13913 (
13914 "invalid role name",
13915 role_versions(&[("Tool_Provider", "v1")]),
13916 ),
13917 ("invalid version", role_versions(&[("tool-provider", "v0")])),
13918 ("nine entries", nine),
13919 ] {
13920 let responses = tokio::time::timeout(
13923 Duration::from_secs(1),
13924 handler.handle_control_frame(
13925 &route_ctx(ConnectionId::new(148)).0,
13926 route_open_frame_with_role_versions(
13927 404,
13928 "aft",
13929 unique_project_root("role-versions-malformed"),
13930 Some(malformed),
13931 ),
13932 ),
13933 )
13934 .await
13935 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13936 .unwrap();
13937 assert_eq!(responses.len(), 1, "{label}");
13938 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13939 let error = parse_error(&responses[0]);
13940 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13941 assert_eq!(
13942 error["detail"]["field"], "role_versions",
13943 "{label}: {error}"
13944 );
13945 assert!(
13946 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13947 "{label}: a malformed declaration is terminal"
13948 );
13949 assert!(
13950 module_rx.try_recv().is_err(),
13951 "{label}: the module must never see a bind"
13952 );
13953 }
13954 }
13955
13956 #[tokio::test]
13960 async fn route_role_versions_capability_is_advertised() {
13961 let handler = ControlHandler::new(Arc::new(Registry::default()));
13962 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13963 let ack = hello_via_sink(
13964 &handler,
13965 &ctx,
13966 &mut rx,
13967 hello_frame("m", PROTOCOL_VERSION, 1),
13968 )
13969 .await;
13970 let ack = parse_ack(&ack);
13971 assert!(
13972 ack.subc_capabilities
13973 .iter()
13974 .any(|c| c == "route-role-versions/v1"),
13975 "{:?}",
13976 ack.subc_capabilities
13977 );
13978
13979 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13980 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13981 let reply = handler
13982 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13983 .await
13984 .unwrap()
13985 .pop()
13986 .unwrap();
13987 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13988 serde_json::from_slice(&reply.body).unwrap()
13989 else {
13990 panic!("not a server.describe reply");
13991 };
13992 assert!(
13993 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13994 "{capabilities:?}"
13995 );
13996 }
13997
13998 #[tokio::test]
14002 async fn catalog_events_capability_is_advertised() {
14003 let handler = ControlHandler::new(Arc::new(Registry::default()));
14004 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14005 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14006 let reply = handler
14007 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
14008 .await
14009 .unwrap()
14010 .pop()
14011 .unwrap();
14012 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14013 serde_json::from_slice(&reply.body).unwrap()
14014 else {
14015 panic!("not a server.describe reply");
14016 };
14017 assert!(
14018 capabilities
14019 .iter()
14020 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
14021 "{capabilities:?}"
14022 );
14023 }
14024
14025 #[tokio::test]
14026 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
14027 let handler = ControlHandler::new(Arc::new(Registry::default()));
14028 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
14029 let ack = hello_via_sink(
14030 &handler,
14031 &module,
14032 &mut module_rx,
14033 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
14034 )
14035 .await;
14036 let ack = parse_ack(&ack);
14037 assert!(
14038 ack.subc_capabilities
14039 .iter()
14040 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
14041 "{:?}",
14042 ack.subc_capabilities
14043 );
14044 assert!(ack
14045 .subc_capabilities
14046 .iter()
14047 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
14048
14049 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
14050 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14051 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14052 let reply = handler
14053 .handle_control_frame(&client, frame)
14054 .await
14055 .unwrap()
14056 .pop()
14057 .unwrap();
14058 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14059 serde_json::from_slice(&reply.body).unwrap()
14060 else {
14061 panic!("not a server.describe reply");
14062 };
14063 assert!(
14064 capabilities
14065 .iter()
14066 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
14067 "{capabilities:?}"
14068 );
14069 assert!(capabilities
14070 .iter()
14071 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
14072 }
14073
14074 #[tokio::test]
14075 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
14076 let handler = ControlHandler::new(Arc::new(Registry::default()));
14077 let (module, mut rx) = route_ctx(ConnectionId::new(156));
14078 let ack = hello_via_sink(
14079 &handler,
14080 &module,
14081 &mut rx,
14082 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
14083 )
14084 .await;
14085 assert!(parse_ack(&ack)
14086 .subc_capabilities
14087 .iter()
14088 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
14089 assert!(parse_ack(&ack)
14090 .subc_capabilities
14091 .iter()
14092 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
14093 let (client, _rx) = route_ctx(ConnectionId::new(157));
14094 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14095 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14096 let reply = handler
14097 .handle_control_frame(&client, frame)
14098 .await
14099 .unwrap()
14100 .pop()
14101 .unwrap();
14102 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14103 serde_json::from_slice(&reply.body).unwrap()
14104 else {
14105 panic!("server.describe reply")
14106 };
14107 assert!(capabilities
14108 .iter()
14109 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
14110 assert!(capabilities
14111 .iter()
14112 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
14113 }
14114
14115 #[tokio::test]
14116 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
14117 let handler = ControlHandler::new(Arc::new(Registry::default()));
14118 let (module, mut rx) = route_ctx(ConnectionId::new(158));
14119 let ack = hello_via_sink(
14120 &handler,
14121 &module,
14122 &mut rx,
14123 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
14124 )
14125 .await;
14126 assert!(parse_ack(&ack)
14127 .subc_capabilities
14128 .iter()
14129 .any(|cap| cap == "scope-workdir/v1"));
14130 let (client, _rx) = route_ctx(ConnectionId::new(159));
14131 let frame = Frame::build(
14132 FrameType::Request,
14133 control_flags(),
14134 0,
14135 0,
14136 5,
14137 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
14138 )
14139 .unwrap();
14140 let reply = handler
14141 .handle_control_frame(&client, frame)
14142 .await
14143 .unwrap()
14144 .remove(0);
14145 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14146 serde_json::from_slice(&reply.body).unwrap()
14147 else {
14148 panic!("server.describe reply")
14149 };
14150 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
14151 }
14152
14153 #[tokio::test]
14154 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
14155 let handler = ControlHandler::new(Arc::new(Registry::default()));
14156 let (module, mut rx) = route_ctx(ConnectionId::new(160));
14157 let ack = hello_via_sink(
14158 &handler,
14159 &module,
14160 &mut rx,
14161 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
14162 )
14163 .await;
14164 assert!(parse_ack(&ack)
14165 .subc_capabilities
14166 .iter()
14167 .any(|cap| cap == "worker-runs/v1"));
14168 let (client, _rx) = route_ctx(ConnectionId::new(161));
14169 let frame = Frame::build(
14170 FrameType::Request,
14171 control_flags(),
14172 0,
14173 0,
14174 5,
14175 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
14176 )
14177 .unwrap();
14178 let reply = handler
14179 .handle_control_frame(&client, frame)
14180 .await
14181 .unwrap()
14182 .remove(0);
14183 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14184 serde_json::from_slice(&reply.body).unwrap()
14185 else {
14186 panic!("server.describe reply")
14187 };
14188 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
14189 }
14190
14191 #[tokio::test]
14192 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
14193 let handler = ControlHandler::new(Arc::new(Registry::default()));
14194 let (module, mut rx) = route_ctx(ConnectionId::new(154));
14195 let ack = hello_via_sink(
14196 &handler,
14197 &module,
14198 &mut rx,
14199 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
14200 )
14201 .await;
14202 assert!(parse_ack(&ack)
14203 .subc_capabilities
14204 .iter()
14205 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
14206 let (client, _rx) = route_ctx(ConnectionId::new(155));
14207 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14208 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14209 let reply = handler
14210 .handle_control_frame(&client, frame)
14211 .await
14212 .unwrap()
14213 .pop()
14214 .unwrap();
14215 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14216 serde_json::from_slice(&reply.body).unwrap()
14217 else {
14218 panic!("not a server.describe reply")
14219 };
14220 assert!(capabilities
14221 .iter()
14222 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
14223 }
14224
14225 #[tokio::test]
14226 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
14227 let registry = Arc::new(Registry::default());
14228 let forwarding = Arc::new(ForwardingTable::default());
14229 let handler =
14230 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14231 .with_health_probe_timeout(Duration::from_secs(5));
14232 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
14233 hello_via_sink(
14234 &handler,
14235 &module_ctx,
14236 &mut module_rx,
14237 non_routable_hello_frame_with_control_ops(
14238 "mcp",
14239 300,
14240 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14241 ),
14242 )
14243 .await;
14244 assert!(registry
14245 .get_module("mcp")
14246 .unwrap()
14247 .unwrap()
14248 .manifest
14249 .provides
14250 .is_empty());
14251
14252 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
14253 let route_response = handler
14254 .handle_control_frame(
14255 &route_client_ctx,
14256 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
14257 )
14258 .await
14259 .unwrap();
14260 assert_eq!(route_response[0].header.ty, FrameType::Error);
14261 assert_eq!(
14262 parse_error(&route_response[0])["code"],
14263 "target_unavailable"
14264 );
14265 assert!(parse_error(&route_response[0])["message"]
14266 .as_str()
14267 .unwrap()
14268 .contains("does not provide the requested target"));
14269 assert!(module_rx.try_recv().is_err());
14270
14271 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
14272 let health_handler = handler.clone();
14273 let health_task = tokio::spawn(async move {
14274 health_handler
14275 .handle_control_frame(
14276 &health_client_ctx,
14277 supervisor_health_probe_frame(302, "mcp"),
14278 )
14279 .await
14280 .unwrap()
14281 });
14282 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14283 .await
14284 .unwrap()
14285 .unwrap();
14286 assert_eq!(
14287 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
14288 ModuleControlRequest::HealthCheck {}
14289 );
14290 handler
14291 .handle_control_frame(
14292 &module_ctx,
14293 health_response(health_frame.header.corr, HealthStatus::Ok),
14294 )
14295 .await
14296 .unwrap();
14297 let health_response = health_task.await.unwrap();
14298 assert_eq!(health_response[0].header.ty, FrameType::Response);
14299 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
14300 ClientControlResponse::SupervisorHealthProbe {
14301 module_id, status, ..
14302 } => {
14303 assert_eq!(module_id, "mcp");
14304 assert_eq!(status, HealthStatus::Ok);
14305 }
14306 other => panic!("unexpected health response: {other:?}"),
14307 }
14308
14309 forwarding
14314 .cleanup_connection(module_ctx.connection_id)
14315 .unwrap();
14316 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
14317 let cleanup_response = tokio::time::timeout(
14318 Duration::from_millis(200),
14319 handler.handle_control_frame(
14320 &cleanup_probe_ctx,
14321 supervisor_health_probe_frame(303, "mcp"),
14322 ),
14323 )
14324 .await
14325 .expect("probe should fail immediately when the control lane is gone")
14326 .unwrap();
14327 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
14328 assert_eq!(
14329 parse_error(&cleanup_response[0])["code"],
14330 "target_unavailable"
14331 );
14332 assert!(parse_error(&cleanup_response[0])["message"]
14333 .as_str()
14334 .unwrap()
14335 .contains("no module connection"));
14336
14337 handler
14338 .cleanup_connection(module_ctx.connection_id)
14339 .unwrap();
14340 }
14341
14342 #[tokio::test]
14343 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
14344 let registry = Arc::new(Registry::default());
14345 let supervisor_handle = SupervisorHandle::new();
14346 let supervisor =
14347 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14348 .with_handle(supervisor_handle.clone())
14349 .with_connection_file_path(
14350 std::env::temp_dir()
14351 .join(format!("subc-route-open-warming-{}", std::process::id())),
14352 );
14353 let module = supervisor
14354 .supervise_configured(
14355 ModuleSpec {
14356 carry_containment: false,
14357 module_id: "warming".to_string(),
14358 program: fake_aft_stub_path(),
14359 args: Vec::new(),
14360 env: Vec::new(),
14361 reserved: false,
14362 reserved_prefixes: Vec::new(),
14363 protocol: ModuleProtocol::Subc,
14364 overlap: Default::default(),
14365 },
14366 true,
14367 )
14368 .unwrap();
14369 assert_eq!(module.state().unwrap(), ModuleState::Running);
14370
14371 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14372 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
14373 let response = handler
14374 .handle_control_frame(
14375 &ctx,
14376 route_open_frame(304, "warming", unique_project_root("warming")),
14377 )
14378 .await
14379 .unwrap();
14380 module.stop().await.unwrap();
14381
14382 assert_eq!(response[0].header.ty, FrameType::Error);
14383 let error = parse_error(&response[0]);
14384 assert_eq!(error["code"], "module_warming");
14385 assert!(error["message"]
14386 .as_str()
14387 .unwrap()
14388 .contains("state=running, enabled=true, live=false"));
14389 }
14390
14391 #[test]
14392 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
14393 let handler = ControlHandler::new(Arc::new(Registry::default()));
14394 let capture = EventCapture::default();
14395 let _subscriber =
14396 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14397 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
14398 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
14399 let pending = (0..limit).collect::<Vec<_>>();
14400 let response = handler
14401 .route_open_capacity_refusal(
14402 &ctx,
14403 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
14404 "busy",
14405 pending.len(),
14406 limit,
14407 )
14408 .unwrap();
14409 assert_eq!(parse_error(&response)["code"], "target_unavailable");
14410 let event = capture
14411 .events()
14412 .into_iter()
14413 .find(|event| {
14414 event.target == "control"
14415 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
14416 })
14417 .expect("connection admission refusal event");
14418 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14419 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14420 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14423 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14424 }
14425
14426 #[test]
14427 fn route_open_target_cap_logs_admission_reason_and_capacity() {
14428 let handler = ControlHandler::new(Arc::new(Registry::default()));
14429 let capture = EventCapture::default();
14430 let _subscriber =
14431 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14432 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
14433 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
14434 let guards = (0..limit)
14435 .map(|_| {
14436 handler
14437 .route_bind_concurrency
14438 .try_admit("busy", limit)
14439 .unwrap()
14440 })
14441 .collect::<Vec<_>>();
14442 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
14443 Err(in_flight) => in_flight,
14444 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
14445 };
14446 let response = handler
14447 .route_open_target_capacity_refusal(
14448 &ctx,
14449 &RouteOpenCaller::unknown(),
14450 &route_open_frame(397, "busy", unique_project_root("target-cap")),
14451 "busy",
14452 in_flight,
14453 )
14454 .unwrap();
14455 assert_eq!(parse_error(&response)["code"], "target_unavailable");
14456 let event = capture
14457 .events()
14458 .into_iter()
14459 .find(|event| {
14460 event.target == "control"
14461 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
14462 })
14463 .expect("target admission refusal event");
14464 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14465 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14466 drop(guards);
14467 }
14468
14469 #[cfg(unix)]
14470 #[tokio::test(start_paused = true)]
14471 async fn supervisor_list_shows_a_failed_modules_next_retry() {
14472 let registry = Arc::new(Registry::default());
14473 let handle = SupervisorHandle::new();
14474 let supervisor =
14475 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
14476 .with_handle(handle.clone());
14477 let module = supervisor
14478 .spawn(crate::ModuleSpec {
14479 carry_containment: false,
14480 module_id: "retry-list".into(),
14481 program: PathBuf::from("/bin/sh"),
14482 args: vec!["-c".into(), "exit 17".into()],
14483 env: vec![],
14484 reserved: false,
14485 reserved_prefixes: vec![],
14486 protocol: ModuleProtocol::None,
14487 overlap: Default::default(),
14488 })
14489 .unwrap();
14490 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14491 while module.state().unwrap() != crate::ModuleState::Failed {
14492 assert!(
14493 std::time::Instant::now() < deadline,
14494 "waiting for budget failure"
14495 );
14496 tokio::task::yield_now().await;
14497 }
14498 let handler = ControlHandler::new(registry).with_supervisor(handle);
14499 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14500 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14501 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14502 assert_eq!(body["modules"][0]["state"], "failed");
14503 let retry = body["modules"][0]["next_retry_at_ms"]
14504 .as_u64()
14505 .expect("scheduled retry on the wire");
14506 let now = std::time::SystemTime::now()
14507 .duration_since(std::time::UNIX_EPOCH)
14508 .unwrap()
14509 .as_millis() as u64;
14510 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
14511 module.set_enabled(false).await.unwrap();
14512 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14513 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14514 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14515 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
14516 }
14517
14518 #[tokio::test]
14519 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
14520 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
14521 let supervisor =
14522 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14523 let module = supervisor
14524 .spawn(crate::ModuleSpec {
14525 carry_containment: false,
14526 module_id: "pid-read-budget".into(),
14527 program: std::env::current_exe().unwrap(),
14528 args: vec![
14529 "--exact".into(),
14530 "control::tests::pid_resource_read_fixture".into(),
14531 ],
14532 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14533 .into_iter()
14534 .map(|key| (key.into(), dir.display().to_string()))
14535 .chain(std::iter::once((
14536 "SUBC_SUBTREE_PID_FIXTURE".into(),
14537 "1".into(),
14538 )))
14539 .collect(),
14540 reserved: false,
14541 reserved_prefixes: Vec::new(),
14542 protocol: ModuleProtocol::None,
14543 overlap: Default::default(),
14544 })
14545 .unwrap();
14546 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14547 while !matches!(
14548 module.child_resource_usage(),
14549 subc_control::ChildResourceUsage::Measured(_)
14550 ) {
14551 assert!(
14552 tokio::time::Instant::now() < ready_deadline,
14553 "fixture process was not measurable"
14554 );
14555 tokio::time::sleep(Duration::from_millis(10)).await;
14556 }
14557 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
14558 let reading = {
14559 let _guard = crate::subtree_resources::fault_read_guard().await;
14560 crate::subtree_resources::read_process(module.clone()).await
14561 };
14562 stop_resource_fixture(&module, &dir).await;
14563 assert!(
14564 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
14565 "{reading:?}"
14566 );
14567 }
14568
14569 #[test]
14570 fn pid_resource_read_fixture() {
14571 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
14572 let dir = std::path::PathBuf::from(std::env::var_os("XDG_DATA_HOME").unwrap());
14573 let deadline = std::time::Instant::now() + Duration::from_secs(60);
14574 while !dir.join("stop-module").exists() && std::time::Instant::now() < deadline {
14575 std::thread::sleep(Duration::from_millis(5));
14576 }
14577 std::fs::write(dir.join("module-stopped"), "").unwrap();
14578 }
14579 }
14580
14581 async fn stop_resource_fixture(
14582 module: &crate::supervise::SupervisedModule,
14583 dir: &std::path::Path,
14584 ) {
14585 let stopping = module.stop();
14586 tokio::pin!(stopping);
14587 tokio::select! {
14588 result = &mut stopping => result.unwrap(),
14589 () = async {
14590 while module.state().unwrap() != crate::supervise::ModuleState::Draining {
14591 tokio::time::sleep(Duration::from_millis(5)).await;
14592 }
14593 std::fs::write(dir.join("stop-module"), "").unwrap();
14594 } => stopping.await.unwrap(),
14595 }
14596 #[cfg(windows)]
14597 assert!(
14598 dir.join("module-stopped").exists(),
14599 "the resource fixture must exit cooperatively, not wait for the drain kill"
14600 );
14601 assert_eq!(
14602 module.state().unwrap(),
14603 crate::supervise::ModuleState::Stopped
14604 );
14605 }
14606
14607 #[tokio::test]
14608 async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
14609 let registry = Arc::new(Registry::default());
14610 let handle = SupervisorHandle::new();
14611 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14612 .with_handle(handle.clone());
14613 supervisor
14614 .supervise_configured(
14615 crate::ModuleSpec {
14616 module_id: "disabled-resource-reading".into(),
14617 program: std::env::current_exe().unwrap(),
14618 args: Vec::new(),
14619 env: Vec::new(),
14620 reserved: false,
14621 reserved_prefixes: Vec::new(),
14622 protocol: ModuleProtocol::None,
14623 overlap: Default::default(),
14624 carry_containment: false,
14625 },
14626 false,
14627 )
14628 .unwrap();
14629 let handler = ControlHandler::new(registry).with_supervisor(handle);
14630 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14631 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14632 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14633 assert!(body["modules"][0].get("tree").is_none());
14634 assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
14635 }
14636
14637 #[tokio::test]
14638 async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
14639 let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
14640 let registry = Arc::new(Registry::default());
14641 let handle = SupervisorHandle::new();
14642 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14643 .with_handle(handle.clone());
14644 let module = supervisor
14645 .spawn(crate::ModuleSpec {
14646 carry_containment: false,
14647 module_id: "list-process-priority".into(),
14648 program: std::env::current_exe().unwrap(),
14649 args: vec![
14650 "--exact".into(),
14651 "control::tests::pid_resource_read_fixture".into(),
14652 ],
14653 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14654 .into_iter()
14655 .map(|key| (key.into(), dir.display().to_string()))
14656 .chain(std::iter::once((
14657 "SUBC_SUBTREE_PID_FIXTURE".into(),
14658 "1".into(),
14659 )))
14660 .collect(),
14661 reserved: false,
14662 reserved_prefixes: Vec::new(),
14663 protocol: ModuleProtocol::None,
14664 overlap: Default::default(),
14665 })
14666 .unwrap();
14667 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14668 loop {
14669 let image = module.running_image_agreement().await;
14670 if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
14671 break;
14672 }
14673 assert!(
14674 tokio::time::Instant::now() < ready_deadline,
14675 "the fixture process never became observable: {image:?}"
14676 );
14677 tokio::time::sleep(Duration::from_millis(10)).await;
14678 }
14679 let guard = crate::subtree_resources::fault_read_guard().await;
14680 let (target, release, _) = crate::subtree_resources::held();
14681 module.set_subtree_target_for_test(target);
14682 let handler = ControlHandler::new(registry).with_supervisor(handle);
14683 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14684 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14685 drop(release);
14686 drop(guard);
14687 stop_resource_fixture(&module, &dir).await;
14688 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14689 assert_eq!(
14690 body["modules"][0]["resources"]["status"], "measured",
14691 "the optional tree timeout must not discard the process reading: {body}"
14692 );
14693 assert!(body["modules"][0].get("tree").is_none());
14694 assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
14695 assert_eq!(
14696 body["modules"][0]["pending_reload"]["image"]["status"], "match",
14697 "the tree timeout must not hide a completed image observation"
14698 );
14699 }
14700
14701 #[cfg(any(target_os = "linux", target_os = "macos"))]
14702 #[tokio::test]
14703 async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
14704 let mut handler = ControlHandler::new(Arc::new(Registry::default()))
14705 .with_daemon_provenance(
14706 std::process::id(),
14707 0,
14708 Some(std::env::current_exe().unwrap()),
14709 None,
14710 None,
14711 );
14712 let (probe, release, began) = ExecutableIdentityProbe::held();
14713 handler.daemon_provenance.probe = probe;
14714 let observe = |handler: ControlHandler, corr| {
14715 tokio::spawn(async move {
14716 let (ctx, _) = route_ctx(ConnectionId::new(corr));
14717 let frame =
14718 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
14719 .unwrap();
14720 handler
14721 .handle_supervisor_provenance(&ctx, frame, None)
14722 .await
14723 .unwrap()
14724 })
14725 };
14726 let explicit = observe(handler.clone(), 114);
14727 began.await.unwrap();
14728 let concurrent = observe(handler.clone(), 115);
14729 tokio::time::sleep(Duration::from_millis(300)).await;
14732 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14733 let list = tokio::time::timeout(
14734 Duration::from_secs(1),
14735 handler.handle_supervisor_list(frame),
14736 )
14737 .await;
14738 let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
14739 drop(release);
14740 let replies = explicit.await.unwrap();
14741 let concurrent_replies = concurrent.await.unwrap();
14742 assert!(
14743 explicit_was_pending,
14744 "explicit provenance must await the full observation"
14745 );
14746 let list = list
14747 .expect("a concurrent explicit probe must not stall list")
14748 .unwrap();
14749 let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
14750 assert_eq!(list_body["op"], "supervisor.list");
14751 for replies in [replies, concurrent_replies] {
14752 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14753 assert_eq!(
14754 body["daemon"]["daemon_observed"]["running_image"]["status"],
14755 "match"
14756 );
14757 }
14758 }
14759
14760 #[tokio::test]
14761 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
14762 let registry = Arc::new(Registry::default());
14763 let handle = SupervisorHandle::new();
14764 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14765 .with_handle(handle.clone());
14766 let module = supervisor
14767 .supervise_configured(
14768 crate::ModuleSpec {
14769 carry_containment: false,
14770 module_id: "memory-read".into(),
14771 program: PathBuf::from("disabled-module"),
14772 args: Vec::new(),
14773 env: Vec::new(),
14774 reserved: false,
14775 reserved_prefixes: Vec::new(),
14776 protocol: ModuleProtocol::Subc,
14777 overlap: Default::default(),
14778 },
14779 true,
14780 )
14781 .unwrap();
14782 let _guard = crate::subtree_resources::fault_read_guard().await;
14783 let (target, release, began) = crate::subtree_resources::held();
14784 let stalled_target = target.clone();
14787 let stalled = tokio::task::spawn_blocking(move || {
14788 stalled_target.tree(&subc_os::ProcessTable::default())
14789 });
14790 began.await.unwrap();
14791 module.set_subtree_target_for_test(target);
14792 let handler = ControlHandler::new(registry).with_supervisor(handle);
14793 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
14794 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14795 let replies = handler
14796 .handle_client_control_request(
14797 &ctx,
14798 frame,
14799 ClientControlRequest::SupervisorList {},
14800 &mut None,
14801 )
14802 .await
14803 .unwrap();
14804 assert!(
14805 replies.is_empty(),
14806 "list reply must be detached from the connection reader"
14807 );
14808 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14809 let replies = handler
14810 .handle_client_control_request(
14811 &ctx,
14812 frame,
14813 ClientControlRequest::SupervisorSpawnSnapshot {},
14814 &mut None,
14815 )
14816 .await
14817 .unwrap();
14818 assert_eq!(
14819 replies.len(),
14820 1,
14821 "another request must complete while the subtree read stalls"
14822 );
14823 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14824 .await
14825 .unwrap()
14826 .unwrap();
14827 release.send(()).unwrap();
14828 drop(release);
14829 stalled.await.unwrap();
14830 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
14831 assert!(
14832 body["modules"][0].get("tree").is_none(),
14833 "an unknown tree reading must be omitted after the bounded read expires"
14834 );
14835 handler
14839 .resource_tracker
14840 .read(handler.supervisor.list(), false, "released_tree_fixture")
14841 .await;
14842 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14843 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14844 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14845 assert_eq!(
14846 body["modules"][0]["tree"]["source"], "process_tree",
14847 "the next request must export the completed refresh"
14848 );
14849 assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
14850 }
14851
14852 #[tokio::test]
14853 async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
14854 use subc_control::{
14855 AgentCommandResources, ChildMemoryKind, ChildResourceReading,
14856 ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
14857 TreeReading, TreeSource,
14858 };
14859 let registry = Arc::new(Registry::default());
14860 let handle = SupervisorHandle::new();
14861 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14862 .with_handle(handle.clone());
14863 supervisor
14864 .supervise_configured(
14865 crate::ModuleSpec {
14866 carry_containment: false,
14867 module_id: "budget-worker".into(),
14868 program: PathBuf::from("disabled-module"),
14869 args: Vec::new(),
14870 env: Vec::new(),
14871 reserved: false,
14872 reserved_prefixes: Vec::new(),
14873 protocol: ModuleProtocol::Subc,
14874 overlap: Default::default(),
14875 },
14876 true,
14877 )
14878 .unwrap();
14879 let handler = ControlHandler::new(registry).with_supervisor(handle);
14880 let now = u64::try_from(
14881 std::time::SystemTime::now()
14882 .duration_since(std::time::UNIX_EPOCH)
14883 .unwrap()
14884 .as_millis(),
14885 )
14886 .unwrap();
14887 let measured = now - 1_000;
14888 let hour = CpuLastHour::new(100, 60_000);
14889 let daemon = DaemonResources::new(7, 1, "test")
14890 .with_measured_at_ms(Some(measured))
14891 .with_resources(Some(ChildResourceUsage::Measured(
14892 ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
14893 )))
14894 .with_cpu_last_hour(Some(hour.clone()));
14895 let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
14896 .with_measured_at_ms(Some(measured))
14897 .with_cpu_last_hour(Some(hour.clone()));
14898 let tree = TreeReading::new(
14899 150,
14900 50,
14901 ChildMemoryKind::ResidentSet,
14902 1,
14903 TreeSource::ProcessTree,
14904 )
14905 .with_measured_at_ms(Some(measured));
14906 handler.resource_tracker.seed_committed_for_test(
14907 &handler.supervisor.list(),
14908 crate::resource_totals::Readings {
14909 modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
14910 daemon: Some(daemon),
14911 agent_commands: Some(vec![agents]),
14912 ..Default::default()
14913 },
14914 now,
14915 );
14916 list_diagnostics::scope(async {
14917 tokio::time::sleep(Duration::from_millis(110)).await;
14918 assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
14919 let request =
14920 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14921 let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
14922 let ClientControlResponse::SupervisorList {
14923 modules,
14924 daemon,
14925 agent_commands,
14926 ..
14927 } = serde_json::from_slice(&replies[0].body).unwrap()
14928 else {
14929 panic!("expected a list response")
14930 };
14931 assert!(matches!(
14932 modules[0].resources,
14933 Some(ChildResourceUsage::Unavailable {
14934 reason: ChildResourceUnavailableReason::Unreadable
14935 })
14936 ));
14937 assert_eq!(
14938 (
14939 modules[0].tree.as_ref().unwrap().cpu_ms,
14940 modules[0].tree.as_ref().unwrap().measured_at_ms
14941 ),
14942 (150, Some(measured))
14943 );
14944 let daemon = daemon.unwrap();
14945 let agents = agent_commands.unwrap();
14946 assert_eq!(daemon.measured_at_ms, Some(measured));
14947 assert_eq!(
14948 (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
14949 (1, 50, Some(measured))
14950 );
14951 for hour in [
14952 modules[0].cpu_last_hour.as_ref(),
14953 daemon.cpu_last_hour.as_ref(),
14954 agents[0].cpu_last_hour.as_ref(),
14955 ] {
14956 let hour = hour.unwrap();
14957 assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
14958 }
14959 })
14960 .await;
14961 }
14962
14963 #[tokio::test]
14964 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14965 let registry = Arc::new(Registry::default());
14966 let handle = SupervisorHandle::new();
14967 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14968 .with_handle(handle.clone());
14969 supervisor
14970 .supervise_configured(
14971 crate::ModuleSpec {
14972 carry_containment: false,
14973 module_id: "drift".into(),
14974 program: PathBuf::from("disabled-module"),
14975 args: Vec::new(),
14976 env: Vec::new(),
14977 reserved: false,
14978 reserved_prefixes: Vec::new(),
14979 protocol: ModuleProtocol::Subc,
14980 overlap: Default::default(),
14981 },
14982 false,
14983 )
14984 .unwrap();
14985 let handler = ControlHandler::new(registry).with_supervisor(handle);
14986 for configured in [None, Some(false), Some(true)] {
14987 handler.capability_evaluator.configure(
14988 configured.map(|enabled| ("drift".to_string(), enabled)),
14989 BTreeMap::new(),
14990 );
14991 let frame =
14992 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14993 let replies = handler
14994 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14995 .await
14996 .unwrap();
14997 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14998 assert!(body.get("configured_enabled").is_none());
14999 assert_eq!(body["op"], "supervisor.ack");
15000 let frame =
15001 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
15002 let replies = handler.handle_supervisor_list(frame).await.unwrap();
15003 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
15004 assert_eq!(
15005 body["modules"][0].get("configured_enabled"),
15006 configured.map(Value::Bool).as_ref()
15007 );
15008 assert!(body["modules"][0].get("tree").is_none());
15009 assert!(body["modules"][0].get("subtree").is_none());
15010 }
15011 }
15012
15013 #[tokio::test(flavor = "current_thread")]
15014 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
15015 let handler = ControlHandler::new(Arc::new(Registry::default()));
15016 let capture = EventCapture::default();
15017 let _subscriber =
15018 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15019 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
15020 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
15021
15022 let _ = handler
15023 .handle_client_control_request(
15024 &ctx,
15025 frame,
15026 ClientControlRequest::SupervisorSetEnabled {
15027 module_id: "broca".to_string(),
15028 enabled: false,
15029 },
15030 &mut None,
15031 )
15032 .await
15033 .unwrap();
15034
15035 let events = capture
15036 .events()
15037 .into_iter()
15038 .filter(|event| {
15039 event.target == "control"
15040 && event.fields.get("message").map(String::as_str)
15041 == Some("supervisor request received")
15042 })
15043 .collect::<Vec<_>>();
15044 assert_eq!(events.len(), 1, "one supervisor request log line");
15045 let event = &events[0];
15046 assert_eq!(event.level, tracing::Level::INFO);
15047 assert_eq!(
15048 event.fields.get("op"),
15049 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
15050 );
15051 assert_eq!(
15052 event.fields.get("module_id"),
15053 Some(&"\"broca\"".to_string())
15054 );
15055 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
15056 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
15057 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
15058 }
15059
15060 #[tokio::test(flavor = "current_thread")]
15061 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
15062 let registry = Arc::new(Registry::default());
15063 let module_connection = ConnectionId::new(303);
15064 registry
15065 .register_with_control_ops(
15066 manifest("aft", PROTOCOL_VERSION),
15067 PROTOCOL_VERSION,
15068 module_connection,
15069 module_baseline_control_ops(),
15070 )
15071 .unwrap();
15072 let handler = ControlHandler::new(registry);
15073 let capture = EventCapture::default();
15074 let _subscriber =
15075 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15076 let (ctx, _rx) = route_ctx(module_connection);
15077 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
15078
15079 let _ = handler
15080 .handle_client_control_request(
15081 &ctx,
15082 frame,
15083 ClientControlRequest::SupervisorSetEnabled {
15084 module_id: "broca".to_string(),
15085 enabled: false,
15086 },
15087 &mut None,
15088 )
15089 .await
15090 .unwrap();
15091
15092 let callers = capture
15093 .events()
15094 .into_iter()
15095 .filter(|event| {
15096 event.fields.get("message").map(String::as_str)
15097 == Some("supervisor request received")
15098 })
15099 .map(|event| event.fields.get("caller").cloned())
15100 .collect::<Vec<_>>();
15101 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
15102 }
15103
15104 #[tokio::test(flavor = "current_thread")]
15105 async fn supervisor_list_does_not_log_request_received() {
15106 let handler = ControlHandler::new(Arc::new(Registry::default()));
15107 let capture = EventCapture::default();
15108 let _subscriber =
15109 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15110 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
15111 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
15112
15113 handler
15114 .handle_client_control_request(
15115 &ctx,
15116 frame,
15117 ClientControlRequest::SupervisorList {},
15118 &mut None,
15119 )
15120 .await
15121 .unwrap();
15122
15123 assert!(capture.events().into_iter().all(|event| {
15124 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
15125 }));
15126 }
15127
15128 #[tokio::test(flavor = "current_thread")]
15129 async fn supervisor_rescan_preview_does_not_log_request_received() {
15130 let handler = ControlHandler::new(Arc::new(Registry::default()));
15131 let capture = EventCapture::default();
15132 let _subscriber =
15133 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15134 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
15135 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
15136
15137 handler
15138 .handle_client_control_request(
15139 &ctx,
15140 frame,
15141 ClientControlRequest::SupervisorRescan { preview: true },
15142 &mut None,
15143 )
15144 .await
15145 .unwrap();
15146
15147 assert!(capture.events().into_iter().all(|event| {
15148 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
15149 }));
15150 }
15151
15152 struct ApproveOperator;
15153 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
15154 fn prompt(
15155 &self,
15156 _: &str,
15157 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
15158 ) -> crate::operator_confirm::ProviderResult {
15159 crate::operator_confirm::ProviderResult::Approved
15160 }
15161 }
15162
15163 #[tokio::test]
15164 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
15165 assert_eq!(
15166 ControlHandler::default().privileged_operator_prompt,
15167 PrivilegedOperatorPrompt::Log
15168 );
15169 for (initial, changed, running) in [
15170 ("log", "enforce", PrivilegedOperatorPrompt::Log),
15171 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
15172 ] {
15173 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
15174 let path = dir.join("subc.jsonc");
15175 std::fs::write(
15176 &path,
15177 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
15178 )
15179 .unwrap();
15180 let config = crate::daemon_config::load(&path).unwrap().unwrap();
15181 let registry = Arc::new(Registry::default());
15182 let handle = SupervisorHandle::new();
15183 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
15184 .with_handle(handle.clone());
15185 let handler = ControlHandler::new(registry)
15186 .with_supervisor(handle)
15187 .with_privileged_operator_prompt(config.privileged_operator_prompt)
15188 .with_supervisor_rescan(supervisor, &path, config.port);
15189 handler.forwarding.operator_confirms().configure(
15190 Arc::new(ApproveOperator),
15191 crate::operator_confirm::OperatorLimits::default(),
15192 );
15193 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
15194
15195 for (disk_mode, needs_restart) in [
15196 (Some(initial), false),
15197 (Some(changed), true),
15198 (Some(changed), true),
15199 (None, running == PrivilegedOperatorPrompt::Enforce),
15200 (Some(initial), false),
15201 ] {
15202 let mut doc = json!({"version":1});
15203 if let Some(mode) = disk_mode {
15204 doc["privileged_operator_prompt"] = json!(mode);
15205 }
15206 std::fs::write(&path, doc.to_string()).unwrap();
15207 for preview in [true, false] {
15208 let request = ClientControlRequest::SupervisorRescan { preview };
15209 let frame = Frame::build(
15210 FrameType::Request,
15211 control_flags(),
15212 0,
15213 0,
15214 1,
15215 serde_json::to_vec(&request).unwrap(),
15216 )
15217 .unwrap();
15218 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
15219 assert_eq!(frames.len(), 1);
15220 let ClientControlResponse::SupervisorRescan { result } =
15221 serde_json::from_slice(&frames[0].body).unwrap()
15222 else {
15223 panic!("expected a rescan response");
15224 };
15225 let expected = if needs_restart {
15226 vec!["privileged_operator_prompt"]
15227 } else {
15228 Vec::new()
15229 };
15230 assert_eq!(result.restart_required, expected);
15231 assert_eq!(handler.privileged_operator_prompt, running);
15232 assert_eq!(
15233 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
15234 running
15235 );
15236 }
15237 }
15238 }
15239 }
15240
15241 #[tokio::test]
15242 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
15243 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
15244 let path = dir.join("subc.jsonc");
15245 let registry = Arc::new(Registry::default());
15246 let handle = SupervisorHandle::new();
15247 let supervisor =
15248 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
15249 let handler = ControlHandler::new(registry)
15250 .with_supervisor(handle)
15251 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
15252 .with_supervisor_rescan(supervisor, &path, None);
15253 handler.forwarding.operator_confirms().configure(
15254 Arc::new(ApproveOperator),
15255 crate::operator_confirm::OperatorLimits::default(),
15256 );
15257 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
15258 for preview in [true, false] {
15259 std::fs::write(
15260 &path,
15261 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
15262 )
15263 .unwrap();
15264 let request = ClientControlRequest::SupervisorRescan { preview };
15265 let frame = Frame::build(
15266 FrameType::Request,
15267 control_flags(),
15268 0,
15269 0,
15270 1,
15271 serde_json::to_vec(&request).unwrap(),
15272 )
15273 .unwrap();
15274 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
15275 assert_eq!(frames.len(), 1);
15276 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
15277 assert_eq!(
15278 handler.privileged_operator_prompt,
15279 PrivilegedOperatorPrompt::Enforce
15280 );
15281 }
15282 }
15283
15284 #[tokio::test]
15289 async fn route_open_refusal_names_the_check_that_refused() {
15290 let handler = ControlHandler::new(Arc::new(Registry::default()));
15291 let capture = EventCapture::default();
15292 let _subscriber =
15293 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15294 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15295 let response = handler
15296 .handle_control_frame(
15297 &ctx,
15298 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
15299 )
15300 .await
15301 .unwrap();
15302
15303 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15304 let event = capture
15305 .events()
15306 .into_iter()
15307 .find(|event| {
15308 event.target == "control"
15309 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
15310 })
15311 .expect("route.open refusal event");
15312 assert_eq!(
15313 event.fields.get("reason"),
15314 Some(&"\"not_registered\"".to_string())
15315 );
15316 }
15317
15318 #[tokio::test]
15319 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
15320 let registry = Arc::new(Registry::default());
15321 let supervisor_handle = SupervisorHandle::new();
15322 let supervisor =
15323 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
15324 .with_handle(supervisor_handle.clone())
15325 .with_connection_file_path(std::env::temp_dir().join(format!(
15326 "subc-route-open-refusal-info-{}",
15327 std::process::id()
15328 )));
15329 let module = supervisor
15330 .supervise_configured(
15331 ModuleSpec {
15332 carry_containment: false,
15333 module_id: "warming".to_string(),
15334 program: fake_aft_stub_path(),
15335 args: Vec::new(),
15336 env: Vec::new(),
15337 reserved: false,
15338 reserved_prefixes: Vec::new(),
15339 protocol: ModuleProtocol::Subc,
15340 overlap: Default::default(),
15341 },
15342 true,
15343 )
15344 .unwrap();
15345 assert_eq!(module.state().unwrap(), ModuleState::Running);
15346
15347 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15348 assert!(handler
15349 .counters()
15350 .snapshot()
15351 .get("route_open_refused_by_code")
15352 .is_none());
15353 let capture = EventCapture::default();
15354 let _subscriber =
15355 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15356 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
15357 let response = handler
15358 .handle_control_frame(
15359 &ctx,
15360 route_open_frame(394, "warming", unique_project_root("refusal-info")),
15361 )
15362 .await
15363 .unwrap();
15364 module.stop().await.unwrap();
15365
15366 assert_eq!(parse_error(&response[0])["code"], "module_warming");
15367 let event = capture
15368 .events()
15369 .into_iter()
15370 .find(|event| {
15371 event.target == "control"
15372 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
15373 })
15374 .expect("route.open refusal event");
15375 assert_eq!(
15376 event.fields.get("module_id"),
15377 Some(&"\"warming\"".to_string())
15378 );
15379 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
15380 assert_eq!(
15381 event.fields.get("reason"),
15382 Some(&"\"supervised_not_registered\"".to_string())
15383 );
15384 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
15385 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
15386 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
15387 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
15388 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
15389 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
15390 assert_eq!(
15391 handler.counters().snapshot()["route_open_refused_by_code"],
15392 json!({ "module_warming": 1 })
15393 );
15394 }
15395
15396 const OUTAGE_START: &str = "route.open refusing module: not serving";
15397 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
15398
15399 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
15400 capture
15401 .events()
15402 .into_iter()
15403 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
15404 .collect()
15405 }
15406
15407 fn supervise_stub(
15408 registry: &Arc<Registry>,
15409 module_id: &str,
15410 enabled: bool,
15411 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
15412 let supervisor_handle = SupervisorHandle::new();
15413 let supervisor =
15414 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
15415 .with_handle(supervisor_handle.clone())
15416 .with_connection_file_path(std::env::temp_dir().join(format!(
15417 "subc-route-outage-{module_id}-{}",
15418 std::process::id()
15419 )));
15420 let module = supervisor
15421 .supervise_configured(
15422 ModuleSpec {
15423 carry_containment: false,
15424 module_id: module_id.to_string(),
15425 program: fake_aft_stub_path(),
15426 args: Vec::new(),
15427 env: Vec::new(),
15428 reserved: false,
15429 reserved_prefixes: Vec::new(),
15430 protocol: ModuleProtocol::Subc,
15431 overlap: Default::default(),
15432 },
15433 enabled,
15434 )
15435 .unwrap();
15436 (supervisor_handle, module)
15437 }
15438
15439 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
15440 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
15441 module_id: module_id.to_string(),
15442 drain_timeout_ms: Some(50),
15443 })
15444 .unwrap();
15445 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
15446 }
15447
15448 #[test]
15452 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
15453 let registry = Arc::new(Registry::default());
15454 let forwarding = Arc::new(ForwardingTable::default());
15455 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15456 let second = ControlHandler::with_forwarding(registry, forwarding);
15457 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
15458 }
15459
15460 #[tokio::test(flavor = "current_thread")]
15464 async fn route_open_unknown_module_refusals_add_no_outage_state() {
15465 let handler = ControlHandler::new(Arc::new(Registry::default()));
15466 let capture = EventCapture::default();
15467 let _subscriber =
15468 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15469 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
15470 for corr in 0..8 {
15471 let response = handler
15472 .handle_control_frame(
15473 &ctx,
15474 route_open_frame(
15475 380 + corr,
15476 &format!("nobody-{corr}"),
15477 unique_project_root("outage-unknown"),
15478 ),
15479 )
15480 .await
15481 .unwrap();
15482 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15483 }
15484
15485 assert_eq!(handler.route_outages.tracked_module_count(), 0);
15486 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
15487 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
15488 }
15489
15490 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15495 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
15496 let registry = Arc::new(Registry::default());
15497 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
15498 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15499 let capture = EventCapture::default();
15500 let _subscriber =
15501 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15502 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
15503 handler.route_outages.record_accepted("outage-restart");
15506
15507 let response = handler
15508 .handle_control_frame(
15509 &ctx,
15510 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
15511 )
15512 .await
15513 .unwrap();
15514 assert_eq!(response[0].header.ty, FrameType::Error);
15515 let starts = outage_lines(&capture, OUTAGE_START);
15516 assert_eq!(starts.len(), 1, "{starts:?}");
15517 assert_eq!(starts[0].level, tracing::Level::WARN);
15518 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
15519 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
15520 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
15521 handler.route_outages.record_accepted("outage-restart");
15522 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
15523
15524 let restart = handler
15525 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
15526 .await
15527 .unwrap();
15528 assert_eq!(
15529 restart[0].header.ty,
15530 FrameType::Response,
15531 "{:?}",
15532 parse_error(&restart[0])
15533 );
15534 handler
15535 .handle_control_frame(
15536 &ctx,
15537 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
15538 )
15539 .await
15540 .unwrap();
15541 module.stop().await.unwrap();
15542
15543 let starts = outage_lines(&capture, OUTAGE_START);
15544 assert_eq!(starts.len(), 2, "{starts:?}");
15545 assert_eq!(starts[1].level, tracing::Level::INFO);
15546 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
15547 }
15548
15549 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15553 async fn failed_operator_restart_leaves_no_operator_mark() {
15554 let registry = Arc::new(Registry::default());
15555 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
15556 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15557 let capture = EventCapture::default();
15558 let _subscriber =
15559 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15560 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
15561 handler.route_outages.record_accepted("outage-disabled");
15562
15563 let restart = handler
15564 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
15565 .await
15566 .unwrap();
15567 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
15568 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
15569
15570 handler
15571 .handle_control_frame(
15572 &ctx,
15573 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
15574 )
15575 .await
15576 .unwrap();
15577 let starts = outage_lines(&capture, OUTAGE_START);
15578 assert_eq!(starts.len(), 1, "{starts:?}");
15579 assert_eq!(starts[0].level, tracing::Level::WARN);
15580 }
15581
15582 #[tokio::test(flavor = "current_thread")]
15583 async fn route_open_unknown_module_escapes_target_module_id() {
15584 let handler = ControlHandler::new(Arc::new(Registry::default()));
15585 let capture = EventCapture::default();
15586 let _subscriber =
15587 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15588 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
15589 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15590 let response = handler
15591 .handle_control_frame(
15592 &ctx,
15593 route_open_frame(
15594 395,
15595 hostile_module_id,
15596 unique_project_root("hostile-target-module-id"),
15597 ),
15598 )
15599 .await
15600 .unwrap();
15601
15602 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15603 let event = capture
15604 .events()
15605 .into_iter()
15606 .find(|event| {
15607 event.target == "control"
15608 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
15609 })
15610 .expect("route.open unknown-module refusal event");
15611 let logged = event.fields.get("module_id").expect("module_id field");
15612 assert!(!logged.bytes().any(|byte| byte < 0x20));
15613 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15614 }
15615
15616 #[tokio::test(flavor = "current_thread")]
15619 async fn route_open_refusal_carries_daemon_origin() {
15620 let handler = ControlHandler::new(Arc::new(Registry::default()));
15621 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15622 let response = handler
15623 .handle_control_frame(
15624 &ctx,
15625 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
15626 )
15627 .await
15628 .unwrap();
15629
15630 assert_eq!(response.len(), 1);
15631 assert_eq!(response[0].header.ty, FrameType::Error);
15632 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15633 assert!(
15634 response[0].header.flags.is_daemon_origin(),
15635 "route.open refusal must carry daemon origin: {:?}",
15636 response[0].header.flags
15637 );
15638 }
15639
15640 #[tokio::test(flavor = "current_thread")]
15644 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
15645 let registry = Arc::new(Registry::default());
15646 let forwarding = Arc::new(ForwardingTable::default());
15647 let handler =
15648 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15649 let module_connection = ConnectionId::new(95);
15650 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15651 hello_via_sink(
15652 &handler,
15653 &module_ctx,
15654 &mut module_rx,
15655 hello_frame("aft", PROTOCOL_VERSION, 395),
15656 )
15657 .await;
15658 forwarding
15659 .begin_module_drain("aft", RouteCloseReason::Restart)
15660 .unwrap()
15661 .expect("the module endpoint drains");
15662
15663 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
15664 let response = handler
15665 .handle_control_frame(
15666 &client_ctx,
15667 route_open_frame(396, "aft", unique_project_root("drain-origin")),
15668 )
15669 .await
15670 .unwrap();
15671
15672 assert_eq!(response.len(), 1);
15673 assert_eq!(response[0].header.ty, FrameType::Error);
15674 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15675 assert!(
15676 response[0].header.flags.is_daemon_origin(),
15677 "drain module_reloading must carry daemon origin: {:?}",
15678 response[0].header.flags
15679 );
15680 }
15681
15682 #[tokio::test(flavor = "current_thread")]
15686 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
15687 let registry = Arc::new(Registry::default());
15688 let forwarding = Arc::new(ForwardingTable::default());
15689 let handler =
15690 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15691 let module_connection = ConnectionId::new(95);
15692 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15693 hello_via_sink(
15694 &handler,
15695 &module_ctx,
15696 &mut module_rx,
15697 hello_frame("aft", PROTOCOL_VERSION, 395),
15698 )
15699 .await;
15700
15701 let client_connection = ConnectionId::new(96);
15702 let (client_ctx, _client_rx) = route_ctx(client_connection);
15703 let (route_task, bind) = relay_route_open(
15704 &handler,
15705 client_connection,
15706 &client_ctx.egress,
15707 &mut module_rx,
15708 396,
15709 "aft",
15710 "spoofed-reloading",
15711 )
15712 .await;
15713 let rejection = Frame::build(
15716 FrameType::Error,
15717 crate::router::daemon_error_flags(),
15718 0,
15719 0,
15720 bind.header.corr,
15721 serde_json::to_vec(&ErrorBody::new(
15722 "module_reloading",
15723 "module refused route.bind",
15724 ))
15725 .unwrap(),
15726 )
15727 .unwrap();
15728 handler
15729 .handle_control_frame(&module_ctx, rejection)
15730 .await
15731 .unwrap();
15732
15733 let response = route_task.await.unwrap();
15734 assert_eq!(response.len(), 1);
15735 assert_eq!(response[0].header.ty, FrameType::Error);
15736 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15737 assert!(
15738 !response[0].header.flags.is_daemon_origin(),
15739 "a module's own rejection must not carry daemon origin: {:?}",
15740 response[0].header.flags
15741 );
15742 }
15743
15744 #[tokio::test(flavor = "current_thread")]
15745 async fn route_open_module_rejection_uses_daemon_counter_key() {
15746 let registry = Arc::new(Registry::default());
15747 let forwarding = Arc::new(ForwardingTable::default());
15748 let handler =
15749 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15750 let module_connection = ConnectionId::new(95);
15751 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15752 hello_via_sink(
15753 &handler,
15754 &module_ctx,
15755 &mut module_rx,
15756 hello_frame("aft", PROTOCOL_VERSION, 395),
15757 )
15758 .await;
15759
15760 let client_connection = ConnectionId::new(96);
15761 let (client_ctx, _client_rx) = route_ctx(client_connection);
15762 let capture = EventCapture::default();
15763 let _subscriber =
15764 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15765 let (route_task, bind) = relay_route_open(
15766 &handler,
15767 client_connection,
15768 &client_ctx.egress,
15769 &mut module_rx,
15770 396,
15771 "aft",
15772 "hostile-module-code",
15773 )
15774 .await;
15775 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
15776 let rejection = Frame::build(
15777 FrameType::Error,
15778 control_flags(),
15779 0,
15780 0,
15781 bind.header.corr,
15782 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
15783 )
15784 .unwrap();
15785 handler
15786 .handle_control_frame(&module_ctx, rejection)
15787 .await
15788 .unwrap();
15789
15790 let response = route_task.await.unwrap();
15791 assert_eq!(parse_error(&response[0])["code"], hostile_code);
15792 let counters = handler.counters().snapshot();
15793 assert_eq!(
15794 counters["route_open_refused_by_code"],
15795 json!({ "module_rejected": 1 })
15796 );
15797 assert!(counters["route_open_refused_by_code"]
15798 .get(hostile_code)
15799 .is_none());
15800
15801 let event = capture
15802 .events()
15803 .into_iter()
15804 .find(|event| {
15805 event.target == "control"
15806 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
15807 })
15808 .expect("route.open module-rejection refusal event");
15809 let logged = event.fields.get("module_code").expect("module_code field");
15810 assert!(!logged.bytes().any(|byte| byte < 0x20));
15811 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15812 }
15813
15814 #[tokio::test]
15815 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
15816 let registry = Arc::new(Registry::default());
15817 let supervisor_handle = SupervisorHandle::new();
15818 let missing_program = std::env::temp_dir().join(format!(
15819 "subc-route-open-missing-program-{}",
15820 std::process::id()
15821 ));
15822 let supervisor =
15823 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
15824 .with_handle(supervisor_handle.clone());
15825 let module = supervisor
15826 .supervise_configured(
15827 ModuleSpec {
15828 carry_containment: false,
15829 module_id: "failed".to_string(),
15830 program: missing_program,
15831 args: Vec::new(),
15832 env: Vec::new(),
15833 reserved: false,
15834 reserved_prefixes: Vec::new(),
15835 protocol: ModuleProtocol::Subc,
15836 overlap: Default::default(),
15837 },
15838 true,
15839 )
15840 .unwrap();
15841 assert_eq!(module.state().unwrap(), ModuleState::Failed);
15842
15843 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15844 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
15845 let response = handler
15846 .handle_control_frame(
15847 &ctx,
15848 route_open_frame(305, "failed", unique_project_root("failed")),
15849 )
15850 .await
15851 .unwrap();
15852
15853 assert_eq!(response[0].header.ty, FrameType::Error);
15854 let error = parse_error(&response[0]);
15855 assert_eq!(error["code"], "target_unavailable");
15856 assert!(error["message"]
15857 .as_str()
15858 .unwrap()
15859 .contains("state=failed, enabled=true, live=false"));
15860 }
15861
15862 #[tokio::test]
15863 async fn route_open_role_mismatch_remains_target_unavailable() {
15864 let registry = Arc::new(Registry::default());
15865 let handler = ControlHandler::new(Arc::clone(®istry));
15866 handler
15867 .handle_control(
15868 ConnectionId::new(41),
15869 non_routable_hello_frame_with_control_ops("health-only", 306, None),
15870 )
15871 .unwrap();
15872
15873 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
15874 let response = handler
15875 .handle_control_frame(
15876 &ctx,
15877 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
15878 )
15879 .await
15880 .unwrap();
15881
15882 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15883 assert!(parse_error(&response[0])["message"]
15884 .as_str()
15885 .unwrap()
15886 .contains("does not provide the requested target"));
15887 }
15888
15889 #[tokio::test]
15890 async fn route_open_inactive_registration_remains_target_unavailable() {
15891 let registry = Arc::new(Registry::default());
15892 let handler = ControlHandler::new(Arc::clone(®istry));
15893 handler
15894 .handle_control(
15895 ConnectionId::new(43),
15896 hello_frame("inactive", PROTOCOL_VERSION, 308),
15897 )
15898 .unwrap();
15899 assert!(registry
15900 .set_module_state_for_test("inactive", ChannelState::Closed)
15901 .unwrap());
15902
15903 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
15904 let response = handler
15905 .handle_control_frame(
15906 &ctx,
15907 route_open_frame(309, "inactive", unique_project_root("inactive")),
15908 )
15909 .await
15910 .unwrap();
15911
15912 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15913 assert!(parse_error(&response[0])["message"]
15914 .as_str()
15915 .unwrap()
15916 .contains("is not active"));
15917 }
15918
15919 #[tokio::test]
15920 async fn late_health_reply_is_recorded_through_the_module_response_path() {
15921 let registry = Arc::new(Registry::default());
15922 let forwarding = Arc::new(ForwardingTable::default());
15923 let supervisor_handle = SupervisorHandle::new();
15924 let supervisor =
15925 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
15926 .with_forwarding(Arc::clone(&forwarding))
15927 .with_handle(supervisor_handle.clone());
15928 let module = supervisor
15929 .supervise_configured(
15930 crate::ModuleSpec {
15931 carry_containment: false,
15932 module_id: "late-health-response".to_string(),
15933 program: PathBuf::from("disabled-module"),
15934 args: Vec::new(),
15935 env: Vec::new(),
15936 reserved: false,
15937 reserved_prefixes: Vec::new(),
15938 protocol: ModuleProtocol::Subc,
15939 overlap: Default::default(),
15940 },
15941 false,
15942 )
15943 .unwrap();
15944 let handler =
15945 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15946 .with_supervisor(supervisor_handle);
15947 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
15948 handler
15949 .handle_control_frame(
15950 &module_ctx,
15951 hello_frame_with_control_ops(
15952 "late-health-response",
15953 PROTOCOL_VERSION,
15954 7,
15955 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15956 ),
15957 )
15958 .await
15959 .unwrap();
15960 let probe_started_at = Instant::now() - Duration::from_millis(80);
15961 let pending = forwarding
15962 .begin_health_probe_rpc_for(
15963 "late-health-response",
15964 MODULE_CONTROL_OP_HEALTH_CHECK,
15965 probe_started_at,
15966 Instant::now() - Duration::from_millis(1),
15967 )
15968 .unwrap();
15969 assert!(forwarding
15970 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15971 .unwrap());
15972
15973 let responses = handler
15974 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15975 .await
15976 .unwrap();
15977
15978 assert!(responses.is_empty());
15979 let health = module.status().unwrap().health;
15980 assert_eq!(health.late_answer_count, 1);
15981 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15982 }
15983
15984 #[tokio::test]
15985 async fn health_probe_timeout_and_module_death_are_typed() {
15986 let registry = Arc::new(Registry::default());
15987 let forwarding = Arc::new(ForwardingTable::default());
15988 let handler =
15989 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15990 .with_health_probe_timeout(Duration::from_millis(50));
15991 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15992 hello_via_sink(
15993 &handler,
15994 &module_ctx,
15995 &mut module_rx,
15996 hello_frame_with_control_ops(
15997 "aft",
15998 PROTOCOL_VERSION,
15999 7,
16000 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
16001 ),
16002 )
16003 .await;
16004
16005 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
16006 let responses = handler
16007 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
16008 .await
16009 .unwrap();
16010 assert_eq!(responses[0].header.ty, FrameType::Error);
16011 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
16012 let _ = module_rx.try_recv();
16013
16014 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
16015 let health_handler = handler.clone();
16016 let death_task = tokio::spawn(async move {
16017 health_handler
16018 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
16019 .await
16020 .unwrap()
16021 });
16022 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
16023 .await
16024 .unwrap()
16025 .unwrap();
16026 handler
16027 .cleanup_connection(module_ctx.connection_id)
16028 .unwrap();
16029 let responses = death_task.await.unwrap();
16030 assert_eq!(responses[0].header.ty, FrameType::Error);
16031 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
16032 }
16033
16034 #[test]
16035 fn hello_requires_exact_protocol_version() {
16036 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
16037 let registry = Arc::new(Registry::default());
16038 let handler = ControlHandler::new(Arc::clone(®istry));
16039 let responses = handler
16040 .handle_control(
16041 ConnectionId::new(connection),
16042 hello_frame("aft", offered, 9),
16043 )
16044 .unwrap();
16045
16046 assert_eq!(responses.len(), 1);
16047 assert_eq!(responses[0].header.ty, FrameType::Error);
16048 let error = parse_error(&responses[0]);
16049 assert_eq!(error["code"], "version_unsupported");
16050 assert!(registry.get_module("aft").unwrap().is_none());
16051 assert_eq!(registry.active_registration_count().unwrap(), 0);
16052 }
16053 }
16054
16055 #[test]
16056 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
16057 let registry = Arc::new(Registry::default());
16058 let forwarding = Arc::new(ForwardingTable::default());
16059 let handler =
16060 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16061 let module_connection = ConnectionId::new(301);
16062 let registration = registry
16063 .register_with_control_ops(
16064 manifest("aft-push", PROTOCOL_VERSION),
16065 PROTOCOL_VERSION,
16066 module_connection,
16067 module_baseline_control_ops(),
16068 )
16069 .unwrap();
16070 let (module_tx, _module_rx) = mpsc::channel(8);
16071 let endpoint = forwarding
16072 .register_module_connection(
16073 module_connection,
16074 "aft-push".to_string(),
16075 PROTOCOL_VERSION,
16076 manifest_concurrency(®istration.manifest),
16077 FrameSink::new(module_tx),
16078 )
16079 .unwrap();
16080
16081 let unknown = Frame::build(
16083 FrameType::Push,
16084 control_flags(),
16085 0,
16086 0,
16087 5,
16088 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
16089 )
16090 .unwrap();
16091 let out = handler.handle_status_update(endpoint, unknown).unwrap();
16092 assert!(
16093 out.is_empty(),
16094 "unknown push op must be ignored, got {out:?}"
16095 );
16096
16097 let malformed = Frame::build(
16099 FrameType::Push,
16100 control_flags(),
16101 0,
16102 0,
16103 6,
16104 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
16105 )
16106 .unwrap();
16107 let out = handler.handle_status_update(endpoint, malformed).unwrap();
16108 assert_eq!(out.len(), 1);
16109 assert_eq!(out[0].header.ty, FrameType::Error);
16110 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
16111 }
16112
16113 #[test]
16114 fn hello_rejected_when_connection_already_owns_client_routes() {
16115 let registry = Arc::new(Registry::default());
16116 let forwarding = Arc::new(ForwardingTable::default());
16117 let handler =
16118 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16119 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
16121 let client_connection = ConnectionId::new(202);
16122
16123 let responses = handler
16126 .handle_control(
16127 client_connection,
16128 hello_frame("aft-second", PROTOCOL_VERSION, 9),
16129 )
16130 .unwrap();
16131 assert_eq!(responses[0].header.ty, FrameType::Error);
16132 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
16133 assert!(registry.get_module("aft-second").unwrap().is_none());
16134 }
16135
16136 #[tokio::test]
16137 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
16138 let registry = Arc::new(Registry::default());
16139 let forwarding = Arc::new(ForwardingTable::default());
16140 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
16141 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
16142 hello_via_sink(
16143 &handler,
16144 &module_ctx,
16145 &mut module_rx,
16146 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
16147 )
16148 .await;
16149 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
16150 let pending = forwarding
16151 .begin_route_bind_relay_for_test(
16152 client_ctx.connection_id,
16153 client_ctx.egress.clone(),
16154 2,
16155 "alpha",
16156 )
16157 .unwrap();
16158 forwarding
16159 .complete_pending_relay(
16160 module_ctx.connection_id,
16161 pending.corr,
16162 RouteBindRelayOutcome::Accepted,
16163 )
16164 .unwrap();
16165 client_rx.try_recv().unwrap();
16166 for module_id in ["beta", "alpha"] {
16167 let replies = handler
16168 .handle_control_frame(
16169 &module_ctx,
16170 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
16171 )
16172 .await
16173 .unwrap();
16174 assert_eq!(replies.len(), 1, "second HELLO must be refused");
16175 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
16176 }
16177 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
16178 assert!(registry.get_module("beta").unwrap().is_none());
16179 assert!(matches!(
16180 forwarding
16181 .lookup_data_route(
16182 client_ctx.connection_id,
16183 pending.client_channel,
16184 pending.client_epoch,
16185 )
16186 .unwrap(),
16187 DataRoute::Client(DataRouteState::Bound(_))
16188 ));
16189 assert!(handler
16190 .hello_launch_nonces
16191 .lock()
16192 .unwrap()
16193 .presented(module_ctx.connection_id, Some("alpha-nonce")));
16194 assert!(module_rx.try_recv().is_err());
16195 }
16196
16197 #[test]
16198 fn reserved_module_hello_requires_matching_launch_nonce() {
16199 let registry = Arc::new(Registry::default());
16200 let supervisor = SupervisorHandle::new();
16201 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
16204 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
16205
16206 let no_nonce = handler
16208 .handle_control(
16209 ConnectionId::new(1),
16210 hello_frame("vault", PROTOCOL_VERSION, 1),
16211 )
16212 .unwrap();
16213 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
16214 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
16215 assert!(registry.get_module("vault").unwrap().is_none());
16216
16217 let wrong = handler
16219 .handle_control(
16220 ConnectionId::new(2),
16221 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
16222 )
16223 .unwrap();
16224 assert_eq!(wrong[0].header.ty, FrameType::Error);
16225 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
16226 assert!(registry.get_module("vault").unwrap().is_none());
16227
16228 let ok = handler
16230 .handle_control(
16231 ConnectionId::new(3),
16232 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
16233 )
16234 .unwrap();
16235 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
16236 assert!(registry.get_module("vault").unwrap().is_some());
16237 }
16238
16239 #[test]
16240 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
16241 let registry = Arc::new(Registry::default());
16242 let supervisor = SupervisorHandle::new();
16243 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
16244 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
16245 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
16246
16247 let squat = handler
16248 .handle_control(
16249 ConnectionId::new(1),
16250 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
16251 )
16252 .unwrap();
16253 assert_eq!(squat[0].header.ty, FrameType::Error);
16254 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
16255 assert!(parse_error(&squat[0])["message"]
16256 .as_str()
16257 .unwrap()
16258 .contains("fed:"));
16259
16260 let accepted_peer = handler
16261 .handle_control(
16262 ConnectionId::new(2),
16263 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
16264 )
16265 .unwrap();
16266 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
16267
16268 let accepted_short = handler
16269 .handle_control(
16270 ConnectionId::new(3),
16271 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
16272 )
16273 .unwrap();
16274 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
16275
16276 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
16277 let response = handler
16278 .handle_control(
16279 ConnectionId::new(conn),
16280 hello_frame(module_id, PROTOCOL_VERSION, conn),
16281 )
16282 .unwrap();
16283 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
16284 }
16285 }
16286
16287 #[test]
16288 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
16289 let registry = Arc::new(Registry::default());
16290 let supervisor = SupervisorHandle::new();
16291 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
16292 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
16293 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
16294 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
16295
16296 let owner_nonce = handler
16297 .handle_control(
16298 ConnectionId::new(1),
16299 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
16300 )
16301 .unwrap();
16302 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
16303 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
16304 assert!(registry.get_module("fed:special").unwrap().is_none());
16305
16306 let exact_nonce = handler
16307 .handle_control(
16308 ConnectionId::new(2),
16309 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
16310 )
16311 .unwrap();
16312 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
16313 assert!(registry.get_module("fed:special").unwrap().is_some());
16314 }
16315
16316 #[test]
16317 fn non_reserved_module_ignores_launch_nonce() {
16318 let registry = Arc::new(Registry::default());
16319 let handler = ControlHandler::new(Arc::clone(®istry));
16322 let no_nonce = handler
16323 .handle_control(
16324 ConnectionId::new(1),
16325 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
16326 )
16327 .unwrap();
16328 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
16329 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
16330
16331 let echoed_nonce = handler
16332 .handle_control(
16333 ConnectionId::new(2),
16334 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
16335 )
16336 .unwrap();
16337 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
16338 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
16339 }
16340
16341 #[test]
16342 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
16343 let handler = ControlHandler::default();
16344 let conn = ConnectionId::new(1);
16345 let malformed = Frame::build(
16346 FrameType::Hello,
16347 control_flags(),
16348 0,
16349 0,
16350 3,
16351 b"{not json".to_vec(),
16352 )
16353 .unwrap();
16354
16355 let error = handler.handle_control(conn, malformed).unwrap();
16356 assert_eq!(error[0].header.ty, FrameType::Error);
16357 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
16358
16359 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
16360 let pong = handler.handle_control(conn, ping).unwrap();
16361 assert_eq!(pong[0].header.ty, FrameType::Pong);
16362 assert_eq!(pong[0].header.corr, 4);
16363 }
16364
16365 #[test]
16366 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
16367 let registry = Arc::new(Registry::default());
16368 let handler = ControlHandler::new(Arc::clone(®istry));
16369
16370 handler
16371 .handle_control(
16372 ConnectionId::new(1),
16373 hello_frame("aft", PROTOCOL_VERSION, 1),
16374 )
16375 .unwrap();
16376 let duplicate = handler
16377 .handle_control(
16378 ConnectionId::new(2),
16379 hello_frame("aft", PROTOCOL_VERSION, 2),
16380 )
16381 .unwrap();
16382
16383 assert_eq!(duplicate[0].header.ty, FrameType::Error);
16384 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16385 let registration = registry.get_module("aft").unwrap().unwrap();
16386 assert_eq!(registration.connection_id, ConnectionId::new(1));
16387 }
16388
16389 #[test]
16390 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
16391 let registry = Arc::new(Registry::default());
16392 let forwarding = Arc::new(ForwardingTable::default());
16393 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
16394 let handler =
16395 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16396 .with_process_liveness(process_liveness);
16397 let (ctx, route_channel, route_epoch) =
16398 bind_liveness_route(®istry, &forwarding, "aft-dead");
16399 let responses = handler
16400 .handle_route_poll(
16401 &ctx,
16402 route_poll_frame(41, PollKind::Liveness, route_channel),
16403 route_channel,
16404 route_epoch,
16405 PollKind::Liveness,
16406 )
16407 .unwrap();
16408
16409 assert_eq!(responses.len(), 1);
16410 assert_eq!(responses[0].header.ty, FrameType::Response);
16411 assert_route_poll_liveness(&responses[0], false);
16412 }
16413
16414 #[test]
16415 fn liveness_poll_without_process_source_uses_bound_route() {
16416 let registry = Arc::new(Registry::default());
16417 let forwarding = Arc::new(ForwardingTable::default());
16418 let handler =
16419 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16420 let (ctx, route_channel, route_epoch) =
16421 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
16422 let responses = handler
16423 .handle_route_poll(
16424 &ctx,
16425 route_poll_frame(42, PollKind::Liveness, route_channel),
16426 route_channel,
16427 route_epoch,
16428 PollKind::Liveness,
16429 )
16430 .unwrap();
16431
16432 assert_route_poll_liveness(&responses[0], true);
16433 }
16434
16435 #[test]
16436 fn liveness_poll_untracked_process_source_uses_bound_route() {
16437 let registry = Arc::new(Registry::default());
16438 let forwarding = Arc::new(ForwardingTable::default());
16439 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
16440 let handler =
16441 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16442 .with_process_liveness(process_liveness);
16443 let (ctx, route_channel, route_epoch) =
16444 bind_liveness_route(®istry, &forwarding, "aft-untracked");
16445 let responses = handler
16446 .handle_route_poll(
16447 &ctx,
16448 route_poll_frame(43, PollKind::Liveness, route_channel),
16449 route_channel,
16450 route_epoch,
16451 PollKind::Liveness,
16452 )
16453 .unwrap();
16454
16455 assert_route_poll_liveness(&responses[0], true);
16456 }
16457
16458 #[tokio::test]
16459 async fn unknown_op_returns_unknown_control_op() {
16460 let handler = ControlHandler::default();
16461 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
16462 let request = Frame::build(
16463 FrameType::Request,
16464 control_flags(),
16465 0,
16466 0,
16467 55,
16468 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
16469 )
16470 .unwrap();
16471
16472 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16473
16474 assert_eq!(response.len(), 1);
16475 assert_eq!(response[0].header.ty, FrameType::Error);
16476 assert_eq!(response[0].header.corr, 55);
16477 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
16478 }
16479
16480 #[tokio::test]
16481 async fn supervisor_provenance_rejects_unknown_exact_module() {
16482 let handler = ControlHandler::default();
16483 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
16484 let request = Frame::build(
16485 FrameType::Request,
16486 control_flags(),
16487 0,
16488 0,
16489 57,
16490 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
16491 )
16492 .unwrap();
16493
16494 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16495
16496 assert_eq!(response.len(), 1);
16497 assert_eq!(response[0].header.ty, FrameType::Error);
16498 assert_eq!(response[0].header.corr, 57);
16499 let error = parse_error(&response[0]);
16500 assert_eq!(error["code"], "unknown_module");
16501 assert_eq!(error["message"], "module_id 'missing' is not supervised");
16502 }
16503
16504 #[test]
16505 fn provenance_probe_override_keeps_handler_tests_deterministic() {
16506 let expected = subc_control::RunningImageAgreement::Unavailable {
16507 reason: subc_control::RunningImageUnavailableReason::HashFailed,
16508 };
16509 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
16510 assert_eq!(handler.provenance_probe_override, Some(expected));
16511 }
16512
16513 #[test]
16514 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
16515 let verdict = reload_verdict(
16516 std::path::Path::new("/bin/new"),
16517 Some(std::path::Path::new("/bin/old")),
16518 subc_control::RunningImageAgreement::Unavailable {
16519 reason: subc_control::RunningImageUnavailableReason::HashFailed,
16520 },
16521 );
16522 assert!(matches!(
16523 verdict.path,
16524 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
16525 if configured == std::path::Path::new("/bin/new")
16526 && spawned_from == std::path::Path::new("/bin/old")
16527 ));
16528 }
16529
16530 #[test]
16531 fn launch_spec_without_running_digest_is_unavailable_not_match() {
16532 use subc_control::{
16533 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
16534 };
16535 let configured = crate::supervise::ModuleSpec {
16536 carry_containment: false,
16537 module_id: "legacy".into(),
16538 program: "/module".into(),
16539 args: Vec::new(),
16540 env: Vec::new(),
16541 reserved: false,
16542 reserved_prefixes: Vec::new(),
16543 protocol: subc_control::ModuleProtocol::Subc,
16544 overlap: Default::default(),
16545 };
16546 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
16547 assert_eq!(
16548 launch_spec_agreement(&configured, false, Some(&digest)),
16549 Agreement::Unavailable {
16550 reason: Reason::NotRunning
16551 }
16552 );
16553 assert_eq!(
16554 launch_spec_agreement(&configured, true, None),
16555 Agreement::Unavailable {
16556 reason: Reason::SpawnedSpecUnavailable
16557 }
16558 );
16559 assert_eq!(
16560 launch_spec_agreement(&configured, true, Some(&digest)),
16561 Agreement::Match
16562 );
16563 }
16564
16565 #[cfg(windows)]
16566 #[test]
16567 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
16568 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
16569 let configured = dir.join("module.exe");
16570 std::fs::write(&configured, b"executable path fixture").unwrap();
16571 let spawned = std::fs::canonicalize(&configured).unwrap();
16572 assert_ne!(
16573 configured, spawned,
16574 "fixture must exercise different path spellings"
16575 );
16576 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
16577 let lower_case = configured.to_string_lossy().to_lowercase();
16578 for spelling in [
16579 &configured,
16580 Path::new(&forward_slashes),
16581 Path::new(&lower_case),
16582 ] {
16583 let verdict = reload_verdict(
16584 spelling,
16585 Some(&spawned),
16586 subc_control::RunningImageAgreement::Unavailable {
16587 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16588 },
16589 );
16590 assert_eq!(
16591 verdict.path,
16592 ReloadPathAgreement::Match,
16593 "configured {spelling:?}, spawned {spawned:?}"
16594 );
16595 }
16596 }
16597
16598 #[cfg(windows)]
16599 #[test]
16600 fn windows_reload_path_keeps_different_existing_program_pending() {
16601 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
16602 let old = dir.join("old.exe");
16603 let new = dir.join("new.exe");
16604 std::fs::write(&old, b"same executable content").unwrap();
16605 std::fs::copy(&old, &new).unwrap();
16606 let spawned = std::fs::canonicalize(&old).unwrap();
16607 let image = subc_control::RunningImageAgreement::Unavailable {
16608 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16609 };
16610 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
16611 assert!(
16612 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
16613 if configured == new && spawned_from == spawned)
16614 );
16615 let missing = dir.join("missing.exe");
16616 assert!(matches!(
16617 reload_verdict(&missing, Some(&spawned), image).path,
16618 ReloadPathAgreement::Mismatch { .. }
16619 ));
16620 }
16621
16622 #[test]
16623 fn reload_verdict_detects_replaced_image_at_same_path() {
16624 let image = subc_control::RunningImageAgreement::Mismatch {
16625 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
16626 digest: "old".into(),
16627 },
16628 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
16629 digest: "new".into(),
16630 },
16631 };
16632 let verdict = reload_verdict(
16633 std::path::Path::new("/bin/same"),
16634 Some(std::path::Path::new("/bin/same")),
16635 image.clone(),
16636 );
16637 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16638 assert_eq!(verdict.image, image);
16639 }
16640
16641 #[test]
16642 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
16643 let image = subc_control::RunningImageAgreement::Unavailable {
16644 reason: subc_control::RunningImageUnavailableReason::NotRunning,
16645 };
16646 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
16647 assert_eq!(
16648 verdict.path,
16649 subc_control::ReloadPathAgreement::Unavailable {
16650 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
16651 }
16652 );
16653 assert_eq!(verdict.image, image);
16654
16655 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
16656 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16657 };
16658 let verdict = reload_verdict(
16659 std::path::Path::new("/bin/same"),
16660 Some(std::path::Path::new("/bin/same")),
16661 unconfirmed.clone(),
16662 );
16663 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16664 assert_eq!(verdict.image, unconfirmed);
16665 }
16666
16667 #[test]
16668 fn reload_verdict_preserves_each_image_unavailability_reason() {
16669 use subc_control::RunningImageUnavailableReason as Reason;
16670
16671 for reason in [
16672 Reason::NotRunning,
16673 Reason::UnsupportedPlatform,
16674 Reason::RunningExecutableUnreadable,
16675 Reason::SpawnedPathUnreadable,
16676 Reason::HashFailed,
16677 Reason::ProcessIdentityUnconfirmed,
16678 Reason::Unknown("future_probe_reason".to_string()),
16679 ] {
16680 let image = subc_control::RunningImageAgreement::Unavailable {
16681 reason: reason.clone(),
16682 };
16683 let verdict = reload_verdict(
16684 std::path::Path::new("/bin/same"),
16685 Some(std::path::Path::new("/bin/same")),
16686 image.clone(),
16687 );
16688 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16689 assert_eq!(verdict.image, image, "{reason:?}");
16690 }
16691 }
16692
16693 #[tokio::test]
16694 async fn malformed_control_bodies_return_invalid_control_body() {
16695 let handler = ControlHandler::default();
16696 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
16697
16698 for (corr, body) in [
16699 (56, br#"{"route_channel":1}"#.as_slice()),
16700 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
16701 (
16702 58,
16703 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
16704 ),
16705 ] {
16706 let request = Frame::build(
16707 FrameType::Request,
16708 control_flags(),
16709 0,
16710 0,
16711 corr,
16712 body.to_vec(),
16713 )
16714 .unwrap();
16715 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16716
16717 assert_eq!(response.len(), 1);
16718 assert_eq!(response[0].header.ty, FrameType::Error);
16719 assert_eq!(response[0].header.corr, corr);
16720 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
16721 }
16722 }
16723
16724 #[tokio::test]
16725 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
16726 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16727 let registry = Arc::new(Registry::default());
16728 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16729 let router = Router::with_control_handler(Arc::clone(&control));
16730 let connection = router.begin_connection();
16731 let (ctx, mut rx) = route_ctx(connection.id());
16732
16733 router
16734 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
16735 .await
16736 .unwrap();
16737 let response = rx.recv().await.unwrap();
16738 let ack = parse_ack(&response);
16739 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16740 let channel = 1;
16741
16742 let goodbye =
16743 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
16744 router.route_for_connection(&ctx, goodbye).await.unwrap();
16745 assert!(rx.try_recv().is_err());
16746 assert!(registry.get_module("aft").unwrap().is_none());
16747
16748 router
16749 .route_for_connection(&ctx, channel_request(channel, 13))
16750 .await
16751 .unwrap();
16752 let error_frame = rx.recv().await.unwrap();
16753 assert_eq!(error_frame.header.ty, FrameType::Error);
16754 assert_eq!(error_frame.header.channel, channel);
16755 let captured = crate::router::test_log::captured_logs(&logs);
16756 assert_eq!(
16757 captured
16758 .lines()
16759 .filter(|line| {
16760 line.contains("module_id=aft")
16761 && line.contains("reason=explicit_goodbye")
16762 && line.contains("module registration ended")
16763 })
16764 .count(),
16765 1,
16766 "unexpected GOODBYE registry log: {captured}"
16767 );
16768 }
16769
16770 #[tokio::test]
16771 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
16772 let registry = Arc::new(Registry::default());
16773 let handler = ControlHandler::new(registry).with_capability_config(
16774 [("prov".to_string(), true), ("cons".to_string(), true)],
16775 BTreeMap::new(),
16776 );
16777 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
16778 register_capability_manifest(
16779 &handler,
16780 &provider_ctx,
16781 &mut provider_rx,
16782 capability_manifest("prov", &["thing/v1"], &[]),
16783 1,
16784 )
16785 .await;
16786 let mut consumer = capability_manifest("cons", &[], &[]);
16787 consumer.capabilities.as_mut().unwrap().requires.push(
16788 subc_protocol::manifest::CapabilityRequirement {
16789 capability: "thing/v1".to_string(),
16790 need: subc_protocol::manifest::CapabilityNeed::Required,
16791 },
16792 );
16793 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
16794 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
16795 assert_eq!(
16796 handler.capability_evaluator.verdict("cons", "thing/v1"),
16797 Some(CapabilityVerdict::Provided)
16798 );
16799 let (mut client_rx, _) = open_route_for_capability_test(
16800 &handler,
16801 &provider_ctx,
16802 &mut provider_rx,
16803 703,
16804 3,
16805 "prov",
16806 None,
16807 )
16808 .await;
16809 let goodbye =
16810 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
16811 handler
16812 .handle_control_frame(&provider_ctx, goodbye)
16813 .await
16814 .unwrap();
16815 assert_eq!(
16816 handler.capability_evaluator.verdict("cons", "thing/v1"),
16817 Some(CapabilityVerdict::NeverProvided)
16818 );
16819 let closed = client_rx
16820 .try_recv()
16821 .expect("GOODBYE pushes route.closed before route GOODBYE");
16822 assert!(
16823 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
16824 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
16825 );
16826 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
16827 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
16828 }
16829
16830 struct RouteClosedFixture {
16831 handler: ControlHandler,
16832 #[cfg(unix)]
16833 supervisor: Supervisor,
16834 module_ctx: RouteCtx,
16835 #[cfg(unix)]
16836 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16837 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16838 route: Arc<crate::forwarding::RouteBinding>,
16839 }
16840
16841 async fn route_closed_fixture() -> RouteClosedFixture {
16842 let registry = Arc::new(Registry::default());
16843 let handler = ControlHandler::new(Arc::clone(®istry));
16844 #[cfg(unix)]
16845 let supervisor = Supervisor::new(registry, RestartPolicy::default())
16846 .with_forwarding(Arc::clone(&handler.forwarding));
16847 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
16848 register_capability_manifest(
16849 &handler,
16850 &module_ctx,
16851 &mut module_rx,
16852 capability_manifest("prov", &[], &[]),
16853 1,
16854 )
16855 .await;
16856 let (client_rx, bind) = open_route_for_capability_test(
16857 &handler,
16858 &module_ctx,
16859 &mut module_rx,
16860 702,
16861 2,
16862 "prov",
16863 None,
16864 )
16865 .await;
16866 let ModuleControlRequest::RouteBind {
16867 route_channel: channel,
16868 epoch: route_epoch,
16869 ..
16870 } = bind
16871 else {
16872 panic!("expected route.bind");
16873 };
16874 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
16875 handler
16876 .forwarding
16877 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
16878 .unwrap()
16879 else {
16880 panic!("the acknowledged route must be bound");
16881 };
16882 RouteClosedFixture {
16883 handler,
16884 #[cfg(unix)]
16885 supervisor,
16886 module_ctx,
16887 #[cfg(unix)]
16888 module_rx,
16889 client_rx,
16890 route,
16891 }
16892 }
16893
16894 impl RouteClosedFixture {
16895 #[cfg(unix)]
16896 async fn cut(&mut self) {
16897 let notices = async {
16898 let mut draining = self.module_rx.recv().await.unwrap();
16899 assert!(matches!(
16900 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
16901 &draining.body
16902 )
16903 .unwrap(),
16904 subc_protocol::session::ModuleControlCommand::Draining {
16905 reason: RouteCloseReason::Disable,
16906 ..
16907 }
16908 ));
16909 draining.flushed.take().unwrap().send(()).unwrap();
16910 let mut closing = self.client_rx.recv().await.unwrap();
16911 assert!(matches!(
16912 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
16913 ClientControlPush::RouteClosing {
16914 reason: RouteCloseReason::Restart,
16915 ..
16916 }
16917 ));
16918 closing.flushed.take().unwrap().send(()).unwrap();
16919 };
16920 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
16921 result.unwrap();
16922 }
16923
16924 fn disconnect(&mut self) -> ClientControlPush {
16925 self.handler
16926 .cleanup_connection(self.module_ctx.connection_id)
16927 .unwrap();
16928 let closed = self.client_rx.try_recv().unwrap();
16929 assert_eq!(closed.header.ty, FrameType::Push);
16930 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
16931 assert!(matches!(
16932 &push,
16933 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
16934 if channels == &[self.route.client_channel]
16935 ));
16936 assert_eq!(
16937 self.client_rx.try_recv().unwrap().header.ty,
16938 FrameType::Goodbye
16939 );
16940 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
16941 push
16942 }
16943 }
16944
16945 #[cfg(unix)]
16946 #[tokio::test(start_paused = true)]
16947 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
16948 let mut fixture = route_closed_fixture().await;
16949 fixture.cut().await;
16950 assert!(matches!(
16951 fixture.disconnect(),
16952 ClientControlPush::RouteClosed {
16953 reason: RouteCloseReason::Restart,
16954 drained: true,
16955 ..
16956 }
16957 ));
16958 }
16959
16960 #[cfg(unix)]
16961 #[tokio::test(start_paused = true)]
16962 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16963 let mut fixture = route_closed_fixture().await;
16964 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16965 let started = tokio::time::Instant::now();
16966 fixture.cut().await;
16967 assert!(started.elapsed() >= Duration::from_secs(2));
16968 assert_eq!(fixture.route.flow.in_flight(), 1);
16969 assert!(matches!(
16970 fixture.disconnect(),
16971 ClientControlPush::RouteClosed {
16972 reason: RouteCloseReason::Restart,
16973 drained: false,
16974 ..
16975 }
16976 ));
16977 }
16978
16979 #[tokio::test]
16980 async fn module_connection_crash_reports_crash_and_not_drained() {
16981 let mut fixture = route_closed_fixture().await;
16982 assert!(matches!(
16986 fixture.disconnect(),
16987 ClientControlPush::RouteClosed {
16988 reason: RouteCloseReason::Crash,
16989 drained: false,
16990 ..
16991 }
16992 ));
16993 }
16994
16995 #[tokio::test]
16996 async fn dropping_router_connection_releases_registration() {
16997 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16998 let registry = Arc::new(Registry::default());
16999 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
17000 let router = Router::with_control_handler(Arc::clone(&control));
17001 let connection = router.begin_connection();
17002 let connection_id = connection.id();
17003 let (ctx, mut rx) = route_ctx(connection_id);
17004
17005 router
17006 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
17007 .await
17008 .unwrap();
17009 let response = rx.recv().await.unwrap();
17010 let ack = parse_ack(&response);
17011 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
17012 assert!(registry.get_module("aft").unwrap().is_some());
17013
17014 drop(connection);
17015
17016 assert!(registry.get_module("aft").unwrap().is_none());
17017 assert_eq!(registry.active_registration_count().unwrap(), 0);
17018
17019 control
17020 .cleanup_connection(ConnectionId::new(u64::MAX))
17021 .unwrap();
17022 let captured = crate::router::test_log::captured_logs(&logs);
17023 println!("captured registry lifecycle logs:\n{captured}");
17024 let events: Vec<_> = captured
17025 .lines()
17026 .filter(|line| {
17027 line.contains("module registered module_id=aft ")
17028 || line.contains("module registration ended")
17029 })
17030 .collect();
17031 assert_eq!(
17032 events.len(),
17033 2,
17034 "unexpected registry lifecycle logs: {captured}"
17035 );
17036 assert!(events[0].contains("module registered module_id=aft "));
17037 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
17038 assert!(events[1].contains(&format!(
17039 "module_id=aft connection_id={}",
17040 connection_id.get()
17041 )));
17042 assert!(events[1].contains("reason=connection_closed"));
17043 assert!(events[1].contains("module registration ended"));
17044 assert_eq!(
17045 captured
17046 .lines()
17047 .filter(|line| line.contains("module registered module_id=aft "))
17048 .count(),
17049 1,
17050 "legacy registration admission line must appear once: {captured}"
17051 );
17052 }
17053
17054 #[tokio::test]
17055 async fn hello_registration_keeps_legacy_module_registered_line_once() {
17056 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
17057 let registry = Arc::new(Registry::default());
17058 let control = ControlHandler::new(Arc::clone(®istry));
17059 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
17060 hello_via_sink(
17061 &control,
17062 &ctx,
17063 &mut rx,
17064 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
17065 )
17066 .await;
17067
17068 let captured = crate::router::test_log::captured_logs(&logs);
17069 let admissions: Vec<_> = captured
17070 .lines()
17071 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
17072 .collect();
17073 assert_eq!(
17074 admissions.len(),
17075 1,
17076 "legacy admission line must remain exactly once: {captured}"
17077 );
17078 assert!(admissions[0].contains("routable_provider=true"));
17079 assert!(admissions[0].contains("connection_id=777"));
17080 }
17081
17082 fn capability_manifest(
17083 module_id: &str,
17084 provides: &[&str],
17085 must_never_reach: &[&str],
17086 ) -> ModuleManifest {
17087 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
17088 manifest.capabilities = Some(CapabilityDeclarations {
17089 provides: provides
17090 .iter()
17091 .map(|capability| (*capability).to_string())
17092 .collect(),
17093 requires: Vec::new(),
17094 must_never_reach: must_never_reach
17095 .iter()
17096 .map(|capability| (*capability).to_string())
17097 .collect(),
17098 });
17099 manifest
17100 }
17101
17102 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
17103 Frame::build(
17104 FrameType::Hello,
17105 control_flags(),
17106 0,
17107 0,
17108 corr,
17109 serde_json::to_vec(&ModuleHelloBody {
17110 protocol_ver: manifest.protocol_ver,
17111 manifest,
17112 control_ops: None,
17113 launch_nonce: None,
17114 })
17115 .expect("capability test HELLO serializes"),
17116 )
17117 .expect("capability test HELLO frame builds")
17118 }
17119
17120 fn catalog_update_with_capabilities_frame(
17121 corr: u64,
17122 capabilities: CapabilityDeclarations,
17123 ) -> Frame {
17124 Frame::build(
17125 FrameType::Request,
17126 control_flags(),
17127 0,
17128 0,
17129 corr,
17130 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17131 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
17132 capabilities: Some(capabilities),
17133 ready: None,
17134 })
17135 .expect("capability catalog.update serializes"),
17136 )
17137 .expect("capability catalog.update frame builds")
17138 }
17139
17140 async fn register_capability_manifest(
17141 handler: &ControlHandler,
17142 ctx: &RouteCtx,
17143 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
17144 manifest: ModuleManifest,
17145 corr: u64,
17146 ) {
17147 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
17148 }
17149
17150 async fn open_route_for_capability_test(
17151 handler: &ControlHandler,
17152 target_ctx: &RouteCtx,
17153 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
17154 client_connection_id: u64,
17155 corr: u64,
17156 target_module_id: &str,
17157 consumer_identity: Option<ConsumerIdentity>,
17158 ) -> (
17159 mpsc::Receiver<crate::router::OutboundFrame>,
17160 ModuleControlRequest,
17161 ) {
17162 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
17163 let route_handler = handler.clone();
17164 let target_module_id = target_module_id.to_string();
17165 let route_task = tokio::spawn(async move {
17166 route_handler
17167 .handle_control_frame(
17168 &client_ctx,
17169 route_open_frame_with_admission_facts(
17170 corr,
17171 &target_module_id,
17172 unique_project_root("admission-facts"),
17173 consumer_identity,
17174 None,
17175 ),
17176 )
17177 .await
17178 .expect("capability test route.open succeeds")
17179 });
17180 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
17181 .await
17182 .expect("capability test route.open must reach route.bind")
17183 .expect("target control receiver stays open");
17184 let bind_request: ModuleControlRequest =
17185 serde_json::from_slice(&bind.body).expect("route.bind decodes");
17186 handler
17187 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
17188 .await
17189 .expect("capability test route.bind ACK succeeds");
17190 assert!(route_task.await.expect("route.open task joins").is_empty());
17191 let opened = client_rx
17192 .recv()
17193 .await
17194 .expect("successful route.open publishes a response");
17195 assert!(matches!(
17196 serde_json::from_slice::<ClientControlResponse>(&opened.body),
17197 Ok(ClientControlResponse::RouteOpen { .. })
17198 ));
17199 (client_rx, bind_request)
17200 }
17201
17202 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
17203 assert_eq!(frame.header.ty, FrameType::Push);
17204 assert_eq!(frame.header.channel, 0);
17205 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
17206 .expect("route.closed control push decodes");
17207 let ClientControlPush::RouteClosed { channels, .. } = &push else {
17208 panic!("expected route.closed");
17209 };
17210 assert_eq!(channels.len(), 1, "exactly one violating route closed");
17211 let closed_channel = channels[0];
17212 let channels = channels.clone();
17213 assert_eq!(
17214 push,
17215 ClientControlPush::RouteClosed {
17216 module_id: target_module_id.to_string(),
17217 channels,
17218 reason: RouteCloseReason::CapabilityDenied,
17219 drained: false,
17220 abandoned: 0,
17221 excluded_subscriptions: 0,
17222 terminal: Some(false),
17223 }
17224 );
17225 closed_channel
17226 }
17227
17228 #[tokio::test]
17229 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
17230 let registry = Arc::new(Registry::default());
17231 let forwarding = Arc::new(ForwardingTable::default());
17232 let supervisor = SupervisorHandle::new();
17233 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17234 let handler =
17235 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17236 .with_supervisor(supervisor);
17237 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
17238 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
17239 register_capability_manifest(
17240 &handler,
17241 &target_ctx,
17242 &mut target_rx,
17243 capability_manifest("target", &["credentials-provider/v1"], &[]),
17244 1,
17245 )
17246 .await;
17247 register_capability_manifest(
17248 &handler,
17249 &opener_ctx,
17250 &mut opener_rx,
17251 capability_manifest("opener", &[], &["credentials-provider/v1"]),
17252 2,
17253 )
17254 .await;
17255
17256 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
17257 let replies = handler
17258 .handle_control_frame(
17259 &client_ctx,
17260 route_open_frame_with_admission_facts(
17261 3,
17262 "target",
17263 unique_project_root("admission-facts"),
17264 Some(ConsumerIdentity {
17265 module_id: "opener".to_string(),
17266 launch_nonce: "opener-nonce".to_string(),
17267 }),
17268 None,
17269 ),
17270 )
17271 .await
17272 .expect("denied route.open returns a typed frame");
17273 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
17274 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17275 assert!(
17276 target_rx.try_recv().is_err(),
17277 "forbidden route.open must not relay route.bind"
17278 );
17279 }
17280
17281 #[tokio::test]
17282 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
17283 let registry = Arc::new(Registry::default());
17284 let forwarding = Arc::new(ForwardingTable::default());
17285 let supervisor = SupervisorHandle::new();
17286 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17287 let handler =
17288 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17289 .with_supervisor(supervisor);
17290 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
17291 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
17292 register_capability_manifest(
17293 &handler,
17294 &target_ctx,
17295 &mut target_rx,
17296 capability_manifest("target", &["credentials-provider/v1"], &[]),
17297 1,
17298 )
17299 .await;
17300 register_capability_manifest(
17301 &handler,
17302 &old_opener_ctx,
17303 &mut old_opener_rx,
17304 capability_manifest("opener", &[], &[]),
17305 2,
17306 )
17307 .await;
17308 let (mut client_rx, _) = open_route_for_capability_test(
17309 &handler,
17310 &target_ctx,
17311 &mut target_rx,
17312 712,
17313 3,
17314 "target",
17315 Some(ConsumerIdentity {
17316 module_id: "opener".to_string(),
17317 launch_nonce: "opener-nonce".to_string(),
17318 }),
17319 )
17320 .await;
17321 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17322
17323 handler
17324 .cleanup_connection(old_opener_ctx.connection_id)
17325 .expect("old opener registration cleans up");
17326 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
17327 register_capability_manifest(
17328 &handler,
17329 &new_opener_ctx,
17330 &mut new_opener_rx,
17331 capability_manifest("opener", &[], &["credentials-provider/v1"]),
17332 4,
17333 )
17334 .await;
17335
17336 let closed_channel = assert_capability_denied_push(
17337 client_rx
17338 .try_recv()
17339 .expect("HELLO deny addition must emit route.closed")
17340 .frame,
17341 "target",
17342 );
17343 let goodbye = client_rx
17344 .try_recv()
17345 .expect("HELLO deny addition must send the client a route GOODBYE")
17346 .frame;
17347 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17348 assert_eq!(goodbye.header.channel, closed_channel);
17349 assert_ne!(goodbye.header.channel, 0);
17350 assert_ne!(goodbye.header.epoch, 0);
17351 assert!(client_rx.try_recv().is_err());
17352 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17353 assert!(matches!(
17354 target_rx.try_recv(),
17355 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
17356 ));
17357 }
17358
17359 #[tokio::test]
17360 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
17361 let registry = Arc::new(Registry::default());
17362 let forwarding = Arc::new(ForwardingTable::default());
17363 let supervisor = SupervisorHandle::new();
17364 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17365 let handler =
17366 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17367 .with_supervisor(supervisor);
17368 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
17369 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
17370 register_capability_manifest(
17371 &handler,
17372 &target_ctx,
17373 &mut target_rx,
17374 capability_manifest("target", &[], &[]),
17375 1,
17376 )
17377 .await;
17378 register_capability_manifest(
17379 &handler,
17380 &opener_ctx,
17381 &mut opener_rx,
17382 capability_manifest("opener", &[], &["credentials-provider/v1"]),
17383 2,
17384 )
17385 .await;
17386 let (mut client_rx, _) = open_route_for_capability_test(
17387 &handler,
17388 &target_ctx,
17389 &mut target_rx,
17390 722,
17391 3,
17392 "target",
17393 Some(ConsumerIdentity {
17394 module_id: "opener".to_string(),
17395 launch_nonce: "opener-nonce".to_string(),
17396 }),
17397 )
17398 .await;
17399 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17400
17401 let replies = handler
17402 .handle_control_frame(
17403 &target_ctx,
17404 catalog_update_with_capabilities_frame(
17405 4,
17406 CapabilityDeclarations {
17407 provides: vec!["credentials-provider/v1".to_string()],
17408 requires: Vec::new(),
17409 must_never_reach: Vec::new(),
17410 },
17411 ),
17412 )
17413 .await
17414 .expect("claim catalog.update succeeds");
17415 assert!(matches!(
17416 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
17417 Ok(ModuleControlResponseToModule::CatalogUpdate {})
17418 ));
17419 let closed_channel = assert_capability_denied_push(
17420 client_rx
17421 .try_recv()
17422 .expect("claim addition must emit route.closed")
17423 .frame,
17424 "target",
17425 );
17426 let goodbye = client_rx
17427 .try_recv()
17428 .expect("claim addition must send the client a route GOODBYE")
17429 .frame;
17430 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17431 assert_eq!(goodbye.header.channel, closed_channel);
17432 assert_ne!(goodbye.header.channel, 0);
17433 assert_ne!(goodbye.header.epoch, 0);
17434 assert!(client_rx.try_recv().is_err());
17435 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17436 assert!(matches!(
17437 target_rx.try_recv(),
17438 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
17439 ));
17440 }
17441
17442 #[tokio::test]
17443 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
17444 let registry = Arc::new(Registry::default());
17445 let forwarding = Arc::new(ForwardingTable::default());
17446 let supervisor = SupervisorHandle::new();
17447 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17448 let handler =
17449 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17450 .with_supervisor(supervisor);
17451 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
17452 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
17453 register_capability_manifest(
17454 &handler,
17455 &target_ctx,
17456 &mut target_rx,
17457 capability_manifest("target", &["credentials-provider/v1"], &[]),
17458 1,
17459 )
17460 .await;
17461 register_capability_manifest(
17462 &handler,
17463 &opener_ctx,
17464 &mut opener_rx,
17465 capability_manifest("opener", &[], &[]),
17466 2,
17467 )
17468 .await;
17469 let (mut client_rx, _) = open_route_for_capability_test(
17470 &handler,
17471 &target_ctx,
17472 &mut target_rx,
17473 732,
17474 3,
17475 "target",
17476 Some(ConsumerIdentity {
17477 module_id: "opener".to_string(),
17478 launch_nonce: "opener-nonce".to_string(),
17479 }),
17480 )
17481 .await;
17482 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17483
17484 handler
17485 .handle_control_frame(
17486 &target_ctx,
17487 catalog_update_with_capabilities_frame(
17488 4,
17489 CapabilityDeclarations {
17490 provides: Vec::new(),
17491 requires: Vec::new(),
17492 must_never_reach: Vec::new(),
17493 },
17494 ),
17495 )
17496 .await
17497 .expect("claim removal catalog.update succeeds");
17498 assert_eq!(
17499 forwarding.active_binding_count().unwrap(),
17500 1,
17501 "removing an attested target claim must leave the route census unchanged"
17502 );
17503 assert!(
17504 client_rx.try_recv().is_err(),
17505 "claim removal must not emit route.closed or a client route GOODBYE"
17506 );
17507 assert!(
17508 target_rx.try_recv().is_err(),
17509 "claim removal must not send the target a route GOODBYE"
17510 );
17511 }
17512
17513 #[tokio::test]
17516 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
17517 let registry = Arc::new(Registry::default());
17518 let forwarding = Arc::new(ForwardingTable::default());
17519 let supervisor = SupervisorHandle::new();
17520 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17521 let handler =
17522 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17523 .with_supervisor(supervisor);
17524 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
17525 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
17526 register_capability_manifest(
17527 &handler,
17528 &target_ctx,
17529 &mut target_rx,
17530 capability_manifest("target", &["credentials-provider/v1"], &[]),
17531 1,
17532 )
17533 .await;
17534 register_capability_manifest(
17535 &handler,
17536 &opener_ctx,
17537 &mut opener_rx,
17538 capability_manifest("opener", &[], &["credentials-provider/v1"]),
17539 2,
17540 )
17541 .await;
17542
17543 let (_client_rx, bind) = open_route_for_capability_test(
17544 &handler,
17545 &target_ctx,
17546 &mut target_rx,
17547 742,
17548 3,
17549 "target",
17550 None,
17551 )
17552 .await;
17553 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
17554 panic!("direct scope-honesty route must bind");
17555 };
17556 assert_eq!(principal, Some(Principal::Direct));
17557 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17558 }
17559
17560 #[tokio::test]
17563 async fn must_never_reach_self_route_is_capability_forbidden() {
17564 let registry = Arc::new(Registry::default());
17565 let forwarding = Arc::new(ForwardingTable::default());
17566 let supervisor = SupervisorHandle::new();
17567 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
17568 let handler =
17569 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17570 .with_supervisor(supervisor);
17571 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
17572 register_capability_manifest(
17573 &handler,
17574 &self_ctx,
17575 &mut self_rx,
17576 capability_manifest(
17577 "self-provider",
17578 &["credentials-provider/v1"],
17579 &["credentials-provider/v1"],
17580 ),
17581 1,
17582 )
17583 .await;
17584
17585 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
17586 let replies = handler
17587 .handle_control_frame(
17588 &client_ctx,
17589 route_open_frame_with_admission_facts(
17590 2,
17591 "self-provider",
17592 unique_project_root("admission-facts"),
17593 Some(ConsumerIdentity {
17594 module_id: "self-provider".to_string(),
17595 launch_nonce: "self-nonce".to_string(),
17596 }),
17597 None,
17598 ),
17599 )
17600 .await
17601 .expect("self-route refusal returns a typed frame");
17602 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
17603 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17604 assert!(
17605 self_rx.try_recv().is_err(),
17606 "self denial must not relay route.bind"
17607 );
17608 }
17609
17610 #[test]
17611 fn unsupported_channel_zero_frame_returns_error() {
17612 let handler = ControlHandler::default();
17613 let request = Frame::build(
17614 FrameType::Request,
17615 control_flags(),
17616 0,
17617 0,
17618 21,
17619 b"opaque".to_vec(),
17620 )
17621 .unwrap();
17622
17623 let response = handler
17624 .handle_control(ConnectionId::new(1), request)
17625 .unwrap();
17626
17627 assert_eq!(response[0].header.ty, FrameType::Error);
17628 assert_eq!(
17629 parse_error(&response[0])["code"],
17630 "unsupported_control_frame"
17631 );
17632 }
17633
17634 mod swap {
17639 use super::*;
17640
17641 const INCUMBENT: ConnectionId = ConnectionId::new(30);
17642 const CANDIDATE: ConnectionId = ConnectionId::new(40);
17643
17644 struct Swap {
17645 registry: Arc<Registry>,
17646 forwarding: Arc<ForwardingTable>,
17647 handler: ControlHandler,
17648 incumbent_ctx: RouteCtx,
17649 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17650 candidate_ctx: RouteCtx,
17651 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17652 }
17653
17654 async fn swap_with_incumbent() -> Swap {
17655 let registry = Arc::new(Registry::default());
17656 let forwarding = Arc::new(ForwardingTable::default());
17657 let handler =
17658 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17659 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
17660 hello_via_sink(
17661 &handler,
17662 &incumbent_ctx,
17663 &mut incumbent_rx,
17664 hello_frame("aft", PROTOCOL_VERSION, 7),
17665 )
17666 .await;
17667 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
17668 Swap {
17669 registry,
17670 forwarding,
17671 handler,
17672 incumbent_ctx,
17673 incumbent_rx,
17674 candidate_ctx,
17675 candidate_rx,
17676 }
17677 }
17678
17679 fn register_candidate(swap: &Swap, ready: Option<bool>) {
17680 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
17681 candidate_manifest.ready = ready;
17682 let registration = swap
17683 .registry
17684 .register_candidate_with_control_ops(
17685 candidate_manifest,
17686 PROTOCOL_VERSION,
17687 CANDIDATE,
17688 module_baseline_control_ops(),
17689 )
17690 .unwrap();
17691 swap.forwarding
17692 .register_candidate_module_connection(
17693 CANDIDATE,
17694 "aft".to_string(),
17695 PROTOCOL_VERSION,
17696 manifest_concurrency(®istration.manifest),
17697 swap.candidate_ctx.egress.clone(),
17698 )
17699 .unwrap();
17700 }
17701
17702 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
17703 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
17704 swap.registry.promote_candidate("aft").unwrap().unwrap();
17705 cutover.incumbent.unwrap()
17706 }
17707
17708 #[tokio::test]
17717 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
17718 let mut swap = swap_with_incumbent().await;
17719 let hold = Arc::new(std::sync::Barrier::new(2));
17720 let handler = swap
17721 .handler
17722 .clone()
17723 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
17724 handler
17725 .supervisor
17726 .open_swap("aft", "candidate-nonce".to_string());
17727 let hello = {
17728 let handler = handler.clone();
17729 let candidate_ctx = swap.candidate_ctx.clone();
17730 std::thread::spawn(move || {
17731 handler
17732 .handle_hello(
17733 candidate_ctx.connection_id,
17734 Some(candidate_ctx.egress.clone()),
17735 hello_frame_with_nonce(
17736 "aft",
17737 PROTOCOL_VERSION,
17738 8,
17739 Some("candidate-nonce"),
17740 ),
17741 )
17742 .unwrap()
17743 })
17744 };
17745
17746 hold.wait();
17747 assert!(swap
17750 .registry
17751 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
17752 .unwrap()
17753 .is_some());
17754 assert!(swap
17755 .registry
17756 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17757 .unwrap()
17758 .is_none());
17759 assert!(swap
17760 .forwarding
17761 .module_endpoint_for_connection(CANDIDATE)
17762 .unwrap()
17763 .is_none());
17764
17765 hold.wait();
17766 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
17767 assert_eq!(
17768 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
17769 FrameType::HelloAck
17770 );
17771 let published = swap
17772 .registry
17773 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17774 .unwrap()
17775 .expect("candidate is published once its HELLO completes");
17776 assert_eq!(published.connection_id, CANDIDATE);
17777 assert!(swap
17778 .forwarding
17779 .module_endpoint_for_connection(CANDIDATE)
17780 .unwrap()
17781 .is_some());
17782 }
17783
17784 fn keyed_total(counters: &Value, key: &str) -> u64 {
17785 counters[key]
17786 .as_object()
17787 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
17788 .unwrap_or(0)
17789 }
17790
17791 #[tokio::test]
17792 async fn replacement_logs_old_end_and_new_admission_once() {
17793 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
17794 let mut swap = swap_with_incumbent().await;
17795 swap.handler
17796 .supervisor
17797 .open_swap("aft", "candidate-nonce".to_string());
17798 hello_via_sink(
17799 &swap.handler,
17800 &swap.candidate_ctx,
17801 &mut swap.candidate_rx,
17802 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
17803 )
17804 .await;
17805 cutover(&swap);
17806 swap.handler.cleanup_connection(INCUMBENT).unwrap();
17807
17808 let captured = crate::router::test_log::captured_logs(&logs);
17809 println!("captured replacement registry logs:\n{captured}");
17810 let new_admissions: Vec<_> = captured
17811 .lines()
17812 .filter(|line| {
17813 line.contains("connection_id=40")
17814 && line.contains("swap candidate registered; not routable until cutover")
17815 })
17816 .collect();
17817 assert_eq!(
17818 new_admissions.len(),
17819 1,
17820 "unexpected admission logs: {captured}"
17821 );
17822 assert!(new_admissions[0].contains("module_id=aft"));
17823 assert!(new_admissions[0].contains("connection_id=40"));
17824 assert!(
17825 new_admissions[0].contains("swap candidate registered; not routable until cutover")
17826 );
17827 assert!(new_admissions[0].contains("ready=true"));
17828
17829 let old_admissions: Vec<_> = captured
17830 .lines()
17831 .filter(|line| {
17832 line.contains("module registered module_id=aft ")
17833 && line.contains("connection_id=30")
17834 })
17835 .collect();
17836 assert_eq!(
17837 old_admissions.len(),
17838 1,
17839 "unexpected incumbent admission: {captured}"
17840 );
17841
17842 let old_ends: Vec<_> = captured
17843 .lines()
17844 .filter(|line| {
17845 line.contains("connection_id=30") && line.contains("module registration ended")
17846 })
17847 .collect();
17848 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
17849 assert!(old_ends[0].contains("module_id=aft"));
17850 assert!(old_ends[0].contains("connection_id=30"));
17851 assert!(old_ends[0].contains("reason=replaced"));
17852 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
17853 assert_eq!(
17854 captured
17855 .lines()
17856 .filter(|line| line.contains("module registration promoted"))
17857 .count(),
17858 1,
17859 "unexpected promotion logs: {captured}"
17860 );
17861 }
17862
17863 #[tokio::test]
17869 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
17870 let mut swap = swap_with_incumbent().await;
17871 let handler = swap.handler.clone();
17872
17873 let cotenant = ConnectionId::new(31);
17875 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
17876 let (cotenant_task, cotenant_bind) = relay_route_open(
17877 &handler,
17878 cotenant,
17879 &cotenant_ctx.egress,
17880 &mut swap.incumbent_rx,
17881 100,
17882 "aft",
17883 "swap-cotenant",
17884 )
17885 .await;
17886 handler
17887 .handle_control_frame(
17888 &swap.incumbent_ctx,
17889 route_bind_ack(cotenant_bind.header.corr),
17890 )
17891 .await
17892 .unwrap();
17893 assert!(cotenant_task.await.unwrap().is_empty());
17894 let (cotenant_channel, cotenant_epoch) =
17895 published_route(&cotenant_rx.recv().await.unwrap());
17896
17897 let caller = ConnectionId::new(32);
17899 let (caller_ctx, mut caller_rx) = route_ctx(caller);
17900 let (caller_task, caller_bind) = relay_route_open(
17901 &handler,
17902 caller,
17903 &caller_ctx.egress,
17904 &mut swap.incumbent_rx,
17905 101,
17906 "aft",
17907 "swap-caller",
17908 )
17909 .await;
17910 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
17911
17912 register_candidate(&swap, None);
17913 cutover(&swap);
17914
17915 let ack = handler
17917 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
17918 .await;
17919 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
17920 if module_loop_error.is_some() {
17921 handler.cleanup_connection(INCUMBENT).unwrap();
17924 }
17925
17926 assert!(
17928 cotenant_rx.try_recv().is_err(),
17929 "the co-tenant route on the incumbent was torn down by one late ack: \
17930 {module_loop_error:?}"
17931 );
17932 assert!(matches!(
17933 swap.forwarding
17934 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
17935 .unwrap(),
17936 DataRoute::Client(DataRouteState::Bound(_))
17937 ));
17938 assert_eq!(module_loop_error, None);
17939 assert!(swap
17940 .registry
17941 .get_module_by_connection(INCUMBENT)
17942 .unwrap()
17943 .is_some());
17944
17945 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
17947 .await
17948 .expect("the incumbent is told to drop the abandoned binding")
17949 .unwrap()
17950 .frame;
17951 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17952 assert_eq!(goodbye.header.channel, abandoned_channel);
17953 assert_eq!(goodbye.header.epoch, abandoned_epoch);
17954 assert!(swap.incumbent_rx.try_recv().is_err());
17955
17956 let response = caller_task.await.unwrap();
17958 assert_eq!(response.len(), 1);
17959 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17960 assert!(caller_rx.try_recv().is_err());
17961
17962 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17966 let counters = handler.counters().snapshot();
17967 assert_eq!(
17968 keyed_total(&counters, "route_open_accepted_by_principal"),
17969 1
17970 );
17971 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17972 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17973 }
17974
17975 #[tokio::test]
17979 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17980 let mut swap = swap_with_incumbent().await;
17981 register_candidate(&swap, None);
17982 let incumbent = cutover(&swap);
17983 swap.forwarding
17984 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17985 .unwrap()
17986 .expect("the incumbent is still registered");
17987
17988 let client = ConnectionId::new(33);
17989 let (client_ctx, mut client_rx) = route_ctx(client);
17990 let route_handler = swap.handler.clone();
17991 let open_ctx = RouteCtx {
17992 connection_id: client,
17993 egress: client_ctx.egress.clone(),
17994 };
17995 let mut route_task = tokio::spawn(async move {
17996 route_handler
17997 .handle_control_frame(
17998 &open_ctx,
17999 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
18000 )
18001 .await
18002 .unwrap()
18003 });
18004 let bind = tokio::select! {
18005 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
18006 response = &mut route_task => {
18007 let response = response.unwrap();
18008 panic!(
18009 "post-cutover route.open was refused instead of relayed to the candidate: {}",
18010 parse_error(&response[0])["code"]
18011 );
18012 }
18013 };
18014 swap.handler
18015 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
18016 .await
18017 .unwrap();
18018 assert!(route_task.await.unwrap().is_empty());
18019 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
18020 match swap
18021 .forwarding
18022 .lookup_data_route(client, channel, epoch)
18023 .unwrap()
18024 {
18025 DataRoute::Client(DataRouteState::Bound(route)) => {
18026 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
18027 }
18028 other => panic!("expected a bound route on the candidate, got {other:?}"),
18029 }
18030 assert!(swap.incumbent_rx.try_recv().is_err());
18031 }
18032
18033 #[tokio::test]
18038 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
18039 let swap = swap_with_incumbent().await;
18040 register_candidate(&swap, Some(false));
18041 let update = Frame::build(
18042 FrameType::Request,
18043 control_flags(),
18044 0,
18045 0,
18046 55,
18047 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
18048 provides: manifest("aft", PROTOCOL_VERSION).provides,
18049 capabilities: None,
18050 ready: Some(true),
18051 })
18052 .unwrap(),
18053 )
18054 .unwrap();
18055
18056 let replies = swap
18057 .handler
18058 .handle_control_frame(&swap.candidate_ctx, update)
18059 .await
18060 .unwrap();
18061
18062 assert_eq!(replies.len(), 1);
18063 assert_eq!(
18064 replies[0].header.ty,
18065 FrameType::Response,
18066 "candidate catalog.update was refused: {:?}",
18067 serde_json::from_slice::<Value>(&replies[0].body).ok()
18068 );
18069 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
18070 assert_eq!(
18071 swap.registry
18072 .get_module("aft")
18073 .unwrap()
18074 .unwrap()
18075 .connection_id,
18076 INCUMBENT
18077 );
18078 }
18079 }
18080
18081 mod swap_admission {
18085 use super::*;
18086
18087 const INCUMBENT_NONCE: &str = "incumbent-nonce";
18088 const CANDIDATE_NONCE: &str = "candidate-nonce";
18089
18090 fn handler_with_incumbent(
18091 module_id: &str,
18092 reserved: bool,
18093 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
18094 let registry = Arc::new(Registry::default());
18095 let supervisor = SupervisorHandle::new();
18096 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
18097 if reserved {
18098 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
18099 }
18100 let handler =
18101 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
18102 let incumbent = handler
18103 .handle_control(
18104 ConnectionId::new(1),
18105 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
18106 )
18107 .unwrap();
18108 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
18109 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
18110 (registry, supervisor, handler)
18111 }
18112
18113 #[test]
18119 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
18120 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
18121
18122 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
18123 let replies = handler
18124 .handle_control(
18125 ConnectionId::new(connection),
18126 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
18127 )
18128 .unwrap();
18129 assert_eq!(replies[0].header.ty, FrameType::Error);
18130 assert_eq!(
18131 parse_error(&replies[0])["code"],
18132 "swap_token_invalid",
18133 "nonce {nonce:?}"
18134 );
18135 }
18136 assert!(registry.get_candidate("aft").unwrap().is_none());
18137 assert_eq!(
18138 registry.get_module("aft").unwrap().unwrap().connection_id,
18139 ConnectionId::new(1)
18140 );
18141
18142 let admitted = handler
18145 .handle_control(
18146 ConnectionId::new(4),
18147 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
18148 )
18149 .unwrap();
18150 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
18151 assert_eq!(
18152 registry
18153 .get_candidate("aft")
18154 .unwrap()
18155 .unwrap()
18156 .connection_id,
18157 ConnectionId::new(4)
18158 );
18159 assert_eq!(
18160 registry.get_module("aft").unwrap().unwrap().connection_id,
18161 ConnectionId::new(1),
18162 "the candidate must not take the active slot"
18163 );
18164 let replayed = handler
18165 .handle_control(
18166 ConnectionId::new(5),
18167 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
18168 )
18169 .unwrap();
18170 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
18171
18172 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
18176 let squatter = handler
18177 .handle_control(
18178 ConnectionId::new(6),
18179 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
18180 )
18181 .unwrap();
18182 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
18183 assert!(
18184 registry.get_module("aft").unwrap().is_none(),
18185 "a squatter took the active slot of an id being swapped"
18186 );
18187 }
18188
18189 #[test]
18194 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
18195 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
18196
18197 let replies = handler
18198 .handle_control(
18199 ConnectionId::new(2),
18200 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
18201 )
18202 .unwrap();
18203
18204 assert_eq!(
18205 replies[0].header.ty,
18206 FrameType::HelloAck,
18207 "reserved candidate refused: {:?}",
18208 serde_json::from_slice::<Value>(&replies[0].body).ok()
18209 );
18210 assert_eq!(
18211 registry
18212 .get_candidate("vault")
18213 .unwrap()
18214 .unwrap()
18215 .connection_id,
18216 ConnectionId::new(2)
18217 );
18218 }
18219
18220 #[test]
18223 fn without_an_open_swap_the_ordinary_gates_decide() {
18224 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
18225 supervisor.close_swap("vault");
18226
18227 let candidate = handler
18228 .handle_control(
18229 ConnectionId::new(2),
18230 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
18231 )
18232 .unwrap();
18233 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
18234 let duplicate = handler
18235 .handle_control(
18236 ConnectionId::new(3),
18237 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
18238 )
18239 .unwrap();
18240 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
18241 assert!(registry.get_candidate("vault").unwrap().is_none());
18242 }
18243 }
18244
18245 mod scopes {
18249 mod idle_tests {
18250 use super::*;
18251 include!("scope_idle_tests.rs");
18252 }
18253 mod route_open_timings {
18254 use super::*;
18255 include!("route_open_timing_tests.rs");
18256 }
18257 mod route_open_refusal_log {
18258 use super::*;
18259 include!("route_open_refusal_log_tests.rs");
18260 }
18261 mod activity {
18262 include!("activity_scope_tests.rs");
18263 }
18264 mod delta {
18265 include!("scope_handler_delta_tests.rs");
18266 }
18267 mod call_modes {
18268 include!("scope_call_mode_tests.rs");
18269 }
18270 mod federated {
18271 include!("scope_federated_tests.rs");
18272 }
18273 mod authorize {
18274 include!("route_authorize_tests.rs");
18275 }
18276 #[tokio::test]
18277 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
18278 let rig = rig().await;
18279 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
18280 "s".into(),
18281 1,
18282 "tool".into(),
18283 ))
18284 .unwrap();
18285 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
18286 let (direct, _rx) = wide_ctx(999);
18287 for (ctx, expected) in [
18288 (&rig.owner, FrameType::Response),
18289 (&direct, FrameType::Error),
18290 ] {
18291 let frame =
18292 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
18293 .unwrap();
18294 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
18295 assert_eq!(replies[0].header.ty, expected);
18296 if expected == FrameType::Response {
18297 let reply: subc_control::ScopeServedByReply =
18298 serde_json::from_slice(&replies[0].body).unwrap();
18299 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
18300 } else {
18301 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
18302 }
18303 }
18304 let reply = call(
18305 &rig.handler,
18306 &rig.owner,
18307 &ModuleControlRequestFromModule::ScopeDescribe {
18308 owner: Principal::Reserved {
18309 module_id: OWNER.into(),
18310 },
18311 scope_ref: "s".into(),
18312 },
18313 )
18314 .await;
18315 assert!(matches!(
18316 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
18317 ModuleControlResponseToModule::ScopeDescribe { .. }
18318 ));
18319 }
18320
18321 use subc_protocol::scope::{
18322 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
18323 ScopeStatus,
18324 };
18325
18326 use super::*;
18327
18328 const OWNER: &str = "prefrontal-core";
18329
18330 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
18331 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
18332 }
18333
18334 async fn call(
18335 handler: &ControlHandler,
18336 ctx: &RouteCtx,
18337 request: &ModuleControlRequestFromModule,
18338 ) -> Frame {
18339 let body = serde_json::to_vec(request).unwrap();
18340 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
18341 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
18342 assert_eq!(replies.len(), 1, "{replies:?}");
18343 replies.pop().unwrap()
18344 }
18345
18346 async fn sync(
18347 handler: &ControlHandler,
18348 ctx: &RouteCtx,
18349 generation: u64,
18350 scopes: Vec<ScopeRecord>,
18351 ) -> Result<ModuleControlResponseToModule, String> {
18352 let reply = call(
18353 handler,
18354 ctx,
18355 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
18356 )
18357 .await;
18358 match reply.header.ty {
18359 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
18360 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
18361 }
18362 }
18363
18364 async fn describe(
18365 handler: &ControlHandler,
18366 ctx: &RouteCtx,
18367 owner: &str,
18368 scope_ref: &str,
18369 ) -> ModuleControlResponseToModule {
18370 let reply = call(
18371 handler,
18372 ctx,
18373 &ModuleControlRequestFromModule::ScopeDescribe {
18374 owner: Principal::Reserved {
18375 module_id: owner.to_string(),
18376 },
18377 scope_ref: scope_ref.to_string(),
18378 },
18379 )
18380 .await;
18381 assert_eq!(
18382 reply.header.ty,
18383 FrameType::Response,
18384 "{:?}",
18385 parse_error(&reply)
18386 );
18387 serde_json::from_slice(&reply.body).unwrap()
18388 }
18389
18390 #[tokio::test]
18391 async fn scope_describe_never_echoes_bind_proof_or_hash() {
18392 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18393 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18394 let mut rig = rig().await;
18395 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18396 .await;
18397 let _bound = rig
18398 .bound(
18399 Some(AFT),
18400 PLEXUS,
18401 Some(
18402 rig.selector("s", Some(1))
18403 .with_bind_proof(Some(PROOF.into())),
18404 ),
18405 )
18406 .await;
18407 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
18408 let output = serde_json::to_string(&reply).unwrap();
18409 assert!(output.contains("\"status\":\"live\""));
18410 assert!(!output.contains(PROOF));
18411 assert!(!output.contains(HASH));
18412 }
18413
18414 async fn module(
18416 handler: &ControlHandler,
18417 connection: u64,
18418 module_id: &str,
18419 nonce: Option<&str>,
18420 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
18421 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
18422 hello_via_sink(
18423 handler,
18424 &ctx,
18425 &mut rx,
18426 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
18427 )
18428 .await;
18429 (ctx, rx)
18430 }
18431
18432 #[tokio::test]
18435 async fn a_client_connection_cannot_sync_or_describe() {
18436 let handler = ControlHandler::new(Arc::new(Registry::default()));
18437 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
18438 for request in [
18439 ModuleControlRequestFromModule::ScopeSync {
18440 generation: 1,
18441 scopes: vec![head("s", 1)],
18442 },
18443 ModuleControlRequestFromModule::ScopeDescribe {
18444 owner: Principal::Direct,
18445 scope_ref: "s".to_string(),
18446 },
18447 ] {
18448 let reply = call(&handler, &ctx, &request).await;
18449 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
18450 }
18451 assert!(
18452 !handler
18453 .scopes
18454 .read()
18455 .unwrap()
18456 .describe(
18457 &Principal::Reserved {
18458 module_id: OWNER.to_string()
18459 },
18460 "s"
18461 )
18462 .owner_synced
18463 );
18464 }
18465
18466 #[tokio::test]
18469 async fn a_module_without_a_supervised_launch_cannot_sync() {
18470 let handler = ControlHandler::new(Arc::new(Registry::default()));
18471 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
18472 assert_eq!(
18473 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
18474 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18475 );
18476 }
18477
18478 #[tokio::test]
18479 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
18480 let supervisor = SupervisorHandle::new();
18481 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18482 let handler = ControlHandler::new(Arc::new(Registry::default()))
18483 .with_supervisor(supervisor.clone());
18484 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
18485 sync(&handler, &incumbent, 1, vec![head("s", 1)])
18486 .await
18487 .expect("the current launch syncs");
18488
18489 supervisor.open_swap(OWNER, "n2".to_string());
18492 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
18493 assert_eq!(
18494 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
18495 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18496 );
18497 sync(&handler, &incumbent, 2, vec![head("s", 1)])
18498 .await
18499 .expect("the serving owner syncs during the swap");
18500
18501 supervisor.close_swap(OWNER);
18504 assert_eq!(
18505 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
18506 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18507 );
18508 sync(&handler, &incumbent, 3, vec![head("s", 1)])
18509 .await
18510 .expect("the serving owner syncs after the rollback");
18511 handler.cleanup_connection(candidate.connection_id).unwrap();
18512
18513 supervisor.open_swap(OWNER, "n3".to_string());
18518 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
18519 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
18520 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
18521 .await
18522 .expect("the promoted launch takes authority");
18523 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18524 panic!("unexpected reply {reply:?}");
18525 };
18526 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
18527 assert_eq!(
18528 sync(&handler, &incumbent, 4, Vec::new()).await,
18529 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18530 );
18531 }
18532
18533 #[tokio::test]
18536 async fn closing_the_authority_connection_frees_sync_authority() {
18537 let supervisor = SupervisorHandle::new();
18538 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18539 let handler = ControlHandler::new(Arc::new(Registry::default()))
18540 .with_supervisor(supervisor.clone());
18541 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
18542 sync(&handler, &first, 10, vec![head("s", 1)])
18543 .await
18544 .unwrap();
18545 handler.cleanup_connection(first.connection_id).unwrap();
18546
18547 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
18548 sync(&handler, &second, 1, vec![head("s", 1)])
18549 .await
18550 .expect("the next connection takes the released authority");
18551 }
18552
18553 #[tokio::test]
18554 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
18555 let supervisor = SupervisorHandle::new();
18556 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18557 let handler =
18558 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
18559 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
18560 sync(&handler, &first, 10, vec![head("s", 1)])
18561 .await
18562 .unwrap();
18563 handler
18564 .handle_control_frame(
18565 &first,
18566 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
18567 )
18568 .await
18569 .unwrap();
18570 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
18571 sync(&handler, &second, 1, vec![head("s", 1)])
18572 .await
18573 .expect("GOODBYE releases authority even if the old socket remains open");
18574 }
18575
18576 #[tokio::test]
18577 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
18578 let registry = Arc::new(Registry::default());
18579 let supervisor_handle = SupervisorHandle::new();
18580 let supervisor =
18581 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
18582 .with_handle(supervisor_handle.clone())
18583 .with_daemon_incarnation("incarnation-7".to_string());
18584 supervisor
18587 .supervise_configured(
18588 ModuleSpec {
18589 carry_containment: false,
18590 module_id: OWNER.to_string(),
18591 program: PathBuf::from("/nonexistent/prefrontal-core"),
18592 args: Vec::new(),
18593 env: Vec::new(),
18594 reserved: false,
18595 reserved_prefixes: Vec::new(),
18596 protocol: ModuleProtocol::Subc,
18597 overlap: Default::default(),
18598 },
18599 false,
18600 )
18601 .unwrap();
18602 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
18603 let handler =
18604 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
18605 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
18606
18607 let ModuleControlResponseToModule::ScopeDescribe {
18609 status,
18610 daemon_incarnation,
18611 owner_synced,
18612 owner_configured,
18613 scope,
18614 ..
18615 } = describe(&handler, &reader, OWNER, "s").await
18616 else {
18617 panic!("not a describe reply");
18618 };
18619 assert_eq!(status, ScopeStatus::NotLive);
18620 assert_eq!(daemon_incarnation, "incarnation-7");
18621 assert!(!owner_synced);
18622 assert!(owner_configured);
18623 assert!(scope.is_none());
18624
18625 let ModuleControlResponseToModule::ScopeDescribe {
18628 status,
18629 owner_configured,
18630 ..
18631 } = describe(&handler, &reader, "ghost", "s").await
18632 else {
18633 panic!("not a describe reply");
18634 };
18635 assert_eq!(status, ScopeStatus::NotLive);
18636 assert!(!owner_configured);
18637
18638 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
18640 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
18641 let ModuleControlResponseToModule::ScopeDescribe {
18642 status,
18643 scope_epoch,
18644 owner_synced,
18645 scope,
18646 ..
18647 } = describe(&handler, &reader, OWNER, "s").await
18648 else {
18649 panic!("not a describe reply");
18650 };
18651 assert_eq!(status, ScopeStatus::Live);
18652 assert_eq!(scope_epoch, Some(4));
18653 assert!(owner_synced);
18654 let stamp = scope.expect("a live scope carries its stamp");
18655 assert!(
18656 stamp.owner_authorized,
18657 "prefrontal-core is the default authority"
18658 );
18659 assert_eq!(stamp.kind, ScopeKind::Head);
18660 }
18661
18662 #[tokio::test]
18663 async fn scope_authority_owners_decides_owner_authorized() {
18664 let supervisor = SupervisorHandle::new();
18665 supervisor.set_spawn_nonce("broca", "b1".to_string());
18666 let handler = ControlHandler::new(Arc::new(Registry::default()))
18667 .with_supervisor(supervisor)
18668 .with_scope_authority_owners(vec!["broca".to_string()]);
18669 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
18670 let mut gated = head("s", 1);
18671 gated.attributes.agent_id = Some("agent".to_string());
18672 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
18673 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
18674 describe(&handler, &broca, "broca", "s").await
18675 else {
18676 panic!("not a describe reply");
18677 };
18678 assert!(scope.unwrap().owner_authorized);
18679 }
18680
18681 #[tokio::test]
18685 async fn scope_ops_and_the_scopes_capability_are_advertised() {
18686 let handler = ControlHandler::new(Arc::new(Registry::default()));
18687 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
18688 let ack = hello_via_sink(
18689 &handler,
18690 &ctx,
18691 &mut rx,
18692 hello_frame("m", PROTOCOL_VERSION, 1),
18693 )
18694 .await;
18695 let ack = parse_ack(&ack);
18696 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
18697 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
18698 }
18699 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
18700
18701 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
18702 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
18703 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
18704 let reply = handler
18705 .handle_control_frame(&client, frame)
18706 .await
18707 .unwrap()
18708 .pop()
18709 .unwrap();
18710 let ClientControlResponse::ServerDescribe { capabilities, .. } =
18711 serde_json::from_slice(&reply.body).unwrap()
18712 else {
18713 panic!("not a server.describe reply");
18714 };
18715 assert!(
18716 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
18717 "{capabilities:?}"
18718 );
18719 }
18720
18721 const PLEXUS: &str = "plexus";
18724 const OTHER: &str = "other";
18725 const AFT: &str = "aft";
18726 const BROCA: &str = "broca";
18727 const MAGIC: &str = "magic-context";
18728
18729 fn nonce(module_id: &str) -> String {
18730 format!("nonce-{module_id}")
18731 }
18732
18733 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
18734 let (tx, rx) = mpsc::channel(64);
18735 (
18736 RouteCtx {
18737 connection_id: ConnectionId::new(connection),
18738 egress: FrameSink::new(tx),
18739 },
18740 rx,
18741 )
18742 }
18743
18744 struct Rig {
18748 handler: ControlHandler,
18749 forwarding: Arc<ForwardingTable>,
18750 owner: RouteCtx,
18751 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18752 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
18753 generation: u64,
18754 next_connection: u64,
18755 _supervisor: Supervisor,
18756 }
18757
18758 async fn rig() -> Rig {
18759 rig_with_flow_support(true).await
18760 }
18761
18762 async fn rig_with_flow_support(flow_support: bool) -> Rig {
18763 rig_with_scope_capabilities(flow_support, false).await
18764 }
18765
18766 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
18767 let mut provides = Vec::new();
18768 if flow_support {
18769 provides.push("flow-scopes/v1");
18770 }
18771 if run_support {
18772 provides.push("agent-run-scopes/v1");
18773 }
18774 rig_with_target_capabilities(&provides).await
18775 }
18776
18777 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
18778 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
18779 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18780 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18783 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
18784 Frame::build(
18785 FrameType::Hello,
18786 control_flags(),
18787 0,
18788 0,
18789 connection,
18790 serde_json::to_vec(&body).unwrap(),
18791 )
18792 .unwrap()
18793 }
18794
18795 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
18796 let registry = Arc::new(Registry::default());
18797 let forwarding = Arc::new(ForwardingTable::default());
18798 let supervisor_handle = SupervisorHandle::new();
18799 let supervisor =
18800 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
18801 .with_handle(supervisor_handle.clone());
18802 supervisor
18803 .supervise_configured(
18804 ModuleSpec {
18805 carry_containment: false,
18806 module_id: OWNER.to_string(),
18807 program: PathBuf::from("/nonexistent/prefrontal-core"),
18808 args: Vec::new(),
18809 env: Vec::new(),
18810 reserved: false,
18811 reserved_prefixes: Vec::new(),
18812 protocol: ModuleProtocol::Subc,
18813 overlap: Default::default(),
18814 },
18815 false,
18816 )
18817 .unwrap();
18818 for module_id in [OWNER, AFT, BROCA, MAGIC] {
18819 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
18820 }
18821 let handler =
18822 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18823 .with_supervisor(supervisor_handle);
18824 let (owner, mut owner_rx) = wide_ctx(1);
18825 hello_via_sink(
18826 &handler,
18827 &owner,
18828 &mut owner_rx,
18829 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
18830 )
18831 .await;
18832 let mut modules = BTreeMap::new();
18833 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
18834 let (ctx, mut rx) = wide_ctx(connection);
18835 let hello = hello_with_capabilities(module_id, connection, provides);
18836 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
18837 modules.insert(module_id.to_string(), (ctx, rx));
18838 }
18839 Rig {
18840 handler,
18841 forwarding,
18842 owner,
18843 _owner_rx: owner_rx,
18844 modules,
18845 generation: 0,
18846 next_connection: 100,
18847 _supervisor: supervisor,
18848 }
18849 }
18850
18851 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
18852 ScopeCarrier::new(Principal::Reserved {
18853 module_id: module_id.to_string(),
18854 })
18855 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
18856 }
18857
18858 fn session(scope_epoch: u64) -> ScopeRecord {
18861 let mut record = head("s", scope_epoch);
18862 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
18863 record.attributes.agent_id = Some("agent-1".to_string());
18864 record.attributes.delegates = true;
18865 record
18866 }
18867
18868 impl Rig {
18869 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
18870 self.generation += 1;
18871 sync(&self.handler, &self.owner, self.generation, scopes)
18872 .await
18873 .expect("the owner's sync is accepted");
18874 }
18875
18876 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18877 ScopeSelector::new(
18878 Principal::Reserved {
18879 module_id: OWNER.to_string(),
18880 },
18881 scope_ref.to_string(),
18882 scope_epoch,
18883 )
18884 }
18885
18886 fn open_frame(
18887 &mut self,
18888 opener: Option<&str>,
18889 target: &str,
18890 scope: Option<ScopeSelector>,
18891 ) -> (
18892 RouteCtx,
18893 mpsc::Receiver<crate::router::OutboundFrame>,
18894 Frame,
18895 ) {
18896 self.next_connection += 1;
18897 let (ctx, rx) = wide_ctx(self.next_connection);
18898 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
18899 busy_refusals: None,
18900 target: RouteTarget::ToolProvider {
18901 module_id: target.to_string(),
18902 },
18903 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
18904 consumer_identity: opener.map(|module_id| ConsumerIdentity {
18905 module_id: module_id.to_string(),
18906 launch_nonce: nonce(module_id),
18907 }),
18908 consumer_capabilities: None,
18909 role_versions: None,
18910 admission_facts: None,
18911 scope,
18912 })
18913 .unwrap();
18914 let frame = Frame::build(
18915 FrameType::Request,
18916 control_flags(),
18917 0,
18918 0,
18919 self.next_connection,
18920 body,
18921 )
18922 .unwrap();
18923 (ctx, rx, frame)
18924 }
18925
18926 async fn refused(
18928 &mut self,
18929 opener: Option<&str>,
18930 target: &str,
18931 scope: Option<ScopeSelector>,
18932 ) -> String {
18933 self.refusal_body(opener, target, scope).await["code"]
18934 .as_str()
18935 .unwrap()
18936 .to_string()
18937 }
18938
18939 async fn refusal_body(
18940 &mut self,
18941 opener: Option<&str>,
18942 target: &str,
18943 scope: Option<ScopeSelector>,
18944 ) -> Value {
18945 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
18946 let replies = tokio::time::timeout(
18947 Duration::from_secs(2),
18948 self.handler.handle_control_frame(&ctx, frame),
18949 )
18950 .await
18951 .expect("the open must be refused before waiting for a bind ack")
18952 .unwrap();
18953 assert_eq!(replies.len(), 1, "{replies:?}");
18954 assert_eq!(replies[0].header.ty, FrameType::Error);
18955 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18956 assert!(
18957 module_rx.try_recv().is_err(),
18958 "a refused open relays nothing"
18959 );
18960 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
18961 parse_error(&replies[0])
18962 }
18963
18964 async fn relayed(
18966 &mut self,
18967 opener: Option<&str>,
18968 target: &str,
18969 scope: Option<ScopeSelector>,
18970 ) -> Relayed {
18971 self.relayed_with_legacy_root(opener, target, scope, None)
18972 .await
18973 }
18974
18975 async fn relayed_with_legacy_root(
18976 &mut self,
18977 opener: Option<&str>,
18978 target: &str,
18979 scope: Option<ScopeSelector>,
18980 legacy_root: Option<&std::path::Path>,
18981 ) -> Relayed {
18982 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18983 if let Some(root) = legacy_root {
18984 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18985 body["identity"]["project_root"] = serde_json::json!(root);
18986 frame.body = serde_json::to_vec(&body).unwrap();
18987 }
18988 let handler = self.handler.clone();
18989 let task_ctx = ctx.clone();
18990 let task = tokio::spawn(async move {
18991 handler
18992 .handle_control_frame(&task_ctx, frame)
18993 .await
18994 .unwrap()
18995 });
18996 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18997 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18998 .await
18999 .expect("the target receives the relayed route.bind")
19000 .unwrap()
19001 .frame;
19002 Relayed {
19003 target: target.to_string(),
19004 client: ctx,
19005 client_rx: rx,
19006 task,
19007 bind,
19008 }
19009 }
19010
19011 async fn ack(&self, relayed: &Relayed) {
19012 let (module, _) = &self.modules[&relayed.target];
19013 self.handler
19014 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
19015 .await
19016 .unwrap();
19017 }
19018
19019 async fn bound(
19021 &mut self,
19022 opener: Option<&str>,
19023 target: &str,
19024 scope: Option<ScopeSelector>,
19025 ) -> Bound {
19026 let relayed = self.relayed(opener, target, scope).await;
19027 self.commit_relayed(relayed).await
19028 }
19029
19030 async fn bound_with_legacy_root(
19031 &mut self,
19032 opener: Option<&str>,
19033 target: &str,
19034 scope: Option<ScopeSelector>,
19035 root: &std::path::Path,
19036 ) -> Bound {
19037 let relayed = self
19038 .relayed_with_legacy_root(opener, target, scope, Some(root))
19039 .await;
19040 self.commit_relayed(relayed).await
19041 }
19042
19043 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
19044 self.ack(&relayed).await;
19045 let Relayed {
19046 target,
19047 client,
19048 mut client_rx,
19049 task,
19050 bind,
19051 } = relayed;
19052 assert!(
19053 task.await.unwrap().is_empty(),
19054 "the open is answered by commit"
19055 );
19056 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
19057 Bound {
19058 target,
19059 client,
19060 client_rx,
19061 channel,
19062 epoch,
19063 bind,
19064 }
19065 }
19066
19067 fn live(&self, route: &Bound) -> bool {
19068 matches!(
19069 self.forwarding
19070 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
19071 .unwrap(),
19072 DataRoute::Client(DataRouteState::Bound(_))
19073 )
19074 }
19075 }
19076
19077 struct Relayed {
19078 target: String,
19079 client: RouteCtx,
19080 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19081 task: tokio::task::JoinHandle<Vec<Frame>>,
19082 bind: Frame,
19083 }
19084
19085 struct Bound {
19086 target: String,
19087 client: RouteCtx,
19088 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19089 channel: u16,
19090 epoch: u32,
19091 bind: Frame,
19092 }
19093
19094 impl Bound {
19095 fn closed_reason(&mut self) -> RouteCloseReason {
19098 let mut reason = None;
19099 let mut goodbye = false;
19100 while let Ok(outbound) = self.client_rx.try_recv() {
19101 let frame = outbound.frame;
19102 match frame.header.ty {
19103 FrameType::Goodbye => {
19104 assert_eq!(
19105 (frame.header.channel, frame.header.epoch),
19106 (self.channel, self.epoch)
19107 );
19108 goodbye = true;
19109 }
19110 FrameType::Push => {
19111 let ClientControlPush::RouteClosed {
19112 reason: r,
19113 module_id,
19114 ..
19115 } = serde_json::from_slice(&frame.body).unwrap()
19116 else {
19117 panic!("unexpected push");
19118 };
19119 assert_eq!(module_id, self.target);
19120 reason = Some(r);
19121 }
19122 other => panic!("unexpected frame {other:?}"),
19123 }
19124 }
19125 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
19126 reason.expect("the client is told why the route closed")
19127 }
19128
19129 fn untouched(&mut self) -> bool {
19130 self.client_rx.try_recv().is_err()
19131 }
19132
19133 fn stamp(&self) -> Option<ScopeStamp> {
19134 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
19135 ModuleControlRequest::RouteBind { scope, .. } => scope,
19136 other => panic!("expected a route.bind, got {other:?}"),
19137 }
19138 }
19139 }
19140
19141 #[tokio::test]
19142 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
19143 ) {
19144 let mut rig = rig().await;
19145 let mut record = session(1);
19146 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
19147 record.child_owners = vec![Principal::Reserved {
19148 module_id: MAGIC.to_string(),
19149 }];
19150 rig.sync(vec![record]).await;
19151 let scope = || Some(rig_selector("s", Some(1)));
19152
19153 rig.bound(Some(OWNER), PLEXUS, scope()).await;
19156 rig.bound(Some(AFT), OTHER, scope()).await;
19157 rig.bound(Some(BROCA), PLEXUS, scope()).await;
19158
19159 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
19163 assert_eq!(
19164 rig.refused(opener, target, scope()).await,
19165 error_codes::SCOPE_NOT_CARRIER,
19166 "{opener:?} -> {target}"
19167 );
19168 }
19169 }
19170
19171 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
19172 ScopeSelector::new(
19173 Principal::Reserved {
19174 module_id: OWNER.to_string(),
19175 },
19176 scope_ref.to_string(),
19177 scope_epoch,
19178 )
19179 }
19180
19181 #[tokio::test]
19182 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
19183 let mut rig = rig().await;
19184 rig.sync(vec![session(1)]).await;
19185 for opener in [OWNER, AFT] {
19186 assert_eq!(
19187 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
19188 .await,
19189 error_codes::SCOPE_EPOCH_REQUIRED,
19190 "{opener}"
19191 );
19192 }
19193 }
19194
19195 #[tokio::test]
19196 async fn admission_separates_not_synced_not_live_and_ended() {
19197 let mut rig = rig().await;
19198 let code = rig
19200 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19201 .await;
19202 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
19203 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
19204
19205 let ghost = ScopeSelector::new(
19207 Principal::Reserved {
19208 module_id: "ghost".to_string(),
19209 },
19210 "s".to_string(),
19211 Some(1),
19212 );
19213 assert_eq!(
19214 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
19215 error_codes::SCOPE_NOT_LIVE
19216 );
19217
19218 rig.sync(vec![session(2)]).await;
19219 assert_eq!(
19220 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
19221 .await,
19222 error_codes::SCOPE_NOT_LIVE
19223 );
19224 for epoch in [1, 3] {
19225 assert_eq!(
19226 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
19227 .await,
19228 error_codes::SCOPE_ENDED,
19229 "epoch {epoch}"
19230 );
19231 }
19232 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
19234 .await;
19235 }
19236
19237 #[tokio::test]
19238 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
19239 let mut rig = rig().await;
19240 rig.sync(vec![session(1)]).await;
19241 let route = rig
19242 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19243 .await;
19244 let stamp = route.stamp().expect("a scoped bind carries the stamp");
19245 assert_eq!(stamp.scope_ref, "s");
19246 assert_eq!(stamp.scope_epoch, 1);
19247 assert_eq!(stamp.kind, ScopeKind::Head);
19248 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
19249 assert!(stamp.attributes.delegates);
19250 assert!(stamp.owner_authorized);
19251 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19252 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
19253
19254 let (broca, mut broca_rx) = wide_ctx(50);
19257 hello_via_sink(
19258 &rig.handler,
19259 &broca,
19260 &mut broca_rx,
19261 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
19262 )
19263 .await;
19264 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
19265 .await
19266 .unwrap();
19267 let own = ScopeSelector::new(
19268 Principal::Reserved {
19269 module_id: BROCA.to_string(),
19270 },
19271 "b".to_string(),
19272 Some(1),
19273 );
19274 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
19275 assert!(!route.stamp().unwrap().owner_authorized);
19276 }
19277
19278 #[tokio::test]
19279 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
19280 let mut rig = rig().await;
19281 let mut record = head("s", 1);
19282 record.carriers = vec![carrier(AFT, None)];
19283 let flow_id = "Flow:run-7/step_2!~";
19284 record.attributes.flow_id = Some(flow_id.to_string());
19285 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
19286 .await
19287 .unwrap();
19288 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
19289 panic!("not a sync reply");
19290 };
19291 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
19292 let route = rig
19293 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19294 .await;
19295 assert!(rig.live(&route), "the stamped bind committed");
19296 let stamp = route.stamp().expect("a flow scope carries a stamp");
19297 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
19298 assert_eq!(stamp.attributes.agent_id, None);
19299 assert!(!stamp.attributes.delegates);
19300 assert!(stamp.owner_authorized);
19301 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19302 assert_eq!(unscoped.stamp(), None);
19303 }
19304
19305 #[tokio::test]
19306 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
19307 let mut rig = rig_with_flow_support(false).await;
19308 let mut record = session(1);
19309 record.attributes.flow_id = Some("flow:7".to_string());
19310 rig.sync(vec![record]).await;
19311 for opener in [OWNER, AFT] {
19312 let body = rig
19313 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
19314 .await;
19315 assert_eq!(body["code"], "target_flow_unsupported");
19316 let message = body["message"].as_str().unwrap();
19317 for required in [PLEXUS, "flow-scopes/v1"] {
19318 assert!(message.contains(required), "{message}");
19319 }
19320 }
19321 }
19322
19323 #[tokio::test]
19324 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
19325 let mut rig = rig_with_flow_support(true).await;
19326 let mut record = session(1);
19327 record.attributes.flow_id = Some("flow:7".to_string());
19328 rig.sync(vec![record]).await;
19329 for opener in [OWNER, AFT] {
19330 let route = rig
19331 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
19332 .await;
19333 assert!(rig.live(&route));
19334 assert_eq!(
19335 route.stamp().unwrap().attributes.flow_id.as_deref(),
19336 Some("flow:7")
19337 );
19338 }
19339 }
19340
19341 #[tokio::test]
19342 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
19343 use std::future::Future;
19344
19345 let mut rig = rig_with_flow_support(true).await;
19346 let mut record = session(1);
19347 record.attributes.flow_id = Some("flow:7".to_string());
19348 rig.sync(vec![record]).await;
19349 let (client, mut client_rx, frame) =
19350 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
19351 for _ in 0..64 {
19354 client.egress.try_send(route_bind_ack(1)).unwrap();
19355 }
19356 let handler = rig.handler.clone();
19357 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
19358 std::future::poll_fn(|cx| {
19359 assert!(open.as_mut().poll(cx).is_pending());
19360 std::task::Poll::Ready(())
19361 })
19362 .await;
19363 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19364
19365 let old_connection = rig.modules[PLEXUS].0.connection_id;
19366 rig.handler.cleanup_connection(old_connection).unwrap();
19367 let (replacement, mut replacement_rx) = wide_ctx(200);
19368 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
19369 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
19370 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
19371 let hello = Frame::build(
19372 FrameType::Hello,
19373 control_flags(),
19374 0,
19375 0,
19376 200,
19377 serde_json::to_vec(&body).unwrap(),
19378 )
19379 .unwrap();
19380 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
19381
19382 client_rx.try_recv().unwrap();
19383 let replies = tokio::time::timeout(Duration::from_secs(2), open)
19384 .await
19385 .expect("the replacement is refused without waiting for a bind ack")
19386 .unwrap();
19387 assert_eq!(replies.len(), 1);
19388 let body = parse_error(&replies[0]);
19389 assert_eq!(body["code"], "target_flow_unsupported");
19390 for required in [PLEXUS, "flow-scopes/v1"] {
19391 assert!(body["message"].as_str().unwrap().contains(required));
19392 }
19393 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
19394 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
19395 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19396 assert!(rig
19397 .handler
19398 .registry
19399 .get_module_by_connection(replacement.connection_id)
19400 .unwrap()
19401 .is_some());
19402 }
19403
19404 #[tokio::test]
19405 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
19406 use std::future::Future;
19407 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
19408 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19409 const NEXT_HASH: &str =
19410 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
19411 let mut rig = rig().await;
19412 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
19413 record.carriers.push(ScopeCarrier::new(Principal::Direct));
19414 rig.sync(vec![record.clone()]).await;
19415 let (client, mut rx, frame) = rig.open_frame(
19416 None,
19417 PLEXUS,
19418 Some(
19419 rig.selector("s", Some(1))
19420 .with_bind_proof(Some(PROOF.into())),
19421 ),
19422 );
19423 for _ in 0..64 {
19426 client.egress.try_send(route_bind_ack(1)).unwrap();
19427 }
19428 let handler = rig.handler.clone();
19429 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
19430 std::future::poll_fn(|cx| {
19431 assert!(open.as_mut().poll(cx).is_pending());
19432 std::task::Poll::Ready(())
19433 })
19434 .await;
19435 record.bind_proof_sha256 = Some(NEXT_HASH.into());
19436 rig.sync(vec![record]).await;
19437 rx.try_recv().unwrap();
19438 let replies = tokio::time::timeout(Duration::from_secs(2), open)
19439 .await
19440 .unwrap()
19441 .unwrap();
19442 assert_eq!(
19443 parse_error(&replies[0])["code"],
19444 error_codes::SCOPE_BIND_PROOF_MISMATCH
19445 );
19446 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
19447 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
19448 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19449 }
19450
19451 #[tokio::test]
19452 async fn scope_bind_proof_reserved_openers_ignore_proof() {
19453 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19454 let mut rig = rig().await;
19455 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
19456 .await;
19457 for opener in [OWNER, AFT] {
19458 for proof in [None, Some("malformed+proof=")] {
19459 let route = rig
19460 .bound(
19461 Some(opener),
19462 PLEXUS,
19463 Some(
19464 rig.selector("s", Some(1))
19465 .with_bind_proof(proof.map(str::to_string)),
19466 ),
19467 )
19468 .await;
19469 assert!(rig.live(&route));
19470 }
19471 }
19472 }
19473
19474 #[tokio::test]
19475 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
19476 use std::future::Future;
19477 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19478 const NEXT_HASH: &str =
19479 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
19480 let mut rig = rig().await;
19481 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
19482 .await;
19483 let (client, mut rx, frame) =
19484 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
19485 for _ in 0..64 {
19486 client.egress.try_send(route_bind_ack(1)).unwrap();
19487 }
19488 let handler = rig.handler.clone();
19489 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
19490 std::future::poll_fn(|cx| {
19491 assert!(open.as_mut().poll(cx).is_pending());
19492 std::task::Poll::Ready(())
19493 })
19494 .await;
19495 rig.sync(vec![
19496 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
19497 ])
19498 .await;
19499 rx.try_recv().unwrap();
19500 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19501 let bind = std::future::poll_fn(|cx| {
19502 assert!(open.as_mut().poll(cx).is_pending());
19503 module_rx.poll_recv(cx)
19504 })
19505 .await
19506 .unwrap()
19507 .frame;
19508 handler
19509 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
19510 .await
19511 .unwrap();
19512 let replies = tokio::time::timeout(Duration::from_secs(2), open)
19513 .await
19514 .unwrap()
19515 .unwrap();
19516 assert_eq!(replies.len(), 1);
19517 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
19520 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19521 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
19522 let route = rig
19523 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
19524 .await;
19525 assert!(rig.live(&route));
19526 }
19527
19528 #[tokio::test]
19529 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
19530 {
19531 let mut rig = rig_with_flow_support(false).await;
19532 rig.sync(vec![session(1)]).await;
19533 let route = rig
19534 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19535 .await;
19536 assert!(rig.live(&route));
19537 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
19538 let mut record = session(1);
19539 record.attributes.flow_id = Some("flow:7".to_string());
19540 rig.sync(vec![record]).await;
19541 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
19542 assert!(rig.live(&unscoped));
19543 assert_eq!(unscoped.stamp(), None);
19544 }
19545
19546 #[tokio::test]
19547 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
19548 let mut rig = rig().await;
19549 let mut record = session(1);
19550 record.attributes.flow_id = Some("flow:7".to_string());
19551 rig.sync(vec![record.clone()]).await;
19552 let before = rig.forwarding.published_scope_tag(OWNER, "s");
19553 let mut owner_route = rig
19554 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19555 .await;
19556 let mut carrier_route = rig
19557 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
19558 .await;
19559 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19560 rig.sync(vec![record.clone()]).await;
19561 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19562 assert!(rig.live(&owner_route) && owner_route.untouched());
19563 assert!(rig.live(&carrier_route) && carrier_route.untouched());
19564
19565 record.attributes.flow_id = Some("flow:8".to_string());
19566 rig.sync(vec![record]).await;
19567 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
19568 let before = before.unwrap();
19569 assert_eq!(after.scope_epoch, before.scope_epoch);
19570 assert!(after.version > before.version);
19571 for route in [&mut owner_route, &mut carrier_route] {
19572 assert!(!rig.live(route));
19573 assert_eq!(
19574 route.closed_reason(),
19575 RouteCloseReason::ScopeDelegationChanged
19576 );
19577 }
19578 assert!(rig.live(&unscoped) && unscoped.untouched());
19579 for target in [PLEXUS, OTHER] {
19582 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
19583 let goodbye = module_rx
19584 .try_recv()
19585 .expect("the provider sees the drain")
19586 .frame;
19587 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19588 assert!(module_rx.try_recv().is_err());
19589 }
19590 let rebound = rig
19591 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19592 .await;
19593 assert_eq!(
19594 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
19595 Some("flow:8")
19596 );
19597 }
19598
19599 #[tokio::test]
19604 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
19605 let mut rig = rig().await;
19606 rig.sync(vec![session(1)]).await;
19607 let mut cotenant = rig
19608 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19609 .await;
19610
19611 let mut changed = session(1);
19612 changed.child_owners.push(Principal::Reserved {
19613 module_id: MAGIC.to_string(),
19614 });
19615 let mut ended = None;
19616 for (code, next) in [
19617 (error_codes::SCOPE_CHANGED, vec![changed]),
19618 (error_codes::SCOPE_ENDED, Vec::new()),
19619 ] {
19620 let relayed = rig
19621 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19622 .await;
19623 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
19624 ended = Some(next.is_empty());
19625 rig.sync(next).await;
19626 rig.ack(&relayed).await;
19627 let replies = relayed.task.await.unwrap();
19628 assert_eq!(replies.len(), 1, "{replies:?}");
19629 assert_eq!(parse_error(&replies[0])["code"], code);
19630 assert_eq!(
19631 subc_protocol::error_codes::is_retryable_route_open(code),
19632 code == error_codes::SCOPE_CHANGED
19633 );
19634 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19635 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19637 let mut goodbyes = Vec::new();
19640 while let Ok(outbound) = plexus_rx.try_recv() {
19641 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
19642 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
19643 }
19644 assert!(
19645 goodbyes.contains(&(bind_channel, bind_epoch)),
19646 "{goodbyes:?}"
19647 );
19648 assert!(rig
19649 .handler
19650 .registry
19651 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
19652 .unwrap()
19653 .is_some());
19654 }
19655 assert_eq!(ended, Some(true));
19656 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
19659 }
19660
19661 #[tokio::test]
19664 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
19665 struct Case {
19666 name: &'static str,
19667 change: fn(&mut ScopeRecord),
19668 closed: [Option<RouteCloseReason>; 4],
19671 }
19672 use RouteCloseReason::*;
19673 let cases = [
19674 Case {
19675 name: "a carrier entry removed",
19676 change: |r| {
19677 r.carriers.retain(|c| {
19678 c.principal
19679 != Principal::Reserved {
19680 module_id: AFT.to_string(),
19681 }
19682 })
19683 },
19684 closed: [None, Some(ScopeCarrierRemoved), None, None],
19685 },
19686 Case {
19687 name: "a target removed from a carrier",
19688 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
19689 closed: [None, None, None, Some(ScopeCarrierRemoved)],
19690 },
19691 Case {
19692 name: "a bare carrier narrowed to targets",
19693 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
19694 closed: [None, Some(ScopeCarrierRemoved), None, None],
19695 },
19696 Case {
19697 name: "delegates turned off",
19698 change: |r| r.attributes.delegates = false,
19699 closed: [Some(ScopeDelegationChanged); 4],
19700 },
19701 Case {
19702 name: "agent_id changed",
19703 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
19704 closed: [Some(ScopeDelegationChanged); 4],
19705 },
19706 Case {
19707 name: "a carrier added, child owners changed, the record re-sent",
19708 change: |r| {
19709 r.carriers.push(carrier(MAGIC, None));
19710 r.child_owners.push(Principal::Reserved {
19711 module_id: MAGIC.to_string(),
19712 });
19713 },
19714 closed: [None; 4],
19715 },
19716 Case {
19717 name: "a target added",
19718 change: |r| {
19719 r.carriers[1]
19720 .targets
19721 .as_mut()
19722 .unwrap()
19723 .push("third".to_string())
19724 },
19725 closed: [None; 4],
19726 },
19727 Case {
19728 name: "delegates turned on",
19729 change: |r| r.attributes.delegates = true,
19730 closed: [None; 4],
19731 },
19732 ];
19733 for case in cases {
19734 let mut rig = rig().await;
19735 rig.sync(vec![session(1)]).await;
19736 let scope = || Some(rig_selector("s", Some(1)));
19737 let mut routes = [
19738 rig.bound(Some(OWNER), PLEXUS, scope()).await,
19739 rig.bound(Some(AFT), PLEXUS, scope()).await,
19740 rig.bound(Some(BROCA), PLEXUS, scope()).await,
19741 rig.bound(Some(BROCA), OTHER, scope()).await,
19742 ];
19743 let mut record = session(1);
19744 (case.change)(&mut record);
19745 rig.sync(vec![record]).await;
19746 for (index, expected) in case.closed.iter().enumerate() {
19747 let route = &mut routes[index];
19748 match expected {
19749 Some(reason) => {
19750 assert!(!rig.live(route), "{}: route {index} still live", case.name);
19751 assert_eq!(
19752 route.closed_reason(),
19753 *reason,
19754 "{}: route {index}",
19755 case.name
19756 );
19757 }
19758 None => {
19759 assert!(rig.live(route), "{}: route {index} closed", case.name);
19760 assert!(
19761 route.untouched(),
19762 "{}: route {index} was told something",
19763 case.name
19764 );
19765 }
19766 }
19767 }
19768 }
19769 }
19770
19771 #[tokio::test]
19772 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
19773 for next in [Vec::new(), vec![session(2)]] {
19775 let mut rig = rig().await;
19776 rig.sync(vec![session(1)]).await;
19777 let mut route = rig
19778 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19779 .await;
19780 rig.sync(next).await;
19781 assert!(!rig.live(&route));
19782 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
19783 }
19784
19785 let mut rig = rig().await;
19788 let mut child = session(1);
19789 child.scope_ref = "child".to_string();
19790 child.kind = ScopeKind::Worker;
19791 child.parent = Some(ScopeParent::new(
19792 Principal::Reserved {
19793 module_id: OWNER.to_string(),
19794 },
19795 "s".to_string(),
19796 1,
19797 ));
19798 rig.sync(vec![session(1), child.clone()]).await;
19799 let mut child_route = rig
19800 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
19801 .await;
19802 assert_eq!(
19803 child_route.stamp().unwrap().parent_state,
19804 Some(ParentState::Linked)
19805 );
19806 rig.sync(vec![child]).await;
19807 assert!(!rig.live(&child_route));
19808 assert_eq!(
19809 child_route.closed_reason(),
19810 RouteCloseReason::ScopeParentEnded
19811 );
19812 }
19813
19814 #[tokio::test]
19815 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
19816 ) {
19817 let mut rig = rig().await;
19818 rig.sync(vec![session(1)]).await;
19819 let mut route = rig
19820 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19821 .await;
19822 let before = rig.forwarding.published_scope_tag(OWNER, "s");
19823 rig.sync(vec![session(1)]).await;
19824 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19825 assert!(rig.live(&route) && route.untouched());
19826
19827 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
19831 .forwarding
19832 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
19833 .unwrap()
19834 else {
19835 panic!("the route is bound");
19836 };
19837 binding.flow.acquire_tagged(9, false).await.unwrap();
19838 let mut widened = session(1);
19839 widened.carriers.push(carrier(MAGIC, None));
19840 rig.sync(vec![widened]).await;
19841 assert!(rig.live(&route) && route.untouched());
19842 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19843 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
19844 assert_eq!(binding.flow.in_flight(), 1);
19847 binding
19848 .flow
19849 .acquire_tagged(10, false)
19850 .await
19851 .expect("the flow is still open");
19852 }
19853
19854 #[tokio::test]
19857 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
19858 let mut rig = rig().await;
19859 rig.sync(vec![session(1)]).await;
19860 let mut on_incumbent = rig
19861 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19862 .await;
19863
19864 let (candidate, _candidate_rx) = wide_ctx(9);
19867 let registration = rig
19868 .handler
19869 .registry
19870 .register_candidate_with_control_ops(
19871 manifest(PLEXUS, PROTOCOL_VERSION),
19872 PROTOCOL_VERSION,
19873 candidate.connection_id,
19874 module_baseline_control_ops(),
19875 )
19876 .unwrap();
19877 rig.forwarding
19878 .register_candidate_module_connection(
19879 candidate.connection_id,
19880 PLEXUS.to_string(),
19881 PROTOCOL_VERSION,
19882 manifest_concurrency(®istration.manifest),
19883 candidate.egress.clone(),
19884 )
19885 .unwrap();
19886 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
19887 rig.handler
19888 .registry
19889 .promote_candidate(PLEXUS)
19890 .unwrap()
19891 .unwrap();
19892 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
19893
19894 rig.sync(Vec::new()).await;
19895 assert!(!rig.live(&on_incumbent));
19896 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
19897 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19898 let goodbye = incumbent_rx
19899 .try_recv()
19900 .expect("the superseded endpoint is told")
19901 .frame;
19902 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19903 }
19904 }
19905}
19906
19907#[cfg(test)]
19908mod concurrency_default_exposure_tests {
19909 use super::*;
19910
19911 fn hello_body(role_json: &str) -> Vec<u8> {
19912 format!(
19913 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
19914 )
19915 .into_bytes()
19916 }
19917
19918 fn manifest_from(body: &[u8]) -> ModuleManifest {
19919 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
19920 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
19921 .expect("manifest parses")
19922 }
19923
19924 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
19925
19926 #[test]
19927 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
19928 let body = hello_body(&format!(
19929 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
19930 ));
19931 let manifest = manifest_from(&body);
19932 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19935 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
19936 }
19937
19938 #[test]
19939 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
19940 let body = hello_body(&format!(
19941 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
19942 ));
19943 let manifest = manifest_from(&body);
19944 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19945 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19946 }
19947
19948 #[test]
19949 fn non_management_roles_are_never_reported() {
19950 let body = hello_body(
19951 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
19952 );
19953 let manifest = manifest_from(&body);
19954 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19955 }
19956}