1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6 fmt,
7 net::SocketAddr,
8 path::{Path, PathBuf},
9 sync::{Arc, Mutex, RwLock},
10 time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26 error_codes,
27 manifest::{
28 validate_hello_capability_grammar, validate_hello_confirm_operations,
29 validate_hello_display_name, validate_hello_event_declarations,
30 validate_hello_flow_operations, validate_hello_operation_deferral,
31 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32 validate_hello_self_signal_declarations, validate_role_declarations_value,
33 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
34 ProviderRole,
35 },
36 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
37 scope::{
38 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
39 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
40 SCOPE_SYNC_OP,
41 },
42 session::{
43 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
44 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
45 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
46 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS, ROLE_VERSIONS_FIELD,
47 },
48 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
49 Priority, RouteTarget, PROTOCOL_VERSION,
50};
51use tokio::time::{timeout_at, Instant};
52use tracing::{debug, info, instrument::WithSubscriber, warn};
53
54use crate::{
55 caller_identity::CallerIdentity,
56 capability_requirements::{
57 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
58 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
59 RequirementStatus, RuntimeModule,
60 },
61 child_roster::DaemonShutdownFlag,
62 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
63 forwarding::{
64 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
65 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
66 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
67 },
68 observability::{
69 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
70 },
71 privileged_prompt::{prompt_text, PrivilegedOperation},
72 provenance::{
73 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
74 },
75 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
76 router::{daemon_error_flags, RouteCtx, RouterError},
77 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
78 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
79 stderr_tail::{CaptureState, TailEntry},
80 supervise::{
81 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
82 SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
83 },
84 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
85};
86
87pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
93
94const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
95const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
96const CAP_PING_PONG: &str = "ping_pong_v1";
97const CAP_SESSION_ATTACH: &str = "session_attach_v1";
98const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
99
100const SUBC_CONTROL_OPS: &[&str] = &[
101 ops::DAEMON_SHUTDOWN_INTENT,
102 ops::ACTIVITY_RECORD,
103 ops::ACTIVITY_QUERY,
104 ops::ACTIVITY_SUBSCRIBE,
105 ops::SERVER_DESCRIBE,
106 ops::CATALOG_LIST,
107 ops::ROUTE_OPEN,
108 ops::ROUTE_POLL,
109 ops::ROUTE_CLOSING,
110 ops::ROUTE_CLOSED,
111 ops::SUPERVISOR_LIST,
112 ops::SUPERVISOR_RESTART,
113 ops::SUPERVISOR_SWAP,
114 ops::SUPERVISOR_RELOAD,
115 ops::SUPERVISOR_RESCAN,
116 ops::SUPERVISOR_RELEASE_RESERVED,
117 ops::SUPERVISOR_SET_ENABLED,
118 ops::SUPERVISOR_HEALTH_PROBE,
119 ops::SUPERVISOR_HEALTH,
120 ops::SUPERVISOR_STDERR_TAIL,
121 ops::SUPERVISOR_TERMINALS,
122 ops::SUPERVISOR_ROUTES,
123 ops::SUPERVISOR_PROVENANCE,
124 ops::SUPERVISOR_SPAWN_SNAPSHOT,
125 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
126];
127
128const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
135
136mod agent_commands;
137mod rescan_checked;
138use rescan_checked::{RescanMode, RescanPlan};
139
140const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
141 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
142 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
143 "supervisor.live_roots",
144 "supervisor.agent_commands",
145 SCOPE_SYNC_OP,
146 SCOPE_APPLY_OP,
147 SCOPE_DESCRIBE_OP,
148 subc_protocol::scope::ROUTE_AUTHORIZE_OP,
149 "operator.confirm",
150];
151
152const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
156
157const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
158
159pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
167
168pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
183
184pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
202
203const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
204const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
205
206fn reload_verdict(
207 configured: &Path,
208 spawned_from: Option<&Path>,
209 image: subc_control::RunningImageAgreement,
210) -> PendingReloadVerdict {
211 let path = match spawned_from {
212 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
213 ReloadPathAgreement::Match
214 }
215 Some(spawned_from) => ReloadPathAgreement::Mismatch {
216 configured: configured.to_path_buf(),
217 spawned_from: spawned_from.to_path_buf(),
218 },
219 None => ReloadPathAgreement::Unavailable {
220 reason: if matches!(
221 image,
222 subc_control::RunningImageAgreement::Unavailable {
223 reason: subc_control::RunningImageUnavailableReason::NotRunning
224 }
225 ) {
226 ReloadPathUnavailableReason::NotRunning
227 } else {
228 ReloadPathUnavailableReason::SpawnedPathUnavailable
229 },
230 },
231 };
232 PendingReloadVerdict::new(path, image)
233}
234
235fn launch_spec_agreement(
236 configured: &crate::supervise::ModuleSpec,
237 running: bool,
238 spawned: Option<&crate::supervise::LaunchSpecDigest>,
239) -> subc_control::LaunchSpecAgreement {
240 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
241 if !running {
242 return Agreement::Unavailable {
243 reason: Reason::NotRunning,
244 };
245 }
246 match spawned {
247 None => Agreement::Unavailable {
248 reason: Reason::SpawnedSpecUnavailable,
249 },
250 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
251 Agreement::Match
252 }
253 Some(_) => Agreement::Mismatch,
254 }
255}
256
257fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
258 if configured == spawned_from {
259 return true;
260 }
261 #[cfg(windows)]
262 {
263 match (
269 std::fs::canonicalize(configured),
270 std::fs::canonicalize(spawned_from),
271 ) {
272 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
273 _ => false,
274 }
275 }
276 #[cfg(not(windows))]
277 {
278 false
279 }
280}
281
282#[derive(Clone)]
283struct DaemonProvenanceFacts {
284 build: DaemonBuildProvenance,
285 pid: Option<u32>,
286 started_at_ms: Option<u64>,
287 start_clock: Option<crate::clock::StartClock>,
288 executable_path: Option<PathBuf>,
289 executable_identity: Option<SpawnedFileIdentity>,
290 process_start_time: Option<u64>,
291 probe: ExecutableIdentityProbe,
292}
293
294impl Default for DaemonProvenanceFacts {
295 fn default() -> Self {
296 Self {
297 build: DaemonBuildProvenance {
298 build_git_sha: None,
299 build_lock_digest: None,
300 },
301 pid: None,
302 started_at_ms: None,
303 start_clock: None,
304 executable_path: None,
305 executable_identity: None,
306 process_start_time: None,
307 probe: ExecutableIdentityProbe::default(),
308 }
309 }
310}
311
312#[derive(Debug, Clone)]
313struct SupervisorRescanContext {
314 supervisor: Supervisor,
315 config_path: PathBuf,
316 configured_port: Option<u16>,
317 storage_config: Option<crate::daemon_config::StorageConfig>,
318 admission_facts_carrier_module_id: Option<String>,
319 admission_facts_targets: Option<Vec<String>>,
320 scope_authority_owners: Vec<String>,
321 federated_scope_owner: String,
322 privileged_operator_prompt: PrivilegedOperatorPrompt,
323 generation: Arc<std::sync::atomic::AtomicU64>,
329}
330
331const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
352 "reloading",
353 "supervisor_not_live",
354 "registration_not_active",
355 "no_forwarding_connection",
356 "relay_send_failed",
357];
358
359#[derive(Clone)]
361pub struct ControlHandler {
362 registry: Arc<Registry>,
363 activity: Option<crate::activity::ActivityLedger>,
364 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
365 forwarding: Arc<ForwardingTable>,
366 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
367 supervisor: SupervisorHandle,
368 pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
369 agent_declarations: agent_commands::Queues,
370 list_admission: Arc<tokio::sync::Semaphore>,
371 subc_capabilities: Arc<[String]>,
372 route_bind_relay_timeout: Duration,
376 route_bind_relay_timeouts: BTreeMap<String, Duration>,
380 route_bind_breakers: RouteBindBreakers,
383 route_bind_concurrency: RouteBindConcurrency,
386 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
391 route_bind_breaker_threshold: u32,
393 route_bind_breaker_cooldown: Duration,
395 health_probe_timeout: Duration,
396 storage_config: Option<crate::daemon_config::StorageConfig>,
399 machine_id: Option<crate::machine_id::MachineId>,
403 admission_facts_carrier_module_id: Option<String>,
404 admission_facts_targets: Option<Vec<String>>,
405 scopes: Arc<RwLock<ScopeTable>>,
409 scope_deadline_added: Arc<tokio::sync::Notify>,
413 scope_authority_owners: Vec<String>,
416 federated_scope_owner: String,
417 privileged_operator_prompt: PrivilegedOperatorPrompt,
419 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
422 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
423 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
424 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
425 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
426 pub(crate) operator_apps:
427 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
428 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
433 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
439 #[cfg(test)]
441 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
442 #[cfg(test)]
443 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
444 #[allow(dead_code)]
448 #[cfg(windows)]
449 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
450 #[allow(dead_code)]
451 #[cfg(windows)]
452 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
453 #[allow(dead_code)]
457 #[cfg(windows)]
458 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
459 #[cfg(all(test, windows))]
462 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
463 #[cfg(test)]
464 caller_lookup: Option<TestCallerLookup>,
465 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
468 rescan: Option<SupervisorRescanContext>,
469 connected_clients: ConnectedClients,
470 counters: DaemonCounters,
471 local_endpoint_unavailable: Option<String>,
474 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
475 daemon_shutdown: DaemonShutdownFlag,
480 daemon_provenance: DaemonProvenanceFacts,
481 #[cfg(test)]
482 control_dispatch_delay: Option<Duration>,
483 #[cfg(test)]
484 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
485 #[cfg(test)]
489 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
490 #[cfg(test)]
496 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
497}
498
499impl fmt::Debug for ControlHandler {
500 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
501 f.debug_struct("ControlHandler")
502 .field("registry", &self.registry)
503 .field("forwarding", &self.forwarding)
504 .field("process_liveness", &self.process_liveness.is_some())
505 .field("supervisor", &self.supervisor)
506 .field("subc_capabilities", &self.subc_capabilities)
507 .finish()
508 }
509}
510
511struct RouteOpenRequest {
512 target: RouteTarget,
513 identity: BindIdentity,
514 consumer_identity: Option<ConsumerIdentity>,
515 consumer_capabilities: Option<Vec<String>>,
516 role_versions: Option<BTreeMap<String, String>>,
517 busy_refusals: Option<bool>,
518 admission_facts: Option<serde_json::Value>,
519 scope: Option<ScopeSelector>,
520}
521
522#[derive(Clone, Copy)]
523enum RouteOpenPhase {
524 Admission,
525 ProjectRoot,
526 EgressReservation,
527 BindSend,
528 BindAckWait,
529}
530
531struct RouteOpenTimings {
535 module_id: String,
536 principal: String,
537 admission: Duration,
538 project_root: Duration,
539 egress_reservation: Duration,
540 bind_send: Duration,
541 bind_ack_wait: Duration,
542 phase: Option<RouteOpenPhase>,
543 phase_started_at: Instant,
544}
545
546impl RouteOpenTimings {
547 fn new() -> Self {
548 Self {
549 module_id: String::new(),
550 principal: "unresolved".to_string(),
552 admission: Duration::ZERO,
553 project_root: Duration::ZERO,
554 egress_reservation: Duration::ZERO,
555 bind_send: Duration::ZERO,
556 bind_ack_wait: Duration::ZERO,
557 phase: Some(RouteOpenPhase::Admission),
558 phase_started_at: Instant::now(),
559 }
560 }
561
562 fn finish(&mut self) {
563 let elapsed = self.phase_started_at.elapsed();
564 match self.phase.take() {
565 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
566 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
567 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
568 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
569 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
570 None => {}
571 }
572 }
573
574 fn enter(&mut self, phase: RouteOpenPhase) {
575 self.finish();
576 self.phase = Some(phase);
577 self.phase_started_at = Instant::now();
578 }
579}
580
581struct RouteBindReservationGuard {
582 forwarding: Arc<ForwardingTable>,
583 endpoint: ModuleEndpointId,
584 relay_corr: u64,
585 armed: bool,
586}
587
588struct ModuleControlRpcGuard {
589 forwarding: Arc<ForwardingTable>,
590 endpoint: ModuleEndpointId,
591 corr: u64,
592 armed: bool,
593}
594
595impl ModuleControlRpcGuard {
596 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
597 Self {
598 forwarding,
599 endpoint,
600 corr,
601 armed: true,
602 }
603 }
604
605 fn disarm(&mut self) {
606 self.armed = false;
607 }
608}
609
610impl Drop for ModuleControlRpcGuard {
611 fn drop(&mut self) {
612 if self.armed {
613 let _ = self
614 .forwarding
615 .cancel_module_control_rpc(self.endpoint, self.corr);
616 }
617 }
618}
619
620impl RouteBindReservationGuard {
621 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
622 Self {
623 forwarding,
624 endpoint,
625 relay_corr,
626 armed: true,
627 }
628 }
629
630 fn release_and_disarm(&mut self) {
631 if !self.armed {
632 return;
633 }
634 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
635 self.endpoint,
636 self.relay_corr,
637 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
638 ) {
639 send_goodbye_target_best_effort(
640 &self.forwarding.counters(),
641 &target,
642 "canceled route.bind",
643 );
644 }
645 self.armed = false;
646 }
647
648 fn disarm(&mut self) {
649 self.armed = false;
650 }
651}
652
653impl Drop for RouteBindReservationGuard {
654 fn drop(&mut self) {
655 self.release_and_disarm();
656 }
657}
658
659#[derive(Debug, Clone, Default)]
689pub(crate) struct RouteBindBreakers {
690 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
691}
692
693#[derive(Debug, Clone, Default)]
694pub(crate) struct RouteBindConcurrency {
695 modules: Arc<Mutex<HashMap<String, usize>>>,
696}
697
698struct RouteBindConcurrencyGuard {
699 concurrency: RouteBindConcurrency,
700 module_id: String,
701}
702
703impl RouteBindConcurrency {
704 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
707 let mut modules = self
708 .modules
709 .lock()
710 .expect("route.bind concurrency mutex poisoned");
711 let in_flight = modules.entry(module_id.to_string()).or_default();
712 if *in_flight >= limit {
713 return Err(*in_flight);
714 }
715 *in_flight += 1;
716 Ok(RouteBindConcurrencyGuard {
717 concurrency: self.clone(),
718 module_id: module_id.to_string(),
719 })
720 }
721}
722
723impl Drop for RouteBindConcurrencyGuard {
724 fn drop(&mut self) {
725 let mut modules = self
726 .concurrency
727 .modules
728 .lock()
729 .expect("route.bind concurrency mutex poisoned");
730 let remove = {
731 let in_flight = modules
732 .get_mut(&self.module_id)
733 .expect("admitted route.bind has a concurrency entry");
734 *in_flight -= 1;
735 *in_flight == 0
736 };
737 if remove {
738 modules.remove(&self.module_id);
739 }
740 }
741}
742
743#[derive(Debug, Default)]
744struct ModuleBreakerState {
745 consecutive_timeouts: u32,
747 cooldown_until: Option<Instant>,
750 probe_in_flight: Option<Arc<()>>,
755}
756
757enum RouteBindAdmission<'a> {
759 Admitted {
760 guard: RouteBindBreakerGuard<'a>,
761 probe: bool,
764 },
765 Refused {
766 consecutive_timeouts: u32,
767 retry_in: Duration,
771 probe_in_flight: bool,
772 },
773}
774
775struct RouteBindBreakerGuard<'a> {
782 breakers: RouteBindBreakers,
783 module_id: &'a str,
784 probe_token: Option<Arc<()>>,
785 settled: bool,
786}
787
788impl RouteBindBreakerGuard<'_> {
789 fn record_accepted(&mut self) -> bool {
793 self.settled = true;
794 self.breakers.record_accepted(self.module_id)
795 }
796
797 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
800 self.settled = true;
801 self.breakers.record_timeout(
802 self.module_id,
803 self.probe_token.as_ref(),
804 threshold,
805 cooldown,
806 )
807 }
808
809 fn record_inconclusive(&mut self) {
818 self.settled = true;
819 self.breakers
820 .record_inconclusive(self.module_id, self.probe_token.as_ref());
821 }
822}
823
824impl Drop for RouteBindBreakerGuard<'_> {
825 fn drop(&mut self) {
826 if !self.settled {
827 self.breakers
828 .record_inconclusive(self.module_id, self.probe_token.as_ref());
829 }
830 }
831}
832
833struct BreakerOpened {
837 consecutive_timeouts: u32,
838 reopened_after_probe: bool,
841}
842
843impl RouteBindBreakers {
844 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
845 self.modules
846 .lock()
847 .expect("route.bind breaker mutex poisoned")
848 }
849
850 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
853 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
854 probe: probe_token.is_some(),
855 guard: RouteBindBreakerGuard {
856 breakers: self.clone(),
857 module_id,
858 probe_token,
859 settled: false,
860 },
861 };
862
863 let mut modules = self.lock();
864 let Some(state) = modules.get_mut(module_id) else {
865 return admitted(None);
866 };
867 let Some(cooldown_until) = state.cooldown_until else {
868 return admitted(None);
869 };
870 if state.probe_in_flight.is_some() {
871 return RouteBindAdmission::Refused {
872 consecutive_timeouts: state.consecutive_timeouts,
873 retry_in: Duration::ZERO,
874 probe_in_flight: true,
875 };
876 }
877 let now = Instant::now();
878 if now < cooldown_until {
879 return RouteBindAdmission::Refused {
880 consecutive_timeouts: state.consecutive_timeouts,
881 retry_in: cooldown_until - now,
882 probe_in_flight: false,
883 };
884 }
885 let token = Arc::new(());
886 state.probe_in_flight = Some(Arc::clone(&token));
887 admitted(Some(token))
888 }
889
890 fn record_accepted(&self, module_id: &str) -> bool {
891 self.lock()
892 .remove(module_id)
893 .is_some_and(|state| state.cooldown_until.is_some())
894 }
895
896 fn record_timeout(
897 &self,
898 module_id: &str,
899 probe_token: Option<&Arc<()>>,
900 threshold: u32,
901 cooldown: Duration,
902 ) -> Option<BreakerOpened> {
903 let mut modules = self.lock();
904 let state = modules.entry(module_id.to_string()).or_default();
905 let was_open = state.cooldown_until.is_some();
906 let was_probe = Self::owns_probe(state, probe_token);
907 if was_probe {
908 state.probe_in_flight = None;
909 }
910 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
911 if state.consecutive_timeouts < threshold {
912 return None;
913 }
914 state.cooldown_until = Some(Instant::now() + cooldown);
915 Some(BreakerOpened {
916 consecutive_timeouts: state.consecutive_timeouts,
917 reopened_after_probe: was_open && was_probe,
918 })
919 }
920
921 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
922 state
926 .probe_in_flight
927 .as_ref()
928 .zip(token)
929 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
930 }
931
932 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
933 if let Some(state) = self.lock().get_mut(module_id) {
934 if Self::owns_probe(state, probe_token) {
935 state.probe_in_flight = None;
936 }
937 }
938 }
939
940 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
955 self.lock()
956 .remove(module_id)
957 .map(|state| state.consecutive_timeouts)
958 .filter(|discarded| *discarded > 0)
959 }
960
961 fn open_snapshot(&self) -> Option<serde_json::Value> {
968 let now = Instant::now();
969 let modules = self.lock();
970 let open = modules
971 .iter()
972 .filter_map(|(module_id, state)| {
973 let cooldown_until = state.cooldown_until?;
974 Some((
975 module_id.clone(),
976 serde_json::json!({
977 "consecutive_timeouts": state.consecutive_timeouts,
978 "cooldown_remaining_ms":
979 cooldown_until.saturating_duration_since(now).as_millis() as u64,
980 "probe_in_flight": state.probe_in_flight.is_some(),
981 }),
982 ))
983 })
984 .collect::<serde_json::Map<String, serde_json::Value>>();
985 (!open.is_empty()).then_some(serde_json::Value::Object(open))
986 }
987}
988
989impl ControlHandler {
990 pub fn new(registry: Arc<Registry>) -> Self {
991 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
992 }
993
994 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
995 let counters = forwarding.counters();
996 let route_bind_breakers = forwarding.route_bind_breakers();
1000 let route_bind_concurrency = forwarding.route_bind_concurrency();
1001 let route_outages = forwarding.route_outages();
1002 Self {
1003 registry,
1004 activity: None,
1005 activity_streams: Arc::new(Mutex::new(HashMap::new())),
1006 forwarding,
1007 process_liveness: None,
1008 supervisor: SupervisorHandle::new(),
1009 resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1010 env!("CARGO_PKG_VERSION").to_owned(),
1011 )),
1012 agent_declarations: agent_commands::Queues::default(),
1013 list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1014 subc_capabilities: Arc::from([
1015 CAP_MANIFEST_REGISTRATION.to_string(),
1016 CAP_CHANNEL_LIFECYCLE.to_string(),
1017 CAP_PING_PONG.to_string(),
1018 CAP_SESSION_ATTACH.to_string(),
1019 CAP_ADMISSION_FACTS_RELAY.to_string(),
1020 CAP_SCOPES_V1.to_string(),
1021 subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1022 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1023 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1024 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1025 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1026 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1027 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1028 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1029 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1030 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1031 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1032 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1033 ]),
1034 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1035 route_bind_relay_timeouts: BTreeMap::new(),
1036 route_bind_breakers,
1037 route_bind_concurrency,
1038 route_outages,
1039 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1040 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1041 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1042 storage_config: None,
1043 machine_id: None,
1044 admission_facts_carrier_module_id: None,
1045 admission_facts_targets: None,
1046 scopes: Arc::new(RwLock::new(ScopeTable::new(
1047 crate::daemon_config::default_scope_authority_owners(),
1048 ))),
1049 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1050 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1051 federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1052 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1053 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1054 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1055 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1056 local_connections: Arc::new(Mutex::new(HashSet::new())),
1057 operator_app_policy: Default::default(),
1058 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1059 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1060 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1061 #[cfg(test)]
1062 app_open_test_platform: None,
1063 #[cfg(test)]
1064 app_open_test_launch: None,
1065 #[cfg(windows)]
1066 windows_verify: None,
1067 #[cfg(windows)]
1068 windows_flash: None,
1069 #[cfg(windows)]
1070 windows_track: None,
1071 #[cfg(all(test, windows))]
1072 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1073 #[cfg(test)]
1074 caller_lookup: None,
1075 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1076 rescan: None,
1077 connected_clients: ConnectedClients::new(),
1078 local_endpoint_unavailable: None,
1079 counters,
1080 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1081 daemon_shutdown: DaemonShutdownFlag::default(),
1082 daemon_provenance: DaemonProvenanceFacts::default(),
1083 #[cfg(test)]
1084 control_dispatch_delay: None,
1085 #[cfg(test)]
1086 provenance_probe_override: None,
1087 #[cfg(test)]
1088 route_bind_relay_sent_hold: None,
1089 #[cfg(test)]
1090 hello_registry_to_forwarding_hold: None,
1091 }
1092 }
1093
1094 pub fn with_storage_config(
1097 mut self,
1098 storage_config: Option<crate::daemon_config::StorageConfig>,
1099 ) -> Self {
1100 self.storage_config = storage_config;
1101 self
1102 }
1103
1104 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1107 self.activity = Some(crate::activity::ActivityLedger::open(
1108 run_dir.join("activity.jsonl"),
1109 incarnation,
1110 ));
1111 self.supervisor.set_activity(self.activity.clone());
1112 self
1113 }
1114
1115 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1118 self.machine_id = machine_id;
1119 self
1120 }
1121
1122 pub fn with_admission_facts_config(
1126 mut self,
1127 carrier_module_id: Option<String>,
1128 targets: Option<Vec<String>>,
1129 ) -> Self {
1130 self.admission_facts_carrier_module_id = carrier_module_id;
1131 self.admission_facts_targets = targets;
1132 self
1133 }
1134
1135 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1139 self.scopes = Arc::new(RwLock::new(
1140 ScopeTable::new(owners.iter().cloned())
1141 .with_federated_owner(self.federated_scope_owner.clone()),
1142 ));
1143 self.scope_authority_owners = owners;
1144 self
1145 }
1146
1147 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1149 self.scopes = Arc::new(RwLock::new(
1150 ScopeTable::new(self.scope_authority_owners.iter().cloned())
1151 .with_federated_owner(owner.clone()),
1152 ));
1153 self.federated_scope_owner = owner;
1154 self
1155 }
1156
1157 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1159 self.privileged_operator_prompt = mode;
1160 self
1161 }
1162
1163 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1166 self.route_bind_relay_timeout = timeout;
1167 self
1168 }
1169
1170 pub fn with_route_bind_relay_timeouts(
1176 mut self,
1177 timeouts: impl IntoIterator<Item = (String, Duration)>,
1178 ) -> Self {
1179 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1180 self
1181 }
1182
1183 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1189 self.route_bind_relay_timeouts
1190 .get(module_id)
1191 .copied()
1192 .unwrap_or(self.route_bind_relay_timeout)
1193 }
1194
1195 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1203 self.route_bind_breaker_threshold = threshold.max(1);
1204 self.route_bind_breaker_cooldown = cooldown;
1205 self
1206 }
1207
1208 #[cfg(test)]
1209 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1210 self.health_probe_timeout = timeout;
1211 self
1212 }
1213
1214 #[cfg(test)]
1215 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1216 self.control_dispatch_delay = Some(delay);
1217 self
1218 }
1219
1220 #[cfg(test)]
1224 pub(crate) fn with_route_bind_relay_sent_hold(
1225 mut self,
1226 hold: Arc<tokio::sync::Barrier>,
1227 ) -> Self {
1228 self.route_bind_relay_sent_hold = Some(hold);
1229 self
1230 }
1231
1232 #[cfg(test)]
1237 pub(crate) fn with_hello_registry_to_forwarding_hold(
1238 mut self,
1239 hold: Arc<std::sync::Barrier>,
1240 ) -> Self {
1241 self.hello_registry_to_forwarding_hold = Some(hold);
1242 self
1243 }
1244
1245 #[cfg(test)]
1246 fn hold_between_registry_and_forwarding(&self) {
1247 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1248 hold.wait();
1249 hold.wait();
1250 }
1251 }
1252
1253 pub fn with_process_liveness(
1254 mut self,
1255 process_liveness: Arc<dyn ModuleProcessLiveness>,
1256 ) -> Self {
1257 self.process_liveness = Some(process_liveness);
1258 self
1259 }
1260
1261 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1262 self.supervisor = supervisor;
1263 self.supervisor.set_activity(self.activity.clone());
1264 self
1265 }
1266
1267 pub fn with_daemon_provenance(
1268 mut self,
1269 pid: u32,
1270 started_at_ms: u64,
1271 executable_path: Option<PathBuf>,
1272 build_git_sha: Option<String>,
1273 build_lock_digest: Option<String>,
1274 ) -> Self {
1275 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1276 let process_start_time = process_start_time(pid);
1277 self.daemon_provenance = DaemonProvenanceFacts {
1278 build: DaemonBuildProvenance {
1279 build_git_sha,
1280 build_lock_digest,
1281 },
1282 pid: Some(pid),
1283 started_at_ms: Some(started_at_ms),
1284 start_clock: None,
1285 executable_path,
1286 executable_identity,
1287 process_start_time,
1288 probe: ExecutableIdentityProbe::default(),
1289 };
1290 self
1291 }
1292
1293 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1294 self.daemon_provenance.start_clock = Some(clock);
1295 self
1296 }
1297
1298 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1300 self.scopes
1301 .write()
1302 .unwrap_or_else(|poisoned| poisoned.into_inner())
1303 .set_wall_clock(Arc::new(clock));
1304 self
1305 }
1306
1307 #[cfg(test)]
1308 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1309 self.provenance_probe_override = Some(result);
1310 self
1311 }
1312
1313 pub fn with_capability_config(
1317 self,
1318 modules: impl IntoIterator<Item = (String, bool)>,
1319 reserved_capabilities: BTreeMap<String, String>,
1320 ) -> Self {
1321 self.capability_evaluator
1322 .configure(modules, reserved_capabilities);
1323 self
1324 }
1325
1326 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1330 self.daemon_shutdown = flag;
1331 self
1332 }
1333
1334 pub fn with_supervisor_rescan(
1335 mut self,
1336 supervisor: Supervisor,
1337 config_path: impl Into<PathBuf>,
1338 configured_port: Option<u16>,
1339 ) -> Self {
1340 self.rescan = Some(SupervisorRescanContext {
1341 supervisor,
1342 config_path: config_path.into(),
1343 configured_port,
1344 storage_config: self.storage_config.clone(),
1345 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1346 admission_facts_targets: self.admission_facts_targets.clone(),
1347 scope_authority_owners: self.scope_authority_owners.clone(),
1348 federated_scope_owner: self.federated_scope_owner.clone(),
1349 privileged_operator_prompt: self.privileged_operator_prompt,
1350 generation: Arc::default(),
1351 });
1352 self
1353 }
1354
1355 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1356 self.connected_clients = connected_clients;
1357 self
1358 }
1359
1360 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1363 self.local_endpoint_unavailable = reason;
1364 self
1365 }
1366
1367 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1368 Arc::clone(&self.forwarding)
1369 }
1370
1371 pub(crate) fn counters(&self) -> DaemonCounters {
1372 self.counters.clone()
1373 }
1374
1375 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1378 tokio::spawn(async move {
1379 loop {
1380 self.capability_evaluator
1381 .wait_for_change_or_deadline()
1382 .await;
1383 self.refresh_capability_requirements();
1384 }
1385 });
1386 }
1387
1388 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1394 tokio::spawn(async move {
1395 loop {
1396 let has_deadlines = self
1397 .scopes
1398 .read()
1399 .unwrap_or_else(|poisoned| poisoned.into_inner())
1400 .has_deadlines();
1401 if !has_deadlines {
1402 self.scope_deadline_added.notified().await;
1403 continue;
1404 }
1405 tokio::time::sleep(Duration::from_secs(1)).await;
1406 if let Err(error) = self.sweep_expired_scopes() {
1407 warn!(%error, "scope expiry sweep failed");
1408 }
1409 }
1410 });
1411 }
1412
1413 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1414 let mut table = self
1415 .scopes
1416 .write()
1417 .unwrap_or_else(|poisoned| poisoned.into_inner());
1418 let swept = table.sweep_expired();
1419 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1420 drop(table);
1421 self.close_scope_drained_routes(drained);
1422 Ok(())
1423 }
1424
1425 fn publish_scope_changes(
1426 &self,
1427 changes: &[crate::scopes::ScopeTagChange],
1428 expired: &[crate::scopes::ScopeExpired],
1429 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1430 if changes.is_empty() {
1431 return Ok(Vec::new());
1432 }
1433 let drained = self
1434 .forwarding
1435 .publish_scope_changes(changes)
1436 .map_err(RouterError::Forwarding)?;
1437 let mut counts = HashMap::new();
1438 for route in &drained {
1439 *counts
1440 .entry((
1441 route.scope.owner.as_str(),
1442 route.scope.scope_ref.as_str(),
1443 route.scope.tag.scope_epoch,
1444 ))
1445 .or_insert(0usize) += 1;
1446 }
1447 for scope in expired {
1448 let routes_closed = counts
1449 .get(&(
1450 scope.owner.as_str(),
1451 scope.scope_ref.as_str(),
1452 scope.scope_epoch,
1453 ))
1454 .copied()
1455 .unwrap_or(0);
1456 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1457 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1458 routes_closed, "scope expired");
1459 }
1460 Ok(drained)
1461 }
1462
1463 fn runtime_capability_snapshot(
1464 &self,
1465 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1466 let runtime = self
1467 .supervisor
1468 .list()
1469 .into_iter()
1470 .map(|module| {
1471 let status = module.status().map_err(|err| {
1472 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1473 })?;
1474 Ok(RuntimeModule {
1475 module_id: status.module_id,
1476 state: status.state,
1477 enabled: status.enabled,
1478 })
1479 })
1480 .collect::<Result<Vec<_>, RouterError>>()?;
1481 let registrations = self.registry.list_registered_modules().map_err(|err| {
1487 RouterError::backend(
1488 0,
1489 0,
1490 format!("failed to list capability registrations: {err}"),
1491 )
1492 })?;
1493 let registrations = registrations
1494 .into_iter()
1495 .map(|registration| RegisteredModule {
1496 module_id: registration.manifest.module_id,
1497 module_version: registration.manifest.module_version,
1498 capabilities: registration.manifest.capabilities,
1499 })
1500 .collect();
1501 Ok((runtime, registrations))
1502 }
1503
1504 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1511 let cached_registration = RegisteredModule {
1512 module_id: registration.manifest.module_id.clone(),
1513 module_version: registration.manifest.module_version.clone(),
1514 capabilities: registration.manifest.capabilities.clone(),
1515 };
1516 let change = self.capability_evaluator.record_hello(&cached_registration);
1517 log_manifest_change(&cached_registration.module_id, &change);
1518 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1519 self.enforce_capability_denies();
1520 }
1521 self.refresh_capability_requirements();
1522 }
1523
1524 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1528 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1529 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1530 self.supervisor.set_swap_promotion_observer(observer);
1531 }
1532
1533 pub fn refresh_capability_requirements(&self) {
1534 match self.runtime_capability_snapshot() {
1535 Ok((runtime, registrations)) => {
1536 log_requirement_events(
1537 self.capability_evaluator
1538 .evaluate_now(&runtime, ®istrations),
1539 &self.daemon_shutdown,
1540 );
1541 }
1542 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1543 }
1544 }
1545
1546 fn enforce_capability_denies(&self) {
1550 let registrations = match self.registry.list_registered_modules() {
1551 Ok(snapshot) => snapshot,
1552 Err(err) => {
1553 warn!(error = %err, "failed to read registrations for capability deny census");
1554 return;
1555 }
1556 };
1557 let manifests = registrations
1558 .into_iter()
1559 .map(|registration| {
1560 (
1561 registration.manifest.module_id.clone(),
1562 registration.manifest,
1563 )
1564 })
1565 .collect::<BTreeMap<_, _>>();
1566 let census = match self.forwarding.route_census(None) {
1567 Ok(census) => census,
1568 Err(err) => {
1569 warn!(error = %err, "failed to read route census for capability deny enforcement");
1570 return;
1571 }
1572 };
1573
1574 for (target_module_id, routes) in census {
1575 let Some(target_manifest) = manifests.get(&target_module_id) else {
1576 continue;
1577 };
1578 let mut closed_routes = Vec::new();
1579 let mut goodbyes = Vec::new();
1580 for route in routes {
1581 let Principal::Reserved {
1582 module_id: opening_module_id,
1583 } = &route.principal
1584 else {
1585 continue;
1586 };
1587 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1588 continue;
1589 };
1590 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1591 continue;
1592 };
1593
1594 match self.forwarding.release_client_route(
1595 route.goodbye_target.connection_id,
1596 route.goodbye_target.channel,
1597 route.goodbye_target.epoch,
1598 ) {
1599 Ok(RouteRelease::Removed(module_goodbye)) => {
1600 warn!(
1601 opening_module_id,
1602 target_module_id,
1603 capability,
1604 "force-closing route because an attested capability deny edge now matches"
1605 );
1606 goodbyes.push(module_goodbye);
1607 goodbyes.push(route.goodbye_target.clone());
1608 closed_routes.push(route);
1609 }
1610 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1611 Err(err) => warn!(
1612 opening_module_id,
1613 target_module_id,
1614 capability,
1615 error = %err,
1616 "failed to force-close capability-denied route"
1617 ),
1618 }
1619 }
1620
1621 if closed_routes.is_empty() {
1622 continue;
1623 }
1624 send_route_control_pushes(
1625 &self.forwarding,
1626 closed_routes,
1627 ClientControlPush::RouteClosed {
1628 module_id: target_module_id,
1629 channels: Vec::new(),
1630 reason: RouteCloseReason::CapabilityDenied,
1631 drained: false,
1632 abandoned: 0,
1633 excluded_subscriptions: 0,
1634 terminal: Some(false),
1635 },
1636 );
1637 self.emit_route_goodbyes(goodbyes);
1640 }
1641 }
1642
1643 fn not_ready_reason(
1649 &self,
1650 registration: &crate::registry::ModuleRegistration,
1651 ) -> Option<NotReadyReason> {
1652 if !registration.ready {
1653 return Some(NotReadyReason {
1654 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1655 capability: None,
1656 });
1657 }
1658 self.first_unprovided_required_capability(registration)
1659 .map(|capability| NotReadyReason {
1660 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1661 capability: Some(capability),
1662 })
1663 }
1664
1665 fn first_unprovided_required_capability(
1683 &self,
1684 registration: &crate::registry::ModuleRegistration,
1685 ) -> Option<String> {
1686 let required = registration
1687 .manifest
1688 .capabilities
1689 .iter()
1690 .flat_map(|declarations| declarations.requires.iter())
1691 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1692 .map(|requirement| requirement.capability.as_str())
1693 .collect::<BTreeSet<_>>();
1694 if required.is_empty() {
1695 return None;
1696 }
1697 let module_id = registration.manifest.module_id.as_str();
1698 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1699 if required
1700 .iter()
1701 .any(|capability| verdict(capability).is_none())
1702 {
1703 self.refresh_capability_requirements();
1704 }
1705 required
1706 .into_iter()
1707 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1708 .map(str::to_string)
1709 }
1710
1711 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1712 self.capability_evaluator
1713 .statuses()
1714 .into_iter()
1715 .map(capability_requirement_status)
1716 .collect()
1717 }
1718
1719 fn deregister_connection(
1726 &self,
1727 connection_id: ConnectionId,
1728 reason: RegistrationEndReason,
1729 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1730 self.registry
1731 .deregister_connection_with_reason(connection_id, reason)
1732 }
1733
1734 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1735 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1736 return None;
1737 }
1738 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1739 parse_client_control_request(&frame.body)
1740 else {
1741 return None;
1742 };
1743 Some(target_module_id(&target).to_string())
1744 }
1745
1746 pub(crate) fn route_open_capacity_refusal(
1747 &self,
1748 ctx: &RouteCtx,
1749 frame: &Frame,
1750 target_module_id: &str,
1751 in_flight: usize,
1752 limit: usize,
1753 ) -> Result<Frame, RouterError> {
1754 let caller = self.route_open_caller_from_frame(frame);
1757 self.route_open_admission_refusal_frame(
1758 ctx,
1759 &caller,
1760 frame,
1761 target_module_id,
1762 "open_admission_full",
1763 (in_flight, limit),
1764 format!(
1765 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1766 ),
1767 )
1768 }
1769
1770 fn route_open_target_capacity_refusal(
1771 &self,
1772 ctx: &RouteCtx,
1773 caller: &RouteOpenCaller,
1774 frame: &Frame,
1775 target_module_id: &str,
1776 in_flight: usize,
1777 ) -> Result<Frame, RouterError> {
1778 self.route_open_admission_refusal_frame(
1779 ctx,
1780 caller,
1781 frame,
1782 target_module_id,
1783 "target_binds_full",
1784 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1785 format!(
1786 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1787 ),
1788 )
1789 }
1790
1791 #[allow(clippy::too_many_arguments)]
1799 fn route_open_admission_refusal_frame(
1800 &self,
1801 ctx: &RouteCtx,
1802 caller: &RouteOpenCaller,
1803 frame: &Frame,
1804 target_module_id: &str,
1805 reason: &'static str,
1806 (in_flight, limit): (usize, usize),
1807 message: impl Into<String>,
1808 ) -> Result<Frame, RouterError> {
1809 let code = error_codes::TARGET_UNAVAILABLE;
1810 self.counters.increment_route_open_refused(code);
1811 info!(
1812 target: "control",
1813 code,
1814 reason,
1815 module_id = ?target_module_id,
1816 connection_id = ctx.connection_id.get(),
1817 principal = %caller.principal,
1818 scope = %caller.scope,
1819 harness = %caller.harness,
1820 in_flight,
1821 limit,
1822 "route.open refused"
1823 );
1824 control_error_frame(frame, code, message.into())
1825 }
1826
1827 #[cfg(test)]
1833 pub fn handle_control(
1834 &self,
1835 connection_id: ConnectionId,
1836 frame: Frame,
1837 ) -> Result<Vec<Frame>, RouterError> {
1838 match frame.header.ty {
1839 FrameType::Ping => Ok(vec![pong(&frame)?]),
1840 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1841 FrameType::Goodbye => self.handle_goodbye(connection_id),
1842 ty => Ok(vec![control_error_frame(
1843 &frame,
1844 "unsupported_control_frame",
1845 format!("unsupported channel-0 frame {ty:?}"),
1846 )?]),
1847 }
1848 }
1849
1850 pub async fn handle_control_frame(
1851 &self,
1852 ctx: &RouteCtx,
1853 frame: Frame,
1854 ) -> Result<Vec<Frame>, RouterError> {
1855 self.handle_control_frame_timed(ctx, frame, None).await
1856 }
1857
1858 #[cfg(any(test, feature = "test-support"))]
1859 pub fn with_test_operator_app_policy(
1860 mut self,
1861 policy: crate::operator_apps::TestOperatorAppPolicy,
1862 ) -> Self {
1863 self.operator_app_policy = policy.into();
1864 self
1865 }
1866
1867 #[cfg(all(windows, any(test, feature = "test-support")))]
1871 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1872 self.windows_verify = Some(verify);
1873 self
1874 }
1875
1876 #[cfg(all(windows, any(test, feature = "test-support")))]
1878 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1879 self.windows_flash = Some(flash);
1880 self
1881 }
1882
1883 #[cfg(all(windows, any(test, feature = "test-support")))]
1887 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1888 self.windows_track = Some(track);
1889 self
1890 }
1891
1892 pub(crate) fn record_captured_peer(
1893 &self,
1894 connection: ConnectionId,
1895 peer: subc_os::local_peer::CapturedPeer,
1896 ) {
1897 self.record_observed_peer(connection, peer.observation().clone());
1898 #[cfg(windows)]
1899 if let Some(stamp) = self.stamp_windows_launch(&peer) {
1900 self.operator_apps
1901 .lock()
1902 .unwrap_or_else(|p| p.into_inner())
1903 .insert(connection, stamp);
1904 }
1905 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1906 self.operator_apps
1907 .lock()
1908 .unwrap_or_else(|p| p.into_inner())
1909 .insert(connection, stamp);
1910 }
1911 }
1912
1913 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1914 self.local_connections
1915 .lock()
1916 .unwrap_or_else(|p| p.into_inner())
1917 .insert(connection);
1918 }
1919
1920 fn connection_operator_app(
1923 &self,
1924 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1925 ) -> Option<subc_protocol::peer::OperatorApp> {
1926 stamp
1927 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1928 .map(|stamp| stamp.app.clone())
1929 }
1930
1931 fn handle_connection_describe(
1932 &self,
1933 ctx: &RouteCtx,
1934 frame: Frame,
1935 ) -> Result<Vec<Frame>, RouterError> {
1936 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1937 return Ok(vec![control_error_frame(
1938 &frame,
1939 "invalid_control_body",
1940 "malformed connection.describe body",
1941 )?]);
1942 }
1943 let local = self
1944 .local_connections
1945 .lock()
1946 .unwrap_or_else(|p| p.into_inner())
1947 .contains(&ctx.connection_id);
1948 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1949 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1950 Ok(vec![control_response_body_frame(
1951 &frame,
1952 &subc_control::ConnectionDescribeReply::new(local, operator_app),
1953 ops::CONNECTION_DESCRIBE,
1954 )?])
1955 }
1956
1957 pub(crate) fn record_observed_peer(
1958 &self,
1959 connection: ConnectionId,
1960 peer: subc_os::local_peer::ObservedPeer,
1961 ) {
1962 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1963 let peer = match peer {
1964 OsPeer::Unobserved => ObservedPeer::unobserved(),
1965 OsPeer::Observed(peer) => {
1966 let observed = match peer.user {
1967 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1968 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1969 };
1970 observed
1971 .with_exe(
1972 peer.exe
1973 .and_then(|path| path.into_os_string().into_string().ok()),
1974 )
1975 .with_signing_id(peer.signing_id)
1976 }
1977 };
1978 self.observed_peers
1979 .lock()
1980 .unwrap_or_else(|p| p.into_inner())
1981 .insert(connection, peer);
1982 }
1983
1984 pub(crate) fn record_caller_endpoints(
1985 &self,
1986 connection: ConnectionId,
1987 endpoints: Option<(SocketAddr, SocketAddr)>,
1988 ) {
1989 let mut callers = self
1990 .caller_endpoints
1991 .lock()
1992 .unwrap_or_else(|p| p.into_inner());
1993 callers.remove(&connection);
1994 if let Some(endpoints) = endpoints {
1995 callers.insert(connection, endpoints);
1996 }
1997 }
1998
1999 #[cfg(test)]
2000 pub(crate) fn with_caller_lookup_for_test(
2001 mut self,
2002 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2003 ) -> Self {
2004 self.caller_lookup = Some(Arc::new(lookup));
2005 self
2006 }
2007
2008 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2009 let endpoints = self
2010 .caller_endpoints
2011 .lock()
2012 .unwrap_or_else(|p| p.into_inner())
2013 .get(&connection)
2014 .copied();
2015 #[cfg(test)]
2016 let lookup = self.caller_lookup.clone();
2017 tokio::task::spawn_blocking(move || {
2018 #[cfg(test)]
2019 if let Some(lookup) = lookup {
2020 return lookup(endpoints);
2021 }
2022 endpoints
2023 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2024 .unwrap_or_default()
2025 })
2026 .await
2027 .unwrap_or_default()
2028 }
2029
2030 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2031 frame.header.channel == 0
2032 && frame.header.ty == FrameType::Request
2033 && self
2034 .forwarding
2035 .module_endpoint_for_connection(ctx.connection_id)
2036 .is_ok_and(|endpoint| endpoint.is_none())
2037 && parse_client_control_request(&frame.body)
2038 .ok()
2039 .is_some_and(|request| privileged_operation(&request).is_some())
2040 }
2041
2042 pub(crate) async fn handle_control_frame_timed(
2043 &self,
2044 ctx: &RouteCtx,
2045 frame: Frame,
2046 dispatch_started_at: Option<StdInstant>,
2047 ) -> Result<Vec<Frame>, RouterError> {
2048 match frame.header.ty {
2049 FrameType::Ping => Ok(vec![pong(&frame)?]),
2050 FrameType::Hello => {
2051 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2052 }
2053 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2054 FrameType::Cancel => {
2055 if self
2058 .agent_declarations
2059 .cancel(ctx.connection_id, frame.header.corr)
2060 || self
2061 .forwarding
2062 .operator_confirms()
2063 .cancel(ctx.connection_id, frame.header.corr)
2064 || self
2065 .supervisor
2066 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2067 || self.cancel_activity_subscription(ctx, &frame)?
2068 {
2069 Ok(Vec::new())
2070 } else {
2071 Ok(vec![control_error_frame(
2072 &frame,
2073 "unknown_subscription",
2074 "no subscription has this correlation id",
2075 )?])
2076 }
2077 }
2078 FrameType::Request => {
2079 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2082 if op
2083 .as_ref()
2084 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2085 {
2086 return self.handle_activity_record(ctx, frame).await;
2087 }
2088 if op.as_ref().is_some_and(|probe| {
2089 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2090 }) {
2091 return self.handle_activity_read(ctx, frame).await;
2092 }
2093 if op
2094 .as_ref()
2095 .is_some_and(|probe| probe.op == "operator.confirm")
2096 {
2097 return self.handle_operator_confirm(ctx, frame);
2098 }
2099 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2100 return self
2101 .observe_control_wait(
2102 ctx,
2103 None,
2104 ops::APP_OPEN,
2105 self.handle_app_open(ctx, frame),
2106 )
2107 .await;
2108 }
2109 if op
2110 .as_ref()
2111 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2112 {
2113 return self.handle_connection_describe(ctx, frame);
2114 }
2115 if op
2116 .as_ref()
2117 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2118 {
2119 return self.handle_scope_served_by(ctx.connection_id, frame);
2120 }
2121 if self
2122 .forwarding
2123 .module_endpoint_for_connection(ctx.connection_id)
2124 .map_err(RouterError::Forwarding)?
2125 .is_some()
2126 {
2127 if !is_known_module_request_op(&frame.body) {
2128 return Ok(vec![control_error_frame(
2129 &frame,
2130 "unsupported_control_frame",
2131 "module-originated channel-0 REQUEST is not supported",
2132 )?]);
2133 }
2134 let request = match parse_module_control_request_from_module(&frame.body) {
2135 Ok(request) => request,
2136 Err((err, ControlRequestBodyError::UnknownOp)) => {
2137 return Ok(vec![control_error_frame(
2138 &frame,
2139 "unsupported_control_frame",
2140 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2141 )?])
2142 }
2143 Err((err, ControlRequestBodyError::InvalidBody)) => {
2144 return Ok(vec![control_error_frame(
2145 &frame,
2146 "invalid_control_body",
2147 format!("malformed module control body: {err}"),
2148 )?])
2149 }
2150 };
2151 let op = module_control_request_op(&request);
2152 let corr = frame.header.corr;
2153 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2154 if let ModuleControlRequestFromModule::AgentCommands {
2155 started,
2156 finished,
2157 known,
2158 } = request
2159 {
2160 return self
2161 .agent_declarations
2162 .enqueue(
2163 self,
2164 ctx,
2165 frame,
2166 crate::resource_totals::Declaration {
2167 started,
2168 finished,
2169 known,
2170 },
2171 )
2172 .await;
2173 }
2174 let result =
2175 self.handle_module_control_request(ctx.connection_id, frame, request);
2176 log_slow_control_dispatch(
2177 dispatch_started_at,
2178 op,
2179 ctx.connection_id,
2180 corr,
2181 None,
2182 );
2183 return result;
2184 }
2185
2186 if is_known_module_request_op(&frame.body) {
2187 let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2188 .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2189 let code = if is_agents {
2190 "not_supervised"
2191 } else if op
2192 .as_ref()
2193 .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2194 {
2195 error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2196 } else {
2197 "not_registered"
2198 };
2199 return Ok(vec![control_error_frame(
2200 &frame,
2201 code,
2202 "module-originated control requests require the module's own registered connection",
2203 )?]);
2204 }
2205
2206 let request = match parse_client_control_request(&frame.body) {
2207 Ok(request) => request,
2208 Err((err, ControlRequestBodyError::UnknownOp)) => {
2209 return Ok(vec![control_error_frame(
2210 &frame,
2211 "unknown_control_op",
2212 format!("unknown client control op: {err}"),
2213 )?])
2214 }
2215 Err((err, ControlRequestBodyError::InvalidBody)) => {
2216 return Ok(vec![control_error_frame(
2217 &frame,
2218 "invalid_control_body",
2219 format!("malformed client control body: {err}"),
2220 )?])
2221 }
2222 };
2223 let op = client_control_request_op(&request);
2224 let corr = frame.header.corr;
2225 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2226 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2227 #[cfg(test)]
2228 if let Some(delay) = self.control_dispatch_delay {
2229 tokio::time::sleep(delay).await;
2230 }
2231 let result = self
2232 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2233 .await;
2234 if let Some(timings) = route_open_timings.as_mut() {
2235 timings.finish();
2236 }
2237 log_slow_control_dispatch(
2238 dispatch_started_at,
2239 op,
2240 ctx.connection_id,
2241 corr,
2242 route_open_timings.as_ref(),
2243 );
2244 result
2245 }
2246 FrameType::Push => {
2247 let Some(endpoint) = self
2248 .forwarding
2249 .module_endpoint_for_connection(ctx.connection_id)
2250 .map_err(RouterError::Forwarding)?
2251 else {
2252 return Ok(vec![control_error_frame(
2253 &frame,
2254 "unsupported_control_frame",
2255 "client-originated channel-0 PUSH is not supported",
2256 )?]);
2257 };
2258 self.handle_status_update(endpoint, frame)
2259 }
2260 FrameType::Response | FrameType::Error
2261 if self
2262 .forwarding
2263 .module_endpoint_for_connection(ctx.connection_id)
2264 .map_err(RouterError::Forwarding)?
2265 .is_some() =>
2266 {
2267 self.handle_module_relay_response(ctx.connection_id, frame)
2268 }
2269 ty => Ok(vec![control_error_frame(
2270 &frame,
2271 "unsupported_control_frame",
2272 format!("unsupported channel-0 frame {ty:?}"),
2273 )?]),
2274 }
2275 }
2276
2277 pub fn cleanup_connection(
2278 &self,
2279 connection_id: ConnectionId,
2280 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2281 self.local_connections
2282 .lock()
2283 .unwrap_or_else(|p| p.into_inner())
2284 .remove(&connection_id);
2285 self.caller_endpoints
2286 .lock()
2287 .unwrap_or_else(|p| p.into_inner())
2288 .remove(&connection_id);
2289 self.build_class_warned
2290 .lock()
2291 .unwrap_or_else(|p| p.into_inner())
2292 .remove(&connection_id);
2293 self.observed_peers
2294 .lock()
2295 .unwrap_or_else(|p| p.into_inner())
2296 .remove(&connection_id);
2297 self.operator_apps
2298 .lock()
2299 .unwrap_or_else(|p| p.into_inner())
2300 .remove(&connection_id);
2301 self.forwarding
2302 .operator_confirms()
2303 .connection_closed(connection_id);
2304 self.cleanup_connection_with_end_reason(
2305 connection_id,
2306 RegistrationEndReason::ConnectionClosed,
2307 )
2308 }
2309
2310 fn cleanup_connection_with_end_reason(
2311 &self,
2312 connection_id: ConnectionId,
2313 requested_reason: RegistrationEndReason,
2314 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2315 self.agent_declarations.close(connection_id);
2316 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2317 self.registry
2318 .get_module_by_connection(connection_id)?
2319 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2320 .and_then(|module| module.registration_end_reason().ok().flatten())
2321 .unwrap_or(requested_reason)
2322 } else {
2323 requested_reason
2324 };
2325 let crash_closed = self
2326 .registry
2327 .get_module_by_connection(connection_id)?
2328 .and_then(|registration| {
2329 self.forwarding
2330 .module_endpoint_for_connection(connection_id)
2331 .ok()
2332 .flatten()
2333 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2334 .map(|routes| (registration.manifest.module_id, routes))
2335 });
2336 let crash_closed = crash_closed.map(|(module_id, routes)| {
2337 let terminal = match self.supervisor.get(&module_id) {
2338 None => false,
2339 Some(module) => match module.will_recover_after_connection_loss() {
2340 Ok(will_recover) => !will_recover,
2341 Err(err) => {
2342 warn!(
2343 %module_id,
2344 error = %err,
2345 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2346 );
2347 false
2348 }
2349 },
2350 };
2351 let (reason, drained) = match self
2355 .forwarding
2356 .daemon_drain_result_for_connection(connection_id)
2357 {
2358 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2359 Ok(None) => (RouteCloseReason::Crash, false),
2360 Err(err) => {
2361 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2362 (RouteCloseReason::Crash, false)
2363 }
2364 };
2365 (module_id, routes, reason, drained, terminal)
2366 });
2367 let registrations = self.deregister_connection(connection_id, end_reason);
2368 let cleanup = if crash_closed.is_some() {
2369 self.forwarding.cleanup_connection_counted(connection_id)
2370 } else {
2371 self.forwarding
2377 .cleanup_connection(connection_id)
2378 .map(|released| crate::forwarding::ConnectionCleanup {
2379 released,
2380 abandoned_relays: 0,
2381 })
2382 };
2383 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2388 let abandoned = cleanup
2389 .as_ref()
2390 .map_or(0, |cleanup| cleanup.abandoned_relays);
2391 send_route_control_pushes(
2392 &self.forwarding,
2393 routes,
2394 ClientControlPush::RouteClosed {
2395 module_id,
2396 channels: Vec::new(),
2397 reason,
2398 drained,
2399 abandoned,
2400 excluded_subscriptions: 0,
2401 terminal: Some(terminal),
2402 },
2403 );
2404 }
2405 if let Ok(cleanup) = cleanup {
2406 self.emit_route_goodbyes(cleanup.released);
2407 }
2408 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2412 crate::supervise::notify_registration_release();
2413 self.capability_evaluator.wake_deadline_loop();
2414 self.refresh_capability_requirements();
2415 }
2416 self.supervisor.remove_spawn_subscribers(connection_id);
2417 self.activity_streams
2418 .lock()
2419 .unwrap_or_else(|p| p.into_inner())
2420 .retain(|(id, _), task| {
2421 if *id == connection_id {
2422 task.abort();
2423 false
2424 } else {
2425 true
2426 }
2427 });
2428 self.hello_launch_nonces
2431 .lock()
2432 .unwrap_or_else(|poisoned| poisoned.into_inner())
2433 .forget(connection_id);
2434 self.scopes
2435 .write()
2436 .unwrap_or_else(|poisoned| poisoned.into_inner())
2437 .release_connection(connection_id);
2438 registrations
2439 }
2440
2441 pub(crate) fn handle_route_goodbye(
2442 &self,
2443 connection_id: ConnectionId,
2444 route_channel: u16,
2445 route_epoch: u32,
2446 ) -> Result<bool, RouterError> {
2447 debug!(
2448 connection_id = connection_id.get(),
2449 route_channel, route_epoch, "handling route GOODBYE"
2450 );
2451 let RouteRelease::Removed(released_route) = self
2452 .forwarding
2453 .release_client_route(connection_id, route_channel, route_epoch)
2454 .map_err(RouterError::Forwarding)?
2455 else {
2456 return Ok(false);
2457 };
2458 self.emit_route_goodbyes(vec![released_route]);
2459 Ok(true)
2460 }
2461
2462 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2463 for released in released_routes {
2464 let frame = match Frame::build_with_version(
2465 released.negotiated_ver,
2466 FrameType::Goodbye,
2467 control_flags(),
2468 released.channel,
2469 released.epoch,
2470 0,
2471 Vec::new(),
2472 ) {
2473 Ok(frame) => frame,
2474 Err(err) => {
2475 warn!(
2476 route_channel = released.channel,
2477 error = %err,
2478 "failed to build route GOODBYE frame"
2479 );
2480 continue;
2481 }
2482 };
2483 if !released.close_on_delivery_failure() {
2484 crate::forwarding::send_module_route_goodbye(
2485 &self.counters,
2486 &released.sink,
2487 frame,
2488 released.module_id.as_deref(),
2489 "client route released",
2490 );
2491 continue;
2492 }
2493 if let Err(err) = released.sink.try_send(frame) {
2494 warn!(
2495 target_connection_id = released.connection_id.get(),
2496 route_channel = released.channel,
2497 error = %err,
2498 "route GOODBYE was not delivered to client; closing target connection"
2499 );
2500 if self
2501 .forwarding
2502 .escalate_client_delivery_failure(
2503 released.connection_id,
2504 released.channel,
2505 released.epoch,
2506 CloseReason::new(
2507 "route_goodbye_delivery_failed",
2508 format!(
2509 "failed to enqueue route GOODBYE for channel {}: {err}",
2510 released.channel
2511 ),
2512 ),
2513 crate::forwarding::UndeliveredFrame {
2514 module_id: released.module_id.as_deref(),
2515 sink: &released.sink,
2516 },
2517 )
2518 .unwrap_or(false)
2519 {
2520 self.counters.increment_goodbye_relay_client_failed();
2521 }
2522 }
2523 }
2524 }
2525
2526 fn send_abandoned_route_bind_goodbye(
2538 &self,
2539 module_sink: &crate::FrameSink,
2540 negotiated_ver: u8,
2541 module_channel: u16,
2542 module_epoch: u32,
2543 ) {
2544 let frame = match Frame::build_with_version(
2545 negotiated_ver,
2546 FrameType::Goodbye,
2547 control_flags(),
2548 module_channel,
2549 module_epoch,
2550 0,
2551 Vec::new(),
2552 ) {
2553 Ok(frame) => frame,
2554 Err(err) => {
2555 warn!(
2556 route_channel = module_channel,
2557 error = %err,
2558 "failed to build GOODBYE for abandoned route.bind"
2559 );
2560 return;
2561 }
2562 };
2563 crate::forwarding::send_module_route_goodbye(
2564 &self.counters,
2565 module_sink,
2566 frame,
2567 None,
2568 "abandoned route.bind",
2569 );
2570 }
2571
2572 fn handle_hello(
2573 &self,
2574 connection_id: ConnectionId,
2575 sink: Option<crate::FrameSink>,
2576 frame: Frame,
2577 ) -> Result<Vec<Frame>, RouterError> {
2578 debug!(
2579 connection_id = connection_id.get(),
2580 corr = frame.header.corr,
2581 "handling HELLO"
2582 );
2583 if self
2587 .registry
2588 .get_module_by_connection(connection_id)
2589 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2590 .is_some()
2591 {
2592 return Ok(vec![control_error_frame(
2593 &frame,
2594 "invalid_hello",
2595 "connection is already registered as a module",
2596 )?]);
2597 }
2598 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2599 Ok(value) => value,
2600 Err(err) => {
2601 return Ok(vec![control_error_frame(
2602 &frame,
2603 "invalid_hello",
2604 format!("malformed HELLO body: {err}"),
2605 )?])
2606 }
2607 };
2608 if let Err(err) = validate_hello_display_name(&hello_value) {
2609 return Ok(vec![control_error_frame(
2610 &frame,
2611 "invalid_manifest",
2612 err.to_string(),
2613 )?]);
2614 }
2615 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2616 return Ok(vec![control_error_frame(
2617 &frame,
2618 "invalid_capability_grammar",
2619 err.to_string(),
2620 )?]);
2621 }
2622 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2623 return Ok(vec![control_error_frame(
2624 &frame,
2625 "invalid_manifest",
2626 err.to_string(),
2627 )?]);
2628 }
2629 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2630 return Ok(vec![control_error_frame(
2631 &frame,
2632 "invalid_event_declaration",
2633 err.to_string(),
2634 )?]);
2635 }
2636 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2637 return Ok(vec![control_error_frame(
2638 &frame,
2639 "invalid_manifest",
2640 err.to_string(),
2641 )?]);
2642 }
2643 if let Some(provenance) = hello_value
2644 .get("manifest")
2645 .and_then(|manifest| manifest.get("provenance"))
2646 {
2647 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2648 return Ok(vec![control_error_frame(
2649 &frame,
2650 "invalid_manifest",
2651 format!("malformed manifest provenance: {err}"),
2652 )?]);
2653 }
2654 }
2655 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2656 let module_id = hello_value["manifest"]["module_id"]
2657 .as_str()
2658 .unwrap_or("<unknown>");
2659 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2660 }
2661 if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
2662 let module_id = hello_value
2663 .get("manifest")
2664 .and_then(|m| m.get("module_id"))
2665 .and_then(serde_json::Value::as_str)
2666 .unwrap_or("unknown");
2667 warn!(module_id, %reason, "dropping operation_deferral from HELLO");
2668 }
2669 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2670 let module_id = hello_value["manifest"]["module_id"]
2671 .as_str()
2672 .unwrap_or("<unknown>");
2673 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2674 }
2675 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2676 let module_id = hello_value["manifest"]["module_id"]
2677 .as_str()
2678 .unwrap_or("<unknown>");
2679 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2680 }
2681 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2682 Ok(hello) => hello,
2683 Err(err) => {
2684 return Ok(vec![control_error_frame(
2685 &frame,
2686 "invalid_hello",
2687 format!("malformed HELLO body: {err}"),
2688 )?])
2689 }
2690 };
2691
2692 if hello.protocol_ver != hello.manifest.protocol_ver {
2693 return Ok(vec![control_error_frame(
2694 &frame,
2695 "invalid_manifest",
2696 format!(
2697 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2698 hello.protocol_ver, hello.manifest.protocol_ver
2699 ),
2700 )?]);
2701 }
2702
2703 if hello.manifest.module_id.trim().is_empty() {
2704 return Ok(vec![control_error_frame(
2705 &frame,
2706 "invalid_manifest",
2707 "manifest module_id must not be empty",
2708 )?]);
2709 }
2710
2711 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2712 Ok(negotiated_ver) => negotiated_ver,
2713 Err(message) => {
2714 return Ok(vec![control_error_frame(
2715 &frame,
2716 "version_unsupported",
2717 message,
2718 )?])
2719 }
2720 };
2721
2722 let swap_admission = self
2730 .supervisor
2731 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2732 if swap_admission == SwapHelloAdmission::Refused {
2733 warn!(
2734 module_id = %hello.manifest.module_id,
2735 connection_id = connection_id.get(),
2736 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2737 );
2738 return Ok(vec![control_error_frame(
2739 &frame,
2740 "swap_token_invalid",
2741 format!(
2742 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2743 hello.manifest.module_id
2744 ),
2745 )?]);
2746 }
2747 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2748
2749 let verification = if swap_candidate {
2754 ReservedHelloVerification {
2755 nonce_verified: true,
2756 prefix_owner: None,
2757 }
2758 } else {
2759 match self.supervisor.reserved_hello_verification(
2760 &hello.manifest.module_id,
2761 hello.launch_nonce.as_deref(),
2762 ) {
2763 Ok(verified) => verified,
2764 Err(rejection) => {
2765 let message = match rejection {
2766 ReservedHelloRejection::Exact { module_id } => format!(
2767 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2768 ),
2769 ReservedHelloRejection::Prefix {
2770 prefix,
2771 owner_module_id,
2772 } => format!(
2773 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2774 hello.manifest.module_id
2775 ),
2776 };
2777 return Ok(vec![control_error_frame(
2778 &frame,
2779 "reserved_module",
2780 message,
2781 )?]);
2782 }
2783 }
2784 };
2785
2786 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2787 &hello.manifest.module_id,
2788 hello.manifest.capabilities.as_ref(),
2789 );
2790 if let Some(refusal) = reserved_capability_refusals.first() {
2791 let capability = refusal.capability.clone();
2792 let bound_module = refusal.claimants[0].clone();
2793 log_duplicate_claim_events(reserved_capability_refusals);
2794 return Ok(vec![control_error_frame(
2795 &frame,
2796 "reserved_capability",
2797 format!(
2798 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2799 capability, bound_module, hello.manifest.module_id
2800 ),
2801 )?]);
2802 }
2803
2804 if self
2807 .forwarding
2808 .connection_has_client_routes(connection_id)
2809 .map_err(RouterError::Forwarding)?
2810 {
2811 return Ok(vec![control_error_frame(
2812 &frame,
2813 "invalid_hello",
2814 "connection has open client routes and cannot also register as a module",
2815 )?]);
2816 }
2817
2818 self.hello_launch_nonces
2823 .lock()
2824 .unwrap_or_else(|poisoned| poisoned.into_inner())
2825 .record(connection_id, hello.launch_nonce.as_deref());
2826 let control_ops = effective_module_control_ops(hello.control_ops);
2827 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2830 if swap_candidate {
2831 return self.register_swap_candidate(
2832 connection_id,
2833 sink,
2834 &frame,
2835 hello.manifest,
2836 negotiated_ver,
2837 control_ops,
2838 hello_ack,
2839 );
2840 }
2841 let registration = match self
2848 .registry
2849 .register_unpublished_verified_with_control_ops(
2850 hello.manifest,
2851 negotiated_ver,
2852 connection_id,
2853 control_ops,
2854 verification.nonce_verified,
2855 ) {
2856 Ok(registration) => registration,
2857 Err(RegistryError::DuplicateModuleId { module_id }) => {
2858 return Ok(vec![control_error_frame(
2859 &frame,
2860 "duplicate_module_id",
2861 format!(
2862 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2863 ),
2864 )?])
2865 }
2866 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2867 return Ok(vec![control_error_frame(
2868 &frame,
2869 "invalid_module_id",
2870 err.to_string(),
2871 )?])
2872 }
2873 Err(err) => {
2874 return Ok(vec![control_error_frame(
2875 &frame,
2876 "registry_error",
2877 err.to_string(),
2878 )?])
2879 }
2880 };
2881 #[cfg(test)]
2882 self.hold_between_registry_and_forwarding();
2883
2884 let reply = if let Some(sink) = sink {
2885 let concurrency = manifest_concurrency(®istration.manifest);
2901 if let Err(err) = self.forwarding.register_module_connection_acked(
2902 connection_id,
2903 registration.manifest.module_id.clone(),
2904 negotiated_ver,
2905 concurrency,
2906 sink,
2907 crate::forwarding::ModuleHelloAck {
2908 frame: hello_ack,
2909 verified_prefix_owner: verification.prefix_owner,
2910 },
2911 ) {
2912 if matches!(
2917 self.deregister_connection(
2918 connection_id,
2919 RegistrationEndReason::RegistrationFailed,
2920 ),
2921 Ok(r) if !r.is_empty()
2922 ) {
2923 crate::supervise::notify_registration_release();
2924 }
2925 return Ok(vec![control_error_frame(
2926 &frame,
2927 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2928 "invalid_hello"
2929 } else {
2930 forwarding_error_code(&err)
2931 },
2932 err.to_string(),
2933 )?]);
2934 }
2935 Vec::new()
2936 } else {
2937 vec![hello_ack]
2940 };
2941
2942 if !self
2947 .registry
2948 .publish_registration(connection_id)
2949 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2950 {
2951 debug!(
2952 module_id = %registration.manifest.module_id,
2953 connection_id = connection_id.get(),
2954 "registration was removed before it could be published"
2955 );
2956 }
2957
2958 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2967 info!(
2968 module_id = %registration.manifest.module_id,
2969 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2970 );
2971 }
2972
2973 self.apply_registration_capabilities(®istration);
2974
2975 self.supervisor
2976 .record_activity_registration(®istration.manifest.module_id);
2977
2978 info!(
2979 module_id = %registration.manifest.module_id,
2980 module_version = %registration.manifest.module_version,
2981 negotiated_ver,
2982 routable_provider = manifest_provides_routable_role(®istration.manifest),
2983 connection_id = connection_id.get(),
2984 "module registered"
2985 );
2986
2987 Ok(reply)
2988 }
2989
2990 #[allow(clippy::too_many_arguments)]
3007 fn register_swap_candidate(
3008 &self,
3009 connection_id: ConnectionId,
3010 sink: Option<crate::FrameSink>,
3011 frame: &Frame,
3012 manifest: ModuleManifest,
3013 negotiated_ver: u8,
3014 control_ops: Vec<String>,
3015 hello_ack: Frame,
3016 ) -> Result<Vec<Frame>, RouterError> {
3017 let module_id = manifest.module_id.clone();
3018 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
3019 manifest,
3020 negotiated_ver,
3021 connection_id,
3022 control_ops,
3023 ) {
3024 Ok(registration) => registration,
3025 Err(RegistryError::DuplicateModuleId { module_id }) => {
3026 return Ok(vec![control_error_frame(
3027 frame,
3028 "duplicate_module_id",
3029 format!(
3030 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
3031 ),
3032 )?])
3033 }
3034 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3035 return Ok(vec![control_error_frame(
3036 frame,
3037 "invalid_module_id",
3038 err.to_string(),
3039 )?])
3040 }
3041 Err(err) => {
3042 return Ok(vec![control_error_frame(
3043 frame,
3044 "registry_error",
3045 err.to_string(),
3046 )?])
3047 }
3048 };
3049 #[cfg(test)]
3050 self.hold_between_registry_and_forwarding();
3051 let reply = if let Some(sink) = sink {
3052 let concurrency = manifest_concurrency(®istration.manifest);
3056 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
3057 connection_id,
3058 module_id.clone(),
3059 negotiated_ver,
3060 concurrency,
3061 sink,
3062 hello_ack,
3063 ) {
3064 if matches!(
3065 self.deregister_connection(
3066 connection_id,
3067 RegistrationEndReason::RegistrationFailed,
3068 ),
3069 Ok(r) if !r.is_empty()
3070 ) {
3071 crate::supervise::notify_registration_release();
3072 }
3073 return Ok(vec![control_error_frame(
3074 frame,
3075 forwarding_error_code(&err),
3076 err.to_string(),
3077 )?]);
3078 }
3079 Vec::new()
3080 } else {
3081 vec![hello_ack]
3082 };
3083 if !self
3084 .registry
3085 .publish_registration(connection_id)
3086 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3087 {
3088 debug!(
3089 module_id = %module_id,
3090 connection_id = connection_id.get(),
3091 "swap candidate registration was removed before it could be published"
3092 );
3093 }
3094 self.supervisor.mark_swap_candidate_admitted(&module_id);
3095 info!(
3096 module_id = %module_id,
3097 module_version = %registration.manifest.module_version,
3098 negotiated_ver,
3099 ready = registration.ready,
3100 connection_id = connection_id.get(),
3101 "swap candidate registered; not routable until cutover"
3102 );
3103 Ok(reply)
3104 }
3105
3106 fn build_hello_ack(
3107 &self,
3108 frame: &Frame,
3109 negotiated_ver: u8,
3110 module_id: &str,
3111 ) -> Result<Frame, RouterError> {
3112 let ack = ModuleHelloAckBody {
3113 negotiated_ver,
3114 subc_ops: module_subc_ops(),
3115 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3116 storage: self
3117 .storage_config
3118 .as_ref()
3119 .map(|cfg| cfg.descriptor_for(module_id)),
3120 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3121 };
3122 let body = serde_json::to_vec(&ack).map_err(|err| {
3123 RouterError::backend(
3124 0,
3125 frame.header.corr,
3126 format!("failed to encode HELLO_ACK: {err}"),
3127 )
3128 })?;
3129
3130 Frame::build_with_version(
3131 negotiated_ver,
3132 FrameType::HelloAck,
3133 control_flags(),
3134 0,
3135 0,
3136 frame.header.corr,
3137 body,
3138 )
3139 .map_err(RouterError::FrameBuild)
3140 }
3141
3142 async fn handle_client_control_request(
3143 &self,
3144 ctx: &RouteCtx,
3145 frame: Frame,
3146 request: ClientControlRequest,
3147 route_open_timings: &mut Option<RouteOpenTimings>,
3148 ) -> Result<Vec<Frame>, RouterError> {
3149 if let Some(operation) = privileged_operation(&request) {
3150 let caller = self.resolve_caller(ctx.connection_id).await;
3151 let op = client_control_request_op(&request);
3152 let text = prompt_text(operation, &caller);
3153 match self.privileged_operator_prompt {
3154 PrivilegedOperatorPrompt::Log => {
3155 info!(target: "subc_daemon::privileged_prompt", operation = op,
3156 target = ?operation, caller = %caller.description(), summary = %text,
3157 "privileged request would prompt the operator");
3158 }
3159 PrivilegedOperatorPrompt::Enforce => {
3160 let key =
3161 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3162 let stamp = self
3163 .operator_apps
3164 .lock()
3165 .unwrap_or_else(|p| p.into_inner())
3166 .get(&ctx.connection_id)
3167 .cloned();
3168 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3169 &self.operator_app_policy,
3170 &self.launch_table,
3171 stamp.as_ref(),
3172 )
3173 .then(|| self.shortcut_context(stamp, None, None));
3174 let outcome = self
3175 .forwarding
3176 .operator_confirms()
3177 .approve_privileged_with_shortcut(
3178 ctx,
3179 &frame,
3180 op,
3181 text,
3182 caller.description(),
3183 key,
3184 shortcut,
3185 )
3186 .await;
3187 if outcome.code != "confirmed" {
3188 return Ok(vec![outcome.frame(&frame)]);
3189 }
3190 }
3191 }
3192 }
3193 match request {
3194 ClientControlRequest::DaemonShutdownIntent { intent } => {
3195 if intent.as_str() != Some("restart") {
3196 return Ok(vec![control_error_frame(
3197 &frame,
3198 "invalid_params",
3199 "intent must be 'restart'",
3200 )?]);
3201 }
3202 let response = ClientControlResponse::DaemonShutdownIntent {
3203 intent: "restart".to_owned(),
3204 expires_at_ms: self.supervisor.announce_daemon_restart(),
3205 };
3206 Ok(vec![control_response_body_frame(
3207 &frame,
3208 &response,
3209 "ClientControlResponse::DaemonShutdownIntent",
3210 )?])
3211 }
3212 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3213 ClientControlRequest::CatalogList { module_id } => {
3214 self.handle_catalog_list(frame, module_id)
3215 }
3216 ClientControlRequest::RouteOpen {
3217 target,
3218 identity,
3219 consumer_identity,
3220 consumer_capabilities,
3221 role_versions,
3222 busy_refusals,
3223 admission_facts,
3224 scope,
3225 ..
3226 } => {
3227 self.handle_route_open(
3228 ctx,
3229 frame,
3230 RouteOpenRequest {
3231 target,
3232 identity,
3233 consumer_identity,
3234 consumer_capabilities,
3235 role_versions,
3236 busy_refusals,
3237 admission_facts,
3238 scope,
3239 },
3240 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3241 )
3242 .await
3243 }
3244 ClientControlRequest::RoutePoll {
3245 route_channel,
3246 route_epoch,
3247 kind,
3248 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3249 ClientControlRequest::SupervisorList {} => {
3250 let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
3251 self.agent_declarations.refuse(
3252 self,
3253 ctx,
3254 control_error_frame(
3255 &frame,
3256 "busy",
3257 "supervisor list work is at capacity; retry",
3258 )?,
3259 );
3260 return Ok(Vec::new());
3261 };
3262 let handler = self.clone();
3264 let sink = ctx.egress.clone();
3265 tokio::spawn(
3266 async move {
3267 let _permit = permit;
3268 let result = handler.handle_supervisor_list(frame.clone()).await;
3269 let replies = match result {
3270 Ok(replies) => replies,
3271 Err(error) => {
3272 control_error_frame(&frame, "backend_error", error.to_string())
3273 .into_iter()
3274 .collect()
3275 }
3276 };
3277 for reply in replies {
3278 if !matches!(
3279 tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
3280 .await,
3281 Ok(Ok(()))
3282 ) {
3283 break;
3284 }
3285 }
3286 }
3287 .with_current_subscriber(),
3288 );
3289 Ok(Vec::new())
3290 }
3291 ClientControlRequest::SupervisorSpawnSnapshot {} => {
3292 self.handle_supervisor_spawn_snapshot(frame)
3293 }
3294 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3295 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3296 }
3297 ClientControlRequest::SupervisorRestart {
3298 module_id,
3299 drain_timeout_ms,
3300 } => {
3301 self.log_supervisor_request_received(
3302 ctx,
3303 frame.header.corr,
3304 ops::SUPERVISOR_RESTART,
3305 Some(&module_id),
3306 None,
3307 )?;
3308 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3309 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3310 .await
3311 }
3312 ClientControlRequest::SupervisorSwap {
3313 module_id,
3314 ready_timeout_ms,
3315 } => {
3316 self.log_supervisor_request_received(
3317 ctx,
3318 frame.header.corr,
3319 ops::SUPERVISOR_SWAP,
3320 Some(&module_id),
3321 None,
3322 )?;
3323 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3324 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3325 .await
3326 }
3327 ClientControlRequest::SupervisorReload { module_id } => {
3328 self.log_supervisor_request_received(
3329 ctx,
3330 frame.header.corr,
3331 ops::SUPERVISOR_RELOAD,
3332 Some(&module_id),
3333 None,
3334 )?;
3335 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3336 self.handle_supervisor_reload(frame, module_id, requester)
3337 .await
3338 }
3339 ClientControlRequest::SupervisorRescan { preview } => {
3340 if !preview {
3341 self.log_supervisor_request_received(
3342 ctx,
3343 frame.header.corr,
3344 ops::SUPERVISOR_RESCAN,
3345 None,
3346 None,
3347 )?;
3348 }
3349 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3350 let operation = self.handle_supervisor_rescan(
3351 frame,
3352 if preview {
3353 RescanMode::Preview
3354 } else {
3355 RescanMode::Execute
3356 },
3357 requester,
3358 );
3359 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3360 .await
3361 }
3362 ClientControlRequest::SupervisorRescanChecked {
3363 preview,
3364 candidate,
3365 expect,
3366 } => {
3367 self.observe_control_wait(
3368 ctx,
3369 None,
3370 ops::SUPERVISOR_RESCAN,
3371 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3372 )
3373 .await
3374 }
3375 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3376 self.log_supervisor_request_received(
3377 ctx,
3378 frame.header.corr,
3379 ops::SUPERVISOR_RELEASE_RESERVED,
3380 Some(&module_id),
3381 None,
3382 )?;
3383 self.handle_supervisor_release_reserved(frame, module_id)
3384 .await
3385 }
3386 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3387 self.log_supervisor_request_received(
3388 ctx,
3389 frame.header.corr,
3390 ops::SUPERVISOR_SET_ENABLED,
3391 Some(&module_id),
3392 Some(enabled),
3393 )?;
3394 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3395 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3396 .await
3397 }
3398 ClientControlRequest::SupervisorHealthProbe { module_id } => {
3399 self.handle_supervisor_health_probe(ctx, frame, module_id)
3400 .await
3401 }
3402 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3403 ClientControlRequest::SupervisorRoutes { module_id } => {
3404 self.handle_supervisor_routes(frame, module_id)
3405 }
3406 ClientControlRequest::SupervisorProvenance { module_id } => {
3407 self.handle_supervisor_provenance(ctx, frame, module_id)
3408 .await
3409 }
3410 ClientControlRequest::SupervisorStderrTail {
3411 module_id,
3412 max_lines,
3413 max_bytes,
3414 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3415 ClientControlRequest::SupervisorTerminals { module_id } => {
3416 self.handle_supervisor_terminals(ctx, frame, module_id)
3417 .await
3418 }
3419 }
3420 }
3421
3422 fn log_supervisor_request_received(
3423 &self,
3424 ctx: &RouteCtx,
3425 corr: u64,
3426 op: &'static str,
3427 module_id: Option<&str>,
3428 enabled: Option<bool>,
3429 ) -> Result<(), RouterError> {
3430 let caller = self.supervisor_requester(ctx, corr)?;
3431 let caller = principal_label(&caller);
3432 info!(
3435 target: "control",
3436 op,
3437 module_id,
3438 enabled,
3439 connection_id = ctx.connection_id.get(),
3440 caller = %caller,
3441 "supervisor request received"
3442 );
3443 Ok(())
3444 }
3445
3446 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3447 Ok(self
3448 .registry
3449 .get_module_by_connection(ctx.connection_id)
3450 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3451 .map(|registration| Principal::Reserved {
3452 module_id: registration.manifest.module_id,
3453 })
3454 .unwrap_or(Principal::Direct))
3455 }
3456
3457 fn handle_module_control_request(
3458 &self,
3459 connection_id: ConnectionId,
3460 frame: Frame,
3461 request: ModuleControlRequestFromModule,
3462 ) -> Result<Vec<Frame>, RouterError> {
3463 match request {
3464 ModuleControlRequestFromModule::RouteAuthorize {
3465 principal,
3466 scope,
3467 target,
3468 expected,
3469 } => self.handle_route_authorize(
3470 connection_id,
3471 frame,
3472 principal,
3473 scope,
3474 target,
3475 expected,
3476 ),
3477 ModuleControlRequestFromModule::CatalogUpdate {
3478 provides,
3479 capabilities,
3480 ready,
3481 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3482 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { role_declarations } => {
3483 self.handle_catalog_update_role_declarations(
3484 connection_id,
3485 frame,
3486 role_declarations,
3487 )
3488 }
3489 ModuleControlRequestFromModule::LiveRoots {} => {
3490 let registered = self
3491 .registry
3492 .get_module_by_connection(connection_id)
3493 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3494 let Some(registration) = registered else {
3495 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3496 };
3497 let response = self
3498 .forwarding
3499 .live_roots(®istration.manifest.module_id)
3500 .map_err(RouterError::Forwarding)?;
3501 Ok(vec![control_response_body_frame(
3502 &frame,
3503 &response,
3504 "ModuleControlResponseToModule::LiveRoots",
3505 )?])
3506 }
3507 ModuleControlRequestFromModule::AgentCommands {
3508 started,
3509 finished,
3510 known,
3511 } => self.handle_agent_commands(connection_id, frame, started, finished, known),
3512 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3513 self.handle_scope_sync(connection_id, frame, generation, scopes)
3514 }
3515 ModuleControlRequestFromModule::ScopeApply {
3516 generation,
3517 upsert,
3518 end,
3519 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3520 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3521 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3522 }
3523 _ => Ok(vec![control_error_frame(
3524 &frame,
3525 "unsupported_control_frame",
3526 "unsupported module control operation",
3527 )?]),
3528 }
3529 }
3530
3531 fn handle_scope_served_by(
3532 &self,
3533 connection_id: ConnectionId,
3534 frame: Frame,
3535 ) -> Result<Vec<Frame>, RouterError> {
3536 let registered = self
3537 .registry
3538 .get_module_by_connection(connection_id)
3539 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3540 let Some(registered) = registered else {
3541 return Ok(vec![control_error_frame(
3542 &frame,
3543 "not_registered",
3544 "scope.served_by requires an active module registration owned by this connection",
3545 )?]);
3546 };
3547 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3548 {
3549 Ok(request) => request,
3550 Err(err) => {
3551 return Ok(vec![control_error_frame(
3552 &frame,
3553 "invalid_control_body",
3554 format!("malformed scope.served_by body: {err}"),
3555 )?])
3556 }
3557 };
3558 let answer = self
3559 .scopes
3560 .read()
3561 .unwrap_or_else(|poisoned| poisoned.into_inner())
3562 .served_by(
3563 ®istered.manifest.module_id,
3564 &request.scope_ref,
3565 request.scope_epoch,
3566 &request.module_id,
3567 );
3568 let reply = subc_control::ScopeServedByReply::new(
3569 answer,
3570 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3571 );
3572 Ok(vec![control_response_body_frame(
3573 &frame,
3574 &reply,
3575 "ScopeServedByReply",
3576 )?])
3577 }
3578
3579 fn handle_operator_confirm(
3580 &self,
3581 ctx: &RouteCtx,
3582 frame: Frame,
3583 ) -> Result<Vec<Frame>, RouterError> {
3584 use crate::operator_confirm::{audit, Outcome};
3585 let registration = self
3586 .registry
3587 .get_module_by_connection(ctx.connection_id)
3588 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3589 let Some(registration) = registration else {
3590 let outcome = Outcome::refusal("not_registered");
3591 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3592 return Ok(vec![outcome.frame(&frame)]);
3593 };
3594 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3595 Ok(request) => request,
3596 Err(_) => {
3597 let outcome = Outcome::refusal("invalid_control_body");
3598 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3599 return Ok(vec![outcome.frame(&frame)]);
3600 }
3601 };
3602 let declared = matches!(
3603 request.kind,
3604 Some(subc_protocol::session::OperatorConfirmKind::Operation)
3605 ) && request.operation.as_ref().is_some_and(|operation| {
3606 registration
3607 .manifest
3608 .confirm_operations
3609 .as_ref()
3610 .is_some_and(|operations| operations.contains(operation))
3611 });
3612 let module_id = registration.manifest.module_id;
3613 let nonce = self
3617 .hello_launch_nonces
3618 .lock()
3619 .unwrap_or_else(|p| p.into_inner())
3620 .nonce(ctx.connection_id)
3621 .map(str::to_owned);
3622 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3623 self.supervisor
3624 .spawned_consumer_authorized(&module_id, nonce)
3625 });
3626 let confirms = self.forwarding.operator_confirms();
3627 self.forwarding
3628 .with_operator_route(
3629 ctx.connection_id,
3630 request.route_channel,
3631 request.route_epoch,
3632 |binding| {
3633 let shortcut = binding
3634 .filter(|binding| {
3635 nonce_proven
3636 && crate::operator_confirm::valid_summary(&request.summary)
3637 && declared
3638 && crate::operator_confirm::ShortcutContext::route_eligible(
3639 &self.operator_app_policy,
3640 &self.launch_table,
3641 binding,
3642 )
3643 })
3644 .map(|binding| {
3645 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3646 &self.operator_app_policy,
3647 binding,
3648 );
3649 let stamp = if phone.is_some() {
3650 None
3651 } else {
3652 binding.operator_app.clone()
3653 };
3654 self.shortcut_context(
3655 stamp,
3656 phone,
3657 Some((request.route_channel, request.route_epoch)),
3658 )
3659 });
3660 confirms.admit_with_shortcut(
3661 ctx,
3662 frame,
3663 module_id,
3664 nonce_proven,
3665 request,
3666 binding,
3667 shortcut,
3668 )
3669 },
3670 )
3671 .map_err(RouterError::Forwarding)
3672 }
3673
3674 fn shortcut_context(
3675 &self,
3676 stamp: Option<crate::operator_apps::OperatorAppStamp>,
3677 phone_fingerprint: Option<String>,
3678 route: Option<(u16, u32)>,
3679 ) -> crate::operator_confirm::ShortcutContext {
3680 crate::operator_confirm::ShortcutContext {
3681 ledger: self.activity.clone(),
3682 forwarding: self.forwarding.clone(),
3683 launches: self.launch_table.clone(),
3684 stamp,
3685 phone_fingerprint,
3686 route,
3687 connections: self.operator_apps.clone(),
3688 #[cfg(test)]
3689 hooks: Default::default(),
3690 }
3691 }
3692
3693 #[cfg(any(test, feature = "test-support"))]
3695 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3696 self.operator_app_policy.phone_enabled = enabled;
3697 self
3698 }
3699
3700 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3703 let mut scopes = ScopeRecords::from(scopes);
3704 for (index, record) in scopes.records.iter_mut().enumerate() {
3705 if let Some(refusal) = Self::canonicalize_workdir(record) {
3706 scopes.refusals.insert(index, refusal);
3707 }
3708 }
3709 scopes
3710 }
3711
3712 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3713 let workdir = record.workdir.as_mut()?;
3714 let path = std::path::Path::new(workdir);
3715 if !path.is_absolute() {
3716 return Some((
3717 error_codes::INVALID_WORKDIR,
3718 "workdir must be absolute".to_string(),
3719 ));
3720 }
3721 match ProjectRootId::from_path_allowing_missing(path) {
3722 Ok(root) => match root.as_path().to_str() {
3723 Some(canonical) => {
3724 *workdir = canonical.to_string();
3725 None
3726 }
3727 None => Some((
3728 error_codes::INVALID_WORKDIR,
3729 "canonical workdir is not UTF-8".to_string(),
3730 )),
3731 },
3732 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3733 }
3734 }
3735
3736 fn handle_scope_sync(
3740 &self,
3741 connection_id: ConnectionId,
3742 frame: Frame,
3743 generation: u64,
3744 scopes: Vec<ScopeRecord>,
3745 ) -> Result<Vec<Frame>, RouterError> {
3746 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3747 }
3748
3749 fn handle_scope_change(
3750 &self,
3751 connection_id: ConnectionId,
3752 frame: Frame,
3753 generation: u64,
3754 scopes: Vec<ScopeRecord>,
3755 end: Option<Vec<ScopeEnd>>,
3756 ) -> Result<Vec<Frame>, RouterError> {
3757 let is_apply = end.is_some();
3758 let Some(registration) = self
3759 .registry
3760 .get_module_by_connection(connection_id)
3761 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3762 else {
3763 return Ok(vec![control_error_frame(
3764 &frame,
3765 "not_registered",
3766 "scope.sync and scope.apply require an active module registration owned by this connection",
3767 )?]);
3768 };
3769 let owner = registration.manifest.module_id;
3770 let scopes = Self::prepare_scope_records(scopes);
3771 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3772 let is_current_launch = |connection: ConnectionId| {
3773 self.hello_launch_nonces
3774 .lock()
3775 .unwrap_or_else(|poisoned| poisoned.into_inner())
3776 .presented(connection, current_nonce.as_deref())
3777 };
3778 let mut table = self
3783 .scopes
3784 .write()
3785 .unwrap_or_else(|poisoned| poisoned.into_inner());
3786 let outcome = match end {
3787 Some(end) => table.apply(
3788 &owner,
3789 connection_id,
3790 is_current_launch,
3791 generation,
3792 scopes,
3793 end,
3794 ),
3795 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3796 };
3797 let drained = match &outcome {
3798 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3799 Err(_) => Vec::new(),
3800 };
3801 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3802 drop(table);
3803 if has_deadlines {
3804 self.scope_deadline_added.notify_one();
3805 }
3806 match outcome {
3807 Ok(applied) => {
3808 let counts = ScopeOutcomeCounts::of(&applied.results);
3809 info!(
3810 owner = %owner,
3811 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3812 generation,
3813 records = applied.results.len(),
3814 created = counts.created,
3815 replaced = counts.replaced,
3816 updated = counts.updated,
3817 unchanged = counts.unchanged,
3818 refused = counts.refused,
3819 ended = applied.ended.len(),
3820 tag_changes = applied.tag_changes.len(),
3821 routes_closed = drained.len(),
3822 "scope change accepted"
3823 );
3824 for refused in applied
3830 .results
3831 .iter()
3832 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3833 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3834 {
3835 warn!(
3836 owner = %owner,
3837 generation,
3838 scope_ref = %refused.scope_ref,
3839 scope_epoch = refused.scope_epoch,
3840 code = refused.code.as_deref().unwrap_or(""),
3841 "scope record refused"
3842 );
3843 }
3844 self.close_scope_drained_routes(drained);
3845 let response = if is_apply {
3846 ModuleControlResponseToModule::ScopeApply {
3847 generation,
3848 results: applied.results,
3849 end_results: applied.end_results,
3850 ended: applied.ended,
3851 }
3852 } else {
3853 ModuleControlResponseToModule::ScopeSync {
3854 generation,
3855 results: applied.results,
3856 ended: applied.ended,
3857 }
3858 };
3859 Ok(vec![control_response_body_frame(
3860 &frame,
3861 &response,
3862 "scope change response",
3863 )?])
3864 }
3865 Err(refusal) => {
3866 info!(
3867 owner = %owner,
3868 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3869 generation,
3870 code = refusal.code,
3871 "scope change refused"
3872 );
3873 Ok(vec![control_error_frame(
3874 &frame,
3875 refusal.code,
3876 refusal.message,
3877 )?])
3878 }
3879 }
3880 }
3881
3882 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3888 if drained.is_empty() {
3889 return;
3890 }
3891 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3892 BTreeMap::new();
3893 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3894 for route in drained {
3895 warn!(
3896 module_id = %route.module_id,
3897 reason = ?route.reason,
3898 client_connection_id = route.client.connection_id.get(),
3899 route_channel = route.client.channel,
3900 "closing route because its scope changed"
3901 );
3902 pushes
3903 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3904 .or_insert_with(|| (route.reason, Vec::new()))
3905 .1
3906 .push(EndpointRoute {
3907 goodbye_target: route.client.clone(),
3908 principal: Principal::Unverified,
3909 bound_at: Instant::now(),
3910 draining: false,
3911 drain_reason: None,
3912 });
3913 goodbyes.push(route.module);
3914 goodbyes.push(route.client);
3915 }
3916 for ((module_id, _), (reason, routes)) in pushes {
3917 send_route_control_pushes(
3918 &self.forwarding,
3919 routes,
3920 ClientControlPush::RouteClosed {
3921 module_id,
3922 channels: Vec::new(),
3923 reason,
3924 drained: false,
3925 abandoned: 0,
3926 excluded_subscriptions: 0,
3927 terminal: Some(false),
3928 },
3929 );
3930 }
3931 self.emit_route_goodbyes(goodbyes);
3932 }
3933
3934 fn handle_route_authorize(
3935 &self,
3936 connection_id: ConnectionId,
3937 frame: Frame,
3938 principal: Principal,
3939 scope: subc_protocol::scope::RouteAuthorizeScope,
3940 target: String,
3941 expected: subc_protocol::scope::RouteAuthorizeExpected,
3942 ) -> Result<Vec<Frame>, RouterError> {
3943 use crate::scopes::admission::{admit_route, AdmissionMode};
3944 use subc_protocol::scope::{
3945 RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
3946 };
3947 let registry = self
3948 .registry
3949 .route_admission_snapshot()
3950 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3951 let owner_configured = match &scope.owner {
3952 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3953 _ => false,
3954 };
3955 let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
3956 let answer = {
3957 let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
3958 let permitted = registry
3959 .iter()
3960 .find(|r| r.connection_id == connection_id)
3961 .is_some_and(|r| {
3962 scopes.is_federated_owner(&Principal::Reserved {
3963 module_id: r.manifest.module_id.clone(),
3964 }) && r.manifest.capabilities.as_ref().is_some_and(|caps| {
3965 caps.provides
3966 .iter()
3967 .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
3968 })
3969 });
3970 if !permitted {
3971 drop(scopes);
3972 return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
3973 "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
3974 }
3975 admit_route(
3976 &scopes,
3977 ®istry,
3978 &principal,
3979 &target,
3980 Some(&selector),
3981 owner_configured,
3982 AdmissionMode::Authorize(&expected),
3983 )
3984 };
3985 let response = match answer {
3986 Ok(admission) => {
3987 let mut response = RouteAuthorizeResponse::new(
3988 RouteAuthorizeOutcome::Authorized,
3989 "route admission authorized",
3990 );
3991 response.content_version = admission.content_version;
3992 response
3993 }
3994 Err(refusal) => refusal.response(),
3995 };
3996 Ok(vec![control_response_body_frame(
3997 &frame,
3998 &response,
3999 "RouteAuthorizeResponse",
4000 )?])
4001 }
4002
4003 fn handle_scope_describe(
4006 &self,
4007 connection_id: ConnectionId,
4008 frame: Frame,
4009 owner: Principal,
4010 scope_ref: String,
4011 ) -> Result<Vec<Frame>, RouterError> {
4012 let registered = self
4013 .registry
4014 .get_module_by_connection(connection_id)
4015 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4016 let Some(registered) = registered else {
4017 return Ok(vec![control_error_frame(
4018 &frame,
4019 "not_registered",
4020 "scope.describe requires an active module registration owned by this connection",
4021 )?]);
4022 };
4023 let scopes = self
4024 .scopes
4025 .read()
4026 .unwrap_or_else(|poisoned| poisoned.into_inner());
4027 if scopes.is_federated_owner(&owner)
4028 && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
4029 {
4030 return Ok(vec![control_error_frame(
4031 &frame,
4032 error_codes::SCOPE_FEDERATED_UNSUPPORTED,
4033 "scope.describe of the federated owner requires scope-federated/v1",
4034 )?]);
4035 }
4036 let mut description = scopes.describe(&owner, &scope_ref);
4037 drop(scopes);
4038 if owner
4039 != (Principal::Reserved {
4040 module_id: registered.manifest.module_id.clone(),
4041 })
4042 {
4043 if let Some(facts) = description
4044 .stamp
4045 .as_mut()
4046 .and_then(|stamp| stamp.attributes.federated.as_mut())
4047 {
4048 facts.stamp_required = false;
4049 }
4050 }
4051 if !registered
4054 .manifest
4055 .capabilities
4056 .as_ref()
4057 .is_some_and(|caps| {
4058 caps.provides
4059 .iter()
4060 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4061 })
4062 {
4063 if let Some(stamp) = description.stamp.as_mut() {
4064 stamp.attributes.on_behalf_of_agent = None;
4065 }
4066 }
4067 if !registered
4068 .manifest
4069 .capabilities
4070 .as_ref()
4071 .is_some_and(|caps| {
4072 caps.provides
4073 .iter()
4074 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
4075 })
4076 {
4077 if let Some(stamp) = description.stamp.as_mut() {
4078 stamp.attributes.flow_version = None;
4079 stamp.attributes.flow_code_hash = None;
4080 stamp.attributes.flow_install_id = None;
4081 }
4082 }
4083 let owner_configured = match &owner {
4084 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4089 _ => false,
4090 };
4091 let response = ModuleControlResponseToModule::ScopeDescribe {
4092 status: description.status,
4093 scope_epoch: description.scope_epoch,
4094 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4095 owner_synced: description.owner_synced,
4096 owner_configured,
4097 scope: description.stamp,
4098 };
4099 Ok(vec![control_response_body_frame(
4100 &frame,
4101 &response,
4102 "ModuleControlResponseToModule::ScopeDescribe",
4103 )?])
4104 }
4105
4106 fn handle_catalog_update_role_declarations(
4107 &self,
4108 connection_id: ConnectionId,
4109 frame: Frame,
4110 role_declarations: serde_json::Value,
4111 ) -> Result<Vec<Frame>, RouterError> {
4112 if let Err(err) = validate_role_declarations_value(Some(&role_declarations)) {
4113 return Ok(vec![control_error_frame(
4114 &frame,
4115 "invalid_role_declarations",
4116 err.to_string(),
4117 )?]);
4118 }
4119 let declarations = serde_json::from_value(role_declarations)
4120 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4121 if self
4122 .registry
4123 .replace_role_declarations_for_connection(connection_id, declarations)
4124 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4125 .is_none()
4126 {
4127 return Ok(vec![control_error_frame(
4128 &frame,
4129 "not_registered",
4130 "catalog.update_role_declarations requires an active module registration owned by this connection",
4131 )?]);
4132 }
4133 control_response_body_frame(
4134 &frame,
4135 &ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {},
4136 "ModuleControlResponseToModule::CatalogUpdateRoleDeclarations",
4137 )
4138 .map(|frame| vec![frame])
4139 }
4140
4141 fn handle_catalog_update(
4142 &self,
4143 connection_id: ConnectionId,
4144 frame: Frame,
4145 provides: Vec<ProviderRole>,
4146 capabilities: Option<CapabilityDeclarations>,
4147 ready: Option<bool>,
4148 ) -> Result<Vec<Frame>, RouterError> {
4149 self.refresh_capability_requirements();
4150 let Some(registration) = self
4151 .registry
4152 .get_module_by_connection(connection_id)
4153 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4154 else {
4155 return Ok(vec![control_error_frame(
4156 &frame,
4157 "not_registered",
4158 "catalog.update requires an active module registration owned by this connection",
4159 )?]);
4160 };
4161
4162 if let Some(message) =
4163 catalog_update_frozen_field_message(®istration.manifest, &provides)
4164 {
4165 return Ok(vec![control_error_frame(
4166 &frame,
4167 "catalog_update_frozen_field",
4168 message,
4169 )?]);
4170 }
4171
4172 let mut candidate = registration.manifest.clone();
4173 candidate.provides = provides.clone();
4174 candidate.capabilities = capabilities
4175 .clone()
4176 .or_else(|| registration.manifest.capabilities.clone());
4177 if let Some(rich_ui) = &candidate.rich_ui {
4178 if let Err(err) = rich_ui.validate() {
4179 return Ok(vec![control_error_frame(
4180 &frame,
4181 "invalid_manifest",
4182 err.to_string(),
4183 )?]);
4184 }
4185 }
4186 if let Err(err) = candidate.validate_capability_grammar() {
4187 return Ok(vec![control_error_frame(
4188 &frame,
4189 "invalid_capability_grammar",
4190 err.to_string(),
4191 )?]);
4192 }
4193
4194 let mut conflicts = self
4197 .capability_evaluator
4198 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
4199 if let Some(conflict) = conflicts.first() {
4200 let message = format!(
4201 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
4202 conflict.capability, conflict.claimants[0], candidate.module_id
4203 );
4204 for conflict in &mut conflicts {
4205 conflict.source = DuplicateClaimSource::CatalogUpdate;
4206 }
4207 log_duplicate_claim_events(conflicts);
4208 return Ok(vec![control_error_frame(
4209 &frame,
4210 "reserved_capability",
4211 message,
4212 )?]);
4213 }
4214
4215 let updated = match self.registry.replace_catalog_for_connection(connection_id, provides, capabilities, ready) {
4216 Err(crate::registry::RegistryError::FederatedCapabilityChanged) => return Ok(vec![control_error_frame(
4217 &frame, error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
4218 "scope-federated/v1 is fixed at HELLO; re-register on a new connection to change it",
4219 )?]),
4220 result => result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?,
4221 };
4222 if updated.is_none() {
4223 return Ok(vec![control_error_frame(
4224 &frame,
4225 "not_registered",
4226 "catalog.update requires an active module registration owned by this connection",
4227 )?]);
4228 }
4229 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
4230 log_duplicate_claim_events(
4231 self.capability_evaluator
4232 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
4233 );
4234 }
4235 if capability_census_trigger(
4236 registration.manifest.capabilities.as_ref(),
4237 updated
4238 .as_ref()
4239 .and_then(|entry| entry.manifest.capabilities.as_ref()),
4240 ) {
4241 self.enforce_capability_denies();
4242 }
4243 self.refresh_capability_requirements();
4244
4245 let response = ModuleControlResponseToModule::CatalogUpdate {};
4246 control_response_body_frame(
4247 &frame,
4248 &response,
4249 "ModuleControlResponseToModule::CatalogUpdate",
4250 )
4251 .map(|frame| vec![frame])
4252 }
4253
4254 pub(crate) fn with_activity_ledger_config(
4255 mut self,
4256 run_dir: &Path,
4257 incarnation: String,
4258 config: crate::activity::ActivityConfig,
4259 ) -> Self {
4260 self.activity = Some(crate::activity::ActivityLedger::open_config(
4261 run_dir.join("activity.jsonl"),
4262 incarnation,
4263 config,
4264 ));
4265 self.supervisor.set_activity(self.activity.clone());
4266 self
4267 }
4268
4269 pub(crate) async fn configure_activity(
4270 &self,
4271 config: crate::activity::ActivityConfig,
4272 ) -> Result<(), &'static str> {
4273 match &self.activity {
4274 Some(ledger) => ledger.configure(config).await,
4275 None => Ok(()),
4276 }
4277 }
4278
4279 fn cancel_activity_subscription(
4280 &self,
4281 ctx: &RouteCtx,
4282 frame: &Frame,
4283 ) -> Result<bool, RouterError> {
4284 let task = self
4285 .activity_streams
4286 .lock()
4287 .unwrap_or_else(|p| p.into_inner())
4288 .remove(&(ctx.connection_id, frame.header.corr));
4289 let Some(task) = task else {
4290 return Ok(false);
4291 };
4292 task.abort();
4293 let end = Frame::build_with_version(
4294 response_version(frame),
4295 FrameType::StreamEnd,
4296 control_flags(),
4297 0,
4298 0,
4299 frame.header.corr,
4300 Vec::new(),
4301 )
4302 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4303 let sink = ctx.egress.clone();
4304 tokio::spawn(async move {
4305 let _ = sink.send(end).await;
4306 });
4307 Ok(true)
4308 }
4309
4310 async fn handle_activity_read(
4311 &self,
4312 ctx: &RouteCtx,
4313 frame: Frame,
4314 ) -> Result<Vec<Frame>, RouterError> {
4315 use subc_control::{
4316 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4317 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4318 };
4319 #[derive(Deserialize)]
4320 struct Identity {
4321 #[serde(default)]
4322 consumer_identity: Option<ConsumerIdentity>,
4323 }
4324 let principal = match self
4328 .forwarding
4329 .module_id_for_connection(ctx.connection_id)
4330 .map_err(RouterError::Forwarding)?
4331 {
4332 Some(module_id) => Ok(Principal::Reserved { module_id }),
4333 None => {
4334 let identity: Identity = match serde_json::from_slice(&frame.body) {
4335 Ok(identity) => identity,
4336 Err(_) => {
4337 return Ok(vec![control_error_frame(
4338 &frame,
4339 ACTIVITY_MALFORMED,
4340 "malformed activity read",
4341 )?])
4342 }
4343 };
4344 self.route_open_principal(&frame, identity.consumer_identity)?
4345 }
4346 };
4347 let admitted = matches!(principal, Ok(Principal::Direct))
4348 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4349 if !admitted {
4350 return Ok(vec![control_error_frame(
4351 &frame,
4352 ACTIVITY_READ_FORBIDDEN,
4353 "activity reads require an operator or trusted mirror principal",
4354 )?]);
4355 }
4356 let Some(ledger) = &self.activity else {
4357 return Ok(vec![control_error_frame(
4358 &frame,
4359 ACTIVITY_UNAVAILABLE,
4360 "activity ledger unavailable",
4361 )?]);
4362 };
4363 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4364 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4365 {
4366 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4367 Ok(request) => ledger.query(request).await,
4368 Err(_) => Err(ACTIVITY_MALFORMED),
4369 };
4370 return match result {
4371 Ok(reply) => Ok(vec![control_response_body_frame(
4372 &frame,
4373 &reply,
4374 ops::ACTIVITY_QUERY,
4375 )?]),
4376 Err(code) => Ok(vec![control_error_frame(
4377 &frame,
4378 code,
4379 "activity query refused",
4380 )?]),
4381 };
4382 }
4383 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4384 Ok(request) => request,
4385 Err(_) => {
4386 return Ok(vec![control_error_frame(
4387 &frame,
4388 ACTIVITY_MALFORMED,
4389 "malformed activity subscription",
4390 )?])
4391 }
4392 };
4393 let mut subscription = match ledger.subscribe(request.since).await {
4394 Ok(subscription) => subscription,
4395 Err(code) => {
4396 return Ok(vec![control_error_frame(
4397 &frame,
4398 code,
4399 "activity subscription refused",
4400 )?])
4401 }
4402 };
4403 let key = (ctx.connection_id, frame.header.corr);
4404 let sink = ctx.egress.clone();
4405 let mut streams = self
4406 .activity_streams
4407 .lock()
4408 .unwrap_or_else(|p| p.into_inner());
4409 streams.retain(|_, task| !task.is_finished());
4410 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4411 return Ok(vec![control_error_frame(
4412 &frame,
4413 "duplicate_subscription",
4414 "activity subscription correlation id already in use",
4415 )?]);
4416 }
4417 let task = tokio::spawn(async move {
4418 loop {
4419 let entry = if let Some(entry) = subscription.replay.pop_front() {
4420 Some(entry)
4421 } else {
4422 subscription.receiver.recv().await
4423 };
4424 let Some(entry) = entry else {
4425 break;
4426 };
4427 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4428 return;
4429 };
4430 let Ok(next) = Frame::build_with_version(
4431 response_version(&frame),
4432 FrameType::StreamData,
4433 control_flags(),
4434 0,
4435 0,
4436 frame.header.corr,
4437 body,
4438 ) else {
4439 return;
4440 };
4441 if sink.send(next).await.is_err() {
4442 return;
4443 }
4444 }
4445 if let Some(code) = subscription.terminal_code() {
4448 if let Ok(end) =
4449 control_error_frame(&frame, code, "activity subscriber fell behind")
4450 {
4451 let _ = sink.send(end).await;
4452 }
4453 }
4454 });
4455 streams.insert(key, task.abort_handle());
4456 Ok(Vec::new())
4457 }
4458
4459 async fn handle_activity_record(
4460 &self,
4461 ctx: &RouteCtx,
4462 frame: Frame,
4463 ) -> Result<Vec<Frame>, RouterError> {
4464 use subc_control::{
4465 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4466 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4467 };
4468 let result = async {
4469 let module = self
4470 .forwarding
4471 .module_id_for_connection(ctx.connection_id)
4472 .ok()
4473 .flatten()
4474 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4475 let request: ActivityRecordRequest =
4476 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4477 crate::activity::validate(&module, &request)?;
4478 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4479 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4480 let at_ms = std::time::SystemTime::now()
4481 .duration_since(std::time::UNIX_EPOCH)
4482 .unwrap_or_default()
4483 .as_millis() as u64;
4484 ledger.record(module, request, actor, at_ms).await
4485 }
4486 .await;
4487 match result {
4488 Ok(entry) => Ok(vec![control_response_body_frame(
4489 &frame,
4490 &ActivityRecordResponse { seq: entry.seq },
4491 "activity.record",
4492 )?]),
4493 Err(code) => Ok(vec![control_error_frame(
4494 &frame,
4495 code,
4496 "activity recording refused",
4497 )?]),
4498 }
4499 }
4500
4501 fn activity_actor(
4502 &self,
4503 connection: ConnectionId,
4504 module: &str,
4505 on: Option<&subc_control::ActivityOn>,
4506 ) -> Result<subc_control::ActivityActor, &'static str> {
4507 use crate::forwarding::{DataRoute, DataRouteState};
4508 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4509 let principal = |id: &str| Principal::Reserved {
4510 module_id: id.to_owned(),
4511 };
4512 let mut actor = ActivityActor {
4513 principal: principal(module),
4514 scope_owner: None,
4515 agent_id: None,
4516 run_id: None,
4517 flow_id: None,
4518 project_root: None,
4519 workdir: None,
4520 project_id: None,
4521 };
4522 let scope = match on {
4523 None => return Ok(actor),
4524 Some(ActivityOn::Scope(scope)) => {
4525 let selector = subc_protocol::scope::ScopeSelector::new(
4526 principal(module),
4527 scope.scope_ref.clone(),
4528 Some(scope.epoch),
4529 );
4530 self.scopes
4531 .read()
4532 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4533 .attribute(&principal(module), module, &selector, true)
4534 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4535 }
4536 Some(ActivityOn::Route(route)) => {
4537 let DataRoute::Module(DataRouteState::Bound(binding)) = self
4538 .forwarding
4539 .lookup_data_route(connection, route.channel, route.epoch)
4540 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4541 else {
4542 return Err(ACTIVITY_ACTOR_UNKNOWN);
4543 };
4544 if binding.module_id != module {
4545 return Err(ACTIVITY_ACTOR_UNKNOWN);
4546 }
4547 actor.principal = binding.principal.clone();
4548 actor.project_root = binding
4549 .project_root
4550 .as_ref()
4551 .map(|root| root.as_path().to_path_buf());
4552 let Some(bound) = &binding.scope else {
4553 return Ok(actor);
4554 };
4555 let owner = principal(&bound.owner);
4556 let selector = subc_protocol::scope::ScopeSelector::new(
4557 owner.clone(),
4558 bound.scope_ref.clone(),
4559 Some(bound.tag.scope_epoch),
4560 );
4561 let admitted = self
4562 .scopes
4563 .read()
4564 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4565 .attribute(&owner, module, &selector, true)
4566 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4567 if admitted.tag != bound.tag {
4568 return Err(ACTIVITY_ACTOR_UNKNOWN);
4569 }
4570 admitted
4571 }
4572 };
4573 actor.scope_owner = Some(scope.stamp.owner);
4574 actor.agent_id = scope.stamp.attributes.agent_id;
4575 actor.run_id = scope.stamp.attributes.run_id;
4576 actor.flow_id = scope.stamp.attributes.flow_id;
4577 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4578 actor.project_id = scope.stamp.project_id;
4579 Ok(actor)
4580 }
4581
4582 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4583 self.refresh_capability_requirements();
4584 let mut counters = self.counters.snapshot();
4590 let losses = self.resource_tracker.loss_diagnostics();
4591 if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
4592 if let Some(obj) = counters.as_object_mut() {
4593 obj.insert("agent_command_declaration_loss_by_module".into(), losses);
4594 }
4595 }
4596 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4597 obj.insert(
4598 "activity_unparseable_lines".into(),
4599 ledger.read_errors().into(),
4600 );
4601 obj.insert(
4602 "activity_lifecycle_append_failures".into(),
4603 ledger.lifecycle_failures().into(),
4604 );
4605 obj.insert(
4606 "activity_started_digest_skips".into(),
4607 ledger.lifecycle_digest_skips().into(),
4608 );
4609 }
4610 if let (Ok((connections_with_routes, max)), Some(obj)) = (
4611 self.forwarding.client_route_concentration(),
4612 counters.as_object_mut(),
4613 ) {
4614 obj.insert(
4615 "client_connections_with_routes".into(),
4616 connections_with_routes.into(),
4617 );
4618 obj.insert("max_routes_on_one_connection".into(), max.into());
4619 }
4620 if let (Some(open_breakers), Some(obj)) = (
4626 self.route_bind_breakers.open_snapshot(),
4627 counters.as_object_mut(),
4628 ) {
4629 obj.insert("route_bind_breakers_open".into(), open_breakers);
4630 }
4631 if let (Some(reason), Some(obj)) = (
4636 self.local_endpoint_unavailable.clone(),
4637 counters.as_object_mut(),
4638 ) {
4639 obj.insert("local_endpoint_unavailable".into(), reason.into());
4640 }
4641 let response = ClientControlResponse::ServerDescribe {
4642 protocol_ver: PROTOCOL_VERSION,
4643 subc_ops: subc_ops(),
4644 capabilities: self.describe_capabilities(),
4645 connected_clients: self.connected_clients.count(),
4646 counters: Some(counters),
4647 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4648 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4649 capability_requirements: self.capability_requirement_statuses(),
4650 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4651 config_path: self
4652 .rescan
4653 .as_ref()
4654 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4655 };
4656 Ok(vec![control_response_body_frame(
4657 &frame,
4658 &response,
4659 "ClientControlResponse::ServerDescribe",
4660 )?])
4661 }
4662
4663 fn handle_catalog_list(
4664 &self,
4665 frame: Frame,
4666 module_id: Option<String>,
4667 ) -> Result<Vec<Frame>, RouterError> {
4668 let (generation, modules) = self
4669 .registry
4670 .list_modules_with_nonce_verification()
4671 .map_err(|err| {
4672 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4673 })?;
4674 let entries = modules
4675 .into_iter()
4676 .filter(|(registration, _)| {
4677 module_id
4678 .as_deref()
4679 .map(|wanted| registration.manifest.module_id == wanted)
4680 .unwrap_or(true)
4681 })
4682 .map(|(registration, nonce_verified)| {
4683 let not_ready = self.not_ready_reason(®istration);
4684 let roles = registration.manifest.provides;
4685 CatalogEntry::new(
4686 registration.manifest.module_id,
4687 roles,
4688 registration.control_ops,
4689 )
4690 .with_ready(not_ready.is_none())
4691 .with_display_name(registration.manifest.display_name)
4692 .with_not_ready(not_ready)
4693 .with_module_version(Some(registration.manifest.module_version))
4694 .with_provenance(registration.manifest.provenance)
4695 .with_capabilities(registration.manifest.capabilities)
4696 .with_self_signals(registration.manifest.self_signals)
4697 .with_events(registration.manifest.events)
4698 .with_operation_delivery(registration.manifest.operation_delivery)
4699 .with_operation_payload(registration.manifest.operation_payload)
4700 .with_flow_operations(registration.manifest.flow_operations)
4701 .with_operation_deferral(registration.manifest.operation_deferral)
4702 .with_confirm_operations(registration.manifest.confirm_operations)
4703 .with_rich_ui(registration.manifest.rich_ui)
4704 .with_role_declarations(registration.manifest.role_declarations)
4705 .with_nonce_verified(nonce_verified)
4706 })
4707 .collect();
4708 let response = ClientControlResponse::CatalogList {
4709 generation,
4710 modules: entries,
4711 subc_ops: subc_ops(),
4712 };
4713 Ok(vec![control_response_body_frame(
4714 &frame,
4715 &response,
4716 "ClientControlResponse::CatalogList",
4717 )?])
4718 }
4719
4720 fn attest_route_open_consumer(
4725 &self,
4726 consumer_identity: Option<&ConsumerIdentity>,
4727 ) -> Option<Principal> {
4728 let Some(consumer_identity) = consumer_identity else {
4729 return Some(Principal::Direct);
4730 };
4731 self.supervisor
4732 .spawned_consumer_authorized(
4733 &consumer_identity.module_id,
4734 &consumer_identity.launch_nonce,
4735 )
4736 .then(|| Principal::Reserved {
4737 module_id: consumer_identity.module_id.clone(),
4738 })
4739 }
4740
4741 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4745 let Ok(ClientControlRequest::RouteOpen {
4746 identity,
4747 consumer_identity,
4748 scope,
4749 ..
4750 }) = parse_client_control_request(&frame.body)
4751 else {
4752 return RouteOpenCaller::unknown();
4753 };
4754 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4755 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4756 }
4757
4758 fn route_open_principal(
4759 &self,
4760 frame: &Frame,
4761 consumer_identity: Option<ConsumerIdentity>,
4762 ) -> Result<Result<Principal, Frame>, RouterError> {
4763 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4764 self.route_open_principal_attested(frame, consumer_identity, attested)
4765 }
4766
4767 fn route_open_principal_attested(
4771 &self,
4772 frame: &Frame,
4773 consumer_identity: Option<ConsumerIdentity>,
4774 attested: Option<Principal>,
4775 ) -> Result<Result<Principal, Frame>, RouterError> {
4776 if let Some(principal) = attested {
4777 return Ok(Ok(principal));
4778 }
4779 let Some(consumer_identity) = consumer_identity else {
4780 return Ok(Ok(Principal::Direct));
4781 };
4782
4783 Ok(Err(control_error_frame(
4784 frame,
4785 "bad_consumer_identity",
4786 format!(
4787 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4788 consumer_identity.module_id
4789 ),
4790 )?))
4791 }
4792
4793 #[allow(clippy::too_many_arguments)]
4800 fn route_open_refusal_frame(
4801 &self,
4802 ctx: &RouteCtx,
4803 caller: &RouteOpenCaller,
4804 frame: &Frame,
4805 module_id: &str,
4806 reason: &'static str,
4807 code: &'static str,
4808 message: impl Into<String>,
4809 ) -> Result<Frame, RouterError> {
4810 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4811 let detail = if matches!(
4812 reason,
4813 "registration_not_active"
4814 | "supervisor_not_live"
4815 | "no_forwarding_connection"
4816 | "reloading"
4817 ) {
4818 let expected_back = (reason == "reloading").then_some(true);
4821 self.route_open_recovery_detail(module_id, frame.header.corr, None, expected_back)?
4822 } else {
4823 None
4824 };
4825 control_error_body_frame(
4826 frame,
4827 ErrorBody {
4828 code: code.to_string(),
4829 message: message.into(),
4830 detail,
4831 },
4832 )
4833 }
4834
4835 fn route_open_recovery_detail(
4836 &self,
4837 module_id: &str,
4838 corr: u64,
4839 detail: Option<serde_json::Value>,
4840 expected_back: Option<bool>,
4841 ) -> Result<Option<serde_json::Value>, RouterError> {
4842 let Some((status, _)) = self.supervisor_status(module_id, corr)? else {
4843 return Ok(detail);
4844 };
4845 let mut detail = detail.unwrap_or_else(|| serde_json::json!({}));
4846 detail["state"] = serde_json::json!(status.state.to_string());
4847 if let Some(expected_back) = expected_back {
4848 detail["expected_back"] = serde_json::json!(expected_back);
4849 } else if let Some(module) = self.supervisor.get(module_id) {
4850 match module.will_recover_after_connection_loss() {
4851 Ok(expected_back) => detail["expected_back"] = serde_json::json!(expected_back),
4852 Err(err) => warn!(
4853 %module_id,
4854 error = %err,
4855 "failed to read route.open recovery verdict; omitting expected_back"
4856 ),
4857 }
4858 }
4859 Ok(Some(detail))
4860 }
4861
4862 #[allow(clippy::too_many_arguments)]
4871 fn route_bind_rejection(
4872 &self,
4873 ctx: &RouteCtx,
4874 caller: &RouteOpenCaller,
4875 frame: &Frame,
4876 target_module_id: &str,
4877 body: ErrorBody,
4878 daemon_authored: bool,
4879 ) -> Result<Vec<Frame>, RouterError> {
4880 if daemon_authored {
4886 let scope_code = match body.code.as_str() {
4887 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4888 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4889 _ => None,
4890 };
4891 if let Some(code) = scope_code {
4892 self.observe_route_open_refusal(
4893 ctx,
4894 caller,
4895 target_module_id,
4896 "scope_changed_before_commit",
4897 code,
4898 );
4899 return Ok(vec![control_error_body_frame(frame, body)?]);
4900 }
4901 }
4902 self.counters
4903 .increment_route_open_refused("module_rejected");
4904 info!(
4905 target: "control",
4906 code = "module_rejected",
4907 module_code = ?body.code,
4908 module_id = ?target_module_id,
4909 connection_id = ctx.connection_id.get(),
4910 principal = %caller.principal,
4911 scope = %caller.scope,
4912 harness = %caller.harness,
4913 "route.open refused"
4914 );
4915 let refusal = if daemon_authored {
4916 control_error_body_frame(frame, body)?
4917 } else {
4918 control_relayed_error_body_frame(frame, body)?
4919 };
4920 Ok(vec![refusal])
4921 }
4922
4923 #[allow(clippy::too_many_arguments)]
4941 fn route_open_breaker_refusal_frame(
4942 &self,
4943 ctx: &RouteCtx,
4944 caller: &RouteOpenCaller,
4945 frame: &Frame,
4946 module_id: &str,
4947 consecutive_timeouts: u32,
4948 retry_in: Duration,
4949 probe_in_flight: bool,
4950 ) -> Result<Frame, RouterError> {
4951 self.counters
4952 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4953 debug!(
4954 target: "control",
4955 code = "module_timeout",
4956 module_id = ?module_id,
4957 connection_id = ctx.connection_id.get(),
4958 principal = %caller.principal,
4959 scope = %caller.scope,
4960 harness = %caller.harness,
4961 consecutive_timeouts,
4962 retry_in_ms = retry_in.as_millis() as u64,
4963 probe_in_flight,
4964 "route.open refused by open bind-relay breaker"
4965 );
4966 let detail = if probe_in_flight {
4973 "one test route is already being tried; retry once it settles".to_string()
4974 } else {
4975 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4976 };
4977 control_error_frame(
4978 frame,
4979 "module_timeout",
4980 format!(
4981 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4982 timed out in a row); {detail}; its established routes are unaffected"
4983 ),
4984 )
4985 }
4986
4987 fn observe_route_open_refusal(
5000 &self,
5001 ctx: &RouteCtx,
5002 caller: &RouteOpenCaller,
5003 module_id: &str,
5004 reason: &'static str,
5005 code: &'static str,
5006 ) {
5007 self.counters.increment_route_open_refused(code);
5008 info!(
5009 target: "control",
5010 code,
5011 reason,
5012 module_id = ?module_id,
5013 connection_id = ctx.connection_id.get(),
5014 principal = %caller.principal,
5015 scope = %caller.scope,
5016 harness = %caller.harness,
5017 "route.open refused"
5018 );
5019 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
5020 self.route_outages.record_not_serving(module_id, reason);
5021 }
5022 }
5023
5024 fn observe_route_open_accept(
5069 &self,
5070 ctx: &RouteCtx,
5071 module_id: &str,
5072 principal: &str,
5073 scope: &str,
5074 ) {
5075 self.route_outages.record_accepted(module_id);
5076 self.counters.increment_route_open_accepted(principal);
5077 info!(
5078 target: "control",
5079 principal,
5080 module_id,
5081 connection_id = ctx.connection_id.get(),
5082 scope = %scope,
5083 "route.open accepted"
5084 );
5085 }
5086
5087 fn supervised_absent_route_open_refusal_frame(
5088 &self,
5089 ctx: &RouteCtx,
5090 caller: &RouteOpenCaller,
5091 frame: &Frame,
5092 module_id: &str,
5093 code: &'static str,
5094 status: &crate::supervise::ModuleStatus,
5095 ) -> Result<Frame, RouterError> {
5096 self.counters.increment_route_open_refused(code);
5097 info!(
5098 target: "control",
5099 code,
5100 reason = "supervised_not_registered",
5101 module_id = ?module_id,
5102 connection_id = ctx.connection_id.get(),
5103 principal = %caller.principal,
5104 scope = %caller.scope,
5105 harness = %caller.harness,
5106 state = %status.state,
5107 enabled = status.enabled,
5108 live = status.live,
5109 "route.open refused"
5110 );
5111 self.route_outages
5115 .record_not_serving(module_id, "supervised_not_registered");
5116 let message = if let Some(pid) = status.waiting_for_previous_pid {
5117 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
5118 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
5119 })
5120 } else {
5121 format!(
5122 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
5123 status.state, status.enabled, status.live
5124 )
5125 };
5126 let expected_back = (code == error_codes::MODULE_WARMING).then_some(true);
5130 control_error_body_frame(
5131 frame,
5132 ErrorBody {
5133 code: code.to_string(),
5134 message,
5135 detail: self.route_open_recovery_detail(
5136 module_id,
5137 frame.header.corr,
5138 None,
5139 expected_back,
5140 )?,
5141 },
5142 )
5143 }
5144
5145 async fn handle_route_open(
5146 &self,
5147 ctx: &RouteCtx,
5148 frame: Frame,
5149 request: RouteOpenRequest,
5150 timings: &mut RouteOpenTimings,
5151 ) -> Result<Vec<Frame>, RouterError> {
5152 let RouteOpenRequest {
5153 target,
5154 mut identity,
5155 consumer_identity,
5156 consumer_capabilities,
5157 role_versions,
5158 busy_refusals,
5159 admission_facts,
5160 scope,
5161 } = request;
5162 let target_module_id = target_module_id(&target).to_string();
5163 timings.module_id.clone_from(&target_module_id);
5164 if self
5165 .registry
5166 .get_module_by_connection(ctx.connection_id)
5167 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5168 .is_some()
5169 {
5170 return Ok(vec![control_error_frame(
5171 &frame,
5172 "invalid_request",
5173 "module connections cannot open client routes",
5174 )?]);
5175 }
5176 debug!(
5177 connection_id = ctx.connection_id.get(),
5178 corr = frame.header.corr,
5179 module_id = %target_module_id,
5180 "handling route.open"
5181 );
5182
5183 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
5188 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
5189
5190 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
5196 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
5197 self.observe_route_open_refusal(
5198 ctx,
5199 &caller,
5200 &target_module_id,
5201 "invalid_role_versions",
5202 error_codes::INVALID_REQUEST,
5203 );
5204 return Ok(vec![control_error_body_frame(
5205 &frame,
5206 ErrorBody {
5207 code: error_codes::INVALID_REQUEST.to_string(),
5208 message: error.to_string(),
5209 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
5210 },
5211 )?]);
5212 }
5213
5214 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
5215 if status.waiting_for_previous_pid.is_some() {
5216 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
5217 ctx,
5218 &caller,
5219 &frame,
5220 &target_module_id,
5221 "module_warming",
5222 &status,
5223 )?]);
5224 }
5225 }
5226
5227 let admission_registry = self
5246 .registry
5247 .route_admission_snapshot()
5248 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5249 let Some(registration) =
5250 crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
5251 else {
5252 if let Some((status, warming)) =
5253 self.supervisor_status(&target_module_id, frame.header.corr)?
5254 {
5255 if status.protocol == ModuleProtocol::None {
5262 return Ok(vec![self.route_open_refusal_frame(
5263 ctx,
5264 &caller,
5265 &frame,
5266 &target_module_id,
5267 "protocol_none",
5268 error_codes::MODULE_NO_PROTOCOL,
5269 format!(
5270 "module_id '{target_module_id}' is declared protocol: none; \
5271 it speaks no subc wire and serves no routes"
5272 ),
5273 )?]);
5274 }
5275 let code = if warming {
5276 "module_warming"
5277 } else {
5278 "target_unavailable"
5279 };
5280 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
5281 ctx,
5282 &caller,
5283 &frame,
5284 &target_module_id,
5285 code,
5286 &status,
5287 )?]);
5288 }
5289 if let Some(removed_ago_ms) =
5290 self.supervisor.removal_tombstone_age_ms(&target_module_id)
5291 {
5292 return Ok(vec![self.route_open_refusal_frame(
5293 ctx,
5294 &caller,
5295 &frame,
5296 &target_module_id,
5297 "removed",
5298 error_codes::MODULE_REMOVED,
5299 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
5300 )?]);
5301 }
5302 return Ok(vec![self.route_open_refusal_frame(
5303 ctx,
5304 &caller,
5305 &frame,
5306 &target_module_id,
5307 "not_registered",
5308 error_codes::UNKNOWN_MODULE,
5309 format!("module_id '{target_module_id}' is not registered"),
5310 )?]);
5311 };
5312
5313 if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
5317 .is_err()
5318 {
5319 self.counters
5320 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
5321 info!(
5322 target: "control",
5323 code = error_codes::MODULE_WARMING,
5324 module_id = ?target_module_id,
5325 connection_id = ctx.connection_id.get(),
5326 principal = %caller.principal,
5327 scope = %caller.scope,
5328 harness = %caller.harness,
5329 reason = "declared_not_ready",
5330 "route.open refused"
5331 );
5332 self.route_outages
5335 .record_not_serving(&target_module_id, "declared_not_ready");
5336 return Ok(vec![control_error_body_frame(
5339 &frame,
5340 ErrorBody {
5341 code: error_codes::MODULE_WARMING.to_string(),
5342 message: format!(
5343 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5344 ),
5345 detail: self.route_open_recovery_detail(
5346 &target_module_id,
5347 frame.header.corr,
5348 Some(serde_json::json!({ "reason": "declared_not_ready" })),
5349 Some(true),
5350 )?,
5351 },
5352 )?]);
5353 }
5354
5355 if let Some(capability) = self.first_unprovided_required_capability(registration) {
5374 self.counters
5375 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5376 info!(
5377 target: "control",
5378 code = error_codes::MODULE_WARMING,
5379 module_id = ?target_module_id,
5380 connection_id = ctx.connection_id.get(),
5381 principal = %caller.principal,
5382 scope = %caller.scope,
5383 harness = %caller.harness,
5384 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5385 capability = %capability,
5386 "route.open refused"
5387 );
5388 return Ok(vec![control_error_body_frame(
5389 &frame,
5390 ErrorBody {
5391 code: error_codes::MODULE_WARMING.to_string(),
5392 message: format!(
5393 "module_id '{target_module_id}' requires capability '{capability}', \
5394 which no registered module provides; retry"
5395 ),
5396 detail: Some(serde_json::json!({
5397 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5398 "capability": capability,
5399 })),
5400 },
5401 )?]);
5402 }
5403
5404 if !target_has_required_role(&target, ®istration.manifest.provides) {
5405 return Ok(vec![self.route_open_refusal_frame(
5406 ctx,
5407 &caller,
5408 &frame,
5409 &target_module_id,
5410 "role_not_provided",
5411 "target_unavailable",
5412 format!("module_id '{target_module_id}' does not provide the requested target"),
5413 )?]);
5414 }
5415
5416 if registration.state != ChannelState::Active {
5417 return Ok(vec![self.route_open_refusal_frame(
5418 ctx,
5419 &caller,
5420 &frame,
5421 &target_module_id,
5422 "registration_not_active",
5423 "target_unavailable",
5424 format!("module_id '{target_module_id}' is not active"),
5425 )?]);
5426 }
5427
5428 if self
5429 .forwarding
5430 .module_is_draining(&target_module_id)
5431 .map_err(RouterError::Forwarding)?
5432 {
5433 return Ok(vec![self.route_open_refusal_frame(
5434 ctx,
5435 &caller,
5436 &frame,
5437 &target_module_id,
5438 "reloading",
5439 "module_reloading",
5440 format!("module_id '{target_module_id}' is reloading"),
5441 )?]);
5442 }
5443
5444 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5445 process_liveness.process_live(&target_module_id) == Some(false)
5446 }) {
5447 if process_liveness.process_replacing(&target_module_id) {
5454 return Ok(vec![self.route_open_refusal_frame(
5455 ctx,
5456 &caller,
5457 &frame,
5458 &target_module_id,
5459 "reloading",
5460 "module_reloading",
5461 format!("module_id '{target_module_id}' is reloading"),
5462 )?]);
5463 }
5464 return Ok(vec![self.route_open_refusal_frame(
5465 ctx,
5466 &caller,
5467 &frame,
5468 &target_module_id,
5469 "supervisor_not_live",
5470 "target_unavailable",
5471 format!("module_id '{target_module_id}' is not live"),
5472 )?]);
5473 }
5474
5475 if !self
5476 .forwarding
5477 .has_live_module_connection(&target_module_id)
5478 .map_err(RouterError::Forwarding)?
5479 {
5480 return Ok(vec![self.route_open_refusal_frame(
5481 ctx,
5482 &caller,
5483 &frame,
5484 &target_module_id,
5485 "no_forwarding_connection",
5486 "target_unavailable",
5487 format!("module_id '{target_module_id}' has no live forwarding connection"),
5488 )?]);
5489 }
5490
5491 if let Some(error) =
5492 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5493 {
5494 self.observe_route_open_refusal(
5495 ctx,
5496 &caller,
5497 &target_module_id,
5498 "op_not_allowed",
5499 "op_not_allowed",
5500 );
5501 return Ok(vec![error]);
5502 }
5503
5504 let principal =
5505 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5506 Ok(principal) => principal,
5507 Err(error) => {
5508 self.observe_route_open_refusal(
5509 ctx,
5510 &caller,
5511 &target_module_id,
5512 "bad_consumer_identity",
5513 "bad_consumer_identity",
5514 );
5515 return Ok(vec![error]);
5516 }
5517 };
5518 timings.principal = principal_label(&principal);
5519
5520 let build_class = identity.build_class.take().filter(|build_class| {
5523 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5524 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5525 } else {
5526 if self
5527 .build_class_warned
5528 .lock()
5529 .unwrap_or_else(|p| p.into_inner())
5530 .insert(ctx.connection_id)
5531 {
5532 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5533 "ignoring build_class from a non-Callosum opener");
5534 }
5535 false
5536 }
5537 });
5538
5539 if let Principal::Reserved {
5543 module_id: opening_module_id,
5544 } = &principal
5545 {
5546 if let Some(opening_registration) = admission_registry
5547 .iter()
5548 .find(|r| r.manifest.module_id == *opening_module_id)
5549 {
5550 if let Some(capability) =
5551 denied_capability(&opening_registration.manifest, ®istration.manifest)
5552 {
5553 warn!(
5554 opening_module_id,
5555 target_module_id,
5556 capability,
5557 "refusing route.open because an attested capability deny edge matches"
5558 );
5559 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5560 &frame,
5561 &target_module_id,
5562 "capability_deny_edge",
5563 "capability_forbidden",
5564 format!(
5565 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5566 ),
5567 )?]);
5568 }
5569 }
5570 }
5571
5572 if admission_facts.is_some() {
5573 let carrier_matches = matches!(
5574 &principal,
5575 Principal::Reserved { module_id }
5576 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5577 );
5578 if !carrier_matches {
5579 return Ok(vec![self.route_open_refusal_frame(
5580 ctx,
5581 &caller,
5582 &frame,
5583 &target_module_id,
5584 "admission_facts_carrier_not_permitted",
5585 "admission_facts_not_permitted",
5586 "admission facts may only be carried by the configured reserved module",
5587 )?]);
5588 }
5589
5590 let target_allowed = self
5591 .admission_facts_targets
5592 .as_ref()
5593 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5594 if !target_allowed {
5595 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5596 &frame,
5597 &target_module_id,
5598 "admission_facts_target_not_listed",
5599 "admission_facts_target_not_allowed",
5600 format!(
5601 "admission facts are not permitted for target module_id '{target_module_id}'"
5602 ),
5603 )?]);
5604 }
5605
5606 }
5610
5611 let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
5617 None => (None, None, false),
5618 Some(selector) => {
5619 let owner_configured = match &selector.owner {
5620 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5626 _ => false,
5627 };
5628 let mut scopes = self
5629 .scopes
5630 .write()
5631 .unwrap_or_else(|poisoned| poisoned.into_inner());
5632 let admitted = crate::scopes::admission::admit_route(
5633 &scopes,
5634 &admission_registry,
5635 &principal,
5636 &target_module_id,
5637 Some(selector),
5638 owner_configured,
5639 crate::scopes::admission::AdmissionMode::Open,
5640 )
5641 .map(|route| {
5642 let admission = route.scope.expect("scoped admission");
5643 let bound = scopes.reserve_bind(
5644 &admission,
5645 &principal,
5646 ctx.connection_id,
5647 selector.hold_presence,
5648 );
5649 (admission, bound)
5650 });
5651 match admitted {
5652 Ok((admission, bound)) => {
5653 (Some(bound), Some(admission.stamp), admission.stamp_required)
5654 }
5655 Err(refusal) => {
5656 return Ok(vec![self.route_open_refusal_frame(
5657 ctx,
5658 &caller,
5659 &frame,
5660 &target_module_id,
5661 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
5662 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
5663 refusal.message,
5664 )?]);
5665 }
5666 }
5667 }
5668 };
5669 if let Some(stamp) = &scope_stamp {
5672 caller.scope = route_open_scope_label(stamp);
5673 }
5674
5675 timings.enter(RouteOpenPhase::ProjectRoot);
5683 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5684 Ok(project) => project,
5685 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5686 };
5687 timings.enter(RouteOpenPhase::Admission);
5688 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5689 Ok(root) => root,
5690 Err(err) => {
5691 return Ok(vec![control_error_frame(
5692 &frame,
5693 "invalid_project_root",
5694 err.to_string(),
5695 )?])
5696 }
5697 };
5698 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5704
5705 let _concurrency_guard = match self
5716 .route_bind_concurrency
5717 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5718 {
5719 Ok(guard) => guard,
5720 Err(in_flight) => {
5721 return Ok(vec![self.route_open_target_capacity_refusal(
5722 ctx,
5723 &caller,
5724 &frame,
5725 &target_module_id,
5726 in_flight,
5727 )?]);
5728 }
5729 };
5730
5731 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5734 RouteBindAdmission::Admitted { guard, probe } => {
5735 if probe {
5736 info!(
5737 module_id = %target_module_id,
5738 connection_id = ctx.connection_id.get(),
5739 "route.bind breaker half-open: admitting one probe"
5740 );
5741 }
5742 guard
5743 }
5744 RouteBindAdmission::Refused {
5745 consecutive_timeouts,
5746 retry_in,
5747 probe_in_flight,
5748 } => {
5749 return Ok(vec![self.route_open_breaker_refusal_frame(
5750 ctx,
5751 &caller,
5752 &frame,
5753 &target_module_id,
5754 consecutive_timeouts,
5755 retry_in,
5756 probe_in_flight,
5757 )?]);
5758 }
5759 };
5760
5761 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5766 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5767 let operator_app = self
5768 .operator_apps
5769 .lock()
5770 .unwrap_or_else(|p| p.into_inner())
5771 .get(&ctx.connection_id)
5772 .cloned();
5773 timings.enter(RouteOpenPhase::EgressReservation);
5774 let pending = match self
5775 .forwarding
5776 .begin_route_bind_relay_for(
5777 ctx.connection_id,
5778 ctx.egress.clone(),
5779 response_version(&frame),
5780 frame.header.corr,
5781 &target_module_id,
5782 principal.clone(),
5783 bound_scope,
5784 project_root,
5785 identity.harness.clone(),
5786 build_class,
5787 operator_app.clone(),
5788 relay_deadline,
5789 busy_refusals == Some(true),
5790 )
5791 .await
5792 {
5793 Ok(pending) => pending,
5794 Err(err) => {
5795 return Ok(vec![self.route_open_refusal_frame(
5796 ctx,
5797 &caller,
5798 &frame,
5799 &target_module_id,
5800 "relay_reservation_failed",
5801 forwarding_error_code(&err),
5802 err.to_string(),
5803 )?])
5804 }
5805 };
5806 timings.enter(RouteOpenPhase::Admission);
5807 let crate::forwarding::PendingRouteBindRelay {
5808 endpoint,
5809 module_sink,
5810 negotiated_ver,
5811 client_channel,
5812 client_epoch,
5813 module_channel,
5814 module_epoch,
5815 corr: relay_corr,
5816 receiver,
5817 } = pending;
5818 let mut reservation =
5819 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5820
5821 let relay_registration = self
5826 .registry
5827 .get_module_by_connection(endpoint.connection_id)
5828 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5829 let relay_capabilities = relay_registration
5830 .as_ref()
5831 .and_then(|registration| registration.manifest.capabilities.as_ref());
5832 if let Some(selector) = &scope {
5833 if let Err(refusal) = self
5836 .scopes
5837 .read()
5838 .unwrap_or_else(|poisoned| poisoned.into_inner())
5839 .check_bind_proof(&principal, selector)
5840 {
5841 reservation.release_and_disarm();
5842 return Ok(vec![self.route_open_refusal_frame(
5843 ctx,
5844 &caller,
5845 &frame,
5846 &target_module_id,
5847 refusal.code,
5848 refusal.code,
5849 refusal.message,
5850 )?]);
5851 }
5852 }
5853 if let Some(stamp) = scope_stamp.as_ref() {
5854 if let Err(refusal) = crate::scopes::check_target_required_stamp(
5855 stamp_required,
5856 &target_module_id,
5857 relay_capabilities,
5858 )
5859 .and_then(|()| {
5860 crate::scopes::check_target_flow_support(
5861 stamp,
5862 &target_module_id,
5863 relay_capabilities,
5864 )
5865 })
5866 .and_then(|()| {
5867 crate::scopes::check_target_agent_run_support(
5868 stamp,
5869 &target_module_id,
5870 relay_capabilities,
5871 )
5872 }) {
5873 reservation.release_and_disarm();
5874 return Ok(vec![self.route_open_refusal_frame(
5875 ctx,
5876 &caller,
5877 &frame,
5878 &target_module_id,
5879 refusal.code,
5880 refusal.code,
5881 refusal.message,
5882 )?]);
5883 }
5884 }
5885
5886 if scope_stamp
5889 .as_ref()
5890 .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
5891 && !crate::scopes::target_supports_federated(relay_capabilities)
5892 {
5893 scope_stamp = None;
5895 }
5896 if !relay_capabilities.is_some_and(|caps| {
5897 caps.provides
5898 .iter()
5899 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5900 }) {
5901 if let Some(stamp) = scope_stamp.as_mut() {
5902 stamp.workdir = None;
5903 stamp.project_id = None;
5904 }
5905 }
5906 let worker_run = scope_stamp
5911 .as_ref()
5912 .is_some_and(crate::scopes::is_worker_run)
5913 && relay_capabilities.is_some_and(|caps| {
5914 caps.provides
5915 .iter()
5916 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5917 });
5918 if !worker_run
5919 && !relay_capabilities.is_some_and(|caps| {
5920 caps.provides
5921 .iter()
5922 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5923 })
5924 {
5925 if let Some(stamp) = scope_stamp.as_mut() {
5926 stamp.attributes.on_behalf_of_agent = None;
5927 }
5928 }
5929 if !relay_capabilities.is_some_and(|caps| {
5930 caps.provides
5931 .iter()
5932 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5933 }) {
5934 if let Some(stamp) = scope_stamp.as_mut() {
5935 stamp.attributes.flow_version = None;
5936 stamp.attributes.flow_code_hash = None;
5937 stamp.attributes.flow_install_id = None;
5938 }
5939 }
5940 let call_mode = if worker_run
5941 || relay_capabilities
5942 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5943 {
5944 Some(derive_call_mode(scope_stamp.as_ref()))
5945 } else {
5946 if let Some(stamp) = scope_stamp.as_mut() {
5947 stamp.expires_at_ms = None;
5948 }
5949 None
5950 };
5951
5952 debug!(
5953 connection_id = ctx.connection_id.get(),
5954 client_channel,
5955 client_epoch,
5956 module_channel,
5957 module_epoch,
5958 "reserved route handle pair"
5959 );
5960 let route_principal_label = principal_label(&principal);
5963 let relay = ModuleControlRequest::RouteBind {
5964 route_channel: module_channel,
5965 epoch: module_epoch,
5966 target,
5967 identity,
5968 principal: Some(principal),
5969 consumer_capabilities,
5970 role_versions,
5971 admission_facts,
5972 scope: scope_stamp,
5973 call_mode,
5974 };
5975 let peer = RouteBindPeer::new()
5976 .with_observed_peer(
5977 relay_capabilities
5978 .is_some_and(|caps| {
5979 caps.provides
5980 .iter()
5981 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5982 })
5983 .then(|| {
5984 self.observed_peers
5985 .lock()
5986 .unwrap_or_else(|p| p.into_inner())
5987 .get(&ctx.connection_id)
5988 .cloned()
5989 .unwrap_or_else(ObservedPeer::unobserved)
5990 }),
5991 )
5992 .with_operator_app(
5993 self.connection_operator_app(operator_app.as_ref())
5994 .filter(|_| {
5995 relay_capabilities.is_some_and(|caps| {
5996 caps.provides
5997 .iter()
5998 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5999 })
6000 }),
6001 );
6002 let relay_body = crate::bind_compat::encode_bind(
6003 &relay,
6004 &peer,
6005 &legacy_bind_project,
6006 relay_capabilities,
6007 )
6008 .map_err(|err| {
6009 RouterError::backend(
6010 0,
6011 frame.header.corr,
6012 format!("failed to encode route.bind request: {err}"),
6013 )
6014 })?;
6015 let relay_frame = Frame::build_with_version(
6016 negotiated_ver,
6017 FrameType::Request,
6018 control_flags(),
6019 0,
6020 0,
6021 relay_corr,
6022 relay_body,
6023 )
6024 .map_err(RouterError::FrameBuild)?;
6025
6026 timings.enter(RouteOpenPhase::BindSend);
6027 let sent = module_sink.send(relay_frame).await;
6028 timings.enter(RouteOpenPhase::Admission);
6029 if let Err(err) = sent {
6030 reservation.release_and_disarm();
6031 return Ok(vec![self.route_open_refusal_frame(
6032 ctx,
6033 &caller,
6034 &frame,
6035 &target_module_id,
6036 "relay_send_failed",
6037 "target_unavailable",
6038 err.to_string(),
6039 )?]);
6040 }
6041
6042 #[cfg(test)]
6043 if let Some(hold) = &self.route_bind_relay_sent_hold {
6044 hold.wait().await;
6048 hold.wait().await;
6049 }
6050
6051 if self
6056 .forwarding
6057 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
6058 .map_err(RouterError::Forwarding)?
6059 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
6060 {
6061 self.send_abandoned_route_bind_goodbye(
6062 &module_sink,
6063 negotiated_ver,
6064 module_channel,
6065 module_epoch,
6066 );
6067 }
6068
6069 timings.enter(RouteOpenPhase::BindAckWait);
6070 let outcome = timeout_at(relay_deadline, receiver).await;
6071 timings.enter(RouteOpenPhase::Admission);
6072 match outcome {
6073 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
6074 reservation.disarm();
6075 if breaker.record_accepted() {
6076 info!(
6077 module_id = %target_module_id,
6078 "route.bind breaker closed: the probe was accepted"
6079 );
6080 }
6081 self.observe_route_open_accept(
6082 ctx,
6083 &target_module_id,
6084 &route_principal_label,
6085 &caller.scope,
6086 );
6087 Ok(Vec::new())
6088 }
6089 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
6090 reservation.release_and_disarm();
6091 breaker.record_inconclusive();
6095 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
6096 }
6097 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
6098 reservation.release_and_disarm();
6099 breaker.record_inconclusive();
6100 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
6101 }
6102 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
6103 reservation.release_and_disarm();
6104 breaker.record_inconclusive();
6105 tracing::warn!(
6112 module_id = %target_module_id,
6113 "route.bind relay abandoned: {message}"
6114 );
6115 Ok(vec![self.route_open_refusal_frame(
6116 ctx,
6117 &caller,
6118 &frame,
6119 &target_module_id,
6120 "relay_abandoned",
6121 "target_unavailable",
6122 message,
6123 )?])
6124 }
6125 Ok(Err(_)) => {
6126 reservation.release_and_disarm();
6127 breaker.record_inconclusive();
6128 Ok(vec![self.route_open_refusal_frame(
6129 ctx,
6130 &caller,
6131 &frame,
6132 &target_module_id,
6133 "relay_waiter_canceled",
6134 "target_unavailable",
6135 "route.bind relay waiter was canceled before the module responded",
6136 )?])
6137 }
6138 Err(_) => {
6139 reservation.release_and_disarm();
6140 if let Some(opened) = breaker.record_timeout(
6144 self.route_bind_breaker_threshold,
6145 self.route_bind_breaker_cooldown,
6146 ) {
6147 warn!(
6148 module_id = %target_module_id,
6149 consecutive_timeouts = opened.consecutive_timeouts,
6150 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
6151 reopened_after_probe = opened.reopened_after_probe,
6152 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
6153 );
6154 }
6155 tracing::warn!(
6163 module_id = %target_module_id,
6164 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
6165 "route.bind relay timed out: module did not ack within budget"
6166 );
6167 Ok(vec![self.route_open_refusal_frame(
6168 ctx,
6169 &caller,
6170 &frame,
6171 &target_module_id,
6172 "relay_timed_out",
6173 "module_timeout",
6174 format!(
6175 "module_id '{target_module_id}' did not answer route.bind within {:?}",
6176 route_bind_relay_timeout
6177 ),
6178 )?])
6179 }
6180 }
6181 }
6182
6183 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6184 let response = ClientControlResponse::SupervisorSpawnSnapshot {
6185 snapshot: self.supervisor.spawn_snapshot(),
6186 };
6187 Ok(vec![control_response_body_frame(
6188 &frame,
6189 &response,
6190 "ClientControlResponse::SupervisorSpawnSnapshot",
6191 )?])
6192 }
6193
6194 fn handle_supervisor_spawn_subscribe(
6195 &self,
6196 ctx: &RouteCtx,
6197 frame: Frame,
6198 since: Option<SpawnCursor>,
6199 ) -> Result<Vec<Frame>, RouterError> {
6200 match self.supervisor.subscribe_spawns(
6201 ctx.connection_id,
6202 frame.header.corr,
6203 response_version(&frame),
6204 since,
6205 ctx.egress.clone(),
6206 ) {
6207 Ok(()) => Ok(Vec::new()),
6208 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
6209 Ok(vec![control_error_body_frame(
6210 &frame,
6211 ErrorBody {
6212 code: "spawn_cursor_incarnation_mismatch".to_string(),
6213 message: "spawn cursor belongs to a different daemon incarnation"
6214 .to_string(),
6215 detail: Some(serde_json::json!({
6216 "current_daemon_incarnation": current
6217 })),
6218 },
6219 )?])
6220 }
6221 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
6222 &frame,
6223 ErrorBody {
6224 code: "spawn_cursor_too_old".to_string(),
6225 message: "spawn cursor predates the retained event ring".to_string(),
6226 detail: Some(serde_json::json!({
6227 "oldest_retained_cursor": oldest
6228 })),
6229 },
6230 )?]),
6231 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
6232 0,
6233 frame.header.corr,
6234 format!("failed to open supervisor spawn subscription: {error}"),
6235 )),
6236 }
6237 }
6238
6239 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6240 let handler = self.clone();
6243 let runtime = tokio::runtime::Handle::current();
6244 let subscriber = tracing::dispatcher::get_default(Clone::clone);
6247 let corr = frame.header.corr;
6248 tokio::task::spawn_blocking(move || {
6249 runtime.block_on(
6250 list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
6251 .with_subscriber(subscriber),
6252 )
6253 })
6254 .await
6255 .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
6256 }
6257
6258 async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6259 let generation = self
6260 .registry
6261 .generation()
6262 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6263 let supervised = self.supervisor.list();
6264 let mut statuses = BTreeMap::new();
6265 let mut not_running = BTreeSet::new();
6266 for module in &supervised {
6267 let status = module.status_for_control("list").map_err(|err| {
6268 RouterError::backend(
6269 0,
6270 frame.header.corr,
6271 format!("failed to read supervisor status: {err}"),
6272 )
6273 })?;
6274 if !status.enabled || !status.process_alive {
6275 not_running.insert(module.module_id().to_owned());
6276 }
6277 statuses.insert(module.module_id().to_owned(), status);
6278 }
6279 let mut process_resources = BTreeMap::new();
6282 for module in &supervised {
6283 process_resources.insert(
6284 module.module_id().to_owned(),
6285 crate::subtree_resources::read_process(module.clone()).await,
6286 );
6287 }
6288 let mut image_readings = BTreeMap::new();
6291 for module in &supervised {
6292 image_readings.insert(
6293 module.module_id().to_owned(),
6294 module.running_image_agreement().await,
6295 );
6296 }
6297 let readings = self.resource_tracker.list(supervised.clone());
6298 let mut modules = Vec::new();
6299 for module in supervised {
6300 let status = statuses
6301 .remove(module.module_id())
6302 .expect("each supervised module has a status snapshot");
6303 let (configured, _) = module.configuration().map_err(|err| {
6304 RouterError::backend(
6305 0,
6306 frame.header.corr,
6307 format!("failed to read module configuration: {err}"),
6308 )
6309 })?;
6310 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
6311 RouterError::backend(
6312 0,
6313 frame.header.corr,
6314 format!("failed to read retry time: {err}"),
6315 )
6316 })?;
6317 let image = image_readings
6318 .remove(module.module_id())
6319 .expect("each supervised module has an image snapshot");
6320 let resources = process_resources.remove(module.module_id());
6321 let mut verdict =
6322 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
6323 verdict.spec = Some(launch_spec_agreement(
6324 &configured,
6325 status.process_alive,
6326 status.launch_spec_digest.as_ref(),
6327 ));
6328 let pending_reload = Some(verdict);
6329 let configured_enabled = self
6330 .capability_evaluator
6331 .configured_enabled(&status.module_id);
6332 modules.push(
6335 SupervisorEntry::new(
6336 status.module_id,
6337 status.state.to_string(),
6338 status.enabled,
6339 status.live,
6340 status.health.status,
6341 )
6342 .with_configured_enabled(configured_enabled)
6343 .with_next_retry_at_ms(next_retry_at_ms)
6344 .with_launch_nonce_env(Some(!cfg!(unix)))
6345 .with_launch_nonce_source(status.launch_nonce_source)
6346 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
6347 .with_protocol(status.protocol)
6348 .with_pending_reload(pending_reload)
6349 .with_last_probe_ms(status.health.last_probe_ms)
6350 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
6351 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
6352 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
6353 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
6354 .with_restart_count(Some(status.restart_count))
6355 .with_max_restarts(Some(status.max_restarts))
6356 .with_lifetime_restarts(Some(status.lifetime_restarts))
6357 .with_spawn_generation(Some(status.spawn_generation))
6358 .with_restart_window_secs(Some(status.restart_window.as_secs()))
6359 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
6360 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
6361 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
6362 .with_resources(resources),
6363 );
6364 }
6365 for module in &mut modules {
6366 if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
6367 module.tree = tree.clone();
6368 module.cpu_last_hour = hour.clone();
6369 }
6370 module.tree_unavailable = readings
6371 .reasons
6372 .get(&module.module_id)
6373 .map(|reason| (*reason).to_owned());
6374 if module.tree.is_none() {
6375 module.tree_unavailable = Some(
6376 if not_running.contains(&module.module_id) {
6377 "not_running"
6378 } else if let Some(reason) = readings.reasons.get(&module.module_id) {
6379 reason
6380 } else {
6381 "not_yet_measured"
6382 }
6383 .to_owned(),
6384 );
6385 }
6386 }
6387 let response = ClientControlResponse::SupervisorList {
6388 generation,
6389 modules,
6390 daemon: readings.daemon,
6391 agent_commands: readings.agent_commands,
6392 };
6393 Ok(vec![control_response_body_frame(
6394 &frame,
6395 &response,
6396 "ClientControlResponse::SupervisorList",
6397 )?])
6398 }
6399
6400 fn handle_supervisor_stderr_tail(
6401 &self,
6402 frame: Frame,
6403 module_id: String,
6404 max_lines: Option<u32>,
6405 max_bytes: Option<u32>,
6406 ) -> Result<Vec<Frame>, RouterError> {
6407 let Some(module) = self.supervisor.get(&module_id) else {
6408 return Ok(vec![control_error_frame(
6409 &frame,
6410 "unknown_module",
6411 format!("module_id '{module_id}' is not supervised"),
6412 )?]);
6413 };
6414
6415 let snapshot = module.stderr_tail(
6416 max_lines.map(|value| value as usize),
6417 max_bytes.map(|value| value as usize),
6418 );
6419
6420 let response = ClientControlResponse::SupervisorStderrTail {
6421 module_id,
6422 tail: StderrTail {
6423 capture: match snapshot.capture {
6424 CaptureState::Captured => StderrCaptureState::Captured,
6425 CaptureState::Incomplete { reason } => {
6426 StderrCaptureState::Incomplete { reason }
6427 }
6428 CaptureState::NotCaptured { reason } => {
6429 StderrCaptureState::NotCaptured { reason }
6430 }
6431 },
6432 entries: snapshot
6433 .entries
6434 .into_iter()
6435 .map(|entry| match entry {
6436 TailEntry::Line {
6437 text,
6438 truncated,
6439 at_ms,
6440 } => StderrTailEntry::Line {
6441 text,
6442 truncated,
6443 at_ms,
6444 },
6445 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6446 })
6447 .collect(),
6448 dropped_lines: snapshot.dropped_lines,
6449 },
6450 };
6451 Ok(vec![control_response_body_frame(
6452 &frame,
6453 &response,
6454 "ClientControlResponse::SupervisorStderrTail",
6455 )?])
6456 }
6457
6458 async fn observe_control_wait<F: std::future::Future>(
6459 &self,
6460 ctx: &RouteCtx,
6461 target: Option<&str>,
6462 kind: &'static str,
6463 future: F,
6464 ) -> F::Output {
6465 self.counters
6466 .observe_reader_wait(
6467 || {
6468 let module = self
6469 .forwarding
6470 .module_id_for_connection(ctx.connection_id)
6471 .ok()
6472 .flatten();
6473 let principal = if module.is_some() { "module" } else { "direct" };
6474 (
6475 self.resolve_reader_wait_target(kind, target)
6476 .or(module)
6477 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6478 principal.into(),
6479 )
6480 },
6481 kind,
6482 future,
6483 )
6484 .await
6485 }
6486
6487 pub(crate) fn resolve_reader_wait_target(
6490 &self,
6491 op: &str,
6492 selector: Option<&str>,
6493 ) -> Option<String> {
6494 let selector = selector?;
6495 match op {
6496 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6497 .supervisor
6498 .get(selector)
6499 .map(|module| module.module_id().to_owned()),
6500 ops::SUPERVISOR_HEALTH_PROBE => self
6501 .registry
6502 .get_module(selector)
6503 .ok()
6504 .flatten()
6505 .map(|registration| registration.manifest.module_id),
6506 _ => None,
6507 }
6508 }
6509
6510 async fn handle_supervisor_terminals(
6511 &self,
6512 ctx: &RouteCtx,
6513 frame: Frame,
6514 module_id: String,
6515 ) -> Result<Vec<Frame>, RouterError> {
6516 let Some(module) = self.supervisor.get(&module_id) else {
6517 return Ok(vec![control_error_frame(
6518 &frame,
6519 "unknown_module",
6520 format!("module_id '{module_id}' is not supervised"),
6521 )?]);
6522 };
6523
6524 let terminals = self
6527 .observe_control_wait(
6528 ctx,
6529 Some(module.module_id()),
6530 ops::SUPERVISOR_TERMINALS,
6531 module.read_durable_terminal_history(),
6532 )
6533 .await
6534 .map_err(|error| {
6535 RouterError::backend(
6536 0,
6537 frame.header.corr,
6538 format!("failed to read terminal history: {error}"),
6539 )
6540 })?;
6541 let response = ClientControlResponse::SupervisorTerminals {
6542 module_id,
6543 terminals,
6544 };
6545 Ok(vec![control_response_body_frame(
6546 &frame,
6547 &response,
6548 "ClientControlResponse::SupervisorTerminals",
6549 )?])
6550 }
6551
6552 fn handle_supervisor_routes(
6553 &self,
6554 frame: Frame,
6555 module_id: Option<String>,
6556 ) -> Result<Vec<Frame>, RouterError> {
6557 let modules = self
6558 .forwarding
6559 .route_census(module_id.as_deref())
6560 .map_err(RouterError::Forwarding)?
6561 .into_iter()
6562 .map(|(module_id, routes)| SupervisorRouteModule {
6563 module_id,
6564 routes: routes
6565 .into_iter()
6566 .map(|route| SupervisorRoute {
6567 consumer: match route.principal {
6568 Principal::Reserved { module_id } => {
6569 SupervisorRouteConsumer::Reserved { module_id }
6570 }
6571 Principal::Direct | Principal::Unverified => {
6572 SupervisorRouteConsumer::Direct {
6573 connection_id: route.goodbye_target.connection_id.get(),
6574 }
6575 }
6576 },
6577 age_ms: Instant::now()
6578 .saturating_duration_since(route.bound_at)
6579 .as_millis()
6580 .try_into()
6581 .unwrap_or(u64::MAX),
6582 draining: route.draining,
6583 drain_reason: route.drain_reason,
6584 })
6585 .collect(),
6586 })
6587 .collect();
6588 let response = ClientControlResponse::SupervisorRoutes { modules };
6589 Ok(vec![control_response_body_frame(
6590 &frame,
6591 &response,
6592 "ClientControlResponse::SupervisorRoutes",
6593 )?])
6594 }
6595
6596 async fn handle_supervisor_provenance(
6597 &self,
6598 ctx: &RouteCtx,
6599 frame: Frame,
6600 module_id: Option<String>,
6601 ) -> Result<Vec<Frame>, RouterError> {
6602 let targeted = module_id.is_some();
6603 let selected = if let Some(module_id) = module_id {
6604 let Some(module) = self.supervisor.get(&module_id) else {
6605 return Ok(vec![control_error_frame(
6606 &frame,
6607 "unknown_module",
6608 format!("module_id '{module_id}' is not supervised"),
6609 )?]);
6610 };
6611 vec![module]
6612 } else {
6613 self.supervisor.list()
6614 };
6615
6616 let target = targeted.then(|| selected[0].module_id());
6617 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6618 let mut modules = Vec::with_capacity(selected.len());
6619 for module in &selected {
6620 let (status, observed_image) = module
6621 .status_and_running_image_agreement()
6622 .await
6623 .map_err(|err| {
6624 RouterError::backend(
6625 0,
6626 frame.header.corr,
6627 format!("failed to read supervisor status: {err}"),
6628 )
6629 })?;
6630 let module_declared = self
6631 .registry
6632 .get_module(&status.module_id)
6633 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6634 .and_then(|registration| registration.manifest.provenance)
6635 .map(|build| ModuleDeclaredProvenance::Reported { build })
6636 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6637 #[cfg(test)]
6638 let running_image = match &self.provenance_probe_override {
6639 Some(result) => result.clone(),
6640 None => observed_image,
6641 };
6642 #[cfg(not(test))]
6643 let running_image = observed_image;
6644 modules.push(SupervisorModuleProvenance {
6645 module_id: status.module_id,
6646 module_declared,
6647 daemon_observed: SupervisorObservedProcess {
6648 launch_nonce_source: status.launch_nonce_source,
6649 pid: status.pid,
6650 spawned_at_ms: status.spawned_at_ms,
6651 spawned_from: status.spawned_from,
6652 running_image,
6653 },
6654 });
6655 }
6656 let daemon = SupervisorDaemonProvenance {
6657 daemon_build: self.daemon_provenance.build.clone(),
6658 daemon_observed: DaemonObservedProcess {
6659 pid: self.daemon_provenance.pid,
6660 started_at_ms: self
6661 .daemon_provenance
6662 .start_clock
6663 .map(|clock| clock.started_at_ms())
6664 .or(self.daemon_provenance.started_at_ms),
6665 running_image: self
6666 .daemon_provenance
6667 .probe
6668 .observe(
6669 self.daemon_provenance.pid,
6670 self.daemon_provenance.executable_path.as_deref(),
6671 self.daemon_provenance.executable_identity.clone(),
6672 self.daemon_provenance.process_start_time,
6673 )
6674 .await,
6675 },
6676 };
6677 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6678 Ok(vec![control_response_body_frame(
6679 &frame,
6680 &response,
6681 "ClientControlResponse::SupervisorProvenance",
6682 )?])
6683 })
6684 .await
6685 }
6686
6687 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6688 self.refresh_capability_requirements();
6689 let generation = self
6690 .registry
6691 .generation()
6692 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6693 let modules = self
6694 .supervisor
6695 .list()
6696 .into_iter()
6697 .map(|module| {
6698 let status = module.status_for_control("health").map_err(|err| {
6699 RouterError::backend(
6700 0,
6701 frame.header.corr,
6702 format!("failed to read supervisor health: {err}"),
6703 )
6704 })?;
6705 let module_id = status.module_id;
6706 let capability_detail = self
6707 .capability_evaluator
6708 .required_problem_detail(&module_id);
6709 Ok(SupervisorHealthEntry {
6710 module_id,
6711 status: status.health.status,
6712 detail: append_capability_problem_detail(
6713 status.health.detail,
6714 capability_detail,
6715 ),
6716 metrics: status.health.metrics,
6717 consecutive_failures: status.health.consecutive_failures,
6718 late_answer_count: status.health.late_answer_count,
6719 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6720 last_action: status.health.last_action,
6721 last_action_ms: status.health.last_action_ms,
6722 last_probe_ms: status.health.last_probe_ms,
6723 })
6724 })
6725 .collect::<Result<Vec<_>, RouterError>>()?;
6726 let response = ClientControlResponse::SupervisorHealth {
6727 generation,
6728 modules,
6729 };
6730 Ok(vec![control_response_body_frame(
6731 &frame,
6732 &response,
6733 "ClientControlResponse::SupervisorHealth",
6734 )?])
6735 }
6736
6737 async fn handle_supervisor_restart(
6738 &self,
6739 frame: Frame,
6740 module_id: String,
6741 drain_timeout_ms: Option<u64>,
6742 requester: Principal,
6743 ) -> Result<Vec<Frame>, RouterError> {
6744 let operation_lock = self.supervisor.operation_lock();
6745 let _operation_guard = operation_lock.lock().await;
6746 let Some(module) = self.supervisor.get(&module_id) else {
6747 return Ok(vec![control_error_frame(
6748 &frame,
6749 "unknown_module",
6750 format!("module_id '{module_id}' is not supervised"),
6751 )?]);
6752 };
6753
6754 self.route_outages.mark_operator_action(&module_id);
6755 if let Err(err) = module
6756 .restart_requested(drain_timeout_ms, Some(requester))
6757 .await
6758 {
6759 self.route_outages
6760 .operator_action_ended_unrefused(&module_id);
6761 let (code, message) = match err {
6762 crate::supervise::SuperviseError::Disabled { .. } => {
6763 ("module_disabled", err.to_string())
6764 }
6765 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6766 ("swap_in_progress", err.to_string())
6767 }
6768 _ => (
6769 "target_unavailable",
6770 format!("failed to restart module_id '{module_id}': {err}"),
6771 ),
6772 };
6773 return Ok(vec![control_error_frame(&frame, code, message)?]);
6774 }
6775
6776 let response = ClientControlResponse::SupervisorAck {
6777 module_id,
6778 applied: true,
6779 };
6780 Ok(vec![control_response_body_frame(
6781 &frame,
6782 &response,
6783 "ClientControlResponse::SupervisorAck",
6784 )?])
6785 }
6786
6787 async fn handle_supervisor_swap(
6791 &self,
6792 frame: Frame,
6793 module_id: String,
6794 ready_timeout_ms: Option<u64>,
6795 requester: Principal,
6796 ) -> Result<Vec<Frame>, RouterError> {
6797 let module = {
6804 let operation_lock = self.supervisor.operation_lock();
6805 let _operation_guard = operation_lock.lock().await;
6806 self.supervisor.get(&module_id)
6807 };
6808 let Some(module) = module else {
6809 return Ok(vec![control_error_frame(
6810 &frame,
6811 "unknown_module",
6812 format!("module_id '{module_id}' is not supervised"),
6813 )?]);
6814 };
6815
6816 self.route_outages.mark_operator_action(&module_id);
6817 if let Err(err) = module
6818 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6819 .await
6820 {
6821 self.route_outages
6822 .operator_action_ended_unrefused(&module_id);
6823 use crate::supervise::SuperviseError;
6824 let message = err.to_string();
6825 let error = match err {
6826 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6827 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6828 code: "swap_refused".to_string(),
6829 message,
6830 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6831 },
6832 SuperviseError::SwapFailed {
6833 arm,
6834 candidate_exit,
6835 ..
6836 } => ErrorBody {
6837 code: "swap_failed".to_string(),
6838 message,
6839 detail: Some(serde_json::json!({
6840 "arm": arm.as_str(),
6841 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6842 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6843 })),
6844 },
6845 _ => ErrorBody::new(
6846 "target_unavailable",
6847 format!("failed to swap module_id '{module_id}': {message}"),
6848 ),
6849 };
6850 return Ok(vec![control_error_body_frame(&frame, error)?]);
6851 }
6852 self.route_outages
6856 .operator_action_ended_unrefused(&module_id);
6857
6858 let response = ClientControlResponse::SupervisorAck {
6859 module_id,
6860 applied: true,
6861 };
6862 Ok(vec![control_response_body_frame(
6863 &frame,
6864 &response,
6865 "ClientControlResponse::SupervisorAck",
6866 )?])
6867 }
6868
6869 async fn handle_supervisor_reload(
6870 &self,
6871 frame: Frame,
6872 module_id: String,
6873 requester: Principal,
6874 ) -> Result<Vec<Frame>, RouterError> {
6875 let operation_lock = self.supervisor.operation_lock();
6876 let _operation_guard = operation_lock.lock().await;
6877 let Some(module) = self.supervisor.get(&module_id) else {
6878 return Ok(vec![control_error_frame(
6879 &frame,
6880 "unknown_module",
6881 format!("module_id '{module_id}' is not supervised"),
6882 )?]);
6883 };
6884
6885 self.route_outages.mark_operator_action(&module_id);
6886 if let Err(err) = module.reload_requested(Some(requester)).await {
6887 self.route_outages
6888 .operator_action_ended_unrefused(&module_id);
6889 let (code, message) = match err {
6890 crate::supervise::SuperviseError::Disabled { .. } => {
6891 ("module_disabled", err.to_string())
6892 }
6893 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6894 ("swap_in_progress", err.to_string())
6895 }
6896 _ => (
6897 "reload_failed",
6898 format!("failed to reload module_id '{module_id}': {err}"),
6899 ),
6900 };
6901 return Ok(vec![control_error_frame(&frame, code, message)?]);
6902 }
6903
6904 let response = ClientControlResponse::SupervisorAck {
6905 module_id,
6906 applied: true,
6907 };
6908 Ok(vec![control_response_body_frame(
6909 &frame,
6910 &response,
6911 "ClientControlResponse::SupervisorAck",
6912 )?])
6913 }
6914
6915 async fn handle_supervisor_rescan(
6919 &self,
6920 frame: Frame,
6921 mode: RescanMode,
6922 requester: Principal,
6923 ) -> Result<Vec<Frame>, RouterError> {
6924 let preview = mode.previews();
6925 let Some(context) = self.rescan.clone() else {
6926 return Ok(vec![control_error_frame(
6927 &frame,
6928 "rescan_unavailable",
6929 "the daemon was not started with a reloadable config path".to_string(),
6930 )?]);
6931 };
6932
6933 let operation_lock = self.supervisor.operation_lock();
6934 let _operation_guard = operation_lock.lock().await;
6935 let loaded = match self.load_rescan_config(&mode, &context) {
6939 Ok(config) => config,
6940 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6941 };
6942 let Some((config, activity_config)) = loaded else {
6950 return Ok(vec![control_error_frame(
6951 &frame,
6952 "invalid_daemon_config",
6953 format!(
6954 "daemon config not found at {}; refusing to rescan (an absent config would \
6955 retire every supervised module)",
6956 context.config_path.display()
6957 ),
6958 )?]);
6959 };
6960 let (
6961 configured_port,
6962 storage_config,
6963 admission_facts_carrier_module_id,
6964 admission_facts_targets,
6965 scope_authority_owners,
6966 federated_scope_owner,
6967 privileged_operator_prompt,
6968 modules,
6969 reserved_capabilities,
6970 ) = (
6971 config.port,
6972 config.storage,
6973 config.admission_facts_carrier_module_id,
6974 config.admission_facts_targets,
6975 config.scope_authority_owners,
6976 config.federated_scope_owner,
6977 config.privileged_operator_prompt,
6978 config.modules,
6979 config.reserved_capabilities,
6980 );
6981
6982 let mut restart_required = Vec::new();
6991 for section in RestartRequiredSection::ALL {
6992 let changed = match section {
6993 RestartRequiredSection::Port => configured_port != context.configured_port,
6994 RestartRequiredSection::Storage => storage_config != context.storage_config,
6995 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6996 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6997 }
6998 RestartRequiredSection::AdmissionFactsTargets => {
6999 admission_facts_targets != context.admission_facts_targets
7000 }
7001 RestartRequiredSection::ScopeAuthorityOwners => {
7002 scope_authority_owners != context.scope_authority_owners
7003 }
7004 RestartRequiredSection::FederatedScopeOwner => {
7005 federated_scope_owner != context.federated_scope_owner
7006 }
7007 RestartRequiredSection::PrivilegedOperatorPrompt => {
7008 privileged_operator_prompt != context.privileged_operator_prompt
7009 }
7010 };
7011 if changed {
7012 restart_required.push(section.label().to_string());
7013 }
7014 }
7015 if !restart_required.is_empty() {
7016 warn!(
7017 config_path = %context.config_path.display(),
7018 sections = %restart_required.join(", "),
7019 "daemon config changed outside the modules section; restart the daemon to apply those changes"
7020 );
7021 }
7022
7023 for configured in &modules {
7024 if let Err(err) = validate_spec(&configured.module_spec()) {
7025 return Ok(vec![control_error_frame(
7026 &frame,
7027 "invalid_daemon_config",
7028 format!("supervisor rescan rejected daemon config: {err}"),
7029 )?]);
7030 }
7031 }
7032
7033 let configured_capabilities = modules
7034 .iter()
7035 .map(|module| (module.module_id.clone(), module.enabled))
7036 .collect::<Vec<_>>();
7037 let preview_capability_warnings = if preview {
7038 let (_, registrations) = self.runtime_capability_snapshot()?;
7039 let current_modules = self
7040 .supervisor
7041 .list()
7042 .into_iter()
7043 .map(|module| module.module_id().to_string())
7044 .collect::<BTreeSet<_>>();
7045 let resulting_modules = configured_capabilities.clone();
7046 let removed = current_modules
7047 .into_iter()
7048 .filter(|module_id| {
7049 !resulting_modules
7050 .iter()
7051 .any(|(configured_id, _)| configured_id == module_id)
7052 })
7053 .collect::<Vec<_>>();
7054 let disabled = match &mode {
7057 RescanMode::Candidate(_) => resulting_modules
7058 .iter()
7059 .filter(|(_, enabled)| !enabled)
7060 .map(|(module_id, _)| module_id.clone())
7061 .collect::<Vec<_>>(),
7062 _ => Vec::new(),
7063 };
7064 self.capability_evaluator.preview_warnings(
7065 resulting_modules,
7066 &removed,
7067 &disabled,
7068 ®istrations,
7069 )
7070 } else {
7071 Vec::new()
7072 };
7073 let reconciled = match &mode {
7074 RescanMode::Candidate(candidate) => self
7075 .preview_candidate(&context, modules, candidate)
7076 .map(|result| (result, Vec::new())),
7077 _ => {
7078 self.reconcile_supervised_modules(
7079 &context.supervisor,
7080 modules,
7081 preview,
7082 requester.clone(),
7083 )
7084 .await
7085 }
7086 };
7087 let (result, changed) = match reconciled {
7088 Ok(result) => result,
7089 Err(message) => {
7090 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
7091 }
7092 };
7093 if !preview {
7094 if let Some(ledger) = &self.activity {
7095 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
7096 ledger.config_changed(&result.added, &result.removed, &changed, requester);
7097 }
7098 if let Err(code) = self.configure_activity(activity_config).await {
7099 warn!(
7100 code,
7101 "activity retention could not be applied during rescan"
7102 );
7103 }
7104 self.capability_evaluator
7105 .configure(configured_capabilities, reserved_capabilities);
7106 self.capability_evaluator.wake_deadline_loop();
7107 self.refresh_capability_requirements();
7108 }
7109 let mut result = result;
7110 result.restart_required = restart_required;
7111 result.capability_warnings = preview_capability_warnings;
7112 let response = ClientControlResponse::SupervisorRescan { result };
7113 Ok(vec![control_response_body_frame(
7114 &frame,
7115 &response,
7116 "ClientControlResponse::SupervisorRescan",
7117 )?])
7118 }
7119
7120 async fn handle_supervisor_release_reserved(
7121 &self,
7122 frame: Frame,
7123 module_id: String,
7124 ) -> Result<Vec<Frame>, RouterError> {
7125 let Some(context) = self.rescan.clone() else {
7126 return Ok(vec![control_error_frame(
7127 &frame,
7128 "release_unavailable",
7129 "reserved-id release requires a daemon started with a reloadable config path",
7130 )?]);
7131 };
7132 let operation_lock = self.supervisor.operation_lock();
7133 let _operation_guard = operation_lock.lock().await;
7134 let loaded = match crate::daemon_config::load(&context.config_path) {
7135 Ok(Some(config)) => config,
7136 Ok(None) => {
7137 return Ok(vec![control_error_frame(
7138 &frame,
7139 "invalid_daemon_config",
7140 format!(
7141 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
7142 context.config_path.display()
7143 ),
7144 )?])
7145 }
7146 Err(err) => {
7147 return Ok(vec![control_error_frame(
7148 &frame,
7149 "invalid_daemon_config",
7150 format!("unable to verify reserved-id release against daemon config: {err}"),
7151 )?])
7152 }
7153 };
7154 if loaded
7155 .modules
7156 .iter()
7157 .any(|configured| configured.module_id == module_id)
7158 {
7159 return Ok(vec![control_error_frame(
7160 &frame,
7161 "reserved_module_configured",
7162 format!(
7163 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
7164 ),
7165 )?]);
7166 }
7167 if !self.supervisor.release_retained_reserved_gate(&module_id) {
7168 return Ok(vec![control_error_frame(
7169 &frame,
7170 "reserved_gate_not_retained",
7171 format!(
7172 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
7173 ),
7174 )?]);
7175 }
7176
7177 let response = ClientControlResponse::SupervisorAck {
7178 module_id,
7179 applied: true,
7180 };
7181 Ok(vec![control_response_body_frame(
7182 &frame,
7183 &response,
7184 "ClientControlResponse::SupervisorAck",
7185 )?])
7186 }
7187
7188 fn plan_rescan(
7192 &self,
7193 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
7194 ) -> Result<RescanPlan, String> {
7195 let mut current = BTreeMap::new();
7196 for module in self.supervisor.list() {
7197 let (spec, health) = module.configuration().map_err(|err| {
7198 format!(
7199 "failed to read configuration for module_id '{}': {err}",
7200 module.module_id()
7201 )
7202 })?;
7203 let enabled = module
7204 .status()
7205 .map_err(|err| {
7206 format!(
7207 "failed to read status for module_id '{}': {err}",
7208 module.module_id()
7209 )
7210 })?
7211 .enabled;
7212 current.insert(
7213 module.module_id().to_string(),
7214 (module, spec, health, enabled),
7215 );
7216 }
7217 let configured = configured_modules
7218 .into_iter()
7219 .map(|module| (module.module_id.clone(), module))
7220 .collect::<BTreeMap<_, _>>();
7221
7222 let added = configured
7223 .keys()
7224 .filter(|module_id| !current.contains_key(*module_id))
7225 .cloned()
7226 .collect::<Vec<_>>();
7227 let removed = current
7228 .keys()
7229 .filter(|module_id| !configured.contains_key(*module_id))
7230 .cloned()
7231 .collect::<Vec<_>>();
7232 let mut changed_pending_reload = Vec::new();
7233 let mut configuration_changes = BTreeSet::new();
7234 let mut enabled_changes = BTreeSet::new();
7235 let mut unchanged = 0_u32;
7236
7237 for (module_id, configured_module) in &configured {
7238 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
7239 else {
7240 continue;
7241 };
7242 let launch_changed = *current_spec != configured_module.module_spec();
7246 let configuration_changed =
7247 launch_changed || *current_health != configured_module.health;
7248 let enabled_changed = *current_enabled != configured_module.enabled
7251 || self
7252 .capability_evaluator
7253 .configured_enabled(module_id)
7254 .is_some_and(|enabled| enabled != configured_module.enabled);
7255 if configuration_changed {
7256 configuration_changes.insert(module_id.clone());
7257 }
7258 if launch_changed {
7259 changed_pending_reload.push(module_id.clone());
7260 }
7261 if enabled_changed {
7262 enabled_changes.insert(module_id.clone());
7263 }
7264 if !configuration_changed && !enabled_changed {
7265 unchanged = unchanged.saturating_add(1);
7266 }
7267 }
7268 Ok(RescanPlan {
7269 current,
7270 configured,
7271 added,
7272 removed,
7273 changed_pending_reload,
7274 configuration_changes,
7275 enabled_changes,
7276 unchanged,
7277 })
7278 }
7279
7280 async fn reconcile_supervised_modules(
7289 &self,
7290 supervisor: &Supervisor,
7291 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
7292 preview: bool,
7293 requester: Principal,
7294 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
7295 let plan = self.plan_rescan(configured_modules)?;
7296
7297 if preview {
7303 return Ok((plan.preview_result(), Vec::new()));
7304 }
7305 let changed = plan
7306 .configuration_changes
7307 .union(&plan.enabled_changes)
7308 .cloned()
7309 .collect::<Vec<_>>();
7310 let RescanPlan {
7311 current,
7312 configured,
7313 added,
7314 removed,
7315 changed_pending_reload,
7316 configuration_changes,
7317 enabled_changes,
7318 unchanged,
7319 } = plan;
7320 if !(added.is_empty()
7325 && removed.is_empty()
7326 && configuration_changes.is_empty()
7327 && enabled_changes.is_empty())
7328 {
7329 if let Some(context) = &self.rescan {
7330 context
7331 .generation
7332 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7333 }
7334 }
7335
7336 for module_id in &removed {
7337 let module = ¤t
7338 .get(module_id)
7339 .expect("removed module came from current supervisor state")
7340 .0;
7341 module
7342 .retire_requested(Some(requester.clone()))
7343 .await
7344 .map_err(|err| {
7345 format!("failed to retire module_id '{module_id}' during rescan: {err}")
7346 })?;
7347 self.supervisor.record_rescan_removal(module_id);
7381 self.supervisor.retire(module_id);
7382 self.route_outages.forget(module_id);
7383 }
7384
7385 for module_id in configured.keys() {
7386 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
7387 continue;
7388 };
7389 let configured_module = configured
7390 .get(module_id)
7391 .expect("configured module id came from configured map");
7392 if configuration_changes.contains(module_id) {
7393 module
7394 .update_configuration(
7395 configured_module.module_spec(),
7396 configured_module.health.clone(),
7397 configured_module.drain_timeout_ms,
7398 )
7399 .await
7400 .map_err(|err| {
7401 format!(
7402 "failed to update module_id '{module_id}' configuration during rescan: {err}"
7403 )
7404 })?;
7405 }
7406 if *current_enabled != configured_module.enabled {
7409 self.route_outages.mark_operator_action(module_id);
7412 module
7413 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
7414 .await
7415 .map_err(|err| {
7416 self.route_outages.operator_action_ended_unrefused(module_id);
7417 format!(
7418 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
7419 configured_module.enabled
7420 )
7421 })?;
7422 }
7423 }
7424
7425 for module_id in &added {
7426 let configured_module = configured
7427 .get(module_id)
7428 .expect("added module id came from configured map");
7429 supervisor
7430 .supervise_configured_requested(
7431 configured_module.module_spec(),
7432 configured_module.enabled,
7433 configured_module.health.clone(),
7434 configured_module.drain_timeout_ms,
7435 configured_module.restart,
7436 configured_module.register_timeout,
7437 Some(requester.clone()),
7438 )
7439 .map_err(|err| {
7440 format!("failed to add module_id '{module_id}' during rescan: {err}")
7441 })?;
7442 }
7443
7444 Ok((
7445 SupervisorRescanResult {
7446 added,
7447 removed,
7448 changed_pending_reload,
7449 enabled_changes: enabled_changes.iter().cloned().collect(),
7450 unchanged,
7451 preview: false,
7452 restart_required: Vec::new(),
7455 capability_warnings: Vec::new(),
7456 module_actions: Vec::new(),
7457 earlier_pending: Vec::new(),
7458 expect: None,
7459 },
7460 changed,
7461 ))
7462 }
7463
7464 async fn handle_supervisor_set_enabled(
7465 &self,
7466 frame: Frame,
7467 module_id: String,
7468 enabled: bool,
7469 requester: Principal,
7470 ) -> Result<Vec<Frame>, RouterError> {
7471 let operation_lock = self.supervisor.operation_lock();
7472 let _operation_guard = operation_lock.lock().await;
7473 let Some(module) = self.supervisor.get(&module_id) else {
7474 return Ok(vec![control_error_frame(
7475 &frame,
7476 "unknown_module",
7477 format!("module_id '{module_id}' is not supervised"),
7478 )?]);
7479 };
7480
7481 self.route_outages.mark_operator_action(&module_id);
7484 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7485 Ok(applied) => applied,
7486 Err(err) => {
7487 self.route_outages
7488 .operator_action_ended_unrefused(&module_id);
7489 return Ok(vec![control_error_frame(
7490 &frame,
7491 "target_unavailable",
7492 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7493 )?]);
7494 }
7495 };
7496 if !applied {
7497 self.route_outages
7500 .operator_action_ended_unrefused(&module_id);
7501 }
7502
7503 self.capability_evaluator.wake_deadline_loop();
7504 self.refresh_capability_requirements();
7505 let response = ClientControlResponse::SupervisorAck { module_id, applied };
7506 Ok(vec![control_response_body_frame(
7507 &frame,
7508 &response,
7509 "ClientControlResponse::SupervisorAck",
7510 )?])
7511 }
7512
7513 async fn handle_supervisor_health_probe(
7514 &self,
7515 ctx: &RouteCtx,
7516 frame: Frame,
7517 module_id: String,
7518 ) -> Result<Vec<Frame>, RouterError> {
7519 self.refresh_capability_requirements();
7520 let Some(registration) = self
7521 .registry
7522 .get_module(&module_id)
7523 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7524 else {
7525 return Ok(vec![control_error_frame(
7526 &frame,
7527 "unknown_module",
7528 format!("module_id '{module_id}' is not registered"),
7529 )?]);
7530 };
7531
7532 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7545 {
7546 return Ok(vec![control_error_frame(
7547 &frame,
7548 "health_not_advertised",
7549 format!("module_id '{module_id}' did not advertise health.check"),
7550 )?]);
7551 }
7552
7553 self.observe_control_wait(
7554 ctx,
7555 Some(®istration.manifest.module_id),
7556 ops::SUPERVISOR_HEALTH_PROBE,
7557 async {
7558 let deadline = Instant::now() + self.health_probe_timeout;
7559 let pending = match self.forwarding.begin_module_control_rpc_for(
7560 &module_id,
7561 MODULE_CONTROL_OP_HEALTH_CHECK,
7562 deadline,
7563 ) {
7564 Ok(pending) => pending,
7565 Err(err) => {
7566 return Ok(vec![control_error_frame(
7567 &frame,
7568 forwarding_error_code(&err),
7569 err.to_string(),
7570 )?])
7571 }
7572 };
7573
7574 let PendingModuleControlRpc {
7575 endpoint,
7576 module_sink,
7577 negotiated_ver,
7578 corr: probe_corr,
7579 receiver,
7580 } = pending;
7581 let mut guard =
7582 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7583 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7584 .map_err(|err| {
7585 RouterError::backend(
7586 0,
7587 frame.header.corr,
7588 format!("failed to encode health.check request: {err}"),
7589 )
7590 })?;
7591 let probe_frame = Frame::build_with_version(
7592 negotiated_ver,
7593 FrameType::Request,
7594 control_flags(),
7595 0,
7596 0,
7597 probe_corr,
7598 probe_body,
7599 )
7600 .map_err(RouterError::FrameBuild)?;
7601
7602 match timeout_at(deadline, module_sink.send(probe_frame)).await {
7605 Ok(Ok(())) => {}
7606 Ok(Err(err)) => {
7607 return Ok(vec![control_error_frame(
7608 &frame,
7609 "target_unavailable",
7610 err.to_string(),
7611 )?]);
7612 }
7613 Err(_) => {
7614 return Ok(vec![control_error_frame(
7615 &frame,
7616 "module_timeout",
7617 format!(
7618 "module_id '{module_id}' did not answer health.check within {:?}",
7619 self.health_probe_timeout
7620 ),
7621 )?]);
7622 }
7623 }
7624
7625 match timeout_at(deadline, receiver).await {
7626 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7627 guard.disarm();
7628 let Some(report) = response.health_report() else {
7629 return Ok(vec![control_error_frame(
7630 &frame,
7631 "invalid_control_body",
7632 "health.check RPC returned a non-health response",
7633 )?]);
7634 };
7635 let HealthReport {
7640 status,
7641 detail,
7642 metrics,
7643 } = report;
7644 let capability_detail = self
7645 .capability_evaluator
7646 .required_problem_detail(&module_id);
7647 let response = ClientControlResponse::SupervisorHealthProbe {
7648 module_id,
7649 status,
7650 detail: append_capability_problem_detail(detail, capability_detail),
7651 metrics,
7652 };
7653 Ok(vec![control_response_body_frame(
7654 &frame,
7655 &response,
7656 "ClientControlResponse::SupervisorHealthProbe",
7657 )?])
7658 }
7659 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7660 guard.disarm();
7661 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7666 }
7667 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7668 guard.disarm();
7669 Ok(vec![control_error_frame(
7670 &frame,
7671 "target_unavailable",
7672 message,
7673 )?])
7674 }
7675 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7676 guard.disarm();
7677 Ok(vec![control_error_frame(
7678 &frame,
7679 "invalid_control_body",
7680 message,
7681 )?])
7682 }
7683 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7684 guard.disarm();
7685 Ok(vec![control_error_frame(
7686 &frame,
7687 "invalid_control_body",
7688 format!("expected module-control op '{expected}', got '{actual}'"),
7689 )?])
7690 }
7691 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7692 guard.disarm();
7693 Ok(vec![control_error_frame(
7694 &frame,
7695 "module_timeout",
7696 format!(
7697 "module_id '{module_id}' answered health.check after {:?}",
7698 self.health_probe_timeout
7699 ),
7700 )?])
7701 }
7702 Ok(Err(_)) => Ok(vec![control_error_frame(
7703 &frame,
7704 "target_unavailable",
7705 "health.check waiter was canceled before the module responded",
7706 )?]),
7707 Err(_) => Ok(vec![control_error_frame(
7708 &frame,
7709 "module_timeout",
7710 format!(
7711 "module_id '{module_id}' did not answer health.check within {:?}",
7712 self.health_probe_timeout
7713 ),
7714 )?]),
7715 }
7716 },
7717 )
7718 .await
7719 }
7720
7721 fn supervisor_status(
7722 &self,
7723 module_id: &str,
7724 corr: u64,
7725 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7726 self.supervisor
7727 .get(module_id)
7728 .map(|module| {
7729 let warming = module.is_warming_for_control("status").map_err(|err| {
7730 RouterError::backend(
7731 0,
7732 corr,
7733 format!(
7734 "failed to read supervisor warming state for module_id '{module_id}': {err}"
7735 ),
7736 )
7737 })?;
7738 module.status_for_control("status").map_err(|err| {
7739 RouterError::backend(
7740 0,
7741 corr,
7742 format!(
7743 "failed to read supervisor status for module_id '{module_id}': {err}"
7744 ),
7745 )
7746 }).map(|status| (status, warming))
7747 })
7748 .transpose()
7749 }
7750
7751 fn guard_module_control_op(
7752 &self,
7753 frame: &Frame,
7754 module_id: &str,
7755 op: &str,
7756 ) -> Result<Option<Frame>, RouterError> {
7757 if self.module_grants_op(module_id, op, frame.header.corr)? {
7758 return Ok(None);
7759 }
7760
7761 Ok(Some(control_error_frame(
7762 frame,
7763 "op_not_allowed",
7764 format!("module_id '{module_id}' did not grant control op '{op}'"),
7765 )?))
7766 }
7767
7768 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7769 let Some(registration) = self
7770 .registry
7771 .get_module(module_id)
7772 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7773 else {
7774 return Ok(false);
7775 };
7776 Ok(module_registration_grants_op(®istration.control_ops, op))
7777 }
7778
7779 fn handle_status_update(
7780 &self,
7781 endpoint: ModuleEndpointId,
7782 frame: Frame,
7783 ) -> Result<Vec<Frame>, RouterError> {
7784 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7785 Ok(update) => update,
7786 Err(err) => {
7787 if is_known_module_push_op(&frame.body) {
7792 return Ok(vec![control_error_frame(
7793 &frame,
7794 "invalid_control_body",
7795 format!("malformed module control push body: {err}"),
7796 )?]);
7797 }
7798 return Ok(Vec::new());
7799 }
7800 };
7801
7802 match update {
7803 ModuleControlPush::RouteStatus {
7804 route_channel,
7805 route_epoch,
7806 status,
7807 } => {
7808 self.forwarding
7809 .cache_status(endpoint, route_channel, route_epoch, status)
7810 .map_err(RouterError::Forwarding)?;
7811 }
7812 }
7813 Ok(Vec::new())
7814 }
7815
7816 fn handle_route_poll(
7817 &self,
7818 ctx: &RouteCtx,
7819 frame: Frame,
7820 route_channel: u16,
7821 route_epoch: u32,
7822 kind: PollKind,
7823 ) -> Result<Vec<Frame>, RouterError> {
7824 let snapshot = self
7825 .forwarding
7826 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7827 .map_err(RouterError::Forwarding)?;
7828 let response = match (kind, snapshot) {
7829 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7830 ClientControlResponse::RoutePoll {
7831 route_channel,
7832 route_epoch,
7833 status,
7834 live: None,
7835 }
7836 }
7837 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7838 route_channel,
7839 route_epoch,
7840 status: None,
7841 live: None,
7842 },
7843 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7844 let live = self
7870 .process_liveness
7871 .as_ref()
7872 .and_then(|source| source.process_live(&module_id))
7873 .unwrap_or(true);
7874 ClientControlResponse::RoutePoll {
7875 route_channel,
7876 route_epoch,
7877 status: None,
7878 live: Some(live),
7879 }
7880 }
7881 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7882 route_channel,
7883 route_epoch,
7884 status: None,
7885 live: Some(false),
7886 },
7887 };
7888
7889 Ok(vec![control_response_body_frame(
7890 &frame,
7891 &response,
7892 "ClientControlResponse::RoutePoll",
7893 )?])
7894 }
7895
7896 pub(crate) fn observe_module_control_completion(
7897 &self,
7898 completion: ModuleControlRpcCompletion,
7899 ) -> bool {
7900 match completion {
7901 ModuleControlRpcCompletion::Unknown => false,
7902 ModuleControlRpcCompletion::Settled => true,
7903 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7904 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7905 info!(
7906 module_id = %module_id,
7907 latency_ms,
7908 "late health.check answer proves the module is alive"
7909 );
7910 match self
7911 .supervisor
7912 .record_late_health_answer(&module_id, latency_ms)
7913 {
7914 Ok(true) => {}
7915 Ok(false) => debug!(
7916 module_id = %module_id,
7917 latency_ms,
7918 "late health.check answer has no active supervisor snapshot"
7919 ),
7920 Err(err) => warn!(
7921 module_id = %module_id,
7922 latency_ms,
7923 error = %err,
7924 "failed to record late health.check answer"
7925 ),
7926 }
7927 true
7928 }
7929 }
7930 }
7931
7932 fn refuse_to_end_module_connection_for_a_client(
7953 &self,
7954 module_connection_id: ConnectionId,
7955 corr: u64,
7956 err: ForwardingError,
7957 ) -> Result<(), RouterError> {
7958 if let ForwardingError::ConnectionClosing { connection_id } = err {
7959 if connection_id != module_connection_id {
7960 warn!(
7961 module_connection_id = module_connection_id.get(),
7962 client_connection_id = connection_id.get(),
7963 corr,
7964 "dropping a route.bind response for a closing client; the module connection keeps serving"
7965 );
7966 return Ok(());
7967 }
7968 }
7969 Err(RouterError::Forwarding(err))
7970 }
7971
7972 fn handle_module_relay_response(
7973 &self,
7974 connection_id: ConnectionId,
7975 frame: Frame,
7976 ) -> Result<Vec<Frame>, RouterError> {
7977 let mut secondary_error = None;
7978 let outcome = match frame.header.ty {
7979 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7980 Ok(probe) if probe.op == "route.bind" => {
7981 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7982 Ok(ModuleControlResponse::RouteBindAck {}) => {
7983 RouteBindRelayOutcome::Accepted
7984 }
7985 Ok(other) => {
7986 let message =
7987 format!("route.bind response carried unexpected body: {other:?}");
7988 secondary_error = Some(control_error_frame(
7989 &frame,
7990 "invalid_control_body",
7991 message.clone(),
7992 )?);
7993 RouteBindRelayOutcome::ModuleGone(message)
7994 }
7995 Err(err) => {
7996 let message = format!("malformed route.bind response body: {err}");
7997 secondary_error = Some(control_error_frame(
7998 &frame,
7999 "invalid_control_body",
8000 message.clone(),
8001 )?);
8002 RouteBindRelayOutcome::ModuleGone(message)
8003 }
8004 }
8005 }
8006 Ok(probe) => {
8007 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
8008 {
8009 Ok(response) => ModuleControlRpcOutcome::Response(response),
8010 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
8011 "malformed {} response body: {err}",
8012 probe.op
8013 )),
8014 };
8015 let completion = self
8016 .forwarding
8017 .complete_module_control_rpc(
8018 connection_id,
8019 frame.header.corr,
8020 Some(&probe.op),
8021 outcome,
8022 )
8023 .map_err(RouterError::Forwarding)?;
8024 if !self.observe_module_control_completion(completion) {
8025 debug!(
8026 connection_id = connection_id.get(),
8027 corr = frame.header.corr,
8028 op = %probe.op,
8029 "dropping late or unknown module-control RPC response"
8030 );
8031 }
8032 return Ok(Vec::new());
8033 }
8034 Err(err) => {
8035 if let Some(expected_op) = self
8036 .forwarding
8037 .pending_module_control_op(connection_id, frame.header.corr)
8038 .map_err(RouterError::Forwarding)?
8039 {
8040 let completion = self
8041 .forwarding
8042 .complete_module_control_rpc(
8043 connection_id,
8044 frame.header.corr,
8045 None,
8046 ModuleControlRpcOutcome::MalformedResponse(format!(
8047 "malformed {expected_op} response body: {err}"
8048 )),
8049 )
8050 .map_err(RouterError::Forwarding)?;
8051 if !self.observe_module_control_completion(completion) {
8052 debug!(
8053 connection_id = connection_id.get(),
8054 corr = frame.header.corr,
8055 "dropping late malformed module-control RPC response"
8056 );
8057 }
8058 return Ok(Vec::new());
8059 }
8060 let message = format!("malformed route.bind response body: {err}");
8061 secondary_error = Some(control_error_frame(
8062 &frame,
8063 "invalid_control_body",
8064 message.clone(),
8065 )?);
8066 RouteBindRelayOutcome::ModuleGone(message)
8067 }
8068 },
8069 FrameType::Error => {
8070 if self
8071 .forwarding
8072 .pending_module_control_op(connection_id, frame.header.corr)
8073 .map_err(RouterError::Forwarding)?
8074 .is_some()
8075 {
8076 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
8077 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
8078 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
8079 "malformed module-control ERROR body: {err}"
8080 )),
8081 };
8082 let completion = self
8083 .forwarding
8084 .complete_module_control_rpc(
8085 connection_id,
8086 frame.header.corr,
8087 None,
8088 outcome,
8089 )
8090 .map_err(RouterError::Forwarding)?;
8091 if !self.observe_module_control_completion(completion) {
8092 debug!(
8093 connection_id = connection_id.get(),
8094 corr = frame.header.corr,
8095 "dropping late or unknown module-control RPC error"
8096 );
8097 }
8098 return Ok(Vec::new());
8099 }
8100 match serde_json::from_slice::<ErrorBody>(&frame.body) {
8101 Ok(body) => RouteBindRelayOutcome::Rejected(body),
8102 Err(err) => {
8103 let message = format!("malformed route.bind ERROR body: {err}");
8104 secondary_error = Some(control_error_frame(
8105 &frame,
8106 "invalid_control_body",
8107 message.clone(),
8108 )?);
8109 RouteBindRelayOutcome::ModuleGone(message)
8110 }
8111 }
8112 }
8113 ty => {
8114 return Ok(vec![control_error_frame(
8115 &frame,
8116 "unsupported_control_frame",
8117 format!("unsupported module channel-0 frame {ty:?}"),
8118 )?])
8119 }
8120 };
8121
8122 let settled =
8123 self.forwarding
8124 .complete_pending_relay(connection_id, frame.header.corr, outcome);
8125 let completion = match settled {
8126 Ok(completion) => completion,
8127 Err(err) => {
8128 self.refuse_to_end_module_connection_for_a_client(
8129 connection_id,
8130 frame.header.corr,
8131 err,
8132 )?;
8133 return Ok(secondary_error.into_iter().collect());
8134 }
8135 };
8136 if let Some(target) = completion.abandoned.as_ref() {
8137 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
8138 }
8139 if !completion.settled {
8140 debug!(
8141 connection_id = connection_id.get(),
8142 corr = frame.header.corr,
8143 frame_type = ?frame.header.ty,
8144 "dropping late or unknown route.bind relay response"
8145 );
8146 }
8147 Ok(secondary_error.into_iter().collect())
8148 }
8149
8150 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
8151 debug!(connection_id = connection_id.get(), "handling GOODBYE");
8152 self.cleanup_connection_with_end_reason(
8156 connection_id,
8157 RegistrationEndReason::ExplicitGoodbye,
8158 )
8159 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
8160 Ok(Vec::new())
8161 }
8162}
8163
8164impl Default for ControlHandler {
8165 fn default() -> Self {
8166 Self::new(Arc::new(Registry::default()))
8167 }
8168}
8169
8170impl crate::supervise::SwapPromotionObserver for ControlHandler {
8171 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
8172 self.apply_registration_capabilities(registration);
8173 }
8174}
8175
8176fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
8177 CapabilityRequirementStatus {
8178 consumer: status.consumer,
8179 capability: status.capability,
8180 need: match status.need {
8181 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
8182 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
8183 },
8184 verdict: status.verdict.as_str().to_string(),
8185 episode_seq: status.episode_seq,
8186 config_satisfiable: status.config_satisfiable,
8187 runtime_available: status.runtime_available,
8188 detail: status.detail,
8189 }
8190}
8191
8192fn append_capability_problem_detail(
8193 detail: Option<String>,
8194 capability_detail: Option<String>,
8195) -> Option<String> {
8196 match (detail, capability_detail) {
8197 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
8198 (Some(detail), None) => Some(detail),
8199 (None, Some(capability_detail)) => Some(capability_detail),
8200 (None, None) => None,
8201 }
8202}
8203
8204fn subc_ops() -> Vec<String> {
8205 SUBC_CONTROL_OPS
8206 .iter()
8207 .map(|op| (*op).to_string())
8208 .collect()
8209}
8210
8211fn module_subc_ops() -> Vec<String> {
8212 SUBC_CONTROL_OPS
8213 .iter()
8214 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
8215 .map(|op| (*op).to_string())
8216 .collect()
8217}
8218
8219#[cfg(test)]
8220fn module_baseline_control_ops() -> Vec<String> {
8221 MODULE_BASELINE_CONTROL_OPS
8222 .iter()
8223 .map(|op| (*op).to_string())
8224 .collect()
8225}
8226
8227fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
8228 let mut seen = HashSet::new();
8229 let mut effective = Vec::new();
8230 for op in MODULE_BASELINE_CONTROL_OPS {
8231 if seen.insert((*op).to_string()) {
8232 effective.push((*op).to_string());
8233 }
8234 }
8235 for op in declared.unwrap_or_default() {
8236 if seen.insert(op.clone()) {
8237 effective.push(op);
8238 }
8239 }
8240 effective
8241}
8242
8243fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
8244 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
8245}
8246
8247fn target_module_id(target: &RouteTarget) -> &str {
8248 match target {
8249 RouteTarget::ToolProvider { module_id }
8250 | RouteTarget::ManagementSurface { module_id }
8251 | RouteTarget::InternalService { module_id, .. } => module_id,
8252 }
8253}
8254
8255fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
8256 roles.iter().any(|role| match (target, role) {
8257 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
8258 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
8259 (
8260 RouteTarget::InternalService { service_id, .. },
8261 ProviderRole::InternalService {
8262 service_id: provided,
8263 ..
8264 },
8265 ) => service_id == provided,
8266 _ => false,
8267 })
8268}
8269
8270fn is_routable_role(role: &ProviderRole) -> bool {
8271 matches!(
8272 role,
8273 ProviderRole::ToolProvider { .. }
8274 | ProviderRole::ManagementSurface { .. }
8275 | ProviderRole::InternalService { .. }
8276 )
8277}
8278
8279#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8280enum ControlRequestBodyError {
8281 UnknownOp,
8282 InvalidBody,
8283}
8284
8285#[derive(Debug, Deserialize)]
8286struct ControlOpProbe {
8287 op: String,
8288}
8289
8290const MODULE_PUSH_OPS: &[&str] = &["route.status"];
8293
8294fn is_known_module_push_op(body: &[u8]) -> bool {
8295 serde_json::from_slice::<ControlOpProbe>(body)
8296 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
8297 .unwrap_or(false)
8298}
8299
8300fn is_known_module_request_op(body: &[u8]) -> bool {
8301 serde_json::from_slice::<ControlOpProbe>(body)
8302 .map(|probe| is_module_to_subc_op(&probe.op))
8303 .unwrap_or(false)
8304}
8305
8306fn is_module_to_subc_op(op: &str) -> bool {
8307 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
8308}
8309
8310fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
8311 debug!(
8312 op = %op,
8313 connection_id = connection_id.get(),
8314 corr,
8315 "control dispatch"
8316 );
8317}
8318
8319fn log_slow_control_dispatch(
8320 dispatch_started_at: Option<StdInstant>,
8321 op: &'static str,
8322 connection_id: ConnectionId,
8323 corr: u64,
8324 route_open_timings: Option<&RouteOpenTimings>,
8325) {
8326 let Some(dispatch_started_at) = dispatch_started_at else {
8327 return;
8328 };
8329 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
8331 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
8332 if let Some(timings) = route_open_timings {
8333 warn!(
8334 op = %op,
8335 connection_id = connection_id.get(),
8336 corr,
8337 elapsed_ms = elapsed.as_millis() as u64,
8338 module_id = ?timings.module_id,
8339 principal = %timings.principal,
8340 admission_ms = timings.admission.as_millis() as u64,
8341 project_root_ms = timings.project_root.as_millis() as u64,
8342 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
8343 bind_send_ms = timings.bind_send.as_millis() as u64,
8344 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
8345 "slow control dispatch"
8346 );
8347 return;
8348 }
8349 warn!(
8350 op = %op,
8351 connection_id = connection_id.get(),
8352 corr,
8353 elapsed_ms = elapsed.as_millis() as u64,
8354 "slow control dispatch"
8355 );
8356 }
8357}
8358
8359#[cfg(test)]
8361type TestCallerLookup =
8362 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
8363
8364fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
8366 match request {
8367 ClientControlRequest::SupervisorRestart { module_id, .. } => {
8368 Some(PrivilegedOperation::Restart { module_id })
8369 }
8370 ClientControlRequest::SupervisorSwap { module_id, .. } => {
8371 Some(PrivilegedOperation::Swap { module_id })
8372 }
8373 ClientControlRequest::SupervisorReload { module_id } => {
8374 Some(PrivilegedOperation::Reload { module_id })
8375 }
8376 ClientControlRequest::SupervisorRescan { preview } => {
8377 if *preview {
8378 None
8379 } else {
8380 Some(PrivilegedOperation::Rescan)
8381 }
8382 }
8383 ClientControlRequest::SupervisorRescanChecked {
8388 preview,
8389 candidate,
8390 expect,
8391 } => (!*preview && candidate.is_none() && expect.is_some())
8392 .then_some(PrivilegedOperation::Rescan),
8393 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
8394 Some(PrivilegedOperation::SetEnabled {
8395 module_id,
8396 enabled: *enabled,
8397 })
8398 }
8399 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
8400 Some(PrivilegedOperation::ReleaseReserved { module_id })
8401 }
8402 ClientControlRequest::ServerDescribe {}
8403 | ClientControlRequest::DaemonShutdownIntent { .. }
8406 | ClientControlRequest::CatalogList { .. }
8407 | ClientControlRequest::RouteOpen { .. }
8408 | ClientControlRequest::RoutePoll { .. }
8409 | ClientControlRequest::SupervisorList {}
8410 | ClientControlRequest::SupervisorHealth {}
8411 | ClientControlRequest::SupervisorHealthProbe { .. }
8412 | ClientControlRequest::SupervisorRoutes { .. }
8413 | ClientControlRequest::SupervisorProvenance { .. }
8414 | ClientControlRequest::SupervisorStderrTail { .. }
8415 | ClientControlRequest::SupervisorTerminals { .. }
8416 | ClientControlRequest::SupervisorSpawnSnapshot {}
8417 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
8418 }
8419}
8420
8421fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
8422 match request {
8423 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
8424 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
8425 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
8426 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
8427 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
8428 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8429 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8430 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8431 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8432 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8433 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8434 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8435 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8436 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8437 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8438 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8439 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8440 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8441 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8442 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8443 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8444 }
8445}
8446
8447struct RouteOpenCaller {
8455 principal: String,
8459 scope: String,
8462 harness: String,
8465}
8466
8467impl RouteOpenCaller {
8468 fn new(
8469 principal: Option<&Principal>,
8470 scope: Option<&ScopeSelector>,
8471 identity: &BindIdentity,
8472 ) -> Self {
8473 Self {
8474 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8475 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8476 harness: bounded_log_text(&identity.harness),
8477 }
8478 }
8479
8480 fn unknown() -> Self {
8483 Self {
8484 principal: "unknown".to_string(),
8485 scope: "unknown".to_string(),
8486 harness: "unknown".to_string(),
8487 }
8488 }
8489}
8490
8491const MAX_LOGGED_TEXT_CHARS: usize = 64;
8494
8495fn bounded_log_text(value: &str) -> String {
8499 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8500 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8501 bounded.push('…');
8502 }
8503 format!("{bounded:?}")
8504}
8505
8506fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8512 use subc_protocol::scope::ScopeKind;
8513 let kind = match stamp.kind {
8514 ScopeKind::Head => "head",
8515 ScopeKind::Worker => "worker",
8516 ScopeKind::Ephemeral => "ephemeral",
8517 ScopeKind::Federated => "federated",
8518 _ => "unknown",
8519 };
8520 format!(
8521 "{kind}/{}/{}@{}",
8522 principal_label(&stamp.owner),
8523 bounded_log_text(&stamp.scope_ref),
8524 stamp.scope_epoch
8525 )
8526}
8527
8528fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8535 let epoch = selector
8536 .scope_epoch
8537 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8538 format!(
8539 "requested/{}/{}@{epoch}",
8540 bounded_log_text(&principal_label(&selector.owner)),
8541 bounded_log_text(&selector.scope_ref),
8542 )
8543}
8544
8545fn principal_label(principal: &Principal) -> String {
8546 match principal {
8547 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8548 Principal::Direct => "direct".to_string(),
8549 other => format!("{other:?}"),
8550 }
8551}
8552
8553fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8554 match request {
8555 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8556 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { .. } => {
8557 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS
8558 }
8559 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8560 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8561 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8562 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8563 ModuleControlRequestFromModule::RouteAuthorize { .. } => {
8564 subc_protocol::scope::ROUTE_AUTHORIZE_OP
8565 }
8566 ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
8567 _ => "unsupported",
8568 }
8569}
8570
8571fn parse_client_control_request(
8572 body: &[u8],
8573) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8574 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8575 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8576 Ok(probe)
8577 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8578 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8579 {
8580 ControlRequestBodyError::InvalidBody
8581 }
8582 Ok(_) => ControlRequestBodyError::UnknownOp,
8583 Err(_) => ControlRequestBodyError::InvalidBody,
8584 };
8585 (err, classification)
8586 })
8587}
8588
8589fn parse_module_control_request_from_module(
8590 body: &[u8],
8591) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8592 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8593 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8594 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8595 Ok(_) => ControlRequestBodyError::UnknownOp,
8596 Err(_) => ControlRequestBodyError::InvalidBody,
8597 };
8598 (err, classification)
8599 })
8600}
8601
8602#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8603enum ProviderRoleKind {
8604 ToolProvider,
8605 PipelineStage,
8606 ManagementSurface,
8607 InternalService,
8608}
8609
8610fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8611 match role {
8612 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8613 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8614 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8615 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8616 }
8617}
8618
8619fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8620 roles.iter().map(provider_role_kind).collect()
8621}
8622
8623const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8626
8627#[derive(Debug, Default, PartialEq, Eq)]
8629struct ScopeOutcomeCounts {
8630 created: usize,
8631 replaced: usize,
8632 updated: usize,
8633 unchanged: usize,
8634 refused: usize,
8635}
8636
8637impl ScopeOutcomeCounts {
8638 fn of(results: &[ScopeRecordResult]) -> Self {
8639 let mut counts = Self::default();
8640 for result in results {
8641 let slot = match result.outcome {
8642 ScopeRecordOutcome::Created => &mut counts.created,
8643 ScopeRecordOutcome::Replaced => &mut counts.replaced,
8644 ScopeRecordOutcome::Updated => &mut counts.updated,
8645 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8646 ScopeRecordOutcome::Refused => &mut counts.refused,
8647 };
8648 *slot += 1;
8649 }
8650 counts
8651 }
8652}
8653
8654#[cfg(test)]
8655mod scope_outcome_count_tests {
8656 use super::*;
8657
8658 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8659 ScopeRecordResult {
8660 scope_ref: "r".to_string(),
8661 scope_epoch: 1,
8662 outcome,
8663 code: None,
8664 message: None,
8665 version: None,
8666 parent_state: None,
8667 }
8668 }
8669
8670 #[test]
8673 fn every_outcome_is_counted_in_its_own_field() {
8674 let results = [
8675 result(ScopeRecordOutcome::Created),
8676 result(ScopeRecordOutcome::Created),
8677 result(ScopeRecordOutcome::Replaced),
8678 result(ScopeRecordOutcome::Updated),
8679 result(ScopeRecordOutcome::Unchanged),
8680 result(ScopeRecordOutcome::Refused),
8681 result(ScopeRecordOutcome::Refused),
8682 result(ScopeRecordOutcome::Refused),
8683 ];
8684 assert_eq!(
8685 ScopeOutcomeCounts::of(&results),
8686 ScopeOutcomeCounts {
8687 created: 2,
8688 replaced: 1,
8689 updated: 1,
8690 unchanged: 1,
8691 refused: 3,
8692 }
8693 );
8694 }
8695}
8696
8697fn capability_census_trigger(
8701 old: Option<&CapabilityDeclarations>,
8702 new: Option<&CapabilityDeclarations>,
8703) -> bool {
8704 let old_provides = old
8705 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8706 .unwrap_or_default();
8707 let old_denies = old
8708 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8709 .unwrap_or_default();
8710 let new = new.cloned().unwrap_or(CapabilityDeclarations {
8711 provides: Vec::new(),
8712 requires: Vec::new(),
8713 must_never_reach: Vec::new(),
8714 });
8715
8716 new.provides
8717 .iter()
8718 .any(|capability| !old_provides.contains(capability))
8719 || new
8720 .must_never_reach
8721 .iter()
8722 .any(|capability| !old_denies.contains(capability))
8723}
8724
8725fn catalog_update_frozen_field_message(
8726 registered: &ModuleManifest,
8727 provides: &[ProviderRole],
8728) -> Option<String> {
8729 let old_has_provides = !registered.provides.is_empty();
8730 let new_has_provides = !provides.is_empty();
8731 if old_has_provides != new_has_provides {
8732 return Some(format!(
8733 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8734 registered.module_id
8735 ));
8736 }
8737
8738 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
8739 return Some(format!(
8740 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8741 registered.module_id
8742 ));
8743 }
8744
8745 let registered_concurrency = manifest_concurrency(registered);
8746 let mut candidate = registered.clone();
8747 candidate.provides = provides.to_vec();
8748 let candidate_concurrency = manifest_concurrency(&candidate);
8749 if candidate_concurrency != registered_concurrency {
8750 return Some(format!(
8751 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8752 registered.module_id, registered_concurrency, candidate_concurrency
8753 ));
8754 }
8755
8756 None
8759}
8760
8761fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8762 manifest.provides.iter().any(is_routable_role)
8763}
8764
8765fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8771 manifest
8772 .provides
8773 .iter()
8774 .find_map(|provider| match provider {
8775 ProviderRole::ToolProvider { concurrency, .. }
8776 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8777 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8778 })
8779 .unwrap_or(Concurrency::ModuleManaged)
8780}
8781
8782fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8789 let has_management_surface = manifest
8790 .provides
8791 .iter()
8792 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8793 if !has_management_surface {
8794 return false;
8795 }
8796 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8797 return false;
8798 };
8799 let Some(provides) = raw
8800 .get("manifest")
8801 .and_then(|manifest| manifest.get("provides"))
8802 .and_then(serde_json::Value::as_array)
8803 else {
8804 return false;
8805 };
8806 provides.iter().any(|role| {
8811 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8812 && role.get("concurrency").is_none()
8813 })
8814}
8815
8816fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8817 if peer_version != PROTOCOL_VERSION {
8818 return Err(format!(
8819 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8820 ));
8821 }
8822 Ok(PROTOCOL_VERSION)
8823}
8824
8825fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8826 Frame::build_with_version(
8827 response_version(frame),
8828 FrameType::Pong,
8829 frame.header.flags,
8830 0,
8831 0,
8832 frame.header.corr,
8833 Vec::new(),
8834 )
8835 .map_err(RouterError::FrameBuild)
8836}
8837
8838pub(crate) fn control_error_frame(
8839 frame: &Frame,
8840 code: &'static str,
8841 message: impl Into<String>,
8842) -> Result<Frame, RouterError> {
8843 control_error_body_frame(
8844 frame,
8845 ErrorBody {
8846 code: code.to_string(),
8847 message: message.into(),
8848 detail: None,
8849 },
8850 )
8851}
8852
8853fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8861 control_error_frame_with_flags(frame, error, daemon_error_flags())
8862}
8863
8864fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8870 control_error_frame_with_flags(frame, error, control_flags())
8871}
8872
8873fn control_error_frame_with_flags(
8874 frame: &Frame,
8875 error: ErrorBody,
8876 flags: Flags,
8877) -> Result<Frame, RouterError> {
8878 let body = serde_json::to_vec(&error).map_err(|err| {
8879 RouterError::backend(
8880 0,
8881 frame.header.corr,
8882 format!("failed to encode control ERROR: {err}"),
8883 )
8884 })?;
8885
8886 Frame::build_with_version(
8887 response_version(frame),
8888 FrameType::Error,
8889 flags,
8890 0,
8891 0,
8892 frame.header.corr,
8893 body,
8894 )
8895 .map_err(RouterError::FrameBuild)
8896}
8897
8898pub(crate) fn control_response_body_frame<T: Serialize>(
8899 frame: &Frame,
8900 reply: &T,
8901 label: &'static str,
8902) -> Result<Frame, RouterError> {
8903 let body = serde_json::to_vec(reply).map_err(|err| {
8904 RouterError::backend(
8905 0,
8906 frame.header.corr,
8907 format!("failed to encode {label}: {err}"),
8908 )
8909 })?;
8910
8911 Frame::build_with_version(
8912 response_version(frame),
8913 FrameType::Response,
8914 control_flags(),
8915 0,
8916 0,
8917 frame.header.corr,
8918 body,
8919 )
8920 .map_err(RouterError::FrameBuild)
8921}
8922
8923fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8953 match err {
8954 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8955 ForwardingError::NoModuleConnection => "target_unavailable",
8956 ForwardingError::ModuleReloading { .. } => "module_reloading",
8957 ForwardingError::ClientRouteChannelExhausted { .. }
8958 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8959 ForwardingError::StaleModuleEndpoint
8960 | ForwardingError::UnknownReservation { .. }
8961 | ForwardingError::ConnectionClosing { .. }
8962 | ForwardingError::ClientEgressClosed { .. }
8963 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8964 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8967 ForwardingError::RelayCorrelationExhausted
8968 | ForwardingError::RouteOpenBuild(_)
8969 | ForwardingError::Poisoned => "forwarding_error",
8970 }
8971}
8972
8973fn response_version(frame: &Frame) -> u8 {
8974 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8975 frame.header.ver
8976 } else {
8977 PROTOCOL_VERSION
8978 }
8979}
8980
8981fn control_flags() -> Flags {
8982 Flags::new(false, Priority::Passive, false)
8983}
8984
8985fn send_goodbye_target_best_effort(
8990 counters: &DaemonCounters,
8991 target: &GoodbyeTarget,
8992 context: &'static str,
8993) {
8994 let Ok(frame) = Frame::build_with_version(
8995 target.negotiated_ver,
8996 FrameType::Goodbye,
8997 control_flags(),
8998 target.channel,
8999 target.epoch,
9000 0,
9001 Vec::new(),
9002 ) else {
9003 return;
9004 };
9005 crate::forwarding::send_module_route_goodbye(
9006 counters,
9007 &target.sink,
9008 frame,
9009 target.module_id.as_deref(),
9010 context,
9011 );
9012}
9013
9014pub(crate) fn send_route_control_pushes(
9015 forwarding: &ForwardingTable,
9016 routes: Vec<EndpointRoute>,
9017 push: ClientControlPush,
9018) {
9019 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
9020 for route in routes {
9021 let target = route.goodbye_target;
9022 if let Some((existing, channels)) = targets
9023 .iter_mut()
9024 .find(|(existing, _)| existing.connection_id == target.connection_id)
9025 {
9026 debug_assert_eq!(
9027 existing.negotiated_ver, target.negotiated_ver,
9028 "one connection cannot negotiate multiple frame versions"
9029 );
9030 if !channels.contains(&target.channel) {
9031 channels.push(target.channel);
9032 }
9033 continue;
9034 }
9035 let channel = target.channel;
9036 targets.push((target, vec![channel]));
9037 }
9038 for (target, mut channels) in targets {
9039 channels.sort_unstable();
9040 let mut push = push.clone();
9041 match &mut push {
9042 ClientControlPush::RouteClosing {
9043 channels: covered, ..
9044 }
9045 | ClientControlPush::RouteClosed {
9046 channels: covered, ..
9047 } => *covered = channels,
9048 }
9049 let body = match serde_json::to_vec(&push) {
9050 Ok(body) => body,
9051 Err(err) => {
9052 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
9053 continue;
9054 }
9055 };
9056 let frame = match Frame::build_with_version(
9057 target.negotiated_ver,
9058 FrameType::Push,
9059 control_flags(),
9060 0,
9061 0,
9062 0,
9063 body.clone(),
9064 ) {
9065 Ok(frame) => frame,
9066 Err(err) => {
9067 warn!(
9068 route_channel = target.channel,
9069 error = %err,
9070 "failed to build route lifecycle control PUSH frame"
9071 );
9072 continue;
9073 }
9074 };
9075 if let Err(err) = target.sink.try_send(frame) {
9076 if target.close_on_delivery_failure() {
9077 warn!(
9078 target_connection_id = target.connection_id.get(),
9079 route_channel = target.channel,
9080 error = %err,
9081 "route lifecycle control PUSH was not delivered to client; closing target connection"
9082 );
9083 let _ = forwarding.escalate_client_delivery_failure(
9084 target.connection_id,
9085 target.channel,
9086 target.epoch,
9087 CloseReason::new(
9088 "route_lifecycle_push_delivery_failed",
9089 format!(
9090 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
9091 target.channel
9092 ),
9093 ),
9094 crate::forwarding::UndeliveredFrame {
9095 module_id: target.module_id.as_deref(),
9096 sink: &target.sink,
9097 },
9098 );
9099 }
9100 }
9101 }
9102}
9103
9104#[cfg(test)]
9105mod tests {
9106 include!("control/agent_commands_tests.rs");
9107 include!("control/resource_revision_tests.rs");
9108 include!("control/route_open_recovery_tests.rs");
9109 #[tokio::test]
9110 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
9111 let registry = Arc::new(Registry::default());
9112 let handler = ControlHandler::new(registry);
9113 assert_eq!(
9114 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
9115 None
9116 );
9117 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
9118 hello_via_sink(
9119 &handler,
9120 &ctx,
9121 &mut rx,
9122 hello_frame("daemon", PROTOCOL_VERSION, 1),
9123 )
9124 .await;
9125 assert_eq!(
9126 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
9127 Some("daemon".into())
9128 );
9129 assert_eq!(
9130 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
9131 None
9132 );
9133 assert_eq!(
9134 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
9135 None,
9136 "a registered module is not necessarily supervised"
9137 );
9138 assert_eq!(
9139 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
9140 None,
9141 "aggregate provenance has no single target"
9142 );
9143 assert_eq!(
9144 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
9145 None
9146 );
9147 }
9148
9149 mod peer_manifest {
9150 include!("peer_manifest_tests.rs");
9151 }
9152 mod activity_read {
9153 include!("activity_read_tests.rs");
9154 }
9155
9156 mod lifecycle_activity {
9157 include!("control_lifecycle_tests.rs");
9158 }
9159 #[cfg(unix)]
9160 #[tokio::test]
9161 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
9162 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
9163 let path = dir.join("subc.jsonc");
9164 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
9165 "program":"/bin/sleep","args":["60"],"protocol":"none",
9166 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
9167 }}}).to_string()).unwrap();
9168 let mut configured = crate::daemon_config::load(&path)
9169 .unwrap()
9170 .unwrap()
9171 .modules
9172 .pop()
9173 .unwrap();
9174 let registry = std::sync::Arc::new(crate::Registry::default());
9175 let handle = crate::SupervisorHandle::new();
9176 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
9177 .with_handle(handle.clone());
9178 let module = supervisor
9179 .supervise_configured_with_health(
9180 configured.module_spec(),
9181 true,
9182 configured.health.clone(),
9183 None,
9184 configured.restart,
9185 )
9186 .unwrap();
9187 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
9188 let before = module.status().unwrap().pid;
9189 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
9190 configured.health.cadence = std::time::Duration::from_secs(3600);
9191 let health_only = handler
9192 .reconcile_supervised_modules(
9193 &supervisor,
9194 vec![configured.clone()],
9195 false,
9196 Principal::Direct,
9197 )
9198 .await
9199 .unwrap();
9200 assert!(
9201 health_only.0.changed_pending_reload.is_empty(),
9202 "health policy is already applied live"
9203 );
9204 assert_eq!(module.status().unwrap().pid, before);
9205 assert_eq!(
9206 module.configuration().unwrap().1.http,
9207 configured.health.http
9208 );
9209 configured.args = vec!["61".into()];
9210 let launch = handler
9211 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
9212 .await
9213 .unwrap();
9214 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
9215 assert_eq!(
9216 module.status().unwrap().pid,
9217 before,
9218 "a launch edit is stored until reload"
9219 );
9220 module.drain().await.unwrap();
9221 }
9222 use cortexkit_test_support::ScratchDir;
9223 use std::{
9224 collections::BTreeMap,
9225 fmt,
9226 path::PathBuf,
9227 sync::{Arc, Mutex},
9228 time::Duration,
9229 };
9230
9231 use serde_json::{json, Value};
9232 use subc_protocol::{
9233 manifest::{
9234 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
9235 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
9236 },
9237 session::HealthStatus,
9238 FrameType,
9239 };
9240
9241 use super::*;
9242 use crate::{
9243 forwarding::{DataRoute, DataRouteState},
9244 registry::ChannelState,
9245 router::FrameSink,
9246 stderr_tail::DEFAULT_MAX_LINE_BYTES,
9247 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
9248 RouteCtx, Router,
9249 };
9250 use tokio::{
9251 sync::mpsc,
9252 time::{sleep, Instant},
9253 };
9254 use tracing::{
9255 field::{Field, Visit},
9256 Event, Subscriber,
9257 };
9258 use tracing_subscriber::{layer::Context, prelude::*, Layer};
9259
9260 fn fake_aft_stub_path() -> PathBuf {
9279 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
9280 assert!(
9281 path.exists(),
9282 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
9283 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
9284 path.display()
9285 );
9286 path
9287 }
9288
9289 fn client_retries(code: &str) -> bool {
9294 subc_protocol::error_codes::is_retryable_route_open(code)
9295 }
9296
9297 #[test]
9308 fn retryability_of_forwarding_codes_matches_the_failure() {
9309 let transient = [
9312 ForwardingError::NoModuleConnection,
9313 ForwardingError::ModuleReloading {
9314 module_id: "m".into(),
9315 },
9316 ForwardingError::StaleModuleEndpoint,
9317 ForwardingError::UnknownReservation {
9318 client_channel: 1,
9319 module_channel: 1,
9320 },
9321 ForwardingError::ConnectionClosing {
9322 connection_id: ConnectionId::new(1),
9323 },
9324 ForwardingError::ClientEgressClosed {
9325 connection_id: ConnectionId::new(1),
9326 },
9327 ForwardingError::ModuleEgressUnavailable {
9328 connection_id: ConnectionId::new(1),
9329 },
9330 ];
9331 for err in transient {
9332 let code = forwarding_error_code(&err);
9333 assert!(
9334 client_retries(code),
9335 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
9336 );
9337 }
9338
9339 let permanent = [
9344 ForwardingError::ConnectionRoleConflict {
9345 connection_id: ConnectionId::new(1),
9346 },
9347 ForwardingError::ClientRouteChannelExhausted {
9348 connection_id: ConnectionId::new(1),
9349 },
9350 ForwardingError::ModuleRouteChannelExhausted {
9351 endpoint: ModuleEndpointId {
9352 connection_id: ConnectionId::new(1),
9353 generation: 1,
9354 },
9355 },
9356 ForwardingError::RelayCorrelationExhausted,
9357 ForwardingError::RouteOpenBuild("x".into()),
9358 ForwardingError::Poisoned,
9359 ];
9360 for err in permanent {
9361 let code = forwarding_error_code(&err);
9362 assert!(
9363 !client_retries(code),
9364 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
9365 );
9366 }
9367 }
9368
9369 #[tokio::test]
9397 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
9398 let handler = ControlHandler::default();
9399 let frame =
9400 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
9401
9402 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
9406 assert_eq!(
9407 stamped,
9408 Principal::Direct,
9409 "a caller that proved nothing must not be stamped as a supervised module"
9410 );
9411
9412 let forged = handler
9417 .route_open_principal(
9418 &frame,
9419 Some(ConsumerIdentity {
9420 module_id: "aft".to_string(),
9421 launch_nonce: "not-a-real-nonce".to_string(),
9422 }),
9423 )
9424 .unwrap();
9425 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9426 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9427 }
9428
9429 #[test]
9446 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9447 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9448 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9449 let ClientControlRequest::RouteOpen {
9450 consumer_identity, ..
9451 } = parsed
9452 else {
9453 panic!("route.open body must parse as RouteOpen");
9454 };
9455 assert_eq!(
9456 consumer_identity,
9457 Some(ConsumerIdentity {
9458 module_id: "aft".to_string(),
9459 launch_nonce: "n".to_string(),
9460 }),
9461 "the wire field name must reach the value route_open_principal reads"
9462 );
9463 }
9464
9465 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9466 ModuleManifest::builder(module_id, "0.1.0")
9467 .protocol_ver(protocol_ver)
9468 .provides(vec![ProviderRole::ToolProvider {
9469 tools: vec![Tool {
9470 name: "read".to_string(),
9471 description: None,
9472 execution_mode: ExecutionMode::Pure,
9473 schema: json!({"type": "object"}),
9474 }],
9475 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9476 concurrency: Concurrency::ModuleManaged,
9477 emits_push: true,
9478 sub_supervises: true,
9479 }])
9480 .build()
9481 }
9482
9483 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9484 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9485 }
9486
9487 fn hello_frame_with_control_ops(
9488 module_id: &str,
9489 protocol_ver: u8,
9490 corr: u64,
9491 control_ops: Option<Vec<String>>,
9492 ) -> Frame {
9493 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9494 }
9495
9496 fn hello_frame_with_nonce(
9497 module_id: &str,
9498 protocol_ver: u8,
9499 corr: u64,
9500 launch_nonce: Option<&str>,
9501 ) -> Frame {
9502 hello_frame_full(
9503 module_id,
9504 protocol_ver,
9505 corr,
9506 None,
9507 launch_nonce.map(ToOwned::to_owned),
9508 )
9509 }
9510
9511 fn hello_frame_full(
9512 module_id: &str,
9513 protocol_ver: u8,
9514 corr: u64,
9515 control_ops: Option<Vec<String>>,
9516 launch_nonce: Option<String>,
9517 ) -> Frame {
9518 let body = serde_json::to_vec(&ModuleHelloBody {
9519 manifest: manifest(module_id, protocol_ver),
9520 protocol_ver,
9521 control_ops,
9522 launch_nonce,
9523 })
9524 .unwrap();
9525 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9526 }
9527
9528 fn hello_frame_with_rich_ui(
9529 module_id: &str,
9530 protocol_ver: u8,
9531 corr: u64,
9532 launch_nonce: Option<&str>,
9533 rich_ui: Value,
9534 ) -> Frame {
9535 let mut body = serde_json::to_value(ModuleHelloBody {
9536 manifest: manifest(module_id, protocol_ver),
9537 protocol_ver,
9538 control_ops: None,
9539 launch_nonce: launch_nonce.map(str::to_string),
9540 })
9541 .unwrap();
9542 body["manifest"]["rich_ui"] = rich_ui;
9543 Frame::build(
9544 FrameType::Hello,
9545 control_flags(),
9546 0,
9547 0,
9548 corr,
9549 serde_json::to_vec(&body).unwrap(),
9550 )
9551 .unwrap()
9552 }
9553
9554 fn non_routable_hello_frame_with_control_ops(
9555 module_id: &str,
9556 corr: u64,
9557 control_ops: Option<Vec<String>>,
9558 ) -> Frame {
9559 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9560 manifest.provides.clear();
9561 let body = serde_json::to_vec(&ModuleHelloBody {
9562 manifest,
9563 protocol_ver: PROTOCOL_VERSION,
9564 control_ops,
9565 launch_nonce: None,
9566 })
9567 .unwrap();
9568 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9569 }
9570
9571 fn capability_grammar_hello_frame(
9572 capabilities: Value,
9573 runtime_computed: Option<Value>,
9574 corr: u64,
9575 ) -> Frame {
9576 let mut body = serde_json::to_value(ModuleHelloBody {
9577 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9578 protocol_ver: PROTOCOL_VERSION,
9579 control_ops: None,
9580 launch_nonce: None,
9581 })
9582 .expect("HELLO body serializes");
9583 body["manifest"]["capabilities"] = capabilities;
9584 if let Some(runtime_computed) = runtime_computed {
9585 body["runtime_computed"] = runtime_computed;
9586 }
9587 Frame::build(
9588 FrameType::Hello,
9589 control_flags(),
9590 0,
9591 0,
9592 corr,
9593 serde_json::to_vec(&body).expect("HELLO body reserializes"),
9594 )
9595 .expect("HELLO frame builds")
9596 }
9597
9598 fn channel_request(channel: u16, corr: u64) -> Frame {
9599 Frame::build(
9600 FrameType::Request,
9601 Flags::new(true, Priority::Interactive, false),
9602 channel,
9603 0,
9604 corr,
9605 b"opaque".to_vec(),
9606 )
9607 .unwrap()
9608 }
9609
9610 fn route_ctx(
9611 connection_id: ConnectionId,
9612 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9613 let (tx, rx) = mpsc::channel(8);
9614 (
9615 RouteCtx {
9616 connection_id,
9617 egress: FrameSink::new(tx),
9618 },
9619 rx,
9620 )
9621 }
9622
9623 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9624 serde_json::from_slice(&frame.body).unwrap()
9625 }
9626
9627 async fn hello_via_sink(
9632 handler: &ControlHandler,
9633 ctx: &RouteCtx,
9634 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9635 hello: Frame,
9636 ) -> Frame {
9637 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9638 assert!(
9639 replies.is_empty(),
9640 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9641 );
9642 let ack = rx
9643 .try_recv()
9644 .expect("HELLO_ACK is queued on the module sink")
9645 .frame;
9646 assert_eq!(ack.header.ty, FrameType::HelloAck);
9647 ack
9648 }
9649
9650 fn parse_error(frame: &Frame) -> Value {
9651 serde_json::from_slice(&frame.body).unwrap()
9652 }
9653
9654 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9655 serde_json::from_slice(&frame.body).unwrap()
9656 }
9657
9658 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9659 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9660 route_channel,
9661 route_epoch: 0,
9662 kind,
9663 })
9664 .unwrap();
9665 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9666 }
9667
9668 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9669 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9670 module_id: module_id.to_string(),
9671 })
9672 .unwrap();
9673 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9674 }
9675
9676 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9677 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9678 }
9679
9680 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9682 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9683 let mut frame = route_open_frame(corr, module_id, root);
9684 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9685 body["identity"]["project_root"] = serde_json::json!(path.as_path());
9686 frame.body = serde_json::to_vec(&body).unwrap();
9687 frame
9688 }
9689
9690 fn route_open_frame_with_consumer_capabilities(
9691 corr: u64,
9692 module_id: &str,
9693 _project_root: ScratchDir,
9694 consumer_capabilities: Option<Vec<String>>,
9695 ) -> Frame {
9696 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9697 busy_refusals: None,
9698 target: RouteTarget::ToolProvider {
9699 module_id: module_id.to_string(),
9700 },
9701 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9702 consumer_identity: None,
9703 consumer_capabilities,
9704 role_versions: None,
9705 admission_facts: None,
9706 scope: None,
9707 })
9708 .unwrap();
9709 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9710 }
9711
9712 fn route_open_frame_with_role_versions(
9713 corr: u64,
9714 module_id: &str,
9715 _project_root: ScratchDir,
9716 role_versions: Option<BTreeMap<String, String>>,
9717 ) -> Frame {
9718 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9719 busy_refusals: None,
9720 target: RouteTarget::ToolProvider {
9721 module_id: module_id.to_string(),
9722 },
9723 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9724 consumer_identity: None,
9725 consumer_capabilities: None,
9726 role_versions,
9727 admission_facts: None,
9728 scope: None,
9729 })
9730 .unwrap();
9731 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9732 }
9733
9734 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9735 entries
9736 .iter()
9737 .map(|(role, version)| (role.to_string(), version.to_string()))
9738 .collect()
9739 }
9740
9741 fn route_open_frame_with_admission_facts(
9742 corr: u64,
9743 module_id: &str,
9744 _project_root: ScratchDir,
9745 consumer_identity: Option<subc_control::ConsumerIdentity>,
9746 facts: Option<Value>,
9747 ) -> Frame {
9748 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9749 busy_refusals: None,
9750 target: RouteTarget::ToolProvider {
9751 module_id: module_id.to_string(),
9752 },
9753 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9754 consumer_identity,
9755 consumer_capabilities: None,
9756 role_versions: None,
9757 admission_facts: facts,
9758 scope: None,
9759 })
9760 .unwrap();
9761 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9762 }
9763
9764 #[derive(Clone, Default)]
9765 struct EventCapture {
9766 events: Arc<Mutex<Vec<CapturedEvent>>>,
9767 }
9768
9769 #[derive(Clone, Debug)]
9770 struct CapturedEvent {
9771 target: String,
9772 level: tracing::Level,
9773 fields: BTreeMap<String, String>,
9774 }
9775
9776 impl EventCapture {
9777 fn events(&self) -> Vec<CapturedEvent> {
9778 self.events.lock().unwrap().clone()
9779 }
9780 }
9781
9782 impl<S> Layer<S> for EventCapture
9783 where
9784 S: Subscriber,
9785 {
9786 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9787 let mut visitor = EventFieldVisitor::default();
9788 event.record(&mut visitor);
9789 self.events.lock().unwrap().push(CapturedEvent {
9790 target: event.metadata().target().to_string(),
9791 level: *event.metadata().level(),
9792 fields: visitor.fields,
9793 });
9794 }
9795 }
9796
9797 #[derive(Default)]
9798 struct EventFieldVisitor {
9799 fields: BTreeMap<String, String>,
9800 }
9801
9802 impl Visit for EventFieldVisitor {
9803 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9804 self.fields
9805 .insert(field.name().to_string(), format!("{value:?}"));
9806 }
9807 }
9808
9809 fn health_response(corr: u64, status: HealthStatus) -> Frame {
9810 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9811 status,
9812 detail: Some("warming".to_string()),
9813 metrics: Some(json!({"queue_depth": 3})),
9814 })
9815 .unwrap();
9816 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9817 }
9818
9819 fn route_bind_ack(corr: u64) -> Frame {
9820 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9821 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9822 }
9823
9824 fn unique_project_root(label: &str) -> ScratchDir {
9825 ScratchDir::new(label)
9826 }
9827
9828 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9829 match parse_route_poll(frame) {
9830 ClientControlResponse::RoutePoll {
9831 status: None,
9832 live: Some(live),
9833 ..
9834 } => assert_eq!(live, expected_live),
9835 other => panic!("unexpected route.poll response: {other:?}"),
9836 }
9837 }
9838
9839 fn bind_liveness_route(
9840 registry: &Registry,
9841 forwarding: &ForwardingTable,
9842 module_id: &str,
9843 ) -> (RouteCtx, u16, u32) {
9844 let module_connection = ConnectionId::new(101);
9845 let client_connection = ConnectionId::new(202);
9846 let registration = registry
9847 .register_with_control_ops(
9848 manifest(module_id, PROTOCOL_VERSION),
9849 PROTOCOL_VERSION,
9850 module_connection,
9851 module_baseline_control_ops(),
9852 )
9853 .unwrap();
9854 let (module_tx, _module_rx) = mpsc::channel(8);
9855 let endpoint = forwarding
9856 .register_module_connection(
9857 module_connection,
9858 module_id.to_string(),
9859 PROTOCOL_VERSION,
9860 manifest_concurrency(®istration.manifest),
9861 FrameSink::new(module_tx),
9862 )
9863 .unwrap();
9864 let (client_ctx, _client_rx) = route_ctx(client_connection);
9865 let pending = forwarding
9866 .begin_route_bind_relay_for_test(
9867 client_connection,
9868 client_ctx.egress.clone(),
9869 1,
9870 module_id,
9871 )
9872 .unwrap();
9873 assert_eq!(pending.endpoint, endpoint);
9874 let route_channel = pending.client_channel;
9875 let route_epoch = pending.client_epoch;
9876 forwarding
9877 .complete_pending_relay(
9878 module_connection,
9879 pending.corr,
9880 RouteBindRelayOutcome::Accepted,
9881 )
9882 .unwrap();
9883 (client_ctx, route_channel, route_epoch)
9884 }
9885
9886 struct FakeProcessLiveness {
9887 live: Option<bool>,
9888 }
9889
9890 impl ModuleProcessLiveness for FakeProcessLiveness {
9891 fn process_live(&self, _module_id: &str) -> Option<bool> {
9892 self.live
9893 }
9894 }
9895
9896 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9897 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9898 {
9899 let registry = Arc::new(Registry::default());
9900 let supervisor_handle = SupervisorHandle::new();
9901 let supervisor = Supervisor::new_for_test(
9902 Arc::clone(®istry),
9903 RestartPolicy::new(1, Duration::from_millis(10)),
9904 )
9905 .with_handle(supervisor_handle.clone());
9906 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9907 let module = supervisor
9908 .spawn(ModuleSpec {
9909 carry_containment: false,
9910 module_id: "stderr-tail-wire".to_string(),
9911 program: fake_aft_stub_path(),
9912 args: Vec::new(),
9913 env: vec![
9914 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9915 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9916 ],
9917 reserved: false,
9918 reserved_prefixes: Vec::new(),
9919 protocol: ModuleProtocol::Subc,
9920 overlap: Default::default(),
9921 })
9922 .unwrap();
9923
9924 let deadline = Instant::now() + Duration::from_secs(5);
9925 loop {
9926 let tail = module.stderr_tail(None, None);
9927 if tail
9928 .entries
9929 .iter()
9930 .any(|entry| matches!(entry, TailEntry::ProcessStart))
9931 && tail.entries.iter().any(|entry| {
9932 matches!(
9933 entry,
9934 TailEntry::Line {
9935 truncated: true,
9936 ..
9937 }
9938 )
9939 })
9940 {
9941 break;
9942 }
9943 assert!(
9944 Instant::now() < deadline,
9945 "module did not produce a truncated line and restart boundary: {tail:?}"
9946 );
9947 sleep(Duration::from_millis(10)).await;
9948 }
9949
9950 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9951 let request = ClientControlRequest::SupervisorStderrTail {
9952 module_id: "stderr-tail-wire".to_string(),
9953 max_lines: None,
9954 max_bytes: None,
9955 };
9956 let frame = Frame::build(
9957 FrameType::Request,
9958 control_flags(),
9959 0,
9960 0,
9961 1,
9962 serde_json::to_vec(&request).unwrap(),
9963 )
9964 .unwrap();
9965 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9966 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9967 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9968 serde_json::from_slice(&responses[0].body).unwrap()
9969 else {
9970 panic!("expected supervisor.stderr_tail response");
9971 };
9972
9973 assert!(
9974 tail.entries
9975 .iter()
9976 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9977 "the control response lost the restart boundary"
9978 );
9979 let Some(StderrTailEntry::Line {
9980 text,
9981 truncated,
9982 at_ms,
9983 }) = tail.entries.iter().find(|entry| {
9984 matches!(
9985 entry,
9986 StderrTailEntry::Line {
9987 truncated: true,
9988 ..
9989 }
9990 )
9991 })
9992 else {
9993 panic!("the control response lost the truncated line");
9994 };
9995 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9996 assert!(*truncated);
9997 assert!(
9998 at_ms.is_some(),
9999 "the control response lost the line's capture time"
10000 );
10001 }
10002
10003 #[tokio::test(flavor = "current_thread")]
10008 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
10009 let dir = ScratchDir::new("terminals-off-worker");
10010 let journal_path = dir.join("terminals.jsonl");
10011 let registry = Arc::new(Registry::default());
10012 let supervisor_handle = SupervisorHandle::new();
10013 let supervisor =
10014 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
10015 .with_handle(supervisor_handle.clone())
10016 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
10017 let module = supervisor
10018 .spawn(ModuleSpec {
10019 carry_containment: false,
10020 module_id: "terminal-off-worker".to_string(),
10021 program: fake_aft_stub_path(),
10022 args: Vec::new(),
10023 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
10024 reserved: false,
10025 reserved_prefixes: Vec::new(),
10026 protocol: ModuleProtocol::Subc,
10027 overlap: Default::default(),
10028 })
10029 .unwrap();
10030 let deadline = Instant::now() + Duration::from_secs(5);
10031 while module.terminal_history().entries.len() != 2 {
10032 assert!(Instant::now() < deadline, "module did not record two exits");
10033 sleep(Duration::from_millis(10)).await;
10034 }
10035
10036 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
10037 let handler =
10038 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
10039 let frame = Frame::build(
10040 FrameType::Request,
10041 control_flags(),
10042 0,
10043 0,
10044 1,
10045 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
10046 module_id: "terminal-off-worker".to_string(),
10047 })
10048 .unwrap(),
10049 )
10050 .unwrap();
10051 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
10052 let spawned_at = std::time::Instant::now();
10053 let read = tokio::spawn({
10054 let handler = Arc::clone(&handler);
10055 async move { handler.handle_control_frame(&ctx, frame).await }
10056 });
10057 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
10060 .await
10061 .unwrap()
10062 .expect("the history read reached its pause");
10063 let elapsed = spawned_at.elapsed();
10064 assert!(
10065 elapsed < Duration::from_secs(2) && !read.is_finished(),
10066 "this task could not run while the history read was paused \
10067 (resumed after {elapsed:?}, read finished: {})",
10068 read.is_finished()
10069 );
10070
10071 drop(release);
10072 let responses = read.await.unwrap().unwrap();
10073 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
10074 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
10075 panic!("expected supervisor.terminals response");
10076 };
10077 assert_eq!(terminals.entries.len(), 2);
10078 assert_eq!(terminals.journal_skipped_lines, 0);
10079 assert_eq!(terminals.journal_read_errors, 0);
10080 }
10081
10082 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10083 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
10084 let registry = Arc::new(Registry::default());
10085 let supervisor_handle = SupervisorHandle::new();
10086 let supervisor =
10087 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
10088 .with_handle(supervisor_handle.clone());
10089 let module = supervisor
10090 .spawn(ModuleSpec {
10091 carry_containment: false,
10092 module_id: "terminal-golden".to_string(),
10093 program: fake_aft_stub_path(),
10094 args: Vec::new(),
10095 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
10096 reserved: false,
10097 reserved_prefixes: Vec::new(),
10098 protocol: ModuleProtocol::Subc,
10099 overlap: Default::default(),
10100 })
10101 .unwrap();
10102
10103 let deadline = Instant::now() + Duration::from_secs(5);
10104 while module.terminal_history().entries.len() != 2 {
10105 assert!(
10106 Instant::now() < deadline,
10107 "module did not retain two terminal exits: {:?}",
10108 module.terminal_history()
10109 );
10110 sleep(Duration::from_millis(10)).await;
10111 }
10112
10113 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10114 let request = ClientControlRequest::SupervisorTerminals {
10115 module_id: "terminal-golden".to_string(),
10116 };
10117 let frame = Frame::build(
10118 FrameType::Request,
10119 control_flags(),
10120 0,
10121 0,
10122 1,
10123 serde_json::to_vec(&request).unwrap(),
10124 )
10125 .unwrap();
10126 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
10127 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
10128 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
10129 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
10130 panic!("expected supervisor.terminals response");
10131 };
10132 assert_eq!(terminals.entries.len(), 2);
10133 assert_eq!(terminals.dropped, 0);
10134
10135 let mut rendered = serde_json::to_value(response).unwrap();
10136 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
10139 for (index, entry) in rendered["entries"]
10140 .as_array_mut()
10141 .expect("terminal response entries array")
10142 .iter_mut()
10143 .enumerate()
10144 {
10145 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
10146 }
10147
10148 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10149 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
10150 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
10151 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10152 std::fs::write(&golden_path, &serialized).unwrap();
10153 }
10154 let expected: Value =
10155 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
10156 assert_eq!(rendered, expected);
10157 }
10158
10159 #[test]
10160 fn hello_registers_manifest_and_returns_ack() {
10161 let registry = Arc::new(Registry::default());
10162 let handler = ControlHandler::new(Arc::clone(®istry));
10163 let conn = ConnectionId::new(1);
10164
10165 let responses = handler
10166 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
10167 .unwrap();
10168
10169 assert_eq!(responses.len(), 1);
10170 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10171 assert_eq!(responses[0].header.channel, 0);
10172 assert_eq!(responses[0].header.corr, 7);
10173 let ack = parse_ack(&responses[0]);
10174 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10175 assert!(ack
10176 .subc_capabilities
10177 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
10178 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
10179 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
10180 assert!(ack
10181 .subc_ops
10182 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
10183 assert!(ack
10184 .subc_ops
10185 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
10186 assert!(ack
10187 .subc_ops
10188 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS.to_string()));
10189
10190 let registration = registry.get_module("aft").unwrap().unwrap();
10191 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
10192 assert_eq!(registration.state, ChannelState::Active);
10193 assert_eq!(registration.connection_id, conn);
10194 assert_eq!(registration.control_ops, module_baseline_control_ops());
10195 assert!(registration.manifest.display_name.is_none());
10196 }
10197
10198 #[test]
10199 fn hello_display_name_refuses_invalid_manifest_by_field() {
10200 let registry = Arc::new(Registry::default());
10201 let handler = ControlHandler::new(Arc::clone(®istry));
10202 for (index, invalid) in [
10203 json!(""),
10204 json!("a".repeat(41)),
10205 json!(" Routing"),
10206 json!("a\nb"),
10207 json!("a\u{202e}b"),
10208 json!(7),
10209 ]
10210 .into_iter()
10211 .enumerate()
10212 {
10213 let module_id = format!("bad-display-name-{index}");
10214 let mut body = serde_json::to_value(ModuleHelloBody {
10215 manifest: manifest(&module_id, PROTOCOL_VERSION),
10216 protocol_ver: PROTOCOL_VERSION,
10217 control_ops: None,
10218 launch_nonce: None,
10219 })
10220 .unwrap();
10221 body["manifest"]["display_name"] = invalid;
10222 let hello = Frame::build(
10223 FrameType::Hello,
10224 control_flags(),
10225 0,
10226 0,
10227 7,
10228 serde_json::to_vec(&body).unwrap(),
10229 )
10230 .unwrap();
10231 let response = handler
10232 .handle_control(ConnectionId::new(100 + index as u64), hello)
10233 .unwrap();
10234 assert_eq!(response[0].header.ty, FrameType::Error);
10235 let error = parse_error(&response[0]);
10236 assert_eq!(error["code"], "invalid_manifest");
10237 assert!(error["message"].as_str().unwrap().contains("display_name"));
10238 assert!(registry.get_module(&module_id).unwrap().is_none());
10239 }
10240 }
10241
10242 #[test]
10243 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
10244 let invalid_identifiers = [
10245 ("case_change", "credentials-Provider/v1"),
10246 ("leading_zero", "credentials-provider/v01"),
10247 ("trailing_hyphen", "credentials-provider-/v1"),
10248 ("consecutive_hyphens", "credentials--provider/v1"),
10249 ("uppercase", "Credentials-provider/v1"),
10250 ("missing_v", "credentials-provider/1"),
10251 ("whitespace", "credentials provider/v1"),
10252 ("zero_version", "credentials-provider/v0"),
10253 ("out_of_range_version", "credentials-provider/v4294967296"),
10254 (
10255 "overlength_name",
10256 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
10257 ),
10258 ];
10259 let mut cases = invalid_identifiers
10260 .into_iter()
10261 .map(|(name, identifier)| {
10262 (
10263 format!("identifier_{name}"),
10264 "capabilities.provides[0]".to_string(),
10265 identifier.to_string(),
10266 json!({ "provides": [identifier] }),
10267 None,
10268 )
10269 })
10270 .collect::<Vec<_>>();
10271 cases.extend([
10272 (
10273 "unknown_need".to_string(),
10274 "capabilities.requires[0].need".to_string(),
10275 "deferred".to_string(),
10276 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
10277 None,
10278 ),
10279 (
10280 "duplicate_provides".to_string(),
10281 "capabilities.provides[1]".to_string(),
10282 "credentials-provider/v1".to_string(),
10283 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
10284 None,
10285 ),
10286 (
10287 "duplicate_must_never_reach".to_string(),
10288 "capabilities.must_never_reach[1]".to_string(),
10289 "credentials-provider/v1".to_string(),
10290 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
10291 None,
10292 ),
10293 (
10294 "duplicate_requires_same_need".to_string(),
10295 "capabilities.requires[1]".to_string(),
10296 "credentials-provider/v1".to_string(),
10297 json!({ "requires": [
10298 { "capability": "credentials-provider/v1", "need": "required" },
10299 { "capability": "credentials-provider/v1", "need": "required" }
10300 ] }),
10301 None,
10302 ),
10303 (
10304 "duplicate_requires_conflicting_need".to_string(),
10305 "capabilities.requires[1]".to_string(),
10306 "credentials-provider/v1".to_string(),
10307 json!({ "requires": [
10308 { "capability": "credentials-provider/v1", "need": "required" },
10309 { "capability": "credentials-provider/v1", "need": "optional" }
10310 ] }),
10311 None,
10312 ),
10313 (
10314 "capabilities_root_pointer".to_string(),
10315 "runtime_computed[0]".to_string(),
10316 "/capabilities".to_string(),
10317 json!({}),
10318 Some(json!(["/capabilities"])),
10319 ),
10320 (
10321 "capabilities_descendant_pointer".to_string(),
10322 "runtime_computed[0]".to_string(),
10323 "/capabilities/provides".to_string(),
10324 json!({}),
10325 Some(json!(["/capabilities/provides"])),
10326 ),
10327 (
10328 "malformed_pointer_without_leading_slash".to_string(),
10329 "runtime_computed[0]".to_string(),
10330 "capabilities".to_string(),
10331 json!({}),
10332 Some(json!(["capabilities"])),
10333 ),
10334 (
10335 "malformed_pointer_escape".to_string(),
10336 "runtime_computed[0]".to_string(),
10337 "/roles/~2/tools".to_string(),
10338 json!({}),
10339 Some(json!(["/roles/~2/tools"])),
10340 ),
10341 (
10342 "unknown_capabilities_field".to_string(),
10343 "capabilities.future".to_string(),
10344 "<array>".to_string(),
10345 json!({ "future": [] }),
10346 None,
10347 ),
10348 ]);
10349
10350 for (index, (name, field, value, capabilities, runtime_computed)) in
10351 cases.into_iter().enumerate()
10352 {
10353 let registry = Arc::new(Registry::default());
10354 let handler = ControlHandler::new(Arc::clone(®istry));
10355 let response = handler
10356 .handle_control(
10357 ConnectionId::new((index + 1) as u64),
10358 capability_grammar_hello_frame(
10359 capabilities,
10360 runtime_computed,
10361 index as u64 + 1,
10362 ),
10363 )
10364 .expect("invalid HELLO returns a refusal");
10365
10366 assert_eq!(response.len(), 1, "{name} must emit one refusal");
10367 let error = parse_error(&response[0]);
10368 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
10369 let message = error["message"]
10370 .as_str()
10371 .expect("error message is a string");
10372 assert!(
10373 message.contains(&field),
10374 "{name}: field missing from {message}"
10375 );
10376 assert!(
10377 message.contains(&value),
10378 "{name}: value missing from {message}"
10379 );
10380 assert_eq!(
10381 registry
10382 .active_registration_count()
10383 .expect("registry reads"),
10384 0,
10385 "{name}: refused HELLO must not create a catalog entry"
10386 );
10387 }
10388 }
10389
10390 #[test]
10391 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
10392 let registry = Arc::new(Registry::default());
10393 let handler = ControlHandler::new(Arc::clone(®istry));
10394 let capabilities = json!({
10395 "provides": ["credentials-provider/v1"],
10396 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
10397 "must_never_reach": ["federation-transport/v1"]
10398 });
10399 let response = handler
10400 .handle_control(
10401 ConnectionId::new(99),
10402 capability_grammar_hello_frame(
10403 capabilities.clone(),
10404 Some(json!(["/roles/0/tools"])),
10405 99,
10406 ),
10407 )
10408 .expect("valid HELLO registers");
10409 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10410
10411 let request = Frame::build(
10412 FrameType::Request,
10413 control_flags(),
10414 0,
10415 0,
10416 100,
10417 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10418 .expect("catalog request serializes"),
10419 )
10420 .expect("catalog request frame builds");
10421 let response = handler
10422 .handle_catalog_list(request, None)
10423 .expect("catalog list succeeds");
10424 let ClientControlResponse::CatalogList { modules, .. } =
10425 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
10426 else {
10427 panic!("catalog request must return catalog.list");
10428 };
10429 assert_eq!(modules.len(), 1);
10430 assert_eq!(
10431 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
10432 capabilities
10433 );
10434 }
10435
10436 #[test]
10437 fn catalog_list_mirrors_management_operation_description() {
10438 let registry = Arc::new(Registry::default());
10439 let handler = ControlHandler::new(Arc::clone(®istry));
10440 let description = "List managed records and return their identifiers and metadata.";
10441 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
10442 manifest.provides = vec![ProviderRole::ManagementSurface {
10443 operations: vec![ManagementOperation {
10444 name: "records.list".to_string(),
10445 kind: ManagementOperationKind::Query,
10446 description: Some(description.to_string()),
10447 }],
10448 config_schema: json!({"type": "object"}),
10449 observability: vec![ObservabilitySurface {
10450 name: "records.stats".to_string(),
10451 kind: ObservabilityKind::Snapshot,
10452 }],
10453 identity_scope: vec![IdentityScope::Project],
10454 concurrency: Concurrency::ModuleManaged,
10455 }];
10456 registry
10457 .register_with_control_ops(
10458 manifest,
10459 PROTOCOL_VERSION,
10460 ConnectionId::new(99),
10461 Vec::new(),
10462 )
10463 .expect("described management manifest registers");
10464
10465 let request = Frame::build(
10466 FrameType::Request,
10467 control_flags(),
10468 0,
10469 0,
10470 100,
10471 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10472 .expect("catalog request serializes"),
10473 )
10474 .expect("catalog request frame builds");
10475 let response = handler
10476 .handle_catalog_list(request, None)
10477 .expect("catalog list succeeds");
10478 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10479 assert_eq!(
10480 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10481 "catalog.list must preserve the declared operation description verbatim"
10482 );
10483 }
10484
10485 #[test]
10486 fn catalog_list_mirrors_registered_operation_delivery() {
10487 let registry = Arc::new(Registry::default());
10488 let handler = ControlHandler::new(Arc::clone(®istry));
10489 let delivery =
10490 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10491 let mut body = serde_json::to_value(ModuleHelloBody {
10492 manifest: manifest("watch-management", PROTOCOL_VERSION),
10493 protocol_ver: PROTOCOL_VERSION,
10494 control_ops: None,
10495 launch_nonce: None,
10496 })
10497 .unwrap();
10498 body["manifest"]["provides"] = json!([{
10499 "role": "management_surface",
10500 "operations": [
10501 {"name": "status", "kind": "query"},
10502 {"name": "future", "kind": "query"},
10503 {"name": "reset", "kind": "mutate"}
10504 ],
10505 "config_schema": {}, "observability": [], "identity_scope": []
10506 }]);
10507 body["manifest"]["operation_delivery"] = delivery.clone();
10508 let hello = Frame::build(
10509 FrameType::Hello,
10510 control_flags(),
10511 0,
10512 0,
10513 99,
10514 serde_json::to_vec(&body).unwrap(),
10515 )
10516 .unwrap();
10517 let response = handler
10518 .handle_control(ConnectionId::new(99), hello)
10519 .unwrap();
10520 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10521
10522 let (_, registrations) = registry.list_modules().unwrap();
10523 let registered = ®istrations[0].manifest;
10524 assert!(registered.is_watch_operation("status"));
10525 assert!(!registered.is_watch_operation("future"));
10526 assert!(!registered.is_watch_operation("reset"));
10527 assert!(!registered.is_watch_operation("missing"));
10528
10529 let request = Frame::build(
10530 FrameType::Request,
10531 control_flags(),
10532 0,
10533 0,
10534 100,
10535 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10536 )
10537 .unwrap();
10538 let response = handler.handle_catalog_list(request, None).unwrap();
10539 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10540 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10541 let ClientControlResponse::CatalogList { modules, .. } =
10542 serde_json::from_slice(&response[0].body).unwrap()
10543 else {
10544 panic!("expected catalog.list");
10545 };
10546 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10547 }
10548
10549 #[test]
10550 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10551 let registry = Arc::new(Registry::default());
10552 let supervisor = SupervisorHandle::new();
10553 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10554 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10555 let descriptor = json!({
10556 "schema": "ckios.richui.manifest/1",
10557 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10558 "bytes": 4096,
10559 "chunks": 4
10560 });
10561 let connection = ConnectionId::new(201);
10562 let hello = handler
10563 .handle_control(
10564 connection,
10565 hello_frame_with_rich_ui(
10566 "vault",
10567 PROTOCOL_VERSION,
10568 201,
10569 Some("the-real-nonce"),
10570 descriptor.clone(),
10571 ),
10572 )
10573 .unwrap();
10574 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10575
10576 let request =
10577 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10578 let provides = registry
10579 .get_module_by_connection(connection)
10580 .unwrap()
10581 .unwrap()
10582 .manifest
10583 .provides;
10584 handler
10585 .handle_catalog_update(connection, request, provides, None, Some(false))
10586 .unwrap();
10587
10588 let request = Frame::build(
10589 FrameType::Request,
10590 control_flags(),
10591 0,
10592 0,
10593 203,
10594 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10595 )
10596 .unwrap();
10597 let response = handler.handle_catalog_list(request, None).unwrap();
10598 let ClientControlResponse::CatalogList { modules, .. } =
10599 serde_json::from_slice(&response[0].body).unwrap()
10600 else {
10601 panic!("expected catalog.list");
10602 };
10603 assert_eq!(
10604 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10605 descriptor
10606 );
10607 assert!(modules[0].nonce_verified);
10608 }
10609
10610 fn hello_with_role_declarations(declarations: Value) -> Frame {
10611 let mut body = serde_json::to_value(ModuleHelloBody {
10612 manifest: manifest("role-test", PROTOCOL_VERSION),
10613 protocol_ver: PROTOCOL_VERSION,
10614 control_ops: None,
10615 launch_nonce: None,
10616 })
10617 .unwrap();
10618 body["manifest"]["role_declarations"] = declarations;
10619 Frame::build(
10620 FrameType::Hello,
10621 control_flags(),
10622 0,
10623 0,
10624 301,
10625 serde_json::to_vec(&body).unwrap(),
10626 )
10627 .unwrap()
10628 }
10629
10630 fn update_role_declarations_frame(declarations: Value) -> Frame {
10631 Frame::build(
10632 FrameType::Request,
10633 control_flags(),
10634 0,
10635 0,
10636 303,
10637 serde_json::to_vec(&json!({
10638 "op": "catalog.update_role_declarations",
10639 "role_declarations": declarations,
10640 }))
10641 .unwrap(),
10642 )
10643 .unwrap()
10644 }
10645
10646 #[tokio::test]
10647 async fn role_declarations_update_replaces_then_clears_catalog() {
10648 let registry = Arc::new(Registry::default());
10649 let handler = ControlHandler::new(Arc::clone(®istry));
10650 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
10651 let initial = json!({"media-provider/v1": {"engines": ["old"]}, "old/v1": {}});
10652 hello_via_sink(
10653 &handler,
10654 &ctx,
10655 &mut rx,
10656 hello_with_role_declarations(initial.clone()),
10657 )
10658 .await;
10659 assert_eq!(
10660 role_declarations_catalog(&handler).0["role_declarations"],
10661 initial
10662 );
10663 let before = registry.get_module("role-test").unwrap().unwrap();
10664 for declarations in [
10665 json!({"media-provider/v1": {"engines": ["new"]}}),
10666 json!({}),
10667 ] {
10668 let response = handler
10669 .handle_control_frame(&ctx, update_role_declarations_frame(declarations.clone()))
10670 .await
10671 .unwrap();
10672 assert_eq!(response[0].header.ty, FrameType::Response);
10673 assert_eq!(
10674 serde_json::from_slice::<Value>(&response[0].body).unwrap(),
10675 json!({"op": "catalog.update_role_declarations"})
10676 );
10677 let (raw, entry) = role_declarations_catalog(&handler);
10678 assert_eq!(
10679 serde_json::to_value(entry.role_declarations).unwrap(),
10680 declarations
10681 );
10682 if declarations.as_object().unwrap().is_empty() {
10683 assert!(raw.get("role_declarations").is_none());
10684 } else {
10685 assert_eq!(raw["role_declarations"], declarations);
10686 }
10687 let after = registry.get_module("role-test").unwrap().unwrap();
10688 let mut expected = before.manifest.clone();
10689 expected.role_declarations = serde_json::from_value(declarations).unwrap();
10690 assert_eq!(after.manifest, expected);
10691 assert_eq!(after.ready, before.ready);
10692 }
10693 }
10694
10695 #[tokio::test]
10696 async fn role_declarations_update_refuses_invalid_maps_without_changing_catalog() {
10697 let registry = Arc::new(Registry::default());
10698 let handler = ControlHandler::new(registry);
10699 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
10700 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
10701 hello_via_sink(
10702 &handler,
10703 &ctx,
10704 &mut rx,
10705 hello_with_role_declarations(initial.clone()),
10706 )
10707 .await;
10708 for (invalid, reason) in [
10709 (json!(null), "role_declarations must be an object"),
10710 (json!([]), "role_declarations must be an object"),
10711 (
10712 json!({"bad key": {}}),
10713 "role_declarations key must use capability-name grammar",
10714 ),
10715 (
10716 json!({"media-provider/v1": []}),
10717 "role_declarations values must be objects",
10718 ),
10719 (
10720 json!({"media-provider/v1": {"padding": "x".repeat(16384)}}),
10721 "role_declarations exceeds 16384 compact JSON bytes",
10722 ),
10723 ] {
10724 let response = handler
10725 .handle_control_frame(&ctx, update_role_declarations_frame(invalid))
10726 .await
10727 .unwrap();
10728 let error = parse_error(&response[0]);
10729 assert_eq!(error["code"], "invalid_role_declarations");
10730 assert_eq!(error["message"], reason);
10731 assert_eq!(
10732 role_declarations_catalog(&handler).0["role_declarations"],
10733 initial
10734 );
10735 }
10736 }
10737
10738 #[tokio::test]
10739 async fn role_declarations_update_is_owned_by_the_registered_connection() {
10740 let registry = Arc::new(Registry::default());
10741 let handler = ControlHandler::new(Arc::clone(®istry));
10742 let (owner, mut owner_rx) = route_ctx(ConnectionId::new(301));
10743 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
10744 hello_via_sink(
10745 &handler,
10746 &owner,
10747 &mut owner_rx,
10748 hello_with_role_declarations(initial.clone()),
10749 )
10750 .await;
10751 let (other, mut other_rx) = route_ctx(ConnectionId::new(302));
10752 let replacement = json!({"media-provider/v1": {"engines": ["other"]}});
10753 let response = handler
10754 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
10755 .await
10756 .unwrap();
10757 assert_eq!(parse_error(&response[0])["code"], "not_registered");
10758 hello_via_sink(
10759 &handler,
10760 &other,
10761 &mut other_rx,
10762 hello_frame("other", PROTOCOL_VERSION, 304),
10763 )
10764 .await;
10765 let response = handler
10766 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
10767 .await
10768 .unwrap();
10769 assert_eq!(response[0].header.ty, FrameType::Response);
10770 assert_eq!(
10771 serde_json::to_value(
10772 registry
10773 .get_module("role-test")
10774 .unwrap()
10775 .unwrap()
10776 .manifest
10777 .role_declarations
10778 )
10779 .unwrap(),
10780 initial
10781 );
10782 assert_eq!(
10783 serde_json::to_value(
10784 registry
10785 .get_module("other")
10786 .unwrap()
10787 .unwrap()
10788 .manifest
10789 .role_declarations
10790 )
10791 .unwrap(),
10792 replacement
10793 );
10794 }
10795
10796 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10797 let request = Frame::build(
10798 FrameType::Request,
10799 control_flags(),
10800 0,
10801 0,
10802 302,
10803 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10804 )
10805 .unwrap();
10806 let response = handler.handle_catalog_list(request, None).unwrap();
10807 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10808 let ClientControlResponse::CatalogList { mut modules, .. } =
10809 serde_json::from_slice(&response[0].body).unwrap()
10810 else {
10811 panic!("expected catalog.list");
10812 };
10813 (raw["modules"][0].clone(), modules.remove(0))
10814 }
10815
10816 #[test]
10817 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10818 let registry = Arc::new(Registry::default());
10819 let handler = ControlHandler::new(Arc::clone(®istry));
10820 let declarations = json!({
10821 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10822 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10823 });
10824 let connection = ConnectionId::new(301);
10825 let response = handler
10826 .handle_control(
10827 connection,
10828 hello_with_role_declarations(declarations.clone()),
10829 )
10830 .unwrap();
10831 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10832 let registered = registry
10833 .get_module_by_connection(connection)
10834 .unwrap()
10835 .unwrap()
10836 .manifest;
10837 assert_eq!(
10841 serde_json::to_value(®istered.role_declarations).unwrap(),
10842 declarations
10843 );
10844 let (raw, entry) = role_declarations_catalog(&handler);
10845 assert_eq!(raw["role_declarations"], declarations);
10846 assert_eq!(entry.role_declarations, registered.role_declarations);
10847
10848 let request =
10849 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10850 handler
10851 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10852 .unwrap();
10853 let (raw, entry) = role_declarations_catalog(&handler);
10854 assert_eq!(raw["role_declarations"], declarations);
10855 assert_eq!(entry.role_declarations, registered.role_declarations);
10856 }
10857
10858 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10859 let registry = Arc::new(Registry::default());
10860 let handler = ControlHandler::new(Arc::clone(®istry));
10861 let capture = EventCapture::default();
10862 let _subscriber =
10863 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10864 let response = handler
10865 .handle_control(
10866 ConnectionId::new(301),
10867 hello_with_role_declarations(declarations),
10868 )
10869 .unwrap();
10870 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10871 assert!(registry
10872 .get_module_by_connection(ConnectionId::new(301))
10873 .unwrap()
10874 .unwrap()
10875 .manifest
10876 .role_declarations
10877 .is_empty());
10878 let (raw, entry) = role_declarations_catalog(&handler);
10879 assert!(raw.get("role_declarations").is_none());
10880 assert!(entry.role_declarations.is_empty());
10881 let warnings: Vec<_> = capture
10882 .events()
10883 .into_iter()
10884 .filter(|event| {
10885 event.fields.get("message").map(String::as_str)
10886 == Some("dropping whole role_declarations map from HELLO")
10887 })
10888 .collect();
10889 assert_eq!(warnings.len(), 1);
10890 assert_eq!(warnings[0].level, tracing::Level::WARN);
10891 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10892 assert_eq!(warnings[0].fields["reason"], reason);
10893 }
10894
10895 #[test]
10896 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10897 for value in [
10898 json!(null),
10899 json!(false),
10900 json!(0),
10901 json!("opaque"),
10902 json!([]),
10903 ] {
10904 assert_hello_drops_role_declarations_with_warning(
10905 json!({"routing/v1": {}, "broken/v1": value}),
10906 "role_declarations values must be objects",
10907 );
10908 }
10909 assert_hello_drops_role_declarations_with_warning(
10910 json!([]),
10911 "role_declarations must be an object",
10912 );
10913 }
10914
10915 #[test]
10916 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10917 assert_hello_drops_role_declarations_with_warning(
10918 json!({"routing/v1": {}, "bad key": {}}),
10919 "role_declarations key must use capability-name grammar",
10920 );
10921 }
10922
10923 #[test]
10924 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10925 assert_hello_drops_role_declarations_with_warning(
10926 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10927 "role_declarations exceeds 16384 compact JSON bytes",
10928 );
10929 }
10930
10931 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10932 let mut body = serde_json::to_value(ModuleHelloBody {
10933 manifest: manifest("flow-ops", PROTOCOL_VERSION),
10934 protocol_ver: PROTOCOL_VERSION,
10935 control_ops: None,
10936 launch_nonce: None,
10937 })
10938 .unwrap();
10939 body["manifest"]["provides"] = json!([{
10940 "role": "management_surface",
10941 "operations": [
10942 {"name": "diagnostics.findings", "kind": "query"},
10943 {"name": "reset", "kind": "mutate"}
10944 ],
10945 "config_schema": {}, "observability": [], "identity_scope": []
10946 }]);
10947 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10948 if let Some(flow_operations) = flow_operations {
10949 body["manifest"]["flow_operations"] = flow_operations;
10950 }
10951 Frame::build(
10952 FrameType::Hello,
10953 control_flags(),
10954 0,
10955 0,
10956 401,
10957 serde_json::to_vec(&body).unwrap(),
10958 )
10959 .unwrap()
10960 }
10961
10962 #[test]
10963 fn catalog_list_mirrors_flow_operations_unchanged() {
10964 for (capability, flow_operations) in [
10967 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10968 ("call-modes/v1", json!(["diagnostics.findings"])),
10969 ("flow-scopes/v1", json!([])),
10970 ] {
10971 let registry = Arc::new(Registry::default());
10972 let handler = ControlHandler::new(Arc::clone(®istry));
10973 let connection = ConnectionId::new(401);
10974 let response = handler
10975 .handle_control(
10976 connection,
10977 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10978 )
10979 .unwrap();
10980 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10981 let registered = registry
10982 .get_module_by_connection(connection)
10983 .unwrap()
10984 .unwrap()
10985 .manifest;
10986 assert_eq!(
10987 serde_json::to_value(®istered.flow_operations).unwrap(),
10988 flow_operations
10989 );
10990 let (raw, entry) = role_declarations_catalog(&handler);
10991 assert_eq!(raw["flow_operations"], flow_operations);
10992 assert_eq!(entry.flow_operations, registered.flow_operations);
10993 }
10994
10995 let registry = Arc::new(Registry::default());
10996 let handler = ControlHandler::new(Arc::clone(®istry));
10997 let response = handler
10998 .handle_control(
10999 ConnectionId::new(401),
11000 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
11001 )
11002 .unwrap();
11003 assert_eq!(response[0].header.ty, FrameType::HelloAck);
11004 let (raw, entry) = role_declarations_catalog(&handler);
11005 assert!(raw.get("flow_operations").is_none());
11006 assert_eq!(entry.flow_operations, None);
11007 }
11008
11009 #[test]
11010 fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
11011 let oversized: serde_json::Map<String, Value> = (0..257)
11012 .map(|n| (format!("tool:t{n}"), json!(true)))
11013 .collect();
11014 for (capabilities, map, valid) in [
11015 (
11016 json!(["scope-federated/v1"]),
11017 json!({"tool:read":true,"management:reset":true}),
11018 true,
11019 ),
11020 (json!(["scope-federated/v1"]), json!({}), true),
11021 (json!([]), json!({"tool:read":true}), false),
11022 (
11023 json!(["scope-federated/v1"]),
11024 json!({"tool:valid":true,"tool:bad":false}),
11025 false,
11026 ),
11027 (
11028 json!(["scope-federated/v1"]),
11029 json!({"tool:valid":true,"management:missing":true}),
11030 false,
11031 ),
11032 (
11033 json!(["scope-federated/v1"]),
11034 Value::Object(oversized),
11035 false,
11036 ),
11037 ] {
11038 let capture = EventCapture::default();
11039 let _subscriber = tracing::subscriber::set_default(
11040 tracing_subscriber::registry().with(capture.clone()),
11041 );
11042 let registry = Arc::new(Registry::default());
11043 let handler = ControlHandler::new(Arc::clone(®istry));
11044 let hello = hello_with_flow_operations(capabilities, None);
11045 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
11046 body["manifest"]["operation_deferral"] = map.clone();
11047 let hello = Frame::build(
11048 FrameType::Hello,
11049 control_flags(),
11050 0,
11051 0,
11052 401,
11053 serde_json::to_vec(&body).unwrap(),
11054 )
11055 .unwrap();
11056 let response = handler
11057 .handle_control(ConnectionId::new(401), hello)
11058 .unwrap();
11059 assert_eq!(response[0].header.ty, FrameType::HelloAck);
11060 let (raw, entry) = role_declarations_catalog(&handler);
11061 if valid {
11062 assert_eq!(raw["operation_deferral"], map);
11063 assert_eq!(
11064 serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
11065 map
11066 );
11067 } else {
11068 assert!(entry.operation_deferral.is_none());
11069 assert!(raw.get("operation_deferral").is_none());
11070 assert_eq!(
11071 capture
11072 .events()
11073 .iter()
11074 .filter(|event| event.fields.get("message").map(String::as_str)
11075 == Some("dropping operation_deferral from HELLO"))
11076 .count(),
11077 1
11078 );
11079 }
11080 }
11081 }
11082
11083 fn assert_hello_drops_flow_operations_with_warning(
11084 capabilities: Value,
11085 flow_operations: Value,
11086 reason: &str,
11087 ) {
11088 let registry = Arc::new(Registry::default());
11089 let handler = ControlHandler::new(Arc::clone(®istry));
11090 let capture = EventCapture::default();
11091 let _subscriber =
11092 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11093 let response = handler
11094 .handle_control(
11095 ConnectionId::new(401),
11096 hello_with_flow_operations(capabilities, Some(flow_operations)),
11097 )
11098 .unwrap();
11099 assert_eq!(response[0].header.ty, FrameType::HelloAck);
11100 let registered = registry
11101 .get_module_by_connection(ConnectionId::new(401))
11102 .unwrap()
11103 .expect("the module still registers")
11104 .manifest;
11105 assert_eq!(registered.module_id, "flow-ops");
11106 assert_eq!(registered.flow_operations, None);
11107 let (raw, entry) = role_declarations_catalog(&handler);
11108 assert_eq!(raw["module_id"], "flow-ops");
11109 assert!(raw.get("flow_operations").is_none());
11110 assert_eq!(entry.flow_operations, None);
11111 let warnings: Vec<_> = capture
11112 .events()
11113 .into_iter()
11114 .filter(|event| {
11115 event.fields.get("message").map(String::as_str)
11116 == Some("dropping flow_operations from HELLO")
11117 })
11118 .collect();
11119 assert_eq!(warnings.len(), 1);
11120 assert_eq!(warnings[0].level, tracing::Level::WARN);
11121 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
11122 assert_eq!(warnings[0].fields["reason"], reason);
11123 }
11124
11125 #[test]
11126 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
11127 assert_hello_drops_flow_operations_with_warning(
11128 json!(["flow-scopes/v1"]),
11129 json!(["diagnostics.findings", "missing"]),
11130 "flow_operations names 'missing', which is not a declared management operation",
11131 );
11132 }
11133
11134 #[test]
11135 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
11136 assert_hello_drops_flow_operations_with_warning(
11137 json!([]),
11138 json!(["diagnostics.findings"]),
11139 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
11140 );
11141 assert_hello_drops_flow_operations_with_warning(
11143 json!(["agent-run-scopes/v1"]),
11144 json!([]),
11145 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
11146 );
11147 }
11148
11149 #[test]
11150 fn hello_malformed_flow_operations_is_dropped_with_warning() {
11151 assert_hello_drops_flow_operations_with_warning(
11152 json!(["flow-scopes/v1"]),
11153 json!(["reset", "reset"]),
11154 "flow_operations names 'reset' more than once",
11155 );
11156 assert_hello_drops_flow_operations_with_warning(
11157 json!(["flow-scopes/v1"]),
11158 json!(vec!["reset"; 257]),
11159 "flow_operations lists 257 names, more than 256",
11160 );
11161 assert_hello_drops_flow_operations_with_warning(
11162 json!(["flow-scopes/v1"]),
11163 json!("reset"),
11164 "flow_operations must be an array of operation names",
11165 );
11166 }
11167
11168 #[test]
11169 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
11170 let registry = Arc::new(Registry::default());
11171 let handler = ControlHandler::new(registry.clone());
11172 let descriptor = json!({
11173 "schema": "ckios.richui.manifest/1",
11174 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
11175 "bytes": 128,
11176 "chunks": 2
11177 });
11178 let malformed = [
11179 ("schema", json!("")),
11180 ("schema", json!("a".repeat(65))),
11181 ("schema", json!("Bad schema")),
11182 (
11183 "manifest_digest",
11184 json!(format!("sha256:{}", "A".repeat(64))),
11185 ),
11186 ("manifest_digest", json!("sha256:abc")),
11187 ("bytes", json!(0)),
11188 ("bytes", json!(16_777_217)),
11189 ("bytes", json!("128")),
11190 ("chunks", json!(0)),
11191 ("chunks", json!(1025)),
11192 ("chunks", json!("2")),
11193 ];
11194 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
11195 let mut invalid = descriptor.clone();
11196 invalid[field] = invalid_value;
11197 let response = handler
11198 .handle_control(
11199 ConnectionId::new(210 + index as u64),
11200 hello_frame_with_rich_ui(
11201 &format!("bad-rich-ui-{index}"),
11202 PROTOCOL_VERSION,
11203 210 + index as u64,
11204 None,
11205 invalid,
11206 ),
11207 )
11208 .unwrap();
11209 assert_eq!(response[0].header.ty, FrameType::Error);
11210 let error = parse_error(&response[0]);
11211 assert_eq!(error["code"], "invalid_manifest");
11212 assert!(error["message"]
11213 .as_str()
11214 .unwrap()
11215 .contains(&format!("rich_ui.{field}")));
11216 assert!(registry
11217 .get_module(&format!("bad-rich-ui-{index}"))
11218 .unwrap()
11219 .is_none());
11220 }
11221 }
11222
11223 #[test]
11224 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
11225 let registry = Arc::new(Registry::default());
11226 let handler = ControlHandler::new(Arc::clone(®istry));
11227 let connection = ConnectionId::new(219);
11228 registry
11229 .register_with_control_ops(
11230 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
11231 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
11232 "bad schema",
11233 format!("sha256:{}", "a".repeat(64)),
11234 128,
11235 2,
11236 )))
11237 .build(),
11238 PROTOCOL_VERSION,
11239 connection,
11240 Vec::new(),
11241 )
11242 .unwrap();
11243 let request =
11244 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
11245 let response = handler
11246 .handle_catalog_update(connection, request, Vec::new(), None, None)
11247 .unwrap();
11248 assert_eq!(response[0].header.ty, FrameType::Error);
11249 let error = parse_error(&response[0]);
11250 assert_eq!(error["code"], "invalid_manifest");
11251 assert!(error["message"]
11252 .as_str()
11253 .unwrap()
11254 .contains("rich_ui.schema"));
11255 }
11256
11257 #[test]
11258 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
11259 let registry = Arc::new(Registry::default());
11260 let supervisor = SupervisorHandle::new();
11261 supervisor.open_swap("provider", "candidate-nonce".to_string());
11262 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11263 let connection = ConnectionId::new(218);
11264 let hello = handler
11265 .handle_control(
11266 connection,
11267 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
11268 )
11269 .unwrap();
11270 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11271 assert!(registry.list_modules().unwrap().1.is_empty());
11272
11273 registry.promote_candidate("provider").unwrap().unwrap();
11274 let request = Frame::build(
11275 FrameType::Request,
11276 control_flags(),
11277 0,
11278 0,
11279 219,
11280 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
11281 )
11282 .unwrap();
11283 let response = handler.handle_catalog_list(request, None).unwrap();
11284 let ClientControlResponse::CatalogList { modules, .. } =
11285 serde_json::from_slice(&response[0].body).unwrap()
11286 else {
11287 panic!("expected catalog.list");
11288 };
11289 assert_eq!(modules.len(), 1);
11290 assert!(modules[0].nonce_verified);
11291 }
11292
11293 #[test]
11294 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
11295 for provenance_json in [
11296 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
11297 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
11298 ] {
11299 let registry = Arc::new(Registry::default());
11300 let handler = ControlHandler::new(registry);
11301 let hello = Frame::build(
11302 FrameType::Hello,
11303 control_flags(),
11304 0,
11305 0,
11306 220,
11307 serde_json::to_vec(&ModuleHelloBody {
11308 manifest: ModuleManifest::builder("built", "0.1.0")
11309 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
11310 .build(),
11311 protocol_ver: PROTOCOL_VERSION,
11312 control_ops: None,
11313 launch_nonce: None,
11314 })
11315 .unwrap(),
11316 )
11317 .unwrap();
11318 assert_eq!(
11319 handler
11320 .handle_control(ConnectionId::new(220), hello)
11321 .unwrap()[0]
11322 .header
11323 .ty,
11324 FrameType::HelloAck
11325 );
11326 let request = Frame::build(
11327 FrameType::Request,
11328 control_flags(),
11329 0,
11330 0,
11331 221,
11332 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
11333 )
11334 .unwrap();
11335 let response = handler.handle_catalog_list(request, None).unwrap();
11336 let ClientControlResponse::CatalogList { modules, .. } =
11337 serde_json::from_slice(&response[0].body).unwrap()
11338 else {
11339 panic!("expected catalog.list");
11340 };
11341 assert_eq!(modules.len(), 1);
11342 assert_eq!(
11343 serde_json::to_vec(&modules[0]).unwrap(),
11344 format!(
11345 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
11346 )
11347 .as_bytes()
11348 );
11349 }
11350 }
11351
11352 #[test]
11353 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
11354 let registry = Arc::new(Registry::default());
11355 let handler = ControlHandler::new(Arc::clone(®istry));
11356 let hello = Frame::build(
11357 FrameType::Hello,
11358 control_flags(),
11359 0,
11360 0,
11361 220,
11362 serde_json::to_vec(&ModuleHelloBody {
11363 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
11364 protocol_ver: PROTOCOL_VERSION,
11365 control_ops: None,
11366 launch_nonce: Some("unverified".to_string()),
11367 })
11368 .unwrap(),
11369 )
11370 .unwrap();
11371 assert_eq!(
11372 handler
11373 .handle_control(ConnectionId::new(220), hello)
11374 .unwrap()[0]
11375 .header
11376 .ty,
11377 FrameType::HelloAck
11378 );
11379 let request = Frame::build(
11380 FrameType::Request,
11381 control_flags(),
11382 0,
11383 0,
11384 221,
11385 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
11386 )
11387 .unwrap();
11388 let response = handler.handle_catalog_list(request, None).unwrap();
11389 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
11390 assert!(wire["modules"][0].get("provenance").is_none());
11391 let ClientControlResponse::CatalogList { modules, .. } =
11392 serde_json::from_slice(&response[0].body).unwrap()
11393 else {
11394 panic!("expected catalog.list");
11395 };
11396 assert_eq!(
11397 serde_json::to_vec(&modules[0]).unwrap(),
11398 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
11399 );
11400 assert!(!modules[0].nonce_verified);
11401 assert!(modules[0].provenance.is_none());
11402 }
11403
11404 #[test]
11405 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
11406 let registry = Arc::new(Registry::default());
11407 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11408 [("vault".to_string(), true), ("squatter".to_string(), true)],
11409 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
11410 );
11411 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
11412 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
11413 provides: vec!["credentials-provider/v1".to_string()],
11414 requires: Vec::new(),
11415 must_never_reach: Vec::new(),
11416 });
11417 let frame = Frame::build(
11418 FrameType::Hello,
11419 control_flags(),
11420 0,
11421 0,
11422 77,
11423 serde_json::to_vec(&ModuleHelloBody {
11424 manifest: squatter,
11425 protocol_ver: PROTOCOL_VERSION,
11426 control_ops: None,
11427 launch_nonce: None,
11428 })
11429 .expect("HELLO serializes"),
11430 )
11431 .expect("HELLO frame builds");
11432 let response = handler
11433 .handle_control(ConnectionId::new(77), frame)
11434 .expect("reserved claim receives a typed refusal");
11435 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
11436 assert_eq!(
11437 registry
11438 .active_registration_count()
11439 .expect("registry reads"),
11440 0,
11441 "a reserved capability refusal must not leave a catalog entry"
11442 );
11443 }
11444
11445 #[test]
11446 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
11447 for settlement in ["timeout", "inconclusive", "drop"] {
11448 let breakers = RouteBindBreakers::default();
11449 let RouteBindAdmission::Admitted {
11450 guard: mut old,
11451 probe: false,
11452 } = breakers.admit("prov")
11453 else {
11454 panic!("ordinary relay admitted")
11455 };
11456 let RouteBindAdmission::Admitted {
11457 guard: mut opener, ..
11458 } = breakers.admit("prov")
11459 else {
11460 panic!("second relay admitted")
11461 };
11462 assert!(
11463 !opener
11464 .record_timeout(1, Duration::ZERO)
11465 .unwrap()
11466 .reopened_after_probe
11467 );
11468 let RouteBindAdmission::Admitted {
11469 guard: mut probe,
11470 probe: true,
11471 } = breakers.admit("prov")
11472 else {
11473 panic!("one cooldown probe admitted")
11474 };
11475 match settlement {
11476 "timeout" => assert!(
11477 !old.record_timeout(1, Duration::ZERO)
11478 .unwrap()
11479 .reopened_after_probe
11480 ),
11481 "inconclusive" => old.record_inconclusive(),
11482 "drop" => drop(old),
11483 _ => unreachable!(),
11484 }
11485 assert!(
11486 matches!(
11487 breakers.admit("prov"),
11488 RouteBindAdmission::Refused {
11489 probe_in_flight: true,
11490 ..
11491 }
11492 ),
11493 "{settlement} of a pre-open relay cannot release the real probe"
11494 );
11495 assert!(
11496 probe
11497 .record_timeout(1, Duration::ZERO)
11498 .unwrap()
11499 .reopened_after_probe
11500 );
11501 assert!(matches!(
11502 breakers.admit("prov"),
11503 RouteBindAdmission::Admitted { probe: true, .. }
11504 ));
11505 }
11506 let breakers = RouteBindBreakers::default();
11507 let admit = || match breakers.admit("prov") {
11508 RouteBindAdmission::Admitted { guard, .. } => guard,
11509 _ => panic!("relay admitted"),
11510 };
11511 admit().record_timeout(1, Duration::ZERO);
11512 let mut old_probe = admit();
11513 breakers.reset_for_new_module_connection("prov");
11514 admit().record_timeout(1, Duration::ZERO);
11515 let _new_probe = admit();
11516 old_probe.record_inconclusive();
11517 assert!(matches!(
11518 breakers.admit("prov"),
11519 RouteBindAdmission::Refused {
11520 probe_in_flight: true,
11521 ..
11522 }
11523 ));
11524 }
11525
11526 #[tokio::test]
11527 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
11528 for candidate in [false, true] {
11529 let registry = Arc::new(Registry::default());
11530 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11531 [("vault".to_string(), true), ("squatter".to_string(), true)],
11532 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
11533 );
11534 let conn = ConnectionId::new(77);
11535 let (ctx, mut rx) = route_ctx(conn);
11536 let initial = capability_manifest("squatter", &[], &[]);
11537 if candidate {
11538 registry
11539 .register_candidate_with_control_ops(
11540 initial.clone(),
11541 PROTOCOL_VERSION,
11542 conn,
11543 module_baseline_control_ops(),
11544 )
11545 .unwrap();
11546 handler
11547 .forwarding
11548 .register_candidate_module_connection(
11549 conn,
11550 "squatter".to_string(),
11551 PROTOCOL_VERSION,
11552 manifest_concurrency(&initial),
11553 ctx.egress.clone(),
11554 )
11555 .unwrap();
11556 } else {
11557 hello_via_sink(
11558 &handler,
11559 &ctx,
11560 &mut rx,
11561 hello_frame_with_manifest(initial.clone(), 1),
11562 )
11563 .await;
11564 }
11565 let response = handler
11566 .handle_control_frame(
11567 &ctx,
11568 catalog_update_with_capabilities_frame(
11569 2,
11570 capability_manifest("squatter", &["credentials-provider/v1"], &[])
11571 .capabilities
11572 .unwrap(),
11573 ),
11574 )
11575 .await
11576 .unwrap();
11577 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
11578 assert_eq!(
11579 registry
11580 .get_module_by_connection(conn)
11581 .unwrap()
11582 .unwrap()
11583 .manifest,
11584 initial
11585 );
11586 }
11587 }
11588
11589 #[test]
11590 fn server_describe_surfaces_required_capability_verdict_fields() {
11591 let registry = Arc::new(Registry::default());
11592 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11593 [
11594 ("consumer".to_string(), true),
11595 ("provider".to_string(), false),
11596 ],
11597 BTreeMap::new(),
11598 );
11599 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11600 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
11601 provides: Vec::new(),
11602 requires: vec![subc_protocol::manifest::CapabilityRequirement {
11603 capability: "credentials-provider/v1".to_string(),
11604 need: subc_protocol::manifest::CapabilityNeed::Required,
11605 }],
11606 must_never_reach: Vec::new(),
11607 });
11608 let hello = Frame::build(
11609 FrameType::Hello,
11610 control_flags(),
11611 0,
11612 0,
11613 78,
11614 serde_json::to_vec(&ModuleHelloBody {
11615 manifest: consumer,
11616 protocol_ver: PROTOCOL_VERSION,
11617 control_ops: None,
11618 launch_nonce: None,
11619 })
11620 .expect("HELLO serializes"),
11621 )
11622 .expect("HELLO frame builds");
11623 handler
11624 .handle_control(ConnectionId::new(78), hello)
11625 .expect("consumer registers");
11626 let describe = Frame::build(
11627 FrameType::Request,
11628 control_flags(),
11629 0,
11630 0,
11631 79,
11632 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
11633 .expect("request serializes"),
11634 )
11635 .expect("describe frame builds");
11636 let response = handler
11637 .handle_server_describe(describe)
11638 .expect("server.describe succeeds");
11639 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
11640 let requirement = &rendered["capability_requirements"][0];
11641 assert_eq!(requirement["consumer"], "consumer");
11642 assert_eq!(requirement["verdict"], "never_provided");
11643 assert_eq!(requirement["episode_seq"], 1);
11644 assert_eq!(requirement["config_satisfiable"], false);
11645 assert_eq!(requirement["runtime_available"], false);
11646 assert!(requirement["detail"]
11647 .as_str()
11648 .expect("detail string")
11649 .contains("credentials-provider/v1"));
11650 }
11651
11652 fn deregister_required_provider(
11658 daemon_shutdown_begun: bool,
11659 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
11660 const CAPABILITY: &str = "project-identity/v1";
11661 let registry = Arc::new(Registry::default());
11662 let daemon_shutdown = DaemonShutdownFlag::default();
11663 let handler = ControlHandler::new(Arc::clone(®istry))
11664 .with_capability_config(
11665 [
11666 ("consumer".to_string(), true),
11667 ("provider".to_string(), true),
11668 ],
11669 BTreeMap::new(),
11670 )
11671 .with_daemon_shutdown_flag(daemon_shutdown.clone());
11672 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11673 consumer.capabilities = Some(CapabilityDeclarations {
11674 provides: Vec::new(),
11675 requires: vec![subc_protocol::manifest::CapabilityRequirement {
11676 capability: CAPABILITY.to_string(),
11677 need: subc_protocol::manifest::CapabilityNeed::Required,
11678 }],
11679 must_never_reach: Vec::new(),
11680 });
11681 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
11682 let provider_conn = ConnectionId::new(182);
11683 for (conn, manifest, corr) in [
11684 (ConnectionId::new(181), consumer, 181),
11685 (provider_conn, provider, 182),
11686 ] {
11687 let hello = handler
11688 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
11689 .expect("module registers");
11690 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11691 }
11692 assert_eq!(
11693 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11694 Some(CapabilityVerdict::Provided),
11695 "the registered provider must satisfy the requirement first"
11696 );
11697 if daemon_shutdown_begun {
11698 daemon_shutdown.set_for_test();
11699 }
11700
11701 let capture = EventCapture::default();
11702 let _subscriber =
11703 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11704 handler
11705 .cleanup_connection(provider_conn)
11706 .expect("provider deregisters");
11707 let events = capture
11708 .events()
11709 .into_iter()
11710 .filter(|event| {
11711 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11712 })
11713 .collect();
11714 (
11715 events,
11716 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11717 )
11718 }
11719
11720 #[test]
11721 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11722 let (events, verdict) = deregister_required_provider(true);
11723 assert_eq!(
11724 verdict,
11725 Some(CapabilityVerdict::NeverProvided),
11726 "shutdown must not change the verdict route.open refusals read"
11727 );
11728 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11729 let event = &events[0];
11730 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11731 assert_eq!(
11732 event.fields.get("cause").map(String::as_str),
11733 Some("\"daemon_shutdown\""),
11734 "{event:?}"
11735 );
11736 assert_eq!(
11737 event.fields.get("verdict").map(String::as_str),
11738 Some("\"never_provided\""),
11739 "{event:?}"
11740 );
11741 }
11742
11743 #[test]
11744 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11745 let (events, verdict) = deregister_required_provider(false);
11746 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11747 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11748 let event = &events[0];
11749 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11750 assert_eq!(event.fields.get("cause"), None, "{event:?}");
11751 assert_eq!(
11752 event.fields.get("verdict").map(String::as_str),
11753 Some("\"never_provided\""),
11754 "{event:?}"
11755 );
11756 }
11757
11758 #[test]
11759 fn catalog_list_omits_capabilities_for_legacy_manifest() {
11760 let registry = Arc::new(Registry::default());
11761 let handler = ControlHandler::new(Arc::clone(®istry));
11762 let hello = handler
11763 .handle_control(
11764 ConnectionId::new(101),
11765 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11766 )
11767 .expect("legacy HELLO registers");
11768 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11769
11770 let request = Frame::build(
11771 FrameType::Request,
11772 control_flags(),
11773 0,
11774 0,
11775 102,
11776 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11777 .expect("catalog request serializes"),
11778 )
11779 .expect("catalog request frame builds");
11780 let response = handler
11781 .handle_catalog_list(request, None)
11782 .expect("catalog list succeeds");
11783 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11784 assert!(
11785 body["modules"][0].get("capabilities").is_none(),
11786 "legacy manifest must retain an absent capabilities field on catalog.list"
11787 );
11788 }
11789
11790 #[test]
11791 fn hello_ack_omits_storage_when_no_storage_config() {
11792 let registry = Arc::new(Registry::default());
11793 let handler = ControlHandler::new(Arc::clone(®istry));
11794 let responses = handler
11795 .handle_control(
11796 ConnectionId::new(1),
11797 hello_frame("aft", PROTOCOL_VERSION, 7),
11798 )
11799 .unwrap();
11800 let ack = parse_ack(&responses[0]);
11801 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11802 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11803 }
11804
11805 #[tokio::test]
11806 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11807 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11808 let registry = Arc::new(Registry::default());
11809 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
11810 let responses = handler
11811 .handle_control(
11812 ConnectionId::new(1),
11813 hello_frame("aft", PROTOCOL_VERSION, 7),
11814 )
11815 .unwrap();
11816 let ack = parse_ack(&responses[0]);
11817 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11818
11819 let described = handler
11820 .handle_control_frame(
11821 &route_ctx(ConnectionId::new(2)).0,
11822 Frame::build(
11823 FrameType::Request,
11824 control_flags(),
11825 0,
11826 0,
11827 9,
11828 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11829 )
11830 .unwrap(),
11831 )
11832 .await
11833 .unwrap();
11834 let ClientControlResponse::ServerDescribe { machine_id, .. } =
11835 serde_json::from_slice(&described[0].body).unwrap()
11836 else {
11837 panic!("server.describe answered with another shape");
11838 };
11839 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11840 }
11841
11842 #[test]
11843 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11844 let registry = Arc::new(Registry::default());
11847 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
11848 crate::daemon_config::StorageConfig::Sqlite {
11849 data_home: std::path::PathBuf::from("/data"),
11850 },
11851 ));
11852
11853 let responses = handler
11854 .handle_control(
11855 ConnectionId::new(1),
11856 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11857 )
11858 .unwrap();
11859 let ack = parse_ack(&responses[0]);
11860 assert_eq!(
11861 ack.storage,
11862 Some(serde_json::json!({
11863 "module_id": "alfonso-routing",
11864 "storage_namespace": "default",
11865 "isolation": { "kind": "module" },
11866 "backend": {
11867 "backend": "sqlite",
11868 "path": "/data/cortexkit/alfonso-routing/store.db"
11869 }
11870 })),
11871 "the delivered descriptor is the module's own sqlite store path"
11872 );
11873 }
11874
11875 #[test]
11876 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11877 let registry = Arc::new(Registry::default());
11878 let handler = ControlHandler::new(Arc::clone(®istry));
11879 let conn = ConnectionId::new(1);
11880 let responses = handler
11881 .handle_control(
11882 conn,
11883 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11884 )
11885 .unwrap();
11886 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11887 let registration = registry.get_module("aft").unwrap().unwrap();
11888 assert_eq!(registration.control_ops, module_baseline_control_ops());
11889
11890 let frame =
11891 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11892 assert!(handler
11893 .guard_module_control_op(&frame, "aft", "route.bind")
11894 .unwrap()
11895 .is_none());
11896 let error = handler
11897 .guard_module_control_op(&frame, "aft", "test.synthetic")
11898 .unwrap()
11899 .expect("synthetic ungranted op should be rejected");
11900 assert_eq!(error.header.ty, FrameType::Error);
11901 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11902 }
11903
11904 #[test]
11905 fn hello_control_ops_some_adds_optional_grants() {
11906 let registry = Arc::new(Registry::default());
11907 let handler = ControlHandler::new(Arc::clone(®istry));
11908 handler
11909 .handle_control(
11910 ConnectionId::new(1),
11911 hello_frame_with_control_ops(
11912 "aft",
11913 PROTOCOL_VERSION,
11914 7,
11915 Some(vec![
11916 "future.synthetic".to_string(),
11917 "route.bind".to_string(),
11918 ]),
11919 ),
11920 )
11921 .unwrap();
11922 let registration = registry.get_module("aft").unwrap().unwrap();
11923 assert_eq!(
11924 registration.control_ops,
11925 vec![
11926 "route.bind".to_string(),
11927 "route.status".to_string(),
11928 "future.synthetic".to_string(),
11929 ]
11930 );
11931 let frame =
11932 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11933 assert!(handler
11934 .guard_module_control_op(&frame, "aft", "future.synthetic")
11935 .unwrap()
11936 .is_none());
11937 }
11938
11939 #[tokio::test]
11940 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11941 let registry = Arc::new(Registry::default());
11942 let forwarding = Arc::new(ForwardingTable::default());
11943 let handler =
11944 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11945 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11946 hello_via_sink(
11947 &handler,
11948 &module_ctx,
11949 &mut module_rx,
11950 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11951 )
11952 .await;
11953
11954 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11955 let responses = handler
11956 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11957 .await
11958 .unwrap();
11959 assert_eq!(responses.len(), 1);
11960 assert_eq!(responses[0].header.ty, FrameType::Error);
11961 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11962 assert!(module_rx.try_recv().is_err());
11963 }
11964
11965 #[tokio::test]
11966 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11967 let registry = Arc::new(Registry::default());
11968 let forwarding = Arc::new(ForwardingTable::default());
11969 let handler =
11970 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11971 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11972 hello_via_sink(
11973 &handler,
11974 &module_ctx,
11975 &mut module_rx,
11976 hello_frame_with_control_ops(
11977 "aft",
11978 PROTOCOL_VERSION,
11979 7,
11980 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11981 ),
11982 )
11983 .await;
11984
11985 let project_root = unique_project_root("demux");
11986 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11987 let route_handler = handler.clone();
11988 let route_task = tokio::spawn(async move {
11989 route_handler
11990 .handle_control_frame(
11991 &route_client_ctx,
11992 route_open_frame(100, "aft", project_root),
11993 )
11994 .await
11995 .unwrap()
11996 });
11997 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11998 .await
11999 .unwrap()
12000 .unwrap();
12001 assert!(matches!(
12002 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
12003 ModuleControlRequest::RouteBind { .. }
12004 ));
12005
12006 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
12007 let health_handler = handler.clone();
12008 let health_task = tokio::spawn(async move {
12009 health_handler
12010 .handle_control_frame(
12011 &health_client_ctx,
12012 supervisor_health_probe_frame(101, "aft"),
12013 )
12014 .await
12015 .unwrap()
12016 });
12017 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12018 .await
12019 .unwrap()
12020 .unwrap();
12021 assert_eq!(
12022 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12023 ModuleControlRequest::HealthCheck {}
12024 );
12025
12026 handler
12027 .handle_control_frame(
12028 &module_ctx,
12029 health_response(health_frame.header.corr, HealthStatus::Degraded),
12030 )
12031 .await
12032 .unwrap();
12033 let health_response = health_task.await.unwrap();
12034 assert_eq!(health_response.len(), 1);
12035 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12036 ClientControlResponse::SupervisorHealthProbe {
12037 module_id,
12038 status,
12039 detail,
12040 metrics,
12041 } => {
12042 assert_eq!(module_id, "aft");
12043 assert_eq!(status, HealthStatus::Degraded);
12044 assert_eq!(detail.as_deref(), Some("warming"));
12045 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
12046 }
12047 other => panic!("unexpected health response: {other:?}"),
12048 }
12049
12050 handler
12051 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12052 .await
12053 .unwrap();
12054 let route_response = route_task.await.unwrap();
12055 assert!(route_response.is_empty());
12056 let published = route_client_rx.recv().await.unwrap();
12057 assert!(matches!(
12058 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12059 ClientControlResponse::RouteOpen { .. }
12060 ));
12061 }
12062
12063 async fn relay_route_open(
12068 handler: &ControlHandler,
12069 client_connection: ConnectionId,
12070 client_egress: &FrameSink,
12071 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12072 corr: u64,
12073 module_id: &str,
12074 project_root_label: &str,
12075 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
12076 let ctx = RouteCtx {
12077 connection_id: client_connection,
12078 egress: client_egress.clone(),
12079 };
12080 let handler = handler.clone();
12081 let project_root = unique_project_root(project_root_label);
12082 let module_id = module_id.to_string();
12083 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
12084 let task = tokio::spawn(async move {
12085 let _guard = tracing::dispatcher::set_default(&dispatch);
12086 handler
12087 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
12088 .await
12089 .unwrap()
12090 });
12091 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
12092 .await
12093 .expect("module receives the relayed route.bind")
12094 .expect("module egress is open");
12095 (task, bind.frame)
12096 }
12097
12098 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
12099 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
12100 ModuleControlRequest::RouteBind {
12101 route_channel,
12102 epoch,
12103 ..
12104 } => (route_channel, epoch),
12105 other => panic!("expected a route.bind request, got {other:?}"),
12106 }
12107 }
12108
12109 fn published_route(frame: &Frame) -> (u16, u32) {
12110 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
12111 ClientControlResponse::RouteOpen {
12112 route_channel,
12113 route_epoch,
12114 ..
12115 } => (route_channel, route_epoch),
12116 other => panic!("expected a route.open response, got {other:?}"),
12117 }
12118 }
12119
12120 #[tokio::test]
12140 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
12141 let registry = Arc::new(Registry::default());
12142 let forwarding = Arc::new(ForwardingTable::default());
12143 let handler =
12144 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12145
12146 let module_connection = ConnectionId::new(30);
12147 let (module_ctx, mut module_rx) = route_ctx(module_connection);
12148 hello_via_sink(
12149 &handler,
12150 &module_ctx,
12151 &mut module_rx,
12152 hello_frame("aft", PROTOCOL_VERSION, 7),
12153 )
12154 .await;
12155
12156 let dying_client = ConnectionId::new(31);
12157 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
12158
12159 let (first_task, first_bind) = relay_route_open(
12162 &handler,
12163 dying_client,
12164 &dying_ctx.egress,
12165 &mut module_rx,
12166 100,
12167 "aft",
12168 "closing-first",
12169 )
12170 .await;
12171 handler
12172 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
12173 .await
12174 .unwrap();
12175 assert!(first_task.await.unwrap().is_empty());
12176 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
12177
12178 let (second_task, second_bind) = relay_route_open(
12180 &handler,
12181 dying_client,
12182 &dying_ctx.egress,
12183 &mut module_rx,
12184 101,
12185 "aft",
12186 "closing-second",
12187 )
12188 .await;
12189 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
12190
12191 assert!(forwarding
12194 .escalate_client_delivery_failure(
12195 dying_client,
12196 first_channel,
12197 first_epoch,
12198 CloseReason::new(
12199 "module_to_client_delivery_failed",
12200 "client egress refused a module frame",
12201 ),
12202 crate::forwarding::UndeliveredFrame {
12203 module_id: None,
12204 sink: &dying_ctx.egress,
12205 },
12206 )
12207 .unwrap());
12208 assert!(!dying_ctx.egress.is_closed());
12209
12210 let ack = handler
12212 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
12213 .await;
12214 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
12215 if module_loop_error.is_some() {
12216 handler.cleanup_connection(module_connection).unwrap();
12220 }
12221 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12225 .await
12226 .ok()
12227 .flatten();
12228
12229 assert!(
12231 registry
12232 .get_module_by_connection(module_connection)
12233 .unwrap()
12234 .is_some(),
12235 "one client's closing connection ended the shared module connection: \
12236 {module_loop_error:?}"
12237 );
12238 let cotenant = ConnectionId::new(32);
12240 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
12241 let (cotenant_task, cotenant_bind) = relay_route_open(
12242 &handler,
12243 cotenant,
12244 &cotenant_ctx.egress,
12245 &mut module_rx,
12246 102,
12247 "aft",
12248 "closing-cotenant",
12249 )
12250 .await;
12251 handler
12252 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
12253 .await
12254 .unwrap();
12255 assert!(cotenant_task.await.unwrap().is_empty());
12256 let (cotenant_channel, cotenant_epoch) =
12257 published_route(&cotenant_rx.recv().await.unwrap());
12258 assert!(matches!(
12259 forwarding
12260 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
12261 .unwrap(),
12262 DataRoute::Client(DataRouteState::Bound(_))
12263 ));
12264
12265 let goodbye = post_ack_module_frame
12268 .expect("module receives a GOODBYE for the abandoned route channel");
12269 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12270 assert_eq!(goodbye.header.channel, abandoned_channel);
12271 assert_eq!(goodbye.header.epoch, abandoned_epoch);
12272
12273 assert!(dying_rx.try_recv().is_err());
12277 let second_response = second_task.await.unwrap();
12278 assert_eq!(second_response.len(), 1);
12279 assert_eq!(
12280 parse_error(&second_response[0])["code"],
12281 "target_unavailable"
12282 );
12283 }
12284
12285 async fn catalog_snapshot(
12288 handler: &ControlHandler,
12289 client_ctx: &RouteCtx,
12290 corr: u64,
12291 ) -> (u64, Vec<(String, bool)>) {
12292 let body =
12293 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
12294 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
12295 let reply = handler
12296 .handle_control_frame(client_ctx, request)
12297 .await
12298 .unwrap()
12299 .pop()
12300 .unwrap();
12301 let ClientControlResponse::CatalogList {
12302 generation,
12303 modules,
12304 ..
12305 } = serde_json::from_slice(&reply.body).unwrap()
12306 else {
12307 panic!("not a catalog.list reply");
12308 };
12309 (
12310 generation,
12311 modules
12312 .into_iter()
12313 .map(|entry| (entry.module_id, entry.ready))
12314 .collect(),
12315 )
12316 }
12317
12318 fn hello_held_before_forwarding() -> (
12324 ControlHandler,
12325 Arc<std::sync::Barrier>,
12326 std::thread::JoinHandle<Vec<Frame>>,
12327 RouteCtx,
12328 mpsc::Receiver<crate::router::OutboundFrame>,
12329 ) {
12330 let hold = Arc::new(std::sync::Barrier::new(2));
12331 let handler = ControlHandler::with_forwarding(
12332 Arc::new(Registry::default()),
12333 Arc::new(ForwardingTable::default()),
12334 )
12335 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
12336 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
12337 let hello = {
12338 let handler = handler.clone();
12339 let module_ctx = module_ctx.clone();
12340 std::thread::spawn(move || {
12341 handler
12342 .handle_hello(
12343 module_ctx.connection_id,
12344 Some(module_ctx.egress.clone()),
12345 hello_frame("aft", PROTOCOL_VERSION, 7),
12346 )
12347 .unwrap()
12348 })
12349 };
12350 (handler, hold, hello, module_ctx, module_rx)
12351 }
12352
12353 #[tokio::test]
12362 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
12363 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
12364 let client = ConnectionId::new(41);
12365 let (client_ctx, mut client_rx) = route_ctx(client);
12366
12367 hold.wait();
12368 assert!(handler.registry.get_module("aft").unwrap().is_some());
12370 assert!(!handler
12371 .forwarding
12372 .has_live_module_connection("aft")
12373 .unwrap());
12374 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
12375 assert!(
12376 listed.is_empty(),
12377 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
12378 );
12379 let refused = handler
12380 .handle_control_frame(
12381 &client_ctx,
12382 route_open_frame(301, "aft", unique_project_root("listed-window")),
12383 )
12384 .await
12385 .unwrap();
12386 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
12387 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
12388 let duplicate = handler
12389 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
12390 .await
12391 .unwrap();
12392 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12393
12394 hold.wait();
12395 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
12396 let ack = module_rx.try_recv().unwrap().frame;
12397 assert_eq!(ack.header.ty, FrameType::HelloAck);
12398 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
12399 assert_eq!(listed, vec![("aft".to_string(), true)]);
12400
12401 let (open_task, bind) = relay_route_open(
12402 &handler,
12403 client,
12404 &client_ctx.egress,
12405 &mut module_rx,
12406 303,
12407 "aft",
12408 "listed-after",
12409 )
12410 .await;
12411 assert!(handler
12412 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12413 .await
12414 .unwrap()
12415 .is_empty());
12416 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
12417 published_route(&client_rx.try_recv().unwrap().frame);
12418 }
12419
12420 #[tokio::test]
12427 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
12428 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
12429 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12430 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
12431
12432 hold.wait();
12433 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
12434 assert!(listed.is_empty());
12435 assert_eq!(
12436 during, before,
12437 "the generation moved while the module was not listed"
12438 );
12439
12440 hold.wait();
12441 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
12442 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
12443 assert_eq!(listed, vec![("aft".to_string(), true)]);
12444 assert_eq!(
12445 after,
12446 before + 1,
12447 "the module appeared without exactly one generation bump"
12448 );
12449 }
12450
12451 #[tokio::test]
12456 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
12457 let registry = Arc::new(Registry::default());
12458 let handler = ControlHandler::with_forwarding(
12459 Arc::clone(®istry),
12460 Arc::new(ForwardingTable::default()),
12461 );
12462 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12463 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
12464 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
12465 drop(module_rx);
12466
12467 let replies = handler
12468 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
12469 .await
12470 .unwrap();
12471 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
12472 assert!(registry.get_module("aft").unwrap().is_none());
12473 assert!(registry
12474 .get_module_by_connection(module_ctx.connection_id)
12475 .unwrap()
12476 .is_none());
12477 assert_eq!(
12478 catalog_snapshot(&handler, &client_ctx, 301).await,
12479 (before, Vec::new())
12480 );
12481 }
12482
12483 async fn route_open_held_after_bind_send() -> (
12487 ControlHandler,
12488 Arc<tokio::sync::Barrier>,
12489 RouteCtx,
12490 mpsc::Receiver<crate::router::OutboundFrame>,
12491 ) {
12492 let hold = Arc::new(tokio::sync::Barrier::new(2));
12493 let handler = ControlHandler::with_forwarding(
12494 Arc::new(Registry::default()),
12495 Arc::new(ForwardingTable::default()),
12496 )
12497 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
12498 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12499 hello_via_sink(
12500 &handler,
12501 &module_ctx,
12502 &mut module_rx,
12503 hello_frame("aft", PROTOCOL_VERSION, 7),
12504 )
12505 .await;
12506 (handler, hold, module_ctx, module_rx)
12507 }
12508
12509 #[tokio::test]
12516 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
12517 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12518 let client = ConnectionId::new(41);
12519 let (client_ctx, mut client_rx) = route_ctx(client);
12520
12521 let (open_task, bind) = relay_route_open(
12522 &handler,
12523 client,
12524 &client_ctx.egress,
12525 &mut module_rx,
12526 200,
12527 "aft",
12528 "acked-before-mark",
12529 )
12530 .await;
12531 let (module_channel, module_epoch) = route_bind_channel(&bind);
12532 hold.wait().await;
12534
12535 assert!(handler
12537 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12538 .await
12539 .unwrap()
12540 .is_empty());
12541 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
12542
12543 hold.wait().await;
12544 assert!(
12545 open_task.await.unwrap().is_empty(),
12546 "route.open answers accepted"
12547 );
12548
12549 match module_rx.try_recv() {
12552 Err(mpsc::error::TryRecvError::Empty) => {}
12553 Ok(outbound) => panic!(
12554 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
12555 outbound.frame.header.ty,
12556 outbound.frame.header.channel,
12557 outbound.frame.header.epoch
12558 ),
12559 Err(err) => panic!("module egress closed: {err}"),
12560 }
12561
12562 let router = Router::with_control_handler(Arc::new(handler.clone()));
12565 let request = Frame::build(
12566 FrameType::Request,
12567 Flags::new(true, Priority::Interactive, false),
12568 client_channel,
12569 client_epoch,
12570 201,
12571 b"first request".to_vec(),
12572 )
12573 .unwrap();
12574 router
12575 .route_for_connection(&client_ctx, request)
12576 .await
12577 .unwrap();
12578 let delivered = module_rx.try_recv().unwrap().frame;
12579 assert_eq!(delivered.header.ty, FrameType::Request);
12580 assert_eq!(
12581 (delivered.header.channel, delivered.header.epoch),
12582 (module_channel, module_epoch)
12583 );
12584 assert_eq!(delivered.body, b"first request");
12585 }
12586
12587 #[tokio::test]
12593 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
12594 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12595 let client = ConnectionId::new(42);
12596 let (client_ctx, _client_rx) = route_ctx(client);
12597
12598 let (open_task, bind) = relay_route_open(
12599 &handler,
12600 client,
12601 &client_ctx.egress,
12602 &mut module_rx,
12603 300,
12604 "aft",
12605 "released-before-mark",
12606 )
12607 .await;
12608 let (module_channel, module_epoch) = route_bind_channel(&bind);
12609 hold.wait().await;
12610
12611 handler.cleanup_connection(client).unwrap();
12612 assert!(
12613 module_rx.try_recv().is_err(),
12614 "client teardown must not GOODBYE a bind route.open is still queueing"
12615 );
12616
12617 hold.wait().await;
12618 let response = open_task.await.unwrap();
12619 assert_eq!(response.len(), 1);
12620 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12621
12622 let goodbye = module_rx
12623 .try_recv()
12624 .expect("module is told to drop the binding the daemon abandoned")
12625 .frame;
12626 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12627 assert_eq!(
12628 (goodbye.header.channel, goodbye.header.epoch),
12629 (module_channel, module_epoch)
12630 );
12631 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12632 }
12633
12634 #[tokio::test]
12639 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
12640 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12641 let client = ConnectionId::new(43);
12642 let (client_ctx, client_rx) = route_ctx(client);
12643
12644 let (open_task, bind) = relay_route_open(
12645 &handler,
12646 client,
12647 &client_ctx.egress,
12648 &mut module_rx,
12649 400,
12650 "aft",
12651 "gone-client-before-mark",
12652 )
12653 .await;
12654 let (module_channel, module_epoch) = route_bind_channel(&bind);
12655 hold.wait().await;
12656
12657 drop(client_rx);
12658 assert!(client_ctx.egress.is_closed());
12659 handler
12660 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12661 .await
12662 .unwrap();
12663
12664 hold.wait().await;
12665 let response = open_task.await.unwrap();
12666 assert_eq!(response.len(), 1);
12667 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12668
12669 let goodbye = module_rx
12670 .try_recv()
12671 .expect("module is told to drop the binding it acked")
12672 .frame;
12673 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12674 assert_eq!(
12675 (goodbye.header.channel, goodbye.header.epoch),
12676 (module_channel, module_epoch)
12677 );
12678 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12679 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12680 }
12681
12682 #[test]
12689 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
12690 let handler = ControlHandler::default();
12691 let module_connection = ConnectionId::new(30);
12692 let client_connection = ConnectionId::new(31);
12693
12694 handler
12695 .refuse_to_end_module_connection_for_a_client(
12696 module_connection,
12697 77,
12698 ForwardingError::ConnectionClosing {
12699 connection_id: client_connection,
12700 },
12701 )
12702 .expect("a closing client must never end the module connection");
12703
12704 assert!(matches!(
12705 handler.refuse_to_end_module_connection_for_a_client(
12706 module_connection,
12707 78,
12708 ForwardingError::ConnectionClosing {
12709 connection_id: module_connection,
12710 },
12711 ),
12712 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12713 connection_id
12714 })) if connection_id == module_connection
12715 ));
12716 assert!(matches!(
12717 handler.refuse_to_end_module_connection_for_a_client(
12718 module_connection,
12719 79,
12720 ForwardingError::Poisoned,
12721 ),
12722 Err(RouterError::Forwarding(ForwardingError::Poisoned))
12723 ));
12724 assert!(matches!(
12725 handler.refuse_to_end_module_connection_for_a_client(
12726 module_connection,
12727 80,
12728 ForwardingError::StaleModuleEndpoint,
12729 ),
12730 Err(RouterError::Forwarding(
12731 ForwardingError::StaleModuleEndpoint
12732 ))
12733 ));
12734 }
12735
12736 #[tokio::test]
12749 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12750 let registry = Arc::new(Registry::default());
12751 let forwarding = Arc::new(ForwardingTable::default());
12752 let supervisor = SupervisorHandle::new();
12753 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12754 let handler =
12755 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12756 .with_supervisor(supervisor);
12757
12758 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12759 hello_via_sink(
12760 &handler,
12761 &target_ctx,
12762 &mut target_rx,
12763 hello_frame("target", PROTOCOL_VERSION, 1),
12764 )
12765 .await;
12766
12767 let wrong_nonce = handler
12771 .handle_control_frame(
12772 &route_ctx(ConnectionId::new(91)).0,
12773 route_open_frame_with_admission_facts(
12774 20,
12775 "target",
12776 unique_project_root("admission-facts"),
12777 Some(subc_control::ConsumerIdentity {
12778 module_id: "fed".to_string(),
12779 launch_nonce: "not-the-real-nonce".to_string(),
12780 }),
12781 None,
12782 ),
12783 )
12784 .await
12785 .unwrap();
12786 assert_eq!(
12787 parse_error(&wrong_nonce[0])["code"],
12788 "bad_consumer_identity",
12789 "a mismatched launch nonce must be refused, not stamped Reserved"
12790 );
12791
12792 let never_spawned = handler
12796 .handle_control_frame(
12797 &route_ctx(ConnectionId::new(92)).0,
12798 route_open_frame_with_admission_facts(
12799 21,
12800 "target",
12801 unique_project_root("admission-facts"),
12802 Some(subc_control::ConsumerIdentity {
12803 module_id: "never-spawned".to_string(),
12804 launch_nonce: "any-nonce".to_string(),
12805 }),
12806 None,
12807 ),
12808 )
12809 .await
12810 .unwrap();
12811 assert_eq!(
12812 parse_error(&never_spawned[0])["code"],
12813 "bad_consumer_identity",
12814 "an unspawned module_id must be refused rather than accepted for lack of a record"
12815 );
12816 }
12817
12818 #[tokio::test]
12841 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12842 let registry = Arc::new(Registry::default());
12843 let forwarding = Arc::new(ForwardingTable::default());
12844 let supervisor = SupervisorHandle::new();
12845 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12846 let handler =
12847 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12848 .with_supervisor(supervisor);
12849
12850 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12851 hello_via_sink(
12852 &handler,
12853 &target_ctx,
12854 &mut target_rx,
12855 hello_frame("target", PROTOCOL_VERSION, 1),
12856 )
12857 .await;
12858
12859 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12860 let route_handler = handler.clone();
12861 let route_task = tokio::spawn(async move {
12862 route_handler
12863 .handle_control_frame(
12864 &client_ctx,
12865 route_open_frame_with_admission_facts(
12866 30,
12867 "target",
12868 unique_project_root("admission-facts"),
12869 Some(subc_control::ConsumerIdentity {
12870 module_id: "fed".to_string(),
12871 launch_nonce: "fed-nonce".to_string(),
12872 }),
12873 None,
12874 ),
12875 )
12876 .await
12877 .unwrap()
12878 });
12879
12880 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12887 .await
12888 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12889 .expect("module control channel closed before route.bind");
12890 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12891 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12892 panic!("expected route.bind")
12893 };
12894 assert_eq!(
12895 principal,
12896 Some(Principal::Reserved {
12897 module_id: "fed".to_string()
12898 }),
12899 "a correctly attested consumer must be stamped Reserved for its own id"
12900 );
12901
12902 handler
12903 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12904 .await
12905 .unwrap();
12906 assert!(route_task.await.unwrap().is_empty());
12907 assert!(
12908 matches!(
12909 serde_json::from_slice::<ClientControlResponse>(
12910 &client_rx.recv().await.unwrap().body
12911 )
12912 .unwrap(),
12913 ClientControlResponse::RouteOpen { .. }
12914 ),
12915 "the route must actually open, not merely avoid an error"
12916 );
12917 }
12918
12919 #[tokio::test(start_paused = true)]
12920 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12921 let registry = Arc::new(Registry::default());
12922 let forwarding = Arc::new(ForwardingTable::default());
12923 let supervisor = SupervisorHandle::new();
12924 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12925 let handler =
12926 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12927 .with_supervisor(supervisor);
12928
12929 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12930 hello_via_sink(
12931 &handler,
12932 &target_ctx,
12933 &mut target_rx,
12934 hello_frame("target", PROTOCOL_VERSION, 1),
12935 )
12936 .await;
12937
12938 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12939 let direct_handler = handler.clone();
12940 let direct_open = tokio::spawn(async move {
12941 direct_handler
12942 .handle_control_frame(
12943 &direct_ctx,
12944 route_open_frame(2, "target", unique_project_root("route-census-direct")),
12945 )
12946 .await
12947 .unwrap()
12948 });
12949 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12950 .await
12951 .expect("no direct route.bind within 5s")
12952 .expect("target control channel closed before direct route.bind");
12953 handler
12954 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12955 .await
12956 .unwrap();
12957 assert!(direct_open.await.unwrap().is_empty());
12958 let _ = direct_rx.recv().await.unwrap();
12959
12960 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12961 let reserved_handler = handler.clone();
12962 let reserved_open = tokio::spawn(async move {
12963 reserved_handler
12964 .handle_control_frame(
12965 &reserved_ctx,
12966 route_open_frame_with_admission_facts(
12967 3,
12968 "target",
12969 unique_project_root("admission-facts"),
12970 Some(ConsumerIdentity {
12971 module_id: "fed".to_string(),
12972 launch_nonce: "fed-nonce".to_string(),
12973 }),
12974 None,
12975 ),
12976 )
12977 .await
12978 .unwrap()
12979 });
12980 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12981 .await
12982 .expect("no reserved route.bind within 5s")
12983 .expect("target control channel closed before reserved route.bind");
12984 handler
12985 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12986 .await
12987 .unwrap();
12988 assert!(reserved_open.await.unwrap().is_empty());
12989 let _ = reserved_rx.recv().await.unwrap();
12990
12991 forwarding
12992 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12993 .unwrap();
12994 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12995 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12996 module_id: Some("target".to_string()),
12997 })
12998 .unwrap();
12999 let census_frame =
13000 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
13001 let response = handler
13002 .handle_control_frame(&census_ctx, census_frame)
13003 .await
13004 .unwrap()
13005 .pop()
13006 .unwrap();
13007 let actual: Value = serde_json::from_slice(&response.body).unwrap();
13008 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
13009 assert!(matches!(
13010 decoded,
13011 ClientControlResponse::SupervisorRoutes { .. }
13012 ));
13013 let routes = actual["modules"][0]["routes"].as_array().unwrap();
13014 assert_eq!(routes.len(), 2);
13015 assert!(routes.iter().all(|route| route["draining"] == true));
13016 assert!(
13019 routes.iter().all(|route| route["drain_reason"] == "reload"),
13020 "draining routes must name the drain's reason: {routes:?}"
13021 );
13022 assert!(routes.iter().any(|route| {
13023 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
13024 }));
13025 assert!(routes.iter().any(|route| {
13026 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
13027 }));
13028
13029 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
13030 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
13031 if std::env::var_os("UPDATE_GOLDEN").is_some() {
13032 std::fs::write(
13033 &golden_path,
13034 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
13035 )
13036 .unwrap();
13037 }
13038 let expected: Value =
13039 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
13040 assert_eq!(actual, expected);
13041 }
13042
13043 async fn query_live_roots(
13044 handler: &ControlHandler,
13045 module_ctx: &RouteCtx,
13046 ) -> ModuleControlResponseToModule {
13047 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
13048 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
13049 let response = handler
13050 .handle_control_frame(module_ctx, frame)
13051 .await
13052 .unwrap()
13053 .pop()
13054 .unwrap();
13055 serde_json::from_slice(&response.body).unwrap()
13056 }
13057
13058 #[tokio::test(start_paused = true)]
13059 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
13060 let registry = Arc::new(Registry::default());
13061 let forwarding = Arc::new(ForwardingTable::default());
13062 let handler = ControlHandler::with_forwarding(registry, forwarding);
13063 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
13064 hello_via_sink(
13065 &handler,
13066 &target_ctx,
13067 &mut target_rx,
13068 hello_frame("target", PROTOCOL_VERSION, 1),
13069 )
13070 .await;
13071 let root = unique_project_root("live-roots-known");
13072 let path = ProjectRootId::from_path_allowing_missing(root.path())
13073 .unwrap()
13074 .as_path()
13075 .to_path_buf();
13076 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
13077 let open_handler = handler.clone();
13078 let opened = tokio::spawn(async move {
13079 open_handler
13080 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
13081 .await
13082 .unwrap()
13083 });
13084 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
13085 .await
13086 .unwrap()
13087 .unwrap();
13088 handler
13089 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
13090 .await
13091 .unwrap();
13092 assert!(opened.await.unwrap().is_empty());
13093 let _ = client_rx.recv().await.unwrap();
13094
13095 let root = unique_project_root("live-roots-pending");
13096 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
13097 .unwrap()
13098 .as_path()
13099 .to_path_buf();
13100 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
13101 let open_handler = handler.clone();
13102 let pending = tokio::spawn(async move {
13103 open_handler
13104 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
13105 .await
13106 .unwrap()
13107 });
13108 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
13109 .await
13110 .unwrap()
13111 .unwrap();
13112 let actual = query_live_roots(&handler, &target_ctx).await;
13113 let ModuleControlResponseToModule::LiveRoots {
13114 roots,
13115 unknown_root_bindings,
13116 total_bindings,
13117 } = actual
13118 else {
13119 panic!("expected live roots")
13120 };
13121 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
13122 assert_eq!(unknown_root_bindings, 0);
13123 assert_eq!(
13124 roots.len(),
13125 2,
13126 "root-known arm must retain each canonical root"
13127 );
13128 assert_eq!(
13129 total_bindings,
13130 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
13131 );
13132 let counts = roots
13133 .iter()
13134 .map(|root| (root.project_root.clone(), root.bound, root.pending))
13135 .collect::<Vec<_>>();
13136 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
13137 expected.sort_by(|a, b| a.0.cmp(&b.0));
13138 assert_eq!(
13139 counts, expected,
13140 "roots must sort by path and count pending separately"
13141 );
13142 handler
13143 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
13144 .await
13145 .unwrap();
13146 assert!(pending.await.unwrap().is_empty());
13147 }
13148
13149 #[tokio::test(start_paused = true)]
13150 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
13151 let forwarding = Arc::new(ForwardingTable::default());
13152 let handler =
13153 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
13154 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
13155 hello_via_sink(
13156 &handler,
13157 &target_ctx,
13158 &mut target_rx,
13159 hello_frame("target", PROTOCOL_VERSION, 1),
13160 )
13161 .await;
13162 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
13163 let pending = forwarding
13164 .begin_route_bind_relay_for_test(
13165 client_ctx.connection_id,
13166 client_ctx.egress.clone(),
13167 2,
13168 "target",
13169 )
13170 .unwrap();
13171 forwarding
13172 .complete_pending_relay(
13173 target_ctx.connection_id,
13174 pending.corr,
13175 RouteBindRelayOutcome::Accepted,
13176 )
13177 .unwrap();
13178 let actual = query_live_roots(&handler, &target_ctx).await;
13179 let ModuleControlResponseToModule::LiveRoots {
13180 roots,
13181 unknown_root_bindings,
13182 total_bindings,
13183 } = actual
13184 else {
13185 panic!("expected live roots")
13186 };
13187 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
13188 assert_eq!(
13189 unknown_root_bindings, 1,
13190 "unknown-root arm must not read as no bindings"
13191 );
13192 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
13193 assert_eq!(
13194 total_bindings,
13195 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
13196 );
13197 }
13198
13199 #[tokio::test(start_paused = true)]
13207 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
13208 let forwarding = Arc::new(ForwardingTable::default());
13209 let handler =
13210 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
13211 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
13212 let replies = handler
13213 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
13214 .await
13215 .unwrap();
13216 let queued_by_hello = module_rx.len();
13217
13218 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
13219 let open_handler = handler.clone();
13220 let open = tokio::spawn(async move {
13221 open_handler
13222 .handle_control_frame(
13223 &client_ctx,
13224 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
13225 )
13226 .await
13227 .unwrap()
13228 });
13229 let mut spins = 0;
13231 while module_rx.len() == queued_by_hello {
13232 spins += 1;
13233 assert!(spins < 10_000, "route.open never queued a route.bind");
13234 tokio::task::yield_now().await;
13235 }
13236
13237 for reply in replies {
13239 module_ctx.egress.send(reply).await.unwrap();
13240 }
13241
13242 let first = module_rx.recv().await.unwrap().frame;
13243 assert_eq!(
13244 first.header.ty,
13245 FrameType::HelloAck,
13246 "the first frame a registering module reads must be its HELLO_ACK"
13247 );
13248 assert_eq!(first.header.corr, 7);
13249 let second = module_rx.recv().await.unwrap().frame;
13250 assert_eq!(second.header.ty, FrameType::Request);
13251 assert!(
13252 matches!(
13253 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
13254 ModuleControlRequest::RouteBind { .. }
13255 ),
13256 "the route.bind follows the ack"
13257 );
13258 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
13259
13260 handler
13261 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
13262 .await
13263 .unwrap();
13264 assert!(open.await.unwrap().is_empty());
13265 let _ = client_rx.recv().await.unwrap();
13266 }
13267
13268 #[tokio::test(start_paused = true)]
13269 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
13270 let handler = ControlHandler::with_forwarding(
13271 Arc::new(Registry::default()),
13272 Arc::new(ForwardingTable::default()),
13273 );
13274 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
13275 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
13276 hello_via_sink(
13277 &handler,
13278 &first_ctx,
13279 &mut first_rx,
13280 hello_frame("first", PROTOCOL_VERSION, 1),
13281 )
13282 .await;
13283 hello_via_sink(
13284 &handler,
13285 &second_ctx,
13286 &mut second_rx,
13287 hello_frame("second", PROTOCOL_VERSION, 2),
13288 )
13289 .await;
13290 let root = unique_project_root("second-only");
13291 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
13292 let cloned = handler.clone();
13293 let open = tokio::spawn(async move {
13294 cloned
13295 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
13296 .await
13297 .unwrap()
13298 });
13299 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
13300 .await
13301 .unwrap()
13302 .unwrap();
13303 let first = query_live_roots(&handler, &first_ctx).await;
13304 let second = query_live_roots(&handler, &second_ctx).await;
13305 assert!(
13306 matches!(
13307 first,
13308 ModuleControlResponseToModule::LiveRoots {
13309 total_bindings: 0,
13310 ..
13311 }
13312 ),
13313 "cross-module scope must not expose another module's roots"
13314 );
13315 assert!(
13316 matches!(
13317 second,
13318 ModuleControlResponseToModule::LiveRoots {
13319 total_bindings: 1,
13320 ..
13321 }
13322 ),
13323 "second module must see its pending route"
13324 );
13325 handler
13326 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
13327 .await
13328 .unwrap();
13329 assert!(open.await.unwrap().is_empty());
13330 }
13331
13332 #[tokio::test(start_paused = true)]
13333 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
13334 let handler = ControlHandler::with_forwarding(
13335 Arc::new(Registry::default()),
13336 Arc::new(ForwardingTable::default()),
13337 );
13338 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
13339 hello_via_sink(
13340 &handler,
13341 &target_ctx,
13342 &mut target_rx,
13343 hello_frame("target", PROTOCOL_VERSION, 1),
13344 )
13345 .await;
13346 let actual = query_live_roots(&handler, &target_ctx).await;
13347 let ModuleControlResponseToModule::LiveRoots {
13348 roots,
13349 unknown_root_bindings,
13350 total_bindings,
13351 } = actual
13352 else {
13353 panic!("expected live roots")
13354 };
13355 assert!(roots.is_empty());
13356 assert_eq!(unknown_root_bindings, 0);
13357 assert_eq!(total_bindings, 0);
13358 assert_eq!(
13359 total_bindings,
13360 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
13361 );
13362 }
13363
13364 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
13371 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
13372 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
13373 let text = std::fs::read_to_string(&path)
13374 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
13375 let vectors: Vec<(String, Value)> = text
13376 .lines()
13377 .filter(|line| !line.trim().is_empty())
13378 .map(|line| {
13379 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
13380 let id = entry["corpus_id"]
13381 .as_str()
13382 .expect("every vector carries a corpus_id")
13383 .to_string();
13384 (id, entry["package"].clone())
13385 })
13386 .collect();
13387 assert_eq!(
13391 vectors.len(),
13392 3,
13393 "vendored fed corpus changed size; re-sync from subc-federation"
13394 );
13395
13396 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
13411 let richest = vectors
13412 .iter()
13413 .filter_map(|(_, package)| package.as_object())
13414 .map(|object| {
13415 object
13416 .keys()
13417 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
13418 .count()
13419 })
13420 .max()
13421 .unwrap_or(0);
13422 assert!(
13423 richest >= 2,
13424 "vendored corpus no longer carries a package with unmodelled fields, \
13425 so the relay test can no longer distinguish a verbatim relay from a lossy one"
13426 );
13427
13428 vectors
13429 }
13430
13431 #[tokio::test]
13439 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
13440 for (corpus_id, package) in fed_admission_facts_vectors() {
13441 let registry = Arc::new(Registry::default());
13442 let forwarding = Arc::new(ForwardingTable::default());
13443 let supervisor = SupervisorHandle::new();
13444 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
13445 let handler =
13446 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13447 .with_supervisor(supervisor)
13448 .with_admission_facts_config(
13449 Some("fed".to_string()),
13450 Some(vec!["target".to_string()]),
13451 );
13452
13453 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
13454 hello_via_sink(
13455 &handler,
13456 &target_ctx,
13457 &mut target_rx,
13458 hello_frame("target", PROTOCOL_VERSION, 1),
13459 )
13460 .await;
13461
13462 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
13463 let route_handler = handler.clone();
13464 let expected = package.clone();
13465 let route_task = tokio::spawn(async move {
13466 route_handler
13467 .handle_control_frame(
13468 &client_ctx,
13469 route_open_frame_with_admission_facts(
13470 20,
13471 "target",
13472 unique_project_root("admission-facts"),
13473 Some(subc_control::ConsumerIdentity {
13474 module_id: "fed".to_string(),
13475 launch_nonce: "fed-nonce".to_string(),
13476 }),
13477 Some(package),
13478 ),
13479 )
13480 .await
13481 .unwrap()
13482 });
13483
13484 let bind_frame = target_rx.recv().await.unwrap();
13485 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13486 let ModuleControlRequest::RouteBind {
13487 admission_facts, ..
13488 } = bind
13489 else {
13490 panic!("{corpus_id}: expected route.bind")
13491 };
13492 assert_eq!(
13493 admission_facts,
13494 Some(expected),
13495 "{corpus_id}: relay must not add, drop or reshape any field"
13496 );
13497
13498 handler
13499 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13500 .await
13501 .unwrap();
13502 route_task.await.unwrap();
13503 }
13504 }
13505
13506 #[tokio::test]
13507 async fn route_bind_never_relays_the_openers_build_class() {
13508 let registry = Arc::new(Registry::default());
13509 let forwarding = Arc::new(ForwardingTable::default());
13510 let handler =
13511 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13512 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
13513 hello_via_sink(
13514 &handler,
13515 &target_ctx,
13516 &mut target_rx,
13517 hello_frame("target", PROTOCOL_VERSION, 1),
13518 )
13519 .await;
13520
13521 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
13522 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
13523 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13524 busy_refusals: None,
13525 target: RouteTarget::ToolProvider {
13526 module_id: "target".to_string(),
13527 },
13528 identity,
13529 consumer_identity: None,
13530 consumer_capabilities: None,
13531 role_versions: None,
13532 admission_facts: None,
13533 scope: None,
13534 })
13535 .unwrap();
13536 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
13537
13538 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
13539 let route_handler = handler.clone();
13540 let route_task = tokio::spawn(async move {
13541 route_handler
13542 .handle_control_frame(&client_ctx, open)
13543 .await
13544 .unwrap()
13545 });
13546 let bind_frame = target_rx.recv().await.unwrap();
13547 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13548 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
13549 panic!("expected route.bind")
13550 };
13551 assert_eq!(
13552 identity.build_class, None,
13553 "route.bind carried the build class"
13554 );
13555 assert!(
13556 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
13557 "the route.bind bytes still name build_class"
13558 );
13559 handler
13560 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13561 .await
13562 .unwrap();
13563 route_task.await.unwrap();
13564 }
13565
13566 #[tokio::test]
13567 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
13568 let registry = Arc::new(Registry::default());
13569 let forwarding = Arc::new(ForwardingTable::default());
13570 let supervisor = SupervisorHandle::new();
13571 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
13572 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
13573 let handler =
13574 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13575 .with_supervisor(supervisor)
13576 .with_admission_facts_config(
13577 Some("fed".to_string()),
13578 Some(vec!["target".to_string()]),
13579 );
13580
13581 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
13582 hello_via_sink(
13583 &handler,
13584 &target_ctx,
13585 &mut target_rx,
13586 hello_frame("target", PROTOCOL_VERSION, 1),
13587 )
13588 .await;
13589 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
13590 hello_via_sink(
13591 &handler,
13592 &other_ctx,
13593 &mut other_rx,
13594 hello_frame("other", PROTOCOL_VERSION, 2),
13595 )
13596 .await;
13597
13598 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
13599 let expected_facts = facts.clone();
13600 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
13601 let route_handler = handler.clone();
13602 let route_task = tokio::spawn(async move {
13603 route_handler
13604 .handle_control_frame(
13605 &client_ctx,
13606 route_open_frame_with_admission_facts(
13607 10,
13608 "target",
13609 unique_project_root("admission-facts"),
13610 Some(subc_control::ConsumerIdentity {
13611 module_id: "fed".to_string(),
13612 launch_nonce: "fed-nonce".to_string(),
13613 }),
13614 Some(facts.clone()),
13615 ),
13616 )
13617 .await
13618 .unwrap()
13619 });
13620 let bind_frame = target_rx.recv().await.unwrap();
13621 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13622 let ModuleControlRequest::RouteBind {
13623 admission_facts, ..
13624 } = bind
13625 else {
13626 panic!("expected route.bind")
13627 };
13628 assert_eq!(admission_facts, Some(expected_facts));
13629 handler
13630 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13631 .await
13632 .unwrap();
13633 assert!(route_task.await.unwrap().is_empty());
13634 assert!(matches!(
13635 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
13636 .unwrap(),
13637 ClientControlResponse::RouteOpen { .. }
13638 ));
13639
13640 let direct = handler
13641 .handle_control_frame(
13642 &route_ctx(ConnectionId::new(73)).0,
13643 route_open_frame_with_admission_facts(
13644 11,
13645 "target",
13646 unique_project_root("admission-facts"),
13647 None,
13648 Some(json!({"x": 1})),
13649 ),
13650 )
13651 .await
13652 .unwrap();
13653 assert_eq!(
13654 parse_error(&direct[0])["code"],
13655 "admission_facts_not_permitted"
13656 );
13657
13658 let different_reserved = handler
13659 .handle_control_frame(
13660 &route_ctx(ConnectionId::new(77)).0,
13661 route_open_frame_with_admission_facts(
13662 15,
13663 "target",
13664 unique_project_root("admission-facts"),
13665 Some(subc_control::ConsumerIdentity {
13666 module_id: "other".to_string(),
13667 launch_nonce: "other-nonce".to_string(),
13668 }),
13669 Some(json!({"x": 1})),
13670 ),
13671 )
13672 .await
13673 .unwrap();
13674 assert_eq!(
13675 parse_error(&different_reserved[0])["code"],
13676 "admission_facts_not_permitted"
13677 );
13678
13679 let other_target = handler
13680 .handle_control_frame(
13681 &route_ctx(ConnectionId::new(74)).0,
13682 route_open_frame_with_admission_facts(
13683 12,
13684 "other",
13685 unique_project_root("admission-facts"),
13686 Some(subc_control::ConsumerIdentity {
13687 module_id: "fed".to_string(),
13688 launch_nonce: "fed-nonce".to_string(),
13689 }),
13690 Some(json!({"x": 1})),
13691 ),
13692 )
13693 .await
13694 .unwrap();
13695 assert_eq!(
13696 parse_error(&other_target[0])["code"],
13697 "admission_facts_target_not_allowed"
13698 );
13699
13700 let nonexistent = handler
13701 .handle_control_frame(
13702 &route_ctx(ConnectionId::new(75)).0,
13703 route_open_frame_with_admission_facts(
13704 13,
13705 "missing",
13706 unique_project_root("admission-facts"),
13707 None,
13708 Some(json!({"x": 1})),
13709 ),
13710 )
13711 .await
13712 .unwrap();
13713 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13714
13715 let described = handler
13716 .handle_control_frame(
13717 &route_ctx(ConnectionId::new(76)).0,
13718 Frame::build(
13719 FrameType::Request,
13720 control_flags(),
13721 0,
13722 0,
13723 14,
13724 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13725 )
13726 .unwrap(),
13727 )
13728 .await
13729 .unwrap();
13730 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13731 serde_json::from_slice(&described[0].body).unwrap()
13732 else {
13733 panic!("expected server.describe response")
13734 };
13735 assert!(capabilities
13736 .iter()
13737 .any(|cap| cap == "admission_facts_relay_v1"));
13738 }
13739
13740 #[tokio::test]
13741 async fn admission_facts_without_configured_carrier_are_rejected() {
13742 let registry = Arc::new(Registry::default());
13743 let forwarding = Arc::new(ForwardingTable::default());
13744 let handler = ControlHandler::with_forwarding(registry, forwarding);
13745 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13746 hello_via_sink(
13747 &handler,
13748 &target_ctx,
13749 &mut target_rx,
13750 hello_frame("target", PROTOCOL_VERSION, 1),
13751 )
13752 .await;
13753
13754 let responses = handler
13755 .handle_control_frame(
13756 &route_ctx(ConnectionId::new(79)).0,
13757 route_open_frame_with_admission_facts(
13758 16,
13759 "target",
13760 unique_project_root("admission-facts"),
13761 None,
13762 Some(json!({"x": 1})),
13763 ),
13764 )
13765 .await
13766 .unwrap();
13767 assert_eq!(
13768 parse_error(&responses[0])["code"],
13769 "admission_facts_not_permitted"
13770 );
13771 }
13772
13773 #[tokio::test]
13774 async fn route_open_relays_consumer_capabilities_verbatim() {
13775 let registry = Arc::new(Registry::default());
13776 let forwarding = Arc::new(ForwardingTable::default());
13777 let handler =
13778 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13779 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13780 hello_via_sink(
13781 &handler,
13782 &module_ctx,
13783 &mut module_rx,
13784 hello_frame("aft", PROTOCOL_VERSION, 7),
13785 )
13786 .await;
13787
13788 let expected = vec!["elicitation".to_string(), "roots".to_string()];
13789 let expected_for_request = expected.clone();
13790 let project_root = unique_project_root("consumer-capabilities-present");
13791 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13792 let route_handler = handler.clone();
13793 let route_task = tokio::spawn(async move {
13794 route_handler
13795 .handle_control_frame(
13796 &client_ctx,
13797 route_open_frame_with_consumer_capabilities(
13798 401,
13799 "aft",
13800 project_root,
13801 Some(expected_for_request),
13802 ),
13803 )
13804 .await
13805 .unwrap()
13806 });
13807 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13808 .await
13809 .unwrap()
13810 .unwrap();
13811 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13812 let ModuleControlRequest::RouteBind {
13813 consumer_capabilities,
13814 ..
13815 } = bind
13816 else {
13817 panic!("expected route.bind request, got {bind:?}");
13818 };
13819 assert_eq!(consumer_capabilities, Some(expected.clone()));
13820
13821 handler
13822 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13823 .await
13824 .unwrap();
13825 let route_response = route_task.await.unwrap();
13826 assert!(route_response.is_empty());
13827 let published = client_rx.recv().await.unwrap();
13828 assert!(matches!(
13829 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13830 ClientControlResponse::RouteOpen { .. }
13831 ));
13832 }
13833
13834 #[tokio::test]
13835 async fn route_open_without_consumer_capabilities_relays_none() {
13836 let registry = Arc::new(Registry::default());
13837 let forwarding = Arc::new(ForwardingTable::default());
13838 let handler =
13839 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13840 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13841 hello_via_sink(
13842 &handler,
13843 &module_ctx,
13844 &mut module_rx,
13845 hello_frame("aft", PROTOCOL_VERSION, 7),
13846 )
13847 .await;
13848
13849 let project_root = unique_project_root("consumer-capabilities-absent");
13850 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13851 let route_handler = handler.clone();
13852 let route_task = tokio::spawn(async move {
13853 route_handler
13854 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13855 .await
13856 .unwrap()
13857 });
13858 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13859 .await
13860 .unwrap()
13861 .unwrap();
13862 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13863 let ModuleControlRequest::RouteBind {
13864 consumer_capabilities,
13865 ..
13866 } = bind
13867 else {
13868 panic!("expected route.bind request, got {bind:?}");
13869 };
13870 assert_eq!(consumer_capabilities, None);
13871
13872 handler
13873 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13874 .await
13875 .unwrap();
13876 let route_response = route_task.await.unwrap();
13877 assert!(route_response.is_empty());
13878 let published = client_rx.recv().await.unwrap();
13879 assert!(matches!(
13880 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13881 ClientControlResponse::RouteOpen { .. }
13882 ));
13883 }
13884
13885 async fn bind_role_versions_for(
13889 sent: Option<BTreeMap<String, String>>,
13890 connection: u64,
13891 ) -> Option<BTreeMap<String, String>> {
13892 let registry = Arc::new(Registry::default());
13893 let forwarding = Arc::new(ForwardingTable::default());
13894 let handler =
13895 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13896 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13897 hello_via_sink(
13898 &handler,
13899 &module_ctx,
13900 &mut module_rx,
13901 hello_frame("aft", PROTOCOL_VERSION, 7),
13902 )
13903 .await;
13904 let project_root = unique_project_root("role-versions");
13905 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13906 let route_handler = handler.clone();
13907 let route_task = tokio::spawn(async move {
13908 route_handler
13909 .handle_control_frame(
13910 &client_ctx,
13911 route_open_frame_with_role_versions(403, "aft", project_root, sent),
13912 )
13913 .await
13914 .unwrap()
13915 });
13916 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13917 .await
13918 .expect("a well-formed route.open reaches the module as a bind")
13919 .unwrap();
13920 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13921 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13922 panic!("expected route.bind request, got {bind:?}");
13923 };
13924 handler
13925 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13926 .await
13927 .unwrap();
13928 assert!(route_task.await.unwrap().is_empty());
13929 let published = client_rx.recv().await.unwrap();
13930 assert!(matches!(
13931 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13932 ClientControlResponse::RouteOpen { .. }
13933 ));
13934 role_versions
13935 }
13936
13937 #[tokio::test]
13938 async fn route_open_relays_role_versions_verbatim() {
13939 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13940 assert_eq!(
13941 bind_role_versions_for(Some(sent.clone()), 141).await,
13942 Some(sent)
13943 );
13944 }
13945
13946 #[tokio::test]
13949 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13950 assert_eq!(bind_role_versions_for(None, 143).await, None);
13951 assert_eq!(
13952 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13953 None
13954 );
13955 }
13956
13957 #[tokio::test]
13958 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13959 let registry = Arc::new(Registry::default());
13960 let forwarding = Arc::new(ForwardingTable::default());
13961 let handler =
13962 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13963 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13964 hello_via_sink(
13965 &handler,
13966 &module_ctx,
13967 &mut module_rx,
13968 hello_frame("aft", PROTOCOL_VERSION, 7),
13969 )
13970 .await;
13971
13972 let nine: BTreeMap<String, String> = (0..9)
13973 .map(|index| (format!("role-{index}"), "v1".to_string()))
13974 .collect();
13975 for (label, malformed) in [
13976 (
13977 "invalid role name",
13978 role_versions(&[("Tool_Provider", "v1")]),
13979 ),
13980 ("invalid version", role_versions(&[("tool-provider", "v0")])),
13981 ("nine entries", nine),
13982 ] {
13983 let responses = tokio::time::timeout(
13986 Duration::from_secs(1),
13987 handler.handle_control_frame(
13988 &route_ctx(ConnectionId::new(148)).0,
13989 route_open_frame_with_role_versions(
13990 404,
13991 "aft",
13992 unique_project_root("role-versions-malformed"),
13993 Some(malformed),
13994 ),
13995 ),
13996 )
13997 .await
13998 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13999 .unwrap();
14000 assert_eq!(responses.len(), 1, "{label}");
14001 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
14002 let error = parse_error(&responses[0]);
14003 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
14004 assert_eq!(
14005 error["detail"]["field"], "role_versions",
14006 "{label}: {error}"
14007 );
14008 assert!(
14009 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
14010 "{label}: a malformed declaration is terminal"
14011 );
14012 assert!(
14013 module_rx.try_recv().is_err(),
14014 "{label}: the module must never see a bind"
14015 );
14016 }
14017 }
14018
14019 #[tokio::test]
14023 async fn route_role_versions_capability_is_advertised() {
14024 let handler = ControlHandler::new(Arc::new(Registry::default()));
14025 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
14026 let ack = hello_via_sink(
14027 &handler,
14028 &ctx,
14029 &mut rx,
14030 hello_frame("m", PROTOCOL_VERSION, 1),
14031 )
14032 .await;
14033 let ack = parse_ack(&ack);
14034 assert!(
14035 ack.subc_capabilities
14036 .iter()
14037 .any(|c| c == "route-role-versions/v1"),
14038 "{:?}",
14039 ack.subc_capabilities
14040 );
14041
14042 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14043 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14044 let reply = handler
14045 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
14046 .await
14047 .unwrap()
14048 .pop()
14049 .unwrap();
14050 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14051 serde_json::from_slice(&reply.body).unwrap()
14052 else {
14053 panic!("not a server.describe reply");
14054 };
14055 assert!(
14056 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
14057 "{capabilities:?}"
14058 );
14059 }
14060
14061 #[tokio::test]
14065 async fn catalog_events_capability_is_advertised() {
14066 let handler = ControlHandler::new(Arc::new(Registry::default()));
14067 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14068 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14069 let reply = handler
14070 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
14071 .await
14072 .unwrap()
14073 .pop()
14074 .unwrap();
14075 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14076 serde_json::from_slice(&reply.body).unwrap()
14077 else {
14078 panic!("not a server.describe reply");
14079 };
14080 assert!(
14081 capabilities
14082 .iter()
14083 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
14084 "{capabilities:?}"
14085 );
14086 }
14087
14088 #[tokio::test]
14089 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
14090 let handler = ControlHandler::new(Arc::new(Registry::default()));
14091 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
14092 let ack = hello_via_sink(
14093 &handler,
14094 &module,
14095 &mut module_rx,
14096 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
14097 )
14098 .await;
14099 let ack = parse_ack(&ack);
14100 assert!(
14101 ack.subc_capabilities
14102 .iter()
14103 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
14104 "{:?}",
14105 ack.subc_capabilities
14106 );
14107 assert!(ack
14108 .subc_capabilities
14109 .iter()
14110 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
14111
14112 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
14113 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14114 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14115 let reply = handler
14116 .handle_control_frame(&client, frame)
14117 .await
14118 .unwrap()
14119 .pop()
14120 .unwrap();
14121 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14122 serde_json::from_slice(&reply.body).unwrap()
14123 else {
14124 panic!("not a server.describe reply");
14125 };
14126 assert!(
14127 capabilities
14128 .iter()
14129 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
14130 "{capabilities:?}"
14131 );
14132 assert!(capabilities
14133 .iter()
14134 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
14135 }
14136
14137 #[tokio::test]
14138 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
14139 let handler = ControlHandler::new(Arc::new(Registry::default()));
14140 let (module, mut rx) = route_ctx(ConnectionId::new(156));
14141 let ack = hello_via_sink(
14142 &handler,
14143 &module,
14144 &mut rx,
14145 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
14146 )
14147 .await;
14148 assert!(parse_ack(&ack)
14149 .subc_capabilities
14150 .iter()
14151 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
14152 assert!(parse_ack(&ack)
14153 .subc_capabilities
14154 .iter()
14155 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
14156 let (client, _rx) = route_ctx(ConnectionId::new(157));
14157 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14158 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14159 let reply = handler
14160 .handle_control_frame(&client, frame)
14161 .await
14162 .unwrap()
14163 .pop()
14164 .unwrap();
14165 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14166 serde_json::from_slice(&reply.body).unwrap()
14167 else {
14168 panic!("server.describe reply")
14169 };
14170 assert!(capabilities
14171 .iter()
14172 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
14173 assert!(capabilities
14174 .iter()
14175 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
14176 }
14177
14178 #[tokio::test]
14179 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
14180 let handler = ControlHandler::new(Arc::new(Registry::default()));
14181 let (module, mut rx) = route_ctx(ConnectionId::new(158));
14182 let ack = hello_via_sink(
14183 &handler,
14184 &module,
14185 &mut rx,
14186 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
14187 )
14188 .await;
14189 assert!(parse_ack(&ack)
14190 .subc_capabilities
14191 .iter()
14192 .any(|cap| cap == "scope-workdir/v1"));
14193 let (client, _rx) = route_ctx(ConnectionId::new(159));
14194 let frame = Frame::build(
14195 FrameType::Request,
14196 control_flags(),
14197 0,
14198 0,
14199 5,
14200 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
14201 )
14202 .unwrap();
14203 let reply = handler
14204 .handle_control_frame(&client, frame)
14205 .await
14206 .unwrap()
14207 .remove(0);
14208 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14209 serde_json::from_slice(&reply.body).unwrap()
14210 else {
14211 panic!("server.describe reply")
14212 };
14213 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
14214 }
14215
14216 #[tokio::test]
14217 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
14218 let handler = ControlHandler::new(Arc::new(Registry::default()));
14219 let (module, mut rx) = route_ctx(ConnectionId::new(160));
14220 let ack = hello_via_sink(
14221 &handler,
14222 &module,
14223 &mut rx,
14224 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
14225 )
14226 .await;
14227 assert!(parse_ack(&ack)
14228 .subc_capabilities
14229 .iter()
14230 .any(|cap| cap == "worker-runs/v1"));
14231 let (client, _rx) = route_ctx(ConnectionId::new(161));
14232 let frame = Frame::build(
14233 FrameType::Request,
14234 control_flags(),
14235 0,
14236 0,
14237 5,
14238 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
14239 )
14240 .unwrap();
14241 let reply = handler
14242 .handle_control_frame(&client, frame)
14243 .await
14244 .unwrap()
14245 .remove(0);
14246 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14247 serde_json::from_slice(&reply.body).unwrap()
14248 else {
14249 panic!("server.describe reply")
14250 };
14251 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
14252 }
14253
14254 #[tokio::test]
14255 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
14256 let handler = ControlHandler::new(Arc::new(Registry::default()));
14257 let (module, mut rx) = route_ctx(ConnectionId::new(154));
14258 let ack = hello_via_sink(
14259 &handler,
14260 &module,
14261 &mut rx,
14262 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
14263 )
14264 .await;
14265 assert!(parse_ack(&ack)
14266 .subc_capabilities
14267 .iter()
14268 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
14269 let (client, _rx) = route_ctx(ConnectionId::new(155));
14270 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14271 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14272 let reply = handler
14273 .handle_control_frame(&client, frame)
14274 .await
14275 .unwrap()
14276 .pop()
14277 .unwrap();
14278 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14279 serde_json::from_slice(&reply.body).unwrap()
14280 else {
14281 panic!("not a server.describe reply")
14282 };
14283 assert!(capabilities
14284 .iter()
14285 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
14286 }
14287
14288 #[tokio::test]
14289 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
14290 let registry = Arc::new(Registry::default());
14291 let forwarding = Arc::new(ForwardingTable::default());
14292 let handler =
14293 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14294 .with_health_probe_timeout(Duration::from_secs(5));
14295 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
14296 hello_via_sink(
14297 &handler,
14298 &module_ctx,
14299 &mut module_rx,
14300 non_routable_hello_frame_with_control_ops(
14301 "mcp",
14302 300,
14303 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14304 ),
14305 )
14306 .await;
14307 assert!(registry
14308 .get_module("mcp")
14309 .unwrap()
14310 .unwrap()
14311 .manifest
14312 .provides
14313 .is_empty());
14314
14315 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
14316 let route_response = handler
14317 .handle_control_frame(
14318 &route_client_ctx,
14319 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
14320 )
14321 .await
14322 .unwrap();
14323 assert_eq!(route_response[0].header.ty, FrameType::Error);
14324 assert_eq!(
14325 parse_error(&route_response[0])["code"],
14326 "target_unavailable"
14327 );
14328 assert!(parse_error(&route_response[0])["message"]
14329 .as_str()
14330 .unwrap()
14331 .contains("does not provide the requested target"));
14332 assert!(module_rx.try_recv().is_err());
14333
14334 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
14335 let health_handler = handler.clone();
14336 let health_task = tokio::spawn(async move {
14337 health_handler
14338 .handle_control_frame(
14339 &health_client_ctx,
14340 supervisor_health_probe_frame(302, "mcp"),
14341 )
14342 .await
14343 .unwrap()
14344 });
14345 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14346 .await
14347 .unwrap()
14348 .unwrap();
14349 assert_eq!(
14350 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
14351 ModuleControlRequest::HealthCheck {}
14352 );
14353 handler
14354 .handle_control_frame(
14355 &module_ctx,
14356 health_response(health_frame.header.corr, HealthStatus::Ok),
14357 )
14358 .await
14359 .unwrap();
14360 let health_response = health_task.await.unwrap();
14361 assert_eq!(health_response[0].header.ty, FrameType::Response);
14362 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
14363 ClientControlResponse::SupervisorHealthProbe {
14364 module_id, status, ..
14365 } => {
14366 assert_eq!(module_id, "mcp");
14367 assert_eq!(status, HealthStatus::Ok);
14368 }
14369 other => panic!("unexpected health response: {other:?}"),
14370 }
14371
14372 forwarding
14377 .cleanup_connection(module_ctx.connection_id)
14378 .unwrap();
14379 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
14380 let cleanup_response = tokio::time::timeout(
14381 Duration::from_millis(200),
14382 handler.handle_control_frame(
14383 &cleanup_probe_ctx,
14384 supervisor_health_probe_frame(303, "mcp"),
14385 ),
14386 )
14387 .await
14388 .expect("probe should fail immediately when the control lane is gone")
14389 .unwrap();
14390 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
14391 assert_eq!(
14392 parse_error(&cleanup_response[0])["code"],
14393 "target_unavailable"
14394 );
14395 assert!(parse_error(&cleanup_response[0])["message"]
14396 .as_str()
14397 .unwrap()
14398 .contains("no module connection"));
14399
14400 handler
14401 .cleanup_connection(module_ctx.connection_id)
14402 .unwrap();
14403 }
14404
14405 #[tokio::test]
14406 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
14407 let registry = Arc::new(Registry::default());
14408 let supervisor_handle = SupervisorHandle::new();
14409 let supervisor =
14410 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14411 .with_handle(supervisor_handle.clone())
14412 .with_connection_file_path(
14413 std::env::temp_dir()
14414 .join(format!("subc-route-open-warming-{}", std::process::id())),
14415 );
14416 let module = supervisor
14417 .supervise_configured(
14418 ModuleSpec {
14419 carry_containment: false,
14420 module_id: "warming".to_string(),
14421 program: fake_aft_stub_path(),
14422 args: Vec::new(),
14423 env: Vec::new(),
14424 reserved: false,
14425 reserved_prefixes: Vec::new(),
14426 protocol: ModuleProtocol::Subc,
14427 overlap: Default::default(),
14428 },
14429 true,
14430 )
14431 .unwrap();
14432 assert_eq!(module.state().unwrap(), ModuleState::Running);
14433
14434 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14435 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
14436 let response = handler
14437 .handle_control_frame(
14438 &ctx,
14439 route_open_frame(304, "warming", unique_project_root("warming")),
14440 )
14441 .await
14442 .unwrap();
14443 module.stop().await.unwrap();
14444
14445 assert_eq!(response[0].header.ty, FrameType::Error);
14446 let error = parse_error(&response[0]);
14447 assert_eq!(error["code"], "module_warming");
14448 assert!(error["message"]
14449 .as_str()
14450 .unwrap()
14451 .contains("state=running, enabled=true, live=false"));
14452 }
14453
14454 #[test]
14455 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
14456 let handler = ControlHandler::new(Arc::new(Registry::default()));
14457 let capture = EventCapture::default();
14458 let _subscriber =
14459 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14460 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
14461 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
14462 let pending = (0..limit).collect::<Vec<_>>();
14463 let response = handler
14464 .route_open_capacity_refusal(
14465 &ctx,
14466 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
14467 "busy",
14468 pending.len(),
14469 limit,
14470 )
14471 .unwrap();
14472 assert_eq!(parse_error(&response)["code"], "target_unavailable");
14473 let event = capture
14474 .events()
14475 .into_iter()
14476 .find(|event| {
14477 event.target == "control"
14478 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
14479 })
14480 .expect("connection admission refusal event");
14481 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14482 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14483 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14486 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14487 }
14488
14489 #[test]
14490 fn route_open_target_cap_logs_admission_reason_and_capacity() {
14491 let handler = ControlHandler::new(Arc::new(Registry::default()));
14492 let capture = EventCapture::default();
14493 let _subscriber =
14494 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14495 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
14496 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
14497 let guards = (0..limit)
14498 .map(|_| {
14499 handler
14500 .route_bind_concurrency
14501 .try_admit("busy", limit)
14502 .unwrap()
14503 })
14504 .collect::<Vec<_>>();
14505 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
14506 Err(in_flight) => in_flight,
14507 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
14508 };
14509 let response = handler
14510 .route_open_target_capacity_refusal(
14511 &ctx,
14512 &RouteOpenCaller::unknown(),
14513 &route_open_frame(397, "busy", unique_project_root("target-cap")),
14514 "busy",
14515 in_flight,
14516 )
14517 .unwrap();
14518 assert_eq!(parse_error(&response)["code"], "target_unavailable");
14519 let event = capture
14520 .events()
14521 .into_iter()
14522 .find(|event| {
14523 event.target == "control"
14524 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
14525 })
14526 .expect("target admission refusal event");
14527 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14528 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14529 drop(guards);
14530 }
14531
14532 #[cfg(unix)]
14533 #[tokio::test(start_paused = true)]
14534 async fn supervisor_list_shows_a_failed_modules_next_retry() {
14535 let registry = Arc::new(Registry::default());
14536 let handle = SupervisorHandle::new();
14537 let supervisor =
14538 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
14539 .with_handle(handle.clone());
14540 let module = supervisor
14541 .spawn(crate::ModuleSpec {
14542 carry_containment: false,
14543 module_id: "retry-list".into(),
14544 program: PathBuf::from("/bin/sh"),
14545 args: vec!["-c".into(), "exit 17".into()],
14546 env: vec![],
14547 reserved: false,
14548 reserved_prefixes: vec![],
14549 protocol: ModuleProtocol::None,
14550 overlap: Default::default(),
14551 })
14552 .unwrap();
14553 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14554 while module.state().unwrap() != crate::ModuleState::Failed {
14555 assert!(
14556 std::time::Instant::now() < deadline,
14557 "waiting for budget failure"
14558 );
14559 tokio::task::yield_now().await;
14560 }
14561 let handler = ControlHandler::new(registry).with_supervisor(handle);
14562 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14563 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14564 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14565 assert_eq!(body["modules"][0]["state"], "failed");
14566 let retry = body["modules"][0]["next_retry_at_ms"]
14567 .as_u64()
14568 .expect("scheduled retry on the wire");
14569 let now = std::time::SystemTime::now()
14570 .duration_since(std::time::UNIX_EPOCH)
14571 .unwrap()
14572 .as_millis() as u64;
14573 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
14574 module.set_enabled(false).await.unwrap();
14575 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14576 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14577 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14578 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
14579 }
14580
14581 #[tokio::test]
14582 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
14583 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
14584 let supervisor =
14585 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14586 let module = supervisor
14587 .spawn(crate::ModuleSpec {
14588 carry_containment: false,
14589 module_id: "pid-read-budget".into(),
14590 program: std::env::current_exe().unwrap(),
14591 args: vec![
14592 "--exact".into(),
14593 "control::tests::pid_resource_read_fixture".into(),
14594 ],
14595 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14596 .into_iter()
14597 .map(|key| (key.into(), dir.display().to_string()))
14598 .chain(std::iter::once((
14599 "SUBC_SUBTREE_PID_FIXTURE".into(),
14600 "1".into(),
14601 )))
14602 .collect(),
14603 reserved: false,
14604 reserved_prefixes: Vec::new(),
14605 protocol: ModuleProtocol::None,
14606 overlap: Default::default(),
14607 })
14608 .unwrap();
14609 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14610 while !matches!(
14611 module.child_resource_usage(),
14612 subc_control::ChildResourceUsage::Measured(_)
14613 ) {
14614 assert!(
14615 tokio::time::Instant::now() < ready_deadline,
14616 "fixture process was not measurable"
14617 );
14618 tokio::time::sleep(Duration::from_millis(10)).await;
14619 }
14620 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
14621 let reading = {
14622 let _guard = crate::subtree_resources::fault_read_guard().await;
14623 crate::subtree_resources::read_process(module.clone()).await
14624 };
14625 stop_resource_fixture(&module, &dir).await;
14626 assert!(
14627 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
14628 "{reading:?}"
14629 );
14630 }
14631
14632 #[test]
14633 fn pid_resource_read_fixture() {
14634 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
14635 let dir = std::path::PathBuf::from(std::env::var_os("XDG_DATA_HOME").unwrap());
14636 let deadline = std::time::Instant::now() + Duration::from_secs(60);
14637 while !dir.join("stop-module").exists() && std::time::Instant::now() < deadline {
14638 std::thread::sleep(Duration::from_millis(5));
14639 }
14640 std::fs::write(dir.join("module-stopped"), "").unwrap();
14641 }
14642 }
14643
14644 async fn stop_resource_fixture(
14645 module: &crate::supervise::SupervisedModule,
14646 dir: &std::path::Path,
14647 ) {
14648 let stopping = module.stop();
14649 tokio::pin!(stopping);
14650 tokio::select! {
14651 result = &mut stopping => result.unwrap(),
14652 () = async {
14653 while module.state().unwrap() != crate::supervise::ModuleState::Draining {
14654 tokio::time::sleep(Duration::from_millis(5)).await;
14655 }
14656 std::fs::write(dir.join("stop-module"), "").unwrap();
14657 } => stopping.await.unwrap(),
14658 }
14659 #[cfg(windows)]
14660 assert!(
14661 dir.join("module-stopped").exists(),
14662 "the resource fixture must exit cooperatively, not wait for the drain kill"
14663 );
14664 assert_eq!(
14665 module.state().unwrap(),
14666 crate::supervise::ModuleState::Stopped
14667 );
14668 }
14669
14670 #[tokio::test]
14671 async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
14672 let registry = Arc::new(Registry::default());
14673 let handle = SupervisorHandle::new();
14674 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14675 .with_handle(handle.clone());
14676 supervisor
14677 .supervise_configured(
14678 crate::ModuleSpec {
14679 module_id: "disabled-resource-reading".into(),
14680 program: std::env::current_exe().unwrap(),
14681 args: Vec::new(),
14682 env: Vec::new(),
14683 reserved: false,
14684 reserved_prefixes: Vec::new(),
14685 protocol: ModuleProtocol::None,
14686 overlap: Default::default(),
14687 carry_containment: false,
14688 },
14689 false,
14690 )
14691 .unwrap();
14692 let handler = ControlHandler::new(registry).with_supervisor(handle);
14693 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14694 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14695 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14696 assert!(body["modules"][0].get("tree").is_none());
14697 assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
14698 }
14699
14700 #[tokio::test]
14701 async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
14702 let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
14703 let registry = Arc::new(Registry::default());
14704 let handle = SupervisorHandle::new();
14705 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14706 .with_handle(handle.clone());
14707 let module = supervisor
14708 .spawn(crate::ModuleSpec {
14709 carry_containment: false,
14710 module_id: "list-process-priority".into(),
14711 program: std::env::current_exe().unwrap(),
14712 args: vec![
14713 "--exact".into(),
14714 "control::tests::pid_resource_read_fixture".into(),
14715 ],
14716 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14717 .into_iter()
14718 .map(|key| (key.into(), dir.display().to_string()))
14719 .chain(std::iter::once((
14720 "SUBC_SUBTREE_PID_FIXTURE".into(),
14721 "1".into(),
14722 )))
14723 .collect(),
14724 reserved: false,
14725 reserved_prefixes: Vec::new(),
14726 protocol: ModuleProtocol::None,
14727 overlap: Default::default(),
14728 })
14729 .unwrap();
14730 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14731 loop {
14732 let image = module.running_image_agreement().await;
14733 if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
14734 break;
14735 }
14736 assert!(
14737 tokio::time::Instant::now() < ready_deadline,
14738 "the fixture process never became observable: {image:?}"
14739 );
14740 tokio::time::sleep(Duration::from_millis(10)).await;
14741 }
14742 let guard = crate::subtree_resources::fault_read_guard().await;
14743 let (target, release, _) = crate::subtree_resources::held();
14744 module.set_subtree_target_for_test(target);
14745 let handler = ControlHandler::new(registry).with_supervisor(handle);
14746 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14747 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14748 drop(release);
14749 drop(guard);
14750 stop_resource_fixture(&module, &dir).await;
14751 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14752 assert_eq!(
14753 body["modules"][0]["resources"]["status"], "measured",
14754 "the optional tree timeout must not discard the process reading: {body}"
14755 );
14756 assert!(body["modules"][0].get("tree").is_none());
14757 assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
14758 assert_eq!(
14759 body["modules"][0]["pending_reload"]["image"]["status"], "match",
14760 "the tree timeout must not hide a completed image observation"
14761 );
14762 }
14763
14764 #[cfg(any(target_os = "linux", target_os = "macos"))]
14765 #[tokio::test]
14766 async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
14767 let mut handler = ControlHandler::new(Arc::new(Registry::default()))
14768 .with_daemon_provenance(
14769 std::process::id(),
14770 0,
14771 Some(std::env::current_exe().unwrap()),
14772 None,
14773 None,
14774 );
14775 let (probe, release, began) = ExecutableIdentityProbe::held();
14776 handler.daemon_provenance.probe = probe;
14777 let observe = |handler: ControlHandler, corr| {
14778 tokio::spawn(async move {
14779 let (ctx, _) = route_ctx(ConnectionId::new(corr));
14780 let frame =
14781 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
14782 .unwrap();
14783 handler
14784 .handle_supervisor_provenance(&ctx, frame, None)
14785 .await
14786 .unwrap()
14787 })
14788 };
14789 let explicit = observe(handler.clone(), 114);
14790 began.await.unwrap();
14791 let concurrent = observe(handler.clone(), 115);
14792 tokio::time::sleep(Duration::from_millis(300)).await;
14795 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14796 let list = tokio::time::timeout(
14797 Duration::from_secs(1),
14798 handler.handle_supervisor_list(frame),
14799 )
14800 .await;
14801 let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
14802 drop(release);
14803 let replies = explicit.await.unwrap();
14804 let concurrent_replies = concurrent.await.unwrap();
14805 assert!(
14806 explicit_was_pending,
14807 "explicit provenance must await the full observation"
14808 );
14809 let list = list
14810 .expect("a concurrent explicit probe must not stall list")
14811 .unwrap();
14812 let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
14813 assert_eq!(list_body["op"], "supervisor.list");
14814 for replies in [replies, concurrent_replies] {
14815 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14816 assert_eq!(
14817 body["daemon"]["daemon_observed"]["running_image"]["status"],
14818 "match"
14819 );
14820 }
14821 }
14822
14823 #[tokio::test]
14824 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
14825 let registry = Arc::new(Registry::default());
14826 let handle = SupervisorHandle::new();
14827 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14828 .with_handle(handle.clone());
14829 let module = supervisor
14830 .supervise_configured(
14831 crate::ModuleSpec {
14832 carry_containment: false,
14833 module_id: "memory-read".into(),
14834 program: PathBuf::from("disabled-module"),
14835 args: Vec::new(),
14836 env: Vec::new(),
14837 reserved: false,
14838 reserved_prefixes: Vec::new(),
14839 protocol: ModuleProtocol::Subc,
14840 overlap: Default::default(),
14841 },
14842 true,
14843 )
14844 .unwrap();
14845 let _guard = crate::subtree_resources::fault_read_guard().await;
14846 let (target, release, began) = crate::subtree_resources::held();
14847 let stalled_target = target.clone();
14850 let stalled = tokio::task::spawn_blocking(move || {
14851 stalled_target.tree(&subc_os::ProcessTable::default())
14852 });
14853 began.await.unwrap();
14854 module.set_subtree_target_for_test(target);
14855 let handler = ControlHandler::new(registry).with_supervisor(handle);
14856 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
14857 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14858 let replies = handler
14859 .handle_client_control_request(
14860 &ctx,
14861 frame,
14862 ClientControlRequest::SupervisorList {},
14863 &mut None,
14864 )
14865 .await
14866 .unwrap();
14867 assert!(
14868 replies.is_empty(),
14869 "list reply must be detached from the connection reader"
14870 );
14871 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14872 let replies = handler
14873 .handle_client_control_request(
14874 &ctx,
14875 frame,
14876 ClientControlRequest::SupervisorSpawnSnapshot {},
14877 &mut None,
14878 )
14879 .await
14880 .unwrap();
14881 assert_eq!(
14882 replies.len(),
14883 1,
14884 "another request must complete while the subtree read stalls"
14885 );
14886 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14887 .await
14888 .unwrap()
14889 .unwrap();
14890 release.send(()).unwrap();
14891 drop(release);
14892 stalled.await.unwrap();
14893 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
14894 assert!(
14895 body["modules"][0].get("tree").is_none(),
14896 "an unknown tree reading must be omitted after the bounded read expires"
14897 );
14898 handler
14902 .resource_tracker
14903 .read(handler.supervisor.list(), false, "released_tree_fixture")
14904 .await;
14905 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14906 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14907 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14908 assert_eq!(
14909 body["modules"][0]["tree"]["source"], "process_tree",
14910 "the next request must export the completed refresh"
14911 );
14912 assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
14913 }
14914
14915 #[tokio::test]
14916 async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
14917 use subc_control::{
14918 AgentCommandResources, ChildMemoryKind, ChildResourceReading,
14919 ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
14920 TreeReading, TreeSource,
14921 };
14922 let registry = Arc::new(Registry::default());
14923 let handle = SupervisorHandle::new();
14924 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14925 .with_handle(handle.clone());
14926 supervisor
14927 .supervise_configured(
14928 crate::ModuleSpec {
14929 carry_containment: false,
14930 module_id: "budget-worker".into(),
14931 program: PathBuf::from("disabled-module"),
14932 args: Vec::new(),
14933 env: Vec::new(),
14934 reserved: false,
14935 reserved_prefixes: Vec::new(),
14936 protocol: ModuleProtocol::Subc,
14937 overlap: Default::default(),
14938 },
14939 true,
14940 )
14941 .unwrap();
14942 let handler = ControlHandler::new(registry).with_supervisor(handle);
14943 let now = u64::try_from(
14944 std::time::SystemTime::now()
14945 .duration_since(std::time::UNIX_EPOCH)
14946 .unwrap()
14947 .as_millis(),
14948 )
14949 .unwrap();
14950 let measured = now - 1_000;
14951 let hour = CpuLastHour::new(100, 60_000);
14952 let daemon = DaemonResources::new(7, 1, "test")
14953 .with_measured_at_ms(Some(measured))
14954 .with_resources(Some(ChildResourceUsage::Measured(
14955 ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
14956 )))
14957 .with_cpu_last_hour(Some(hour.clone()));
14958 let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
14959 .with_measured_at_ms(Some(measured))
14960 .with_cpu_last_hour(Some(hour.clone()));
14961 let tree = TreeReading::new(
14962 150,
14963 50,
14964 ChildMemoryKind::ResidentSet,
14965 1,
14966 TreeSource::ProcessTree,
14967 )
14968 .with_measured_at_ms(Some(measured));
14969 handler.resource_tracker.seed_committed_for_test(
14970 &handler.supervisor.list(),
14971 crate::resource_totals::Readings {
14972 modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
14973 daemon: Some(daemon),
14974 agent_commands: Some(vec![agents]),
14975 ..Default::default()
14976 },
14977 now,
14978 );
14979 list_diagnostics::scope(async {
14980 tokio::time::sleep(Duration::from_millis(110)).await;
14981 assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
14982 let request =
14983 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14984 let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
14985 let ClientControlResponse::SupervisorList {
14986 modules,
14987 daemon,
14988 agent_commands,
14989 ..
14990 } = serde_json::from_slice(&replies[0].body).unwrap()
14991 else {
14992 panic!("expected a list response")
14993 };
14994 assert!(matches!(
14995 modules[0].resources,
14996 Some(ChildResourceUsage::Unavailable {
14997 reason: ChildResourceUnavailableReason::Unreadable
14998 })
14999 ));
15000 assert_eq!(
15001 (
15002 modules[0].tree.as_ref().unwrap().cpu_ms,
15003 modules[0].tree.as_ref().unwrap().measured_at_ms
15004 ),
15005 (150, Some(measured))
15006 );
15007 let daemon = daemon.unwrap();
15008 let agents = agent_commands.unwrap();
15009 assert_eq!(daemon.measured_at_ms, Some(measured));
15010 assert_eq!(
15011 (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
15012 (1, 50, Some(measured))
15013 );
15014 for hour in [
15015 modules[0].cpu_last_hour.as_ref(),
15016 daemon.cpu_last_hour.as_ref(),
15017 agents[0].cpu_last_hour.as_ref(),
15018 ] {
15019 let hour = hour.unwrap();
15020 assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
15021 }
15022 })
15023 .await;
15024 }
15025
15026 #[tokio::test]
15027 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
15028 let registry = Arc::new(Registry::default());
15029 let handle = SupervisorHandle::new();
15030 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
15031 .with_handle(handle.clone());
15032 supervisor
15033 .supervise_configured(
15034 crate::ModuleSpec {
15035 carry_containment: false,
15036 module_id: "drift".into(),
15037 program: PathBuf::from("disabled-module"),
15038 args: Vec::new(),
15039 env: Vec::new(),
15040 reserved: false,
15041 reserved_prefixes: Vec::new(),
15042 protocol: ModuleProtocol::Subc,
15043 overlap: Default::default(),
15044 },
15045 false,
15046 )
15047 .unwrap();
15048 let handler = ControlHandler::new(registry).with_supervisor(handle);
15049 for configured in [None, Some(false), Some(true)] {
15050 handler.capability_evaluator.configure(
15051 configured.map(|enabled| ("drift".to_string(), enabled)),
15052 BTreeMap::new(),
15053 );
15054 let frame =
15055 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
15056 let replies = handler
15057 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
15058 .await
15059 .unwrap();
15060 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
15061 assert!(body.get("configured_enabled").is_none());
15062 assert_eq!(body["op"], "supervisor.ack");
15063 let frame =
15064 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
15065 let replies = handler.handle_supervisor_list(frame).await.unwrap();
15066 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
15067 assert_eq!(
15068 body["modules"][0].get("configured_enabled"),
15069 configured.map(Value::Bool).as_ref()
15070 );
15071 assert!(body["modules"][0].get("tree").is_none());
15072 assert!(body["modules"][0].get("subtree").is_none());
15073 }
15074 }
15075
15076 #[tokio::test(flavor = "current_thread")]
15077 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
15078 let handler = ControlHandler::new(Arc::new(Registry::default()));
15079 let capture = EventCapture::default();
15080 let _subscriber =
15081 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15082 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
15083 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
15084
15085 let _ = handler
15086 .handle_client_control_request(
15087 &ctx,
15088 frame,
15089 ClientControlRequest::SupervisorSetEnabled {
15090 module_id: "broca".to_string(),
15091 enabled: false,
15092 },
15093 &mut None,
15094 )
15095 .await
15096 .unwrap();
15097
15098 let events = capture
15099 .events()
15100 .into_iter()
15101 .filter(|event| {
15102 event.target == "control"
15103 && event.fields.get("message").map(String::as_str)
15104 == Some("supervisor request received")
15105 })
15106 .collect::<Vec<_>>();
15107 assert_eq!(events.len(), 1, "one supervisor request log line");
15108 let event = &events[0];
15109 assert_eq!(event.level, tracing::Level::INFO);
15110 assert_eq!(
15111 event.fields.get("op"),
15112 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
15113 );
15114 assert_eq!(
15115 event.fields.get("module_id"),
15116 Some(&"\"broca\"".to_string())
15117 );
15118 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
15119 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
15120 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
15121 }
15122
15123 #[tokio::test(flavor = "current_thread")]
15124 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
15125 let registry = Arc::new(Registry::default());
15126 let module_connection = ConnectionId::new(303);
15127 registry
15128 .register_with_control_ops(
15129 manifest("aft", PROTOCOL_VERSION),
15130 PROTOCOL_VERSION,
15131 module_connection,
15132 module_baseline_control_ops(),
15133 )
15134 .unwrap();
15135 let handler = ControlHandler::new(registry);
15136 let capture = EventCapture::default();
15137 let _subscriber =
15138 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15139 let (ctx, _rx) = route_ctx(module_connection);
15140 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
15141
15142 let _ = handler
15143 .handle_client_control_request(
15144 &ctx,
15145 frame,
15146 ClientControlRequest::SupervisorSetEnabled {
15147 module_id: "broca".to_string(),
15148 enabled: false,
15149 },
15150 &mut None,
15151 )
15152 .await
15153 .unwrap();
15154
15155 let callers = capture
15156 .events()
15157 .into_iter()
15158 .filter(|event| {
15159 event.fields.get("message").map(String::as_str)
15160 == Some("supervisor request received")
15161 })
15162 .map(|event| event.fields.get("caller").cloned())
15163 .collect::<Vec<_>>();
15164 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
15165 }
15166
15167 #[tokio::test(flavor = "current_thread")]
15168 async fn supervisor_list_does_not_log_request_received() {
15169 let handler = ControlHandler::new(Arc::new(Registry::default()));
15170 let capture = EventCapture::default();
15171 let _subscriber =
15172 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15173 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
15174 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
15175
15176 handler
15177 .handle_client_control_request(
15178 &ctx,
15179 frame,
15180 ClientControlRequest::SupervisorList {},
15181 &mut None,
15182 )
15183 .await
15184 .unwrap();
15185
15186 assert!(capture.events().into_iter().all(|event| {
15187 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
15188 }));
15189 }
15190
15191 #[tokio::test(flavor = "current_thread")]
15192 async fn supervisor_rescan_preview_does_not_log_request_received() {
15193 let handler = ControlHandler::new(Arc::new(Registry::default()));
15194 let capture = EventCapture::default();
15195 let _subscriber =
15196 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15197 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
15198 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
15199
15200 handler
15201 .handle_client_control_request(
15202 &ctx,
15203 frame,
15204 ClientControlRequest::SupervisorRescan { preview: true },
15205 &mut None,
15206 )
15207 .await
15208 .unwrap();
15209
15210 assert!(capture.events().into_iter().all(|event| {
15211 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
15212 }));
15213 }
15214
15215 struct ApproveOperator;
15216 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
15217 fn prompt(
15218 &self,
15219 _: &str,
15220 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
15221 ) -> crate::operator_confirm::ProviderResult {
15222 crate::operator_confirm::ProviderResult::Approved
15223 }
15224 }
15225
15226 #[tokio::test]
15227 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
15228 assert_eq!(
15229 ControlHandler::default().privileged_operator_prompt,
15230 PrivilegedOperatorPrompt::Log
15231 );
15232 for (initial, changed, running) in [
15233 ("log", "enforce", PrivilegedOperatorPrompt::Log),
15234 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
15235 ] {
15236 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
15237 let path = dir.join("subc.jsonc");
15238 std::fs::write(
15239 &path,
15240 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
15241 )
15242 .unwrap();
15243 let config = crate::daemon_config::load(&path).unwrap().unwrap();
15244 let registry = Arc::new(Registry::default());
15245 let handle = SupervisorHandle::new();
15246 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
15247 .with_handle(handle.clone());
15248 let handler = ControlHandler::new(registry)
15249 .with_supervisor(handle)
15250 .with_privileged_operator_prompt(config.privileged_operator_prompt)
15251 .with_supervisor_rescan(supervisor, &path, config.port);
15252 handler.forwarding.operator_confirms().configure(
15253 Arc::new(ApproveOperator),
15254 crate::operator_confirm::OperatorLimits::default(),
15255 );
15256 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
15257
15258 for (disk_mode, needs_restart) in [
15259 (Some(initial), false),
15260 (Some(changed), true),
15261 (Some(changed), true),
15262 (None, running == PrivilegedOperatorPrompt::Enforce),
15263 (Some(initial), false),
15264 ] {
15265 let mut doc = json!({"version":1});
15266 if let Some(mode) = disk_mode {
15267 doc["privileged_operator_prompt"] = json!(mode);
15268 }
15269 std::fs::write(&path, doc.to_string()).unwrap();
15270 for preview in [true, false] {
15271 let request = ClientControlRequest::SupervisorRescan { preview };
15272 let frame = Frame::build(
15273 FrameType::Request,
15274 control_flags(),
15275 0,
15276 0,
15277 1,
15278 serde_json::to_vec(&request).unwrap(),
15279 )
15280 .unwrap();
15281 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
15282 assert_eq!(frames.len(), 1);
15283 let ClientControlResponse::SupervisorRescan { result } =
15284 serde_json::from_slice(&frames[0].body).unwrap()
15285 else {
15286 panic!("expected a rescan response");
15287 };
15288 let expected = if needs_restart {
15289 vec!["privileged_operator_prompt"]
15290 } else {
15291 Vec::new()
15292 };
15293 assert_eq!(result.restart_required, expected);
15294 assert_eq!(handler.privileged_operator_prompt, running);
15295 assert_eq!(
15296 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
15297 running
15298 );
15299 }
15300 }
15301 }
15302 }
15303
15304 #[tokio::test]
15305 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
15306 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
15307 let path = dir.join("subc.jsonc");
15308 let registry = Arc::new(Registry::default());
15309 let handle = SupervisorHandle::new();
15310 let supervisor =
15311 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
15312 let handler = ControlHandler::new(registry)
15313 .with_supervisor(handle)
15314 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
15315 .with_supervisor_rescan(supervisor, &path, None);
15316 handler.forwarding.operator_confirms().configure(
15317 Arc::new(ApproveOperator),
15318 crate::operator_confirm::OperatorLimits::default(),
15319 );
15320 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
15321 for preview in [true, false] {
15322 std::fs::write(
15323 &path,
15324 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
15325 )
15326 .unwrap();
15327 let request = ClientControlRequest::SupervisorRescan { preview };
15328 let frame = Frame::build(
15329 FrameType::Request,
15330 control_flags(),
15331 0,
15332 0,
15333 1,
15334 serde_json::to_vec(&request).unwrap(),
15335 )
15336 .unwrap();
15337 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
15338 assert_eq!(frames.len(), 1);
15339 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
15340 assert_eq!(
15341 handler.privileged_operator_prompt,
15342 PrivilegedOperatorPrompt::Enforce
15343 );
15344 }
15345 }
15346
15347 #[tokio::test]
15352 async fn route_open_refusal_names_the_check_that_refused() {
15353 let handler = ControlHandler::new(Arc::new(Registry::default()));
15354 let capture = EventCapture::default();
15355 let _subscriber =
15356 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15357 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15358 let response = handler
15359 .handle_control_frame(
15360 &ctx,
15361 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
15362 )
15363 .await
15364 .unwrap();
15365
15366 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15367 let event = capture
15368 .events()
15369 .into_iter()
15370 .find(|event| {
15371 event.target == "control"
15372 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
15373 })
15374 .expect("route.open refusal event");
15375 assert_eq!(
15376 event.fields.get("reason"),
15377 Some(&"\"not_registered\"".to_string())
15378 );
15379 }
15380
15381 #[tokio::test]
15382 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
15383 let registry = Arc::new(Registry::default());
15384 let supervisor_handle = SupervisorHandle::new();
15385 let supervisor =
15386 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
15387 .with_handle(supervisor_handle.clone())
15388 .with_connection_file_path(std::env::temp_dir().join(format!(
15389 "subc-route-open-refusal-info-{}",
15390 std::process::id()
15391 )));
15392 let module = supervisor
15393 .supervise_configured(
15394 ModuleSpec {
15395 carry_containment: false,
15396 module_id: "warming".to_string(),
15397 program: fake_aft_stub_path(),
15398 args: Vec::new(),
15399 env: Vec::new(),
15400 reserved: false,
15401 reserved_prefixes: Vec::new(),
15402 protocol: ModuleProtocol::Subc,
15403 overlap: Default::default(),
15404 },
15405 true,
15406 )
15407 .unwrap();
15408 assert_eq!(module.state().unwrap(), ModuleState::Running);
15409
15410 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15411 assert!(handler
15412 .counters()
15413 .snapshot()
15414 .get("route_open_refused_by_code")
15415 .is_none());
15416 let capture = EventCapture::default();
15417 let _subscriber =
15418 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15419 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
15420 let response = handler
15421 .handle_control_frame(
15422 &ctx,
15423 route_open_frame(394, "warming", unique_project_root("refusal-info")),
15424 )
15425 .await
15426 .unwrap();
15427 module.stop().await.unwrap();
15428
15429 assert_eq!(parse_error(&response[0])["code"], "module_warming");
15430 let event = capture
15431 .events()
15432 .into_iter()
15433 .find(|event| {
15434 event.target == "control"
15435 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
15436 })
15437 .expect("route.open refusal event");
15438 assert_eq!(
15439 event.fields.get("module_id"),
15440 Some(&"\"warming\"".to_string())
15441 );
15442 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
15443 assert_eq!(
15444 event.fields.get("reason"),
15445 Some(&"\"supervised_not_registered\"".to_string())
15446 );
15447 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
15448 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
15449 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
15450 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
15451 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
15452 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
15453 assert_eq!(
15454 handler.counters().snapshot()["route_open_refused_by_code"],
15455 json!({ "module_warming": 1 })
15456 );
15457 }
15458
15459 const OUTAGE_START: &str = "route.open refusing module: not serving";
15460 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
15461
15462 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
15463 capture
15464 .events()
15465 .into_iter()
15466 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
15467 .collect()
15468 }
15469
15470 fn supervise_stub(
15471 registry: &Arc<Registry>,
15472 module_id: &str,
15473 enabled: bool,
15474 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
15475 let supervisor_handle = SupervisorHandle::new();
15476 let supervisor =
15477 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
15478 .with_handle(supervisor_handle.clone())
15479 .with_connection_file_path(std::env::temp_dir().join(format!(
15480 "subc-route-outage-{module_id}-{}",
15481 std::process::id()
15482 )));
15483 let module = supervisor
15484 .supervise_configured(
15485 ModuleSpec {
15486 carry_containment: false,
15487 module_id: module_id.to_string(),
15488 program: fake_aft_stub_path(),
15489 args: Vec::new(),
15490 env: Vec::new(),
15491 reserved: false,
15492 reserved_prefixes: Vec::new(),
15493 protocol: ModuleProtocol::Subc,
15494 overlap: Default::default(),
15495 },
15496 enabled,
15497 )
15498 .unwrap();
15499 (supervisor_handle, module)
15500 }
15501
15502 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
15503 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
15504 module_id: module_id.to_string(),
15505 drain_timeout_ms: Some(50),
15506 })
15507 .unwrap();
15508 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
15509 }
15510
15511 #[test]
15515 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
15516 let registry = Arc::new(Registry::default());
15517 let forwarding = Arc::new(ForwardingTable::default());
15518 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15519 let second = ControlHandler::with_forwarding(registry, forwarding);
15520 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
15521 }
15522
15523 #[tokio::test(flavor = "current_thread")]
15527 async fn route_open_unknown_module_refusals_add_no_outage_state() {
15528 let handler = ControlHandler::new(Arc::new(Registry::default()));
15529 let capture = EventCapture::default();
15530 let _subscriber =
15531 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15532 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
15533 for corr in 0..8 {
15534 let response = handler
15535 .handle_control_frame(
15536 &ctx,
15537 route_open_frame(
15538 380 + corr,
15539 &format!("nobody-{corr}"),
15540 unique_project_root("outage-unknown"),
15541 ),
15542 )
15543 .await
15544 .unwrap();
15545 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15546 }
15547
15548 assert_eq!(handler.route_outages.tracked_module_count(), 0);
15549 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
15550 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
15551 }
15552
15553 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15558 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
15559 let registry = Arc::new(Registry::default());
15560 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
15561 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15562 let capture = EventCapture::default();
15563 let _subscriber =
15564 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15565 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
15566 handler.route_outages.record_accepted("outage-restart");
15569
15570 let response = handler
15571 .handle_control_frame(
15572 &ctx,
15573 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
15574 )
15575 .await
15576 .unwrap();
15577 assert_eq!(response[0].header.ty, FrameType::Error);
15578 let starts = outage_lines(&capture, OUTAGE_START);
15579 assert_eq!(starts.len(), 1, "{starts:?}");
15580 assert_eq!(starts[0].level, tracing::Level::WARN);
15581 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
15582 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
15583 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
15584 handler.route_outages.record_accepted("outage-restart");
15585 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
15586
15587 let restart = handler
15588 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
15589 .await
15590 .unwrap();
15591 assert_eq!(
15592 restart[0].header.ty,
15593 FrameType::Response,
15594 "{:?}",
15595 parse_error(&restart[0])
15596 );
15597 handler
15598 .handle_control_frame(
15599 &ctx,
15600 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
15601 )
15602 .await
15603 .unwrap();
15604 module.stop().await.unwrap();
15605
15606 let starts = outage_lines(&capture, OUTAGE_START);
15607 assert_eq!(starts.len(), 2, "{starts:?}");
15608 assert_eq!(starts[1].level, tracing::Level::INFO);
15609 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
15610 }
15611
15612 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15616 async fn failed_operator_restart_leaves_no_operator_mark() {
15617 let registry = Arc::new(Registry::default());
15618 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
15619 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15620 let capture = EventCapture::default();
15621 let _subscriber =
15622 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15623 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
15624 handler.route_outages.record_accepted("outage-disabled");
15625
15626 let restart = handler
15627 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
15628 .await
15629 .unwrap();
15630 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
15631 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
15632
15633 handler
15634 .handle_control_frame(
15635 &ctx,
15636 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
15637 )
15638 .await
15639 .unwrap();
15640 let starts = outage_lines(&capture, OUTAGE_START);
15641 assert_eq!(starts.len(), 1, "{starts:?}");
15642 assert_eq!(starts[0].level, tracing::Level::WARN);
15643 }
15644
15645 #[tokio::test(flavor = "current_thread")]
15646 async fn route_open_unknown_module_escapes_target_module_id() {
15647 let handler = ControlHandler::new(Arc::new(Registry::default()));
15648 let capture = EventCapture::default();
15649 let _subscriber =
15650 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15651 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
15652 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15653 let response = handler
15654 .handle_control_frame(
15655 &ctx,
15656 route_open_frame(
15657 395,
15658 hostile_module_id,
15659 unique_project_root("hostile-target-module-id"),
15660 ),
15661 )
15662 .await
15663 .unwrap();
15664
15665 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15666 let event = capture
15667 .events()
15668 .into_iter()
15669 .find(|event| {
15670 event.target == "control"
15671 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
15672 })
15673 .expect("route.open unknown-module refusal event");
15674 let logged = event.fields.get("module_id").expect("module_id field");
15675 assert!(!logged.bytes().any(|byte| byte < 0x20));
15676 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15677 }
15678
15679 #[tokio::test(flavor = "current_thread")]
15682 async fn route_open_refusal_carries_daemon_origin() {
15683 let handler = ControlHandler::new(Arc::new(Registry::default()));
15684 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15685 let response = handler
15686 .handle_control_frame(
15687 &ctx,
15688 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
15689 )
15690 .await
15691 .unwrap();
15692
15693 assert_eq!(response.len(), 1);
15694 assert_eq!(response[0].header.ty, FrameType::Error);
15695 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15696 assert!(
15697 response[0].header.flags.is_daemon_origin(),
15698 "route.open refusal must carry daemon origin: {:?}",
15699 response[0].header.flags
15700 );
15701 }
15702
15703 #[tokio::test(flavor = "current_thread")]
15707 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
15708 let registry = Arc::new(Registry::default());
15709 let forwarding = Arc::new(ForwardingTable::default());
15710 let handler =
15711 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15712 let module_connection = ConnectionId::new(95);
15713 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15714 hello_via_sink(
15715 &handler,
15716 &module_ctx,
15717 &mut module_rx,
15718 hello_frame("aft", PROTOCOL_VERSION, 395),
15719 )
15720 .await;
15721 forwarding
15722 .begin_module_drain("aft", RouteCloseReason::Restart)
15723 .unwrap()
15724 .expect("the module endpoint drains");
15725
15726 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
15727 let response = handler
15728 .handle_control_frame(
15729 &client_ctx,
15730 route_open_frame(396, "aft", unique_project_root("drain-origin")),
15731 )
15732 .await
15733 .unwrap();
15734
15735 assert_eq!(response.len(), 1);
15736 assert_eq!(response[0].header.ty, FrameType::Error);
15737 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15738 assert!(
15739 response[0].header.flags.is_daemon_origin(),
15740 "drain module_reloading must carry daemon origin: {:?}",
15741 response[0].header.flags
15742 );
15743 }
15744
15745 #[tokio::test(flavor = "current_thread")]
15749 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
15750 let registry = Arc::new(Registry::default());
15751 let forwarding = Arc::new(ForwardingTable::default());
15752 let handler =
15753 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15754 let module_connection = ConnectionId::new(95);
15755 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15756 hello_via_sink(
15757 &handler,
15758 &module_ctx,
15759 &mut module_rx,
15760 hello_frame("aft", PROTOCOL_VERSION, 395),
15761 )
15762 .await;
15763
15764 let client_connection = ConnectionId::new(96);
15765 let (client_ctx, _client_rx) = route_ctx(client_connection);
15766 let (route_task, bind) = relay_route_open(
15767 &handler,
15768 client_connection,
15769 &client_ctx.egress,
15770 &mut module_rx,
15771 396,
15772 "aft",
15773 "spoofed-reloading",
15774 )
15775 .await;
15776 let rejection = Frame::build(
15779 FrameType::Error,
15780 crate::router::daemon_error_flags(),
15781 0,
15782 0,
15783 bind.header.corr,
15784 serde_json::to_vec(&ErrorBody::new(
15785 "module_reloading",
15786 "module refused route.bind",
15787 ))
15788 .unwrap(),
15789 )
15790 .unwrap();
15791 handler
15792 .handle_control_frame(&module_ctx, rejection)
15793 .await
15794 .unwrap();
15795
15796 let response = route_task.await.unwrap();
15797 assert_eq!(response.len(), 1);
15798 assert_eq!(response[0].header.ty, FrameType::Error);
15799 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15800 assert!(
15801 !response[0].header.flags.is_daemon_origin(),
15802 "a module's own rejection must not carry daemon origin: {:?}",
15803 response[0].header.flags
15804 );
15805 }
15806
15807 #[tokio::test(flavor = "current_thread")]
15808 async fn route_open_module_rejection_uses_daemon_counter_key() {
15809 let registry = Arc::new(Registry::default());
15810 let forwarding = Arc::new(ForwardingTable::default());
15811 let handler =
15812 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15813 let module_connection = ConnectionId::new(95);
15814 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15815 hello_via_sink(
15816 &handler,
15817 &module_ctx,
15818 &mut module_rx,
15819 hello_frame("aft", PROTOCOL_VERSION, 395),
15820 )
15821 .await;
15822
15823 let client_connection = ConnectionId::new(96);
15824 let (client_ctx, _client_rx) = route_ctx(client_connection);
15825 let capture = EventCapture::default();
15826 static INTEREST_ANCHOR: std::sync::OnceLock<tracing::Dispatch> = std::sync::OnceLock::new();
15833 INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
15834 let _subscriber =
15835 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15836 let (route_task, bind) = relay_route_open(
15837 &handler,
15838 client_connection,
15839 &client_ctx.egress,
15840 &mut module_rx,
15841 396,
15842 "aft",
15843 "hostile-module-code",
15844 )
15845 .await;
15846 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
15847 let rejection = Frame::build(
15848 FrameType::Error,
15849 control_flags(),
15850 0,
15851 0,
15852 bind.header.corr,
15853 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
15854 )
15855 .unwrap();
15856 handler
15857 .handle_control_frame(&module_ctx, rejection)
15858 .await
15859 .unwrap();
15860
15861 let response = route_task.await.unwrap();
15862 assert_eq!(parse_error(&response[0])["code"], hostile_code);
15863 let counters = handler.counters().snapshot();
15864 assert_eq!(
15865 counters["route_open_refused_by_code"],
15866 json!({ "module_rejected": 1 })
15867 );
15868 assert!(counters["route_open_refused_by_code"]
15869 .get(hostile_code)
15870 .is_none());
15871
15872 let event = capture
15873 .events()
15874 .into_iter()
15875 .find(|event| {
15876 event.target == "control"
15877 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
15878 })
15879 .expect("route.open module-rejection refusal event");
15880 let logged = event.fields.get("module_code").expect("module_code field");
15881 assert!(!logged.bytes().any(|byte| byte < 0x20));
15882 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15883 }
15884
15885 #[tokio::test]
15886 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
15887 let registry = Arc::new(Registry::default());
15888 let supervisor_handle = SupervisorHandle::new();
15889 let missing_program = std::env::temp_dir().join(format!(
15890 "subc-route-open-missing-program-{}",
15891 std::process::id()
15892 ));
15893 let supervisor =
15894 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
15895 .with_handle(supervisor_handle.clone());
15896 let module = supervisor
15897 .supervise_configured(
15898 ModuleSpec {
15899 carry_containment: false,
15900 module_id: "failed".to_string(),
15901 program: missing_program,
15902 args: Vec::new(),
15903 env: Vec::new(),
15904 reserved: false,
15905 reserved_prefixes: Vec::new(),
15906 protocol: ModuleProtocol::Subc,
15907 overlap: Default::default(),
15908 },
15909 true,
15910 )
15911 .unwrap();
15912 assert_eq!(module.state().unwrap(), ModuleState::Failed);
15913
15914 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15915 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
15916 let response = handler
15917 .handle_control_frame(
15918 &ctx,
15919 route_open_frame(305, "failed", unique_project_root("failed")),
15920 )
15921 .await
15922 .unwrap();
15923
15924 assert_eq!(response[0].header.ty, FrameType::Error);
15925 let error = parse_error(&response[0]);
15926 assert_eq!(error["code"], "target_unavailable");
15927 assert!(error["message"]
15928 .as_str()
15929 .unwrap()
15930 .contains("state=failed, enabled=true, live=false"));
15931 }
15932
15933 #[tokio::test]
15934 async fn route_open_role_mismatch_remains_target_unavailable() {
15935 let registry = Arc::new(Registry::default());
15936 let handler = ControlHandler::new(Arc::clone(®istry));
15937 handler
15938 .handle_control(
15939 ConnectionId::new(41),
15940 non_routable_hello_frame_with_control_ops("health-only", 306, None),
15941 )
15942 .unwrap();
15943
15944 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
15945 let response = handler
15946 .handle_control_frame(
15947 &ctx,
15948 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
15949 )
15950 .await
15951 .unwrap();
15952
15953 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15954 assert!(parse_error(&response[0])["message"]
15955 .as_str()
15956 .unwrap()
15957 .contains("does not provide the requested target"));
15958 }
15959
15960 #[tokio::test]
15961 async fn route_open_inactive_registration_remains_target_unavailable() {
15962 let registry = Arc::new(Registry::default());
15963 let handler = ControlHandler::new(Arc::clone(®istry));
15964 handler
15965 .handle_control(
15966 ConnectionId::new(43),
15967 hello_frame("inactive", PROTOCOL_VERSION, 308),
15968 )
15969 .unwrap();
15970 assert!(registry
15971 .set_module_state_for_test("inactive", ChannelState::Closed)
15972 .unwrap());
15973
15974 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
15975 let response = handler
15976 .handle_control_frame(
15977 &ctx,
15978 route_open_frame(309, "inactive", unique_project_root("inactive")),
15979 )
15980 .await
15981 .unwrap();
15982
15983 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15984 assert!(parse_error(&response[0])["message"]
15985 .as_str()
15986 .unwrap()
15987 .contains("is not active"));
15988 }
15989
15990 #[tokio::test]
15991 async fn late_health_reply_is_recorded_through_the_module_response_path() {
15992 let registry = Arc::new(Registry::default());
15993 let forwarding = Arc::new(ForwardingTable::default());
15994 let supervisor_handle = SupervisorHandle::new();
15995 let supervisor =
15996 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
15997 .with_forwarding(Arc::clone(&forwarding))
15998 .with_handle(supervisor_handle.clone());
15999 let module = supervisor
16000 .supervise_configured(
16001 crate::ModuleSpec {
16002 carry_containment: false,
16003 module_id: "late-health-response".to_string(),
16004 program: PathBuf::from("disabled-module"),
16005 args: Vec::new(),
16006 env: Vec::new(),
16007 reserved: false,
16008 reserved_prefixes: Vec::new(),
16009 protocol: ModuleProtocol::Subc,
16010 overlap: Default::default(),
16011 },
16012 false,
16013 )
16014 .unwrap();
16015 let handler =
16016 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16017 .with_supervisor(supervisor_handle);
16018 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
16019 handler
16020 .handle_control_frame(
16021 &module_ctx,
16022 hello_frame_with_control_ops(
16023 "late-health-response",
16024 PROTOCOL_VERSION,
16025 7,
16026 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
16027 ),
16028 )
16029 .await
16030 .unwrap();
16031 let probe_started_at = Instant::now() - Duration::from_millis(80);
16032 let pending = forwarding
16033 .begin_health_probe_rpc_for(
16034 "late-health-response",
16035 MODULE_CONTROL_OP_HEALTH_CHECK,
16036 probe_started_at,
16037 Instant::now() - Duration::from_millis(1),
16038 )
16039 .unwrap();
16040 assert!(forwarding
16041 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
16042 .unwrap());
16043
16044 let responses = handler
16045 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
16046 .await
16047 .unwrap();
16048
16049 assert!(responses.is_empty());
16050 let health = module.status().unwrap().health;
16051 assert_eq!(health.late_answer_count, 1);
16052 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
16053 }
16054
16055 #[tokio::test]
16056 async fn health_probe_timeout_and_module_death_are_typed() {
16057 let registry = Arc::new(Registry::default());
16058 let forwarding = Arc::new(ForwardingTable::default());
16059 let handler =
16060 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16061 .with_health_probe_timeout(Duration::from_millis(50));
16062 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
16063 hello_via_sink(
16064 &handler,
16065 &module_ctx,
16066 &mut module_rx,
16067 hello_frame_with_control_ops(
16068 "aft",
16069 PROTOCOL_VERSION,
16070 7,
16071 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
16072 ),
16073 )
16074 .await;
16075
16076 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
16077 let responses = handler
16078 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
16079 .await
16080 .unwrap();
16081 assert_eq!(responses[0].header.ty, FrameType::Error);
16082 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
16083 let _ = module_rx.try_recv();
16084
16085 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
16086 let health_handler = handler.clone();
16087 let death_task = tokio::spawn(async move {
16088 health_handler
16089 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
16090 .await
16091 .unwrap()
16092 });
16093 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
16094 .await
16095 .unwrap()
16096 .unwrap();
16097 handler
16098 .cleanup_connection(module_ctx.connection_id)
16099 .unwrap();
16100 let responses = death_task.await.unwrap();
16101 assert_eq!(responses[0].header.ty, FrameType::Error);
16102 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
16103 }
16104
16105 #[test]
16106 fn hello_requires_exact_protocol_version() {
16107 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
16108 let registry = Arc::new(Registry::default());
16109 let handler = ControlHandler::new(Arc::clone(®istry));
16110 let responses = handler
16111 .handle_control(
16112 ConnectionId::new(connection),
16113 hello_frame("aft", offered, 9),
16114 )
16115 .unwrap();
16116
16117 assert_eq!(responses.len(), 1);
16118 assert_eq!(responses[0].header.ty, FrameType::Error);
16119 let error = parse_error(&responses[0]);
16120 assert_eq!(error["code"], "version_unsupported");
16121 assert!(registry.get_module("aft").unwrap().is_none());
16122 assert_eq!(registry.active_registration_count().unwrap(), 0);
16123 }
16124 }
16125
16126 #[test]
16127 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
16128 let registry = Arc::new(Registry::default());
16129 let forwarding = Arc::new(ForwardingTable::default());
16130 let handler =
16131 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16132 let module_connection = ConnectionId::new(301);
16133 let registration = registry
16134 .register_with_control_ops(
16135 manifest("aft-push", PROTOCOL_VERSION),
16136 PROTOCOL_VERSION,
16137 module_connection,
16138 module_baseline_control_ops(),
16139 )
16140 .unwrap();
16141 let (module_tx, _module_rx) = mpsc::channel(8);
16142 let endpoint = forwarding
16143 .register_module_connection(
16144 module_connection,
16145 "aft-push".to_string(),
16146 PROTOCOL_VERSION,
16147 manifest_concurrency(®istration.manifest),
16148 FrameSink::new(module_tx),
16149 )
16150 .unwrap();
16151
16152 let unknown = Frame::build(
16154 FrameType::Push,
16155 control_flags(),
16156 0,
16157 0,
16158 5,
16159 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
16160 )
16161 .unwrap();
16162 let out = handler.handle_status_update(endpoint, unknown).unwrap();
16163 assert!(
16164 out.is_empty(),
16165 "unknown push op must be ignored, got {out:?}"
16166 );
16167
16168 let malformed = Frame::build(
16170 FrameType::Push,
16171 control_flags(),
16172 0,
16173 0,
16174 6,
16175 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
16176 )
16177 .unwrap();
16178 let out = handler.handle_status_update(endpoint, malformed).unwrap();
16179 assert_eq!(out.len(), 1);
16180 assert_eq!(out[0].header.ty, FrameType::Error);
16181 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
16182 }
16183
16184 #[test]
16185 fn hello_rejected_when_connection_already_owns_client_routes() {
16186 let registry = Arc::new(Registry::default());
16187 let forwarding = Arc::new(ForwardingTable::default());
16188 let handler =
16189 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16190 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
16192 let client_connection = ConnectionId::new(202);
16193
16194 let responses = handler
16197 .handle_control(
16198 client_connection,
16199 hello_frame("aft-second", PROTOCOL_VERSION, 9),
16200 )
16201 .unwrap();
16202 assert_eq!(responses[0].header.ty, FrameType::Error);
16203 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
16204 assert!(registry.get_module("aft-second").unwrap().is_none());
16205 }
16206
16207 #[tokio::test]
16208 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
16209 let registry = Arc::new(Registry::default());
16210 let forwarding = Arc::new(ForwardingTable::default());
16211 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
16212 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
16213 hello_via_sink(
16214 &handler,
16215 &module_ctx,
16216 &mut module_rx,
16217 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
16218 )
16219 .await;
16220 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
16221 let pending = forwarding
16222 .begin_route_bind_relay_for_test(
16223 client_ctx.connection_id,
16224 client_ctx.egress.clone(),
16225 2,
16226 "alpha",
16227 )
16228 .unwrap();
16229 forwarding
16230 .complete_pending_relay(
16231 module_ctx.connection_id,
16232 pending.corr,
16233 RouteBindRelayOutcome::Accepted,
16234 )
16235 .unwrap();
16236 client_rx.try_recv().unwrap();
16237 for module_id in ["beta", "alpha"] {
16238 let replies = handler
16239 .handle_control_frame(
16240 &module_ctx,
16241 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
16242 )
16243 .await
16244 .unwrap();
16245 assert_eq!(replies.len(), 1, "second HELLO must be refused");
16246 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
16247 }
16248 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
16249 assert!(registry.get_module("beta").unwrap().is_none());
16250 assert!(matches!(
16251 forwarding
16252 .lookup_data_route(
16253 client_ctx.connection_id,
16254 pending.client_channel,
16255 pending.client_epoch,
16256 )
16257 .unwrap(),
16258 DataRoute::Client(DataRouteState::Bound(_))
16259 ));
16260 assert!(handler
16261 .hello_launch_nonces
16262 .lock()
16263 .unwrap()
16264 .presented(module_ctx.connection_id, Some("alpha-nonce")));
16265 assert!(module_rx.try_recv().is_err());
16266 }
16267
16268 #[test]
16269 fn reserved_module_hello_requires_matching_launch_nonce() {
16270 let registry = Arc::new(Registry::default());
16271 let supervisor = SupervisorHandle::new();
16272 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
16275 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
16276
16277 let no_nonce = handler
16279 .handle_control(
16280 ConnectionId::new(1),
16281 hello_frame("vault", PROTOCOL_VERSION, 1),
16282 )
16283 .unwrap();
16284 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
16285 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
16286 assert!(registry.get_module("vault").unwrap().is_none());
16287
16288 let wrong = handler
16290 .handle_control(
16291 ConnectionId::new(2),
16292 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
16293 )
16294 .unwrap();
16295 assert_eq!(wrong[0].header.ty, FrameType::Error);
16296 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
16297 assert!(registry.get_module("vault").unwrap().is_none());
16298
16299 let ok = handler
16301 .handle_control(
16302 ConnectionId::new(3),
16303 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
16304 )
16305 .unwrap();
16306 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
16307 assert!(registry.get_module("vault").unwrap().is_some());
16308 }
16309
16310 #[test]
16311 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
16312 let registry = Arc::new(Registry::default());
16313 let supervisor = SupervisorHandle::new();
16314 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
16315 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
16316 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
16317
16318 let squat = handler
16319 .handle_control(
16320 ConnectionId::new(1),
16321 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
16322 )
16323 .unwrap();
16324 assert_eq!(squat[0].header.ty, FrameType::Error);
16325 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
16326 assert!(parse_error(&squat[0])["message"]
16327 .as_str()
16328 .unwrap()
16329 .contains("fed:"));
16330
16331 let accepted_peer = handler
16332 .handle_control(
16333 ConnectionId::new(2),
16334 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
16335 )
16336 .unwrap();
16337 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
16338
16339 let accepted_short = handler
16340 .handle_control(
16341 ConnectionId::new(3),
16342 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
16343 )
16344 .unwrap();
16345 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
16346
16347 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
16348 let response = handler
16349 .handle_control(
16350 ConnectionId::new(conn),
16351 hello_frame(module_id, PROTOCOL_VERSION, conn),
16352 )
16353 .unwrap();
16354 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
16355 }
16356 }
16357
16358 #[test]
16359 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
16360 let registry = Arc::new(Registry::default());
16361 let supervisor = SupervisorHandle::new();
16362 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
16363 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
16364 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
16365 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
16366
16367 let owner_nonce = handler
16368 .handle_control(
16369 ConnectionId::new(1),
16370 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
16371 )
16372 .unwrap();
16373 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
16374 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
16375 assert!(registry.get_module("fed:special").unwrap().is_none());
16376
16377 let exact_nonce = handler
16378 .handle_control(
16379 ConnectionId::new(2),
16380 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
16381 )
16382 .unwrap();
16383 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
16384 assert!(registry.get_module("fed:special").unwrap().is_some());
16385 }
16386
16387 #[test]
16388 fn non_reserved_module_ignores_launch_nonce() {
16389 let registry = Arc::new(Registry::default());
16390 let handler = ControlHandler::new(Arc::clone(®istry));
16393 let no_nonce = handler
16394 .handle_control(
16395 ConnectionId::new(1),
16396 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
16397 )
16398 .unwrap();
16399 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
16400 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
16401
16402 let echoed_nonce = handler
16403 .handle_control(
16404 ConnectionId::new(2),
16405 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
16406 )
16407 .unwrap();
16408 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
16409 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
16410 }
16411
16412 #[test]
16413 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
16414 let handler = ControlHandler::default();
16415 let conn = ConnectionId::new(1);
16416 let malformed = Frame::build(
16417 FrameType::Hello,
16418 control_flags(),
16419 0,
16420 0,
16421 3,
16422 b"{not json".to_vec(),
16423 )
16424 .unwrap();
16425
16426 let error = handler.handle_control(conn, malformed).unwrap();
16427 assert_eq!(error[0].header.ty, FrameType::Error);
16428 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
16429
16430 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
16431 let pong = handler.handle_control(conn, ping).unwrap();
16432 assert_eq!(pong[0].header.ty, FrameType::Pong);
16433 assert_eq!(pong[0].header.corr, 4);
16434 }
16435
16436 #[test]
16437 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
16438 let registry = Arc::new(Registry::default());
16439 let handler = ControlHandler::new(Arc::clone(®istry));
16440
16441 handler
16442 .handle_control(
16443 ConnectionId::new(1),
16444 hello_frame("aft", PROTOCOL_VERSION, 1),
16445 )
16446 .unwrap();
16447 let duplicate = handler
16448 .handle_control(
16449 ConnectionId::new(2),
16450 hello_frame("aft", PROTOCOL_VERSION, 2),
16451 )
16452 .unwrap();
16453
16454 assert_eq!(duplicate[0].header.ty, FrameType::Error);
16455 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16456 let registration = registry.get_module("aft").unwrap().unwrap();
16457 assert_eq!(registration.connection_id, ConnectionId::new(1));
16458 }
16459
16460 #[test]
16461 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
16462 let registry = Arc::new(Registry::default());
16463 let forwarding = Arc::new(ForwardingTable::default());
16464 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
16465 let handler =
16466 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16467 .with_process_liveness(process_liveness);
16468 let (ctx, route_channel, route_epoch) =
16469 bind_liveness_route(®istry, &forwarding, "aft-dead");
16470 let responses = handler
16471 .handle_route_poll(
16472 &ctx,
16473 route_poll_frame(41, PollKind::Liveness, route_channel),
16474 route_channel,
16475 route_epoch,
16476 PollKind::Liveness,
16477 )
16478 .unwrap();
16479
16480 assert_eq!(responses.len(), 1);
16481 assert_eq!(responses[0].header.ty, FrameType::Response);
16482 assert_route_poll_liveness(&responses[0], false);
16483 }
16484
16485 #[test]
16486 fn liveness_poll_without_process_source_uses_bound_route() {
16487 let registry = Arc::new(Registry::default());
16488 let forwarding = Arc::new(ForwardingTable::default());
16489 let handler =
16490 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16491 let (ctx, route_channel, route_epoch) =
16492 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
16493 let responses = handler
16494 .handle_route_poll(
16495 &ctx,
16496 route_poll_frame(42, PollKind::Liveness, route_channel),
16497 route_channel,
16498 route_epoch,
16499 PollKind::Liveness,
16500 )
16501 .unwrap();
16502
16503 assert_route_poll_liveness(&responses[0], true);
16504 }
16505
16506 #[test]
16507 fn liveness_poll_untracked_process_source_uses_bound_route() {
16508 let registry = Arc::new(Registry::default());
16509 let forwarding = Arc::new(ForwardingTable::default());
16510 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
16511 let handler =
16512 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16513 .with_process_liveness(process_liveness);
16514 let (ctx, route_channel, route_epoch) =
16515 bind_liveness_route(®istry, &forwarding, "aft-untracked");
16516 let responses = handler
16517 .handle_route_poll(
16518 &ctx,
16519 route_poll_frame(43, PollKind::Liveness, route_channel),
16520 route_channel,
16521 route_epoch,
16522 PollKind::Liveness,
16523 )
16524 .unwrap();
16525
16526 assert_route_poll_liveness(&responses[0], true);
16527 }
16528
16529 #[tokio::test]
16530 async fn unknown_op_returns_unknown_control_op() {
16531 let handler = ControlHandler::default();
16532 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
16533 let request = Frame::build(
16534 FrameType::Request,
16535 control_flags(),
16536 0,
16537 0,
16538 55,
16539 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
16540 )
16541 .unwrap();
16542
16543 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16544
16545 assert_eq!(response.len(), 1);
16546 assert_eq!(response[0].header.ty, FrameType::Error);
16547 assert_eq!(response[0].header.corr, 55);
16548 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
16549 }
16550
16551 #[tokio::test]
16552 async fn supervisor_provenance_rejects_unknown_exact_module() {
16553 let handler = ControlHandler::default();
16554 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
16555 let request = Frame::build(
16556 FrameType::Request,
16557 control_flags(),
16558 0,
16559 0,
16560 57,
16561 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
16562 )
16563 .unwrap();
16564
16565 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16566
16567 assert_eq!(response.len(), 1);
16568 assert_eq!(response[0].header.ty, FrameType::Error);
16569 assert_eq!(response[0].header.corr, 57);
16570 let error = parse_error(&response[0]);
16571 assert_eq!(error["code"], "unknown_module");
16572 assert_eq!(error["message"], "module_id 'missing' is not supervised");
16573 }
16574
16575 #[test]
16576 fn provenance_probe_override_keeps_handler_tests_deterministic() {
16577 let expected = subc_control::RunningImageAgreement::Unavailable {
16578 reason: subc_control::RunningImageUnavailableReason::HashFailed,
16579 };
16580 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
16581 assert_eq!(handler.provenance_probe_override, Some(expected));
16582 }
16583
16584 #[test]
16585 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
16586 let verdict = reload_verdict(
16587 std::path::Path::new("/bin/new"),
16588 Some(std::path::Path::new("/bin/old")),
16589 subc_control::RunningImageAgreement::Unavailable {
16590 reason: subc_control::RunningImageUnavailableReason::HashFailed,
16591 },
16592 );
16593 assert!(matches!(
16594 verdict.path,
16595 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
16596 if configured == std::path::Path::new("/bin/new")
16597 && spawned_from == std::path::Path::new("/bin/old")
16598 ));
16599 }
16600
16601 #[test]
16602 fn launch_spec_without_running_digest_is_unavailable_not_match() {
16603 use subc_control::{
16604 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
16605 };
16606 let configured = crate::supervise::ModuleSpec {
16607 carry_containment: false,
16608 module_id: "legacy".into(),
16609 program: "/module".into(),
16610 args: Vec::new(),
16611 env: Vec::new(),
16612 reserved: false,
16613 reserved_prefixes: Vec::new(),
16614 protocol: subc_control::ModuleProtocol::Subc,
16615 overlap: Default::default(),
16616 };
16617 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
16618 assert_eq!(
16619 launch_spec_agreement(&configured, false, Some(&digest)),
16620 Agreement::Unavailable {
16621 reason: Reason::NotRunning
16622 }
16623 );
16624 assert_eq!(
16625 launch_spec_agreement(&configured, true, None),
16626 Agreement::Unavailable {
16627 reason: Reason::SpawnedSpecUnavailable
16628 }
16629 );
16630 assert_eq!(
16631 launch_spec_agreement(&configured, true, Some(&digest)),
16632 Agreement::Match
16633 );
16634 }
16635
16636 #[cfg(windows)]
16637 #[test]
16638 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
16639 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
16640 let configured = dir.join("module.exe");
16641 std::fs::write(&configured, b"executable path fixture").unwrap();
16642 let spawned = std::fs::canonicalize(&configured).unwrap();
16643 assert_ne!(
16644 configured, spawned,
16645 "fixture must exercise different path spellings"
16646 );
16647 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
16648 let lower_case = configured.to_string_lossy().to_lowercase();
16649 for spelling in [
16650 &configured,
16651 Path::new(&forward_slashes),
16652 Path::new(&lower_case),
16653 ] {
16654 let verdict = reload_verdict(
16655 spelling,
16656 Some(&spawned),
16657 subc_control::RunningImageAgreement::Unavailable {
16658 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16659 },
16660 );
16661 assert_eq!(
16662 verdict.path,
16663 ReloadPathAgreement::Match,
16664 "configured {spelling:?}, spawned {spawned:?}"
16665 );
16666 }
16667 }
16668
16669 #[cfg(windows)]
16670 #[test]
16671 fn windows_reload_path_keeps_different_existing_program_pending() {
16672 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
16673 let old = dir.join("old.exe");
16674 let new = dir.join("new.exe");
16675 std::fs::write(&old, b"same executable content").unwrap();
16676 std::fs::copy(&old, &new).unwrap();
16677 let spawned = std::fs::canonicalize(&old).unwrap();
16678 let image = subc_control::RunningImageAgreement::Unavailable {
16679 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16680 };
16681 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
16682 assert!(
16683 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
16684 if configured == new && spawned_from == spawned)
16685 );
16686 let missing = dir.join("missing.exe");
16687 assert!(matches!(
16688 reload_verdict(&missing, Some(&spawned), image).path,
16689 ReloadPathAgreement::Mismatch { .. }
16690 ));
16691 }
16692
16693 #[test]
16694 fn reload_verdict_detects_replaced_image_at_same_path() {
16695 let image = subc_control::RunningImageAgreement::Mismatch {
16696 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
16697 digest: "old".into(),
16698 },
16699 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
16700 digest: "new".into(),
16701 },
16702 };
16703 let verdict = reload_verdict(
16704 std::path::Path::new("/bin/same"),
16705 Some(std::path::Path::new("/bin/same")),
16706 image.clone(),
16707 );
16708 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16709 assert_eq!(verdict.image, image);
16710 }
16711
16712 #[test]
16713 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
16714 let image = subc_control::RunningImageAgreement::Unavailable {
16715 reason: subc_control::RunningImageUnavailableReason::NotRunning,
16716 };
16717 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
16718 assert_eq!(
16719 verdict.path,
16720 subc_control::ReloadPathAgreement::Unavailable {
16721 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
16722 }
16723 );
16724 assert_eq!(verdict.image, image);
16725
16726 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
16727 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16728 };
16729 let verdict = reload_verdict(
16730 std::path::Path::new("/bin/same"),
16731 Some(std::path::Path::new("/bin/same")),
16732 unconfirmed.clone(),
16733 );
16734 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16735 assert_eq!(verdict.image, unconfirmed);
16736 }
16737
16738 #[test]
16739 fn reload_verdict_preserves_each_image_unavailability_reason() {
16740 use subc_control::RunningImageUnavailableReason as Reason;
16741
16742 for reason in [
16743 Reason::NotRunning,
16744 Reason::UnsupportedPlatform,
16745 Reason::RunningExecutableUnreadable,
16746 Reason::SpawnedPathUnreadable,
16747 Reason::HashFailed,
16748 Reason::ProcessIdentityUnconfirmed,
16749 Reason::Unknown("future_probe_reason".to_string()),
16750 ] {
16751 let image = subc_control::RunningImageAgreement::Unavailable {
16752 reason: reason.clone(),
16753 };
16754 let verdict = reload_verdict(
16755 std::path::Path::new("/bin/same"),
16756 Some(std::path::Path::new("/bin/same")),
16757 image.clone(),
16758 );
16759 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16760 assert_eq!(verdict.image, image, "{reason:?}");
16761 }
16762 }
16763
16764 #[tokio::test]
16765 async fn malformed_control_bodies_return_invalid_control_body() {
16766 let handler = ControlHandler::default();
16767 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
16768
16769 for (corr, body) in [
16770 (56, br#"{"route_channel":1}"#.as_slice()),
16771 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
16772 (
16773 58,
16774 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
16775 ),
16776 ] {
16777 let request = Frame::build(
16778 FrameType::Request,
16779 control_flags(),
16780 0,
16781 0,
16782 corr,
16783 body.to_vec(),
16784 )
16785 .unwrap();
16786 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16787
16788 assert_eq!(response.len(), 1);
16789 assert_eq!(response[0].header.ty, FrameType::Error);
16790 assert_eq!(response[0].header.corr, corr);
16791 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
16792 }
16793 }
16794
16795 #[tokio::test]
16796 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
16797 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16798 let registry = Arc::new(Registry::default());
16799 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16800 let router = Router::with_control_handler(Arc::clone(&control));
16801 let connection = router.begin_connection();
16802 let (ctx, mut rx) = route_ctx(connection.id());
16803
16804 router
16805 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
16806 .await
16807 .unwrap();
16808 let response = rx.recv().await.unwrap();
16809 let ack = parse_ack(&response);
16810 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16811 let channel = 1;
16812
16813 let goodbye =
16814 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
16815 router.route_for_connection(&ctx, goodbye).await.unwrap();
16816 assert!(rx.try_recv().is_err());
16817 assert!(registry.get_module("aft").unwrap().is_none());
16818
16819 router
16820 .route_for_connection(&ctx, channel_request(channel, 13))
16821 .await
16822 .unwrap();
16823 let error_frame = rx.recv().await.unwrap();
16824 assert_eq!(error_frame.header.ty, FrameType::Error);
16825 assert_eq!(error_frame.header.channel, channel);
16826 let captured = crate::router::test_log::captured_logs(&logs);
16827 assert_eq!(
16828 captured
16829 .lines()
16830 .filter(|line| {
16831 line.contains("module_id=aft")
16832 && line.contains("reason=explicit_goodbye")
16833 && line.contains("module registration ended")
16834 })
16835 .count(),
16836 1,
16837 "unexpected GOODBYE registry log: {captured}"
16838 );
16839 }
16840
16841 #[tokio::test]
16842 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
16843 let registry = Arc::new(Registry::default());
16844 let handler = ControlHandler::new(registry).with_capability_config(
16845 [("prov".to_string(), true), ("cons".to_string(), true)],
16846 BTreeMap::new(),
16847 );
16848 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
16849 register_capability_manifest(
16850 &handler,
16851 &provider_ctx,
16852 &mut provider_rx,
16853 capability_manifest("prov", &["thing/v1"], &[]),
16854 1,
16855 )
16856 .await;
16857 let mut consumer = capability_manifest("cons", &[], &[]);
16858 consumer.capabilities.as_mut().unwrap().requires.push(
16859 subc_protocol::manifest::CapabilityRequirement {
16860 capability: "thing/v1".to_string(),
16861 need: subc_protocol::manifest::CapabilityNeed::Required,
16862 },
16863 );
16864 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
16865 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
16866 assert_eq!(
16867 handler.capability_evaluator.verdict("cons", "thing/v1"),
16868 Some(CapabilityVerdict::Provided)
16869 );
16870 let (mut client_rx, _) = open_route_for_capability_test(
16871 &handler,
16872 &provider_ctx,
16873 &mut provider_rx,
16874 703,
16875 3,
16876 "prov",
16877 None,
16878 )
16879 .await;
16880 let goodbye =
16881 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
16882 handler
16883 .handle_control_frame(&provider_ctx, goodbye)
16884 .await
16885 .unwrap();
16886 assert_eq!(
16887 handler.capability_evaluator.verdict("cons", "thing/v1"),
16888 Some(CapabilityVerdict::NeverProvided)
16889 );
16890 let closed = client_rx
16891 .try_recv()
16892 .expect("GOODBYE pushes route.closed before route GOODBYE");
16893 assert!(
16894 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
16895 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
16896 );
16897 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
16898 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
16899 }
16900
16901 struct RouteClosedFixture {
16902 handler: ControlHandler,
16903 #[cfg(unix)]
16904 supervisor: Supervisor,
16905 module_ctx: RouteCtx,
16906 #[cfg(unix)]
16907 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16908 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16909 route: Arc<crate::forwarding::RouteBinding>,
16910 }
16911
16912 async fn route_closed_fixture() -> RouteClosedFixture {
16913 let registry = Arc::new(Registry::default());
16914 let handler = ControlHandler::new(Arc::clone(®istry));
16915 #[cfg(unix)]
16916 let supervisor = Supervisor::new(registry, RestartPolicy::default())
16917 .with_forwarding(Arc::clone(&handler.forwarding));
16918 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
16919 register_capability_manifest(
16920 &handler,
16921 &module_ctx,
16922 &mut module_rx,
16923 capability_manifest("prov", &[], &[]),
16924 1,
16925 )
16926 .await;
16927 let (client_rx, bind) = open_route_for_capability_test(
16928 &handler,
16929 &module_ctx,
16930 &mut module_rx,
16931 702,
16932 2,
16933 "prov",
16934 None,
16935 )
16936 .await;
16937 let ModuleControlRequest::RouteBind {
16938 route_channel: channel,
16939 epoch: route_epoch,
16940 ..
16941 } = bind
16942 else {
16943 panic!("expected route.bind");
16944 };
16945 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
16946 handler
16947 .forwarding
16948 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
16949 .unwrap()
16950 else {
16951 panic!("the acknowledged route must be bound");
16952 };
16953 RouteClosedFixture {
16954 handler,
16955 #[cfg(unix)]
16956 supervisor,
16957 module_ctx,
16958 #[cfg(unix)]
16959 module_rx,
16960 client_rx,
16961 route,
16962 }
16963 }
16964
16965 impl RouteClosedFixture {
16966 #[cfg(unix)]
16967 async fn cut(&mut self) {
16968 let notices = async {
16969 let mut draining = self.module_rx.recv().await.unwrap();
16970 assert!(matches!(
16971 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
16972 &draining.body
16973 )
16974 .unwrap(),
16975 subc_protocol::session::ModuleControlCommand::Draining {
16976 reason: RouteCloseReason::Disable,
16977 ..
16978 }
16979 ));
16980 draining.flushed.take().unwrap().send(()).unwrap();
16981 let mut closing = self.client_rx.recv().await.unwrap();
16982 assert!(matches!(
16983 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
16984 ClientControlPush::RouteClosing {
16985 reason: RouteCloseReason::Restart,
16986 ..
16987 }
16988 ));
16989 closing.flushed.take().unwrap().send(()).unwrap();
16990 };
16991 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
16992 result.unwrap();
16993 }
16994
16995 fn disconnect(&mut self) -> ClientControlPush {
16996 self.handler
16997 .cleanup_connection(self.module_ctx.connection_id)
16998 .unwrap();
16999 let closed = self.client_rx.try_recv().unwrap();
17000 assert_eq!(closed.header.ty, FrameType::Push);
17001 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
17002 assert!(matches!(
17003 &push,
17004 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
17005 if channels == &[self.route.client_channel]
17006 ));
17007 assert_eq!(
17008 self.client_rx.try_recv().unwrap().header.ty,
17009 FrameType::Goodbye
17010 );
17011 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
17012 push
17013 }
17014 }
17015
17016 #[cfg(unix)]
17017 #[tokio::test(start_paused = true)]
17018 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
17019 let mut fixture = route_closed_fixture().await;
17020 fixture.cut().await;
17021 assert!(matches!(
17022 fixture.disconnect(),
17023 ClientControlPush::RouteClosed {
17024 reason: RouteCloseReason::Restart,
17025 drained: true,
17026 ..
17027 }
17028 ));
17029 }
17030
17031 #[cfg(unix)]
17032 #[tokio::test(start_paused = true)]
17033 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
17034 let mut fixture = route_closed_fixture().await;
17035 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
17036 let started = tokio::time::Instant::now();
17037 fixture.cut().await;
17038 assert!(started.elapsed() >= Duration::from_secs(2));
17039 assert_eq!(fixture.route.flow.in_flight(), 1);
17040 assert!(matches!(
17041 fixture.disconnect(),
17042 ClientControlPush::RouteClosed {
17043 reason: RouteCloseReason::Restart,
17044 drained: false,
17045 ..
17046 }
17047 ));
17048 }
17049
17050 #[tokio::test]
17051 async fn module_connection_crash_reports_crash_and_not_drained() {
17052 let mut fixture = route_closed_fixture().await;
17053 assert!(matches!(
17057 fixture.disconnect(),
17058 ClientControlPush::RouteClosed {
17059 reason: RouteCloseReason::Crash,
17060 drained: false,
17061 ..
17062 }
17063 ));
17064 }
17065
17066 #[tokio::test]
17067 async fn dropping_router_connection_releases_registration() {
17068 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
17069 let registry = Arc::new(Registry::default());
17070 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
17071 let router = Router::with_control_handler(Arc::clone(&control));
17072 let connection = router.begin_connection();
17073 let connection_id = connection.id();
17074 let (ctx, mut rx) = route_ctx(connection_id);
17075
17076 router
17077 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
17078 .await
17079 .unwrap();
17080 let response = rx.recv().await.unwrap();
17081 let ack = parse_ack(&response);
17082 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
17083 assert!(registry.get_module("aft").unwrap().is_some());
17084
17085 drop(connection);
17086
17087 assert!(registry.get_module("aft").unwrap().is_none());
17088 assert_eq!(registry.active_registration_count().unwrap(), 0);
17089
17090 control
17091 .cleanup_connection(ConnectionId::new(u64::MAX))
17092 .unwrap();
17093 let captured = crate::router::test_log::captured_logs(&logs);
17094 println!("captured registry lifecycle logs:\n{captured}");
17095 let events: Vec<_> = captured
17096 .lines()
17097 .filter(|line| {
17098 line.contains("module registered module_id=aft ")
17099 || line.contains("module registration ended")
17100 })
17101 .collect();
17102 assert_eq!(
17103 events.len(),
17104 2,
17105 "unexpected registry lifecycle logs: {captured}"
17106 );
17107 assert!(events[0].contains("module registered module_id=aft "));
17108 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
17109 assert!(events[1].contains(&format!(
17110 "module_id=aft connection_id={}",
17111 connection_id.get()
17112 )));
17113 assert!(events[1].contains("reason=connection_closed"));
17114 assert!(events[1].contains("module registration ended"));
17115 assert_eq!(
17116 captured
17117 .lines()
17118 .filter(|line| line.contains("module registered module_id=aft "))
17119 .count(),
17120 1,
17121 "legacy registration admission line must appear once: {captured}"
17122 );
17123 }
17124
17125 #[tokio::test]
17126 async fn hello_registration_keeps_legacy_module_registered_line_once() {
17127 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
17128 let registry = Arc::new(Registry::default());
17129 let control = ControlHandler::new(Arc::clone(®istry));
17130 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
17131 hello_via_sink(
17132 &control,
17133 &ctx,
17134 &mut rx,
17135 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
17136 )
17137 .await;
17138
17139 let captured = crate::router::test_log::captured_logs(&logs);
17140 let admissions: Vec<_> = captured
17141 .lines()
17142 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
17143 .collect();
17144 assert_eq!(
17145 admissions.len(),
17146 1,
17147 "legacy admission line must remain exactly once: {captured}"
17148 );
17149 assert!(admissions[0].contains("routable_provider=true"));
17150 assert!(admissions[0].contains("connection_id=777"));
17151 }
17152
17153 fn capability_manifest(
17154 module_id: &str,
17155 provides: &[&str],
17156 must_never_reach: &[&str],
17157 ) -> ModuleManifest {
17158 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
17159 manifest.capabilities = Some(CapabilityDeclarations {
17160 provides: provides
17161 .iter()
17162 .map(|capability| (*capability).to_string())
17163 .collect(),
17164 requires: Vec::new(),
17165 must_never_reach: must_never_reach
17166 .iter()
17167 .map(|capability| (*capability).to_string())
17168 .collect(),
17169 });
17170 manifest
17171 }
17172
17173 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
17174 Frame::build(
17175 FrameType::Hello,
17176 control_flags(),
17177 0,
17178 0,
17179 corr,
17180 serde_json::to_vec(&ModuleHelloBody {
17181 protocol_ver: manifest.protocol_ver,
17182 manifest,
17183 control_ops: None,
17184 launch_nonce: None,
17185 })
17186 .expect("capability test HELLO serializes"),
17187 )
17188 .expect("capability test HELLO frame builds")
17189 }
17190
17191 fn catalog_update_with_capabilities_frame(
17192 corr: u64,
17193 capabilities: CapabilityDeclarations,
17194 ) -> Frame {
17195 Frame::build(
17196 FrameType::Request,
17197 control_flags(),
17198 0,
17199 0,
17200 corr,
17201 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17202 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
17203 capabilities: Some(capabilities),
17204 ready: None,
17205 })
17206 .expect("capability catalog.update serializes"),
17207 )
17208 .expect("capability catalog.update frame builds")
17209 }
17210
17211 async fn register_capability_manifest(
17212 handler: &ControlHandler,
17213 ctx: &RouteCtx,
17214 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
17215 manifest: ModuleManifest,
17216 corr: u64,
17217 ) {
17218 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
17219 }
17220
17221 async fn open_route_for_capability_test(
17222 handler: &ControlHandler,
17223 target_ctx: &RouteCtx,
17224 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
17225 client_connection_id: u64,
17226 corr: u64,
17227 target_module_id: &str,
17228 consumer_identity: Option<ConsumerIdentity>,
17229 ) -> (
17230 mpsc::Receiver<crate::router::OutboundFrame>,
17231 ModuleControlRequest,
17232 ) {
17233 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
17234 let route_handler = handler.clone();
17235 let target_module_id = target_module_id.to_string();
17236 let route_task = tokio::spawn(async move {
17237 route_handler
17238 .handle_control_frame(
17239 &client_ctx,
17240 route_open_frame_with_admission_facts(
17241 corr,
17242 &target_module_id,
17243 unique_project_root("admission-facts"),
17244 consumer_identity,
17245 None,
17246 ),
17247 )
17248 .await
17249 .expect("capability test route.open succeeds")
17250 });
17251 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
17252 .await
17253 .expect("capability test route.open must reach route.bind")
17254 .expect("target control receiver stays open");
17255 let bind_request: ModuleControlRequest =
17256 serde_json::from_slice(&bind.body).expect("route.bind decodes");
17257 handler
17258 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
17259 .await
17260 .expect("capability test route.bind ACK succeeds");
17261 assert!(route_task.await.expect("route.open task joins").is_empty());
17262 let opened = client_rx
17263 .recv()
17264 .await
17265 .expect("successful route.open publishes a response");
17266 assert!(matches!(
17267 serde_json::from_slice::<ClientControlResponse>(&opened.body),
17268 Ok(ClientControlResponse::RouteOpen { .. })
17269 ));
17270 (client_rx, bind_request)
17271 }
17272
17273 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
17274 assert_eq!(frame.header.ty, FrameType::Push);
17275 assert_eq!(frame.header.channel, 0);
17276 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
17277 .expect("route.closed control push decodes");
17278 let ClientControlPush::RouteClosed { channels, .. } = &push else {
17279 panic!("expected route.closed");
17280 };
17281 assert_eq!(channels.len(), 1, "exactly one violating route closed");
17282 let closed_channel = channels[0];
17283 let channels = channels.clone();
17284 assert_eq!(
17285 push,
17286 ClientControlPush::RouteClosed {
17287 module_id: target_module_id.to_string(),
17288 channels,
17289 reason: RouteCloseReason::CapabilityDenied,
17290 drained: false,
17291 abandoned: 0,
17292 excluded_subscriptions: 0,
17293 terminal: Some(false),
17294 }
17295 );
17296 closed_channel
17297 }
17298
17299 #[tokio::test]
17300 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
17301 let registry = Arc::new(Registry::default());
17302 let forwarding = Arc::new(ForwardingTable::default());
17303 let supervisor = SupervisorHandle::new();
17304 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17305 let handler =
17306 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17307 .with_supervisor(supervisor);
17308 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
17309 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
17310 register_capability_manifest(
17311 &handler,
17312 &target_ctx,
17313 &mut target_rx,
17314 capability_manifest("target", &["credentials-provider/v1"], &[]),
17315 1,
17316 )
17317 .await;
17318 register_capability_manifest(
17319 &handler,
17320 &opener_ctx,
17321 &mut opener_rx,
17322 capability_manifest("opener", &[], &["credentials-provider/v1"]),
17323 2,
17324 )
17325 .await;
17326
17327 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
17328 let replies = handler
17329 .handle_control_frame(
17330 &client_ctx,
17331 route_open_frame_with_admission_facts(
17332 3,
17333 "target",
17334 unique_project_root("admission-facts"),
17335 Some(ConsumerIdentity {
17336 module_id: "opener".to_string(),
17337 launch_nonce: "opener-nonce".to_string(),
17338 }),
17339 None,
17340 ),
17341 )
17342 .await
17343 .expect("denied route.open returns a typed frame");
17344 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
17345 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17346 assert!(
17347 target_rx.try_recv().is_err(),
17348 "forbidden route.open must not relay route.bind"
17349 );
17350 }
17351
17352 #[tokio::test]
17353 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
17354 let registry = Arc::new(Registry::default());
17355 let forwarding = Arc::new(ForwardingTable::default());
17356 let supervisor = SupervisorHandle::new();
17357 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17358 let handler =
17359 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17360 .with_supervisor(supervisor);
17361 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
17362 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
17363 register_capability_manifest(
17364 &handler,
17365 &target_ctx,
17366 &mut target_rx,
17367 capability_manifest("target", &["credentials-provider/v1"], &[]),
17368 1,
17369 )
17370 .await;
17371 register_capability_manifest(
17372 &handler,
17373 &old_opener_ctx,
17374 &mut old_opener_rx,
17375 capability_manifest("opener", &[], &[]),
17376 2,
17377 )
17378 .await;
17379 let (mut client_rx, _) = open_route_for_capability_test(
17380 &handler,
17381 &target_ctx,
17382 &mut target_rx,
17383 712,
17384 3,
17385 "target",
17386 Some(ConsumerIdentity {
17387 module_id: "opener".to_string(),
17388 launch_nonce: "opener-nonce".to_string(),
17389 }),
17390 )
17391 .await;
17392 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17393
17394 handler
17395 .cleanup_connection(old_opener_ctx.connection_id)
17396 .expect("old opener registration cleans up");
17397 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
17398 register_capability_manifest(
17399 &handler,
17400 &new_opener_ctx,
17401 &mut new_opener_rx,
17402 capability_manifest("opener", &[], &["credentials-provider/v1"]),
17403 4,
17404 )
17405 .await;
17406
17407 let closed_channel = assert_capability_denied_push(
17408 client_rx
17409 .try_recv()
17410 .expect("HELLO deny addition must emit route.closed")
17411 .frame,
17412 "target",
17413 );
17414 let goodbye = client_rx
17415 .try_recv()
17416 .expect("HELLO deny addition must send the client a route GOODBYE")
17417 .frame;
17418 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17419 assert_eq!(goodbye.header.channel, closed_channel);
17420 assert_ne!(goodbye.header.channel, 0);
17421 assert_ne!(goodbye.header.epoch, 0);
17422 assert!(client_rx.try_recv().is_err());
17423 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17424 assert!(matches!(
17425 target_rx.try_recv(),
17426 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
17427 ));
17428 }
17429
17430 #[tokio::test]
17431 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
17432 let registry = Arc::new(Registry::default());
17433 let forwarding = Arc::new(ForwardingTable::default());
17434 let supervisor = SupervisorHandle::new();
17435 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17436 let handler =
17437 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17438 .with_supervisor(supervisor);
17439 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
17440 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
17441 register_capability_manifest(
17442 &handler,
17443 &target_ctx,
17444 &mut target_rx,
17445 capability_manifest("target", &[], &[]),
17446 1,
17447 )
17448 .await;
17449 register_capability_manifest(
17450 &handler,
17451 &opener_ctx,
17452 &mut opener_rx,
17453 capability_manifest("opener", &[], &["credentials-provider/v1"]),
17454 2,
17455 )
17456 .await;
17457 let (mut client_rx, _) = open_route_for_capability_test(
17458 &handler,
17459 &target_ctx,
17460 &mut target_rx,
17461 722,
17462 3,
17463 "target",
17464 Some(ConsumerIdentity {
17465 module_id: "opener".to_string(),
17466 launch_nonce: "opener-nonce".to_string(),
17467 }),
17468 )
17469 .await;
17470 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17471
17472 let replies = handler
17473 .handle_control_frame(
17474 &target_ctx,
17475 catalog_update_with_capabilities_frame(
17476 4,
17477 CapabilityDeclarations {
17478 provides: vec!["credentials-provider/v1".to_string()],
17479 requires: Vec::new(),
17480 must_never_reach: Vec::new(),
17481 },
17482 ),
17483 )
17484 .await
17485 .expect("claim catalog.update succeeds");
17486 assert!(matches!(
17487 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
17488 Ok(ModuleControlResponseToModule::CatalogUpdate {})
17489 ));
17490 let closed_channel = assert_capability_denied_push(
17491 client_rx
17492 .try_recv()
17493 .expect("claim addition must emit route.closed")
17494 .frame,
17495 "target",
17496 );
17497 let goodbye = client_rx
17498 .try_recv()
17499 .expect("claim addition must send the client a route GOODBYE")
17500 .frame;
17501 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17502 assert_eq!(goodbye.header.channel, closed_channel);
17503 assert_ne!(goodbye.header.channel, 0);
17504 assert_ne!(goodbye.header.epoch, 0);
17505 assert!(client_rx.try_recv().is_err());
17506 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17507 assert!(matches!(
17508 target_rx.try_recv(),
17509 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
17510 ));
17511 }
17512
17513 #[tokio::test]
17514 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
17515 let registry = Arc::new(Registry::default());
17516 let forwarding = Arc::new(ForwardingTable::default());
17517 let supervisor = SupervisorHandle::new();
17518 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17519 let handler =
17520 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17521 .with_supervisor(supervisor);
17522 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
17523 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
17524 register_capability_manifest(
17525 &handler,
17526 &target_ctx,
17527 &mut target_rx,
17528 capability_manifest("target", &["credentials-provider/v1"], &[]),
17529 1,
17530 )
17531 .await;
17532 register_capability_manifest(
17533 &handler,
17534 &opener_ctx,
17535 &mut opener_rx,
17536 capability_manifest("opener", &[], &[]),
17537 2,
17538 )
17539 .await;
17540 let (mut client_rx, _) = open_route_for_capability_test(
17541 &handler,
17542 &target_ctx,
17543 &mut target_rx,
17544 732,
17545 3,
17546 "target",
17547 Some(ConsumerIdentity {
17548 module_id: "opener".to_string(),
17549 launch_nonce: "opener-nonce".to_string(),
17550 }),
17551 )
17552 .await;
17553 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17554
17555 handler
17556 .handle_control_frame(
17557 &target_ctx,
17558 catalog_update_with_capabilities_frame(
17559 4,
17560 CapabilityDeclarations {
17561 provides: Vec::new(),
17562 requires: Vec::new(),
17563 must_never_reach: Vec::new(),
17564 },
17565 ),
17566 )
17567 .await
17568 .expect("claim removal catalog.update succeeds");
17569 assert_eq!(
17570 forwarding.active_binding_count().unwrap(),
17571 1,
17572 "removing an attested target claim must leave the route census unchanged"
17573 );
17574 assert!(
17575 client_rx.try_recv().is_err(),
17576 "claim removal must not emit route.closed or a client route GOODBYE"
17577 );
17578 assert!(
17579 target_rx.try_recv().is_err(),
17580 "claim removal must not send the target a route GOODBYE"
17581 );
17582 }
17583
17584 #[tokio::test]
17587 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
17588 let registry = Arc::new(Registry::default());
17589 let forwarding = Arc::new(ForwardingTable::default());
17590 let supervisor = SupervisorHandle::new();
17591 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17592 let handler =
17593 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17594 .with_supervisor(supervisor);
17595 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
17596 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
17597 register_capability_manifest(
17598 &handler,
17599 &target_ctx,
17600 &mut target_rx,
17601 capability_manifest("target", &["credentials-provider/v1"], &[]),
17602 1,
17603 )
17604 .await;
17605 register_capability_manifest(
17606 &handler,
17607 &opener_ctx,
17608 &mut opener_rx,
17609 capability_manifest("opener", &[], &["credentials-provider/v1"]),
17610 2,
17611 )
17612 .await;
17613
17614 let (_client_rx, bind) = open_route_for_capability_test(
17615 &handler,
17616 &target_ctx,
17617 &mut target_rx,
17618 742,
17619 3,
17620 "target",
17621 None,
17622 )
17623 .await;
17624 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
17625 panic!("direct scope-honesty route must bind");
17626 };
17627 assert_eq!(principal, Some(Principal::Direct));
17628 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17629 }
17630
17631 #[tokio::test]
17634 async fn must_never_reach_self_route_is_capability_forbidden() {
17635 let registry = Arc::new(Registry::default());
17636 let forwarding = Arc::new(ForwardingTable::default());
17637 let supervisor = SupervisorHandle::new();
17638 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
17639 let handler =
17640 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17641 .with_supervisor(supervisor);
17642 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
17643 register_capability_manifest(
17644 &handler,
17645 &self_ctx,
17646 &mut self_rx,
17647 capability_manifest(
17648 "self-provider",
17649 &["credentials-provider/v1"],
17650 &["credentials-provider/v1"],
17651 ),
17652 1,
17653 )
17654 .await;
17655
17656 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
17657 let replies = handler
17658 .handle_control_frame(
17659 &client_ctx,
17660 route_open_frame_with_admission_facts(
17661 2,
17662 "self-provider",
17663 unique_project_root("admission-facts"),
17664 Some(ConsumerIdentity {
17665 module_id: "self-provider".to_string(),
17666 launch_nonce: "self-nonce".to_string(),
17667 }),
17668 None,
17669 ),
17670 )
17671 .await
17672 .expect("self-route refusal returns a typed frame");
17673 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
17674 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17675 assert!(
17676 self_rx.try_recv().is_err(),
17677 "self denial must not relay route.bind"
17678 );
17679 }
17680
17681 #[test]
17682 fn unsupported_channel_zero_frame_returns_error() {
17683 let handler = ControlHandler::default();
17684 let request = Frame::build(
17685 FrameType::Request,
17686 control_flags(),
17687 0,
17688 0,
17689 21,
17690 b"opaque".to_vec(),
17691 )
17692 .unwrap();
17693
17694 let response = handler
17695 .handle_control(ConnectionId::new(1), request)
17696 .unwrap();
17697
17698 assert_eq!(response[0].header.ty, FrameType::Error);
17699 assert_eq!(
17700 parse_error(&response[0])["code"],
17701 "unsupported_control_frame"
17702 );
17703 }
17704
17705 mod swap {
17710 use super::*;
17711
17712 const INCUMBENT: ConnectionId = ConnectionId::new(30);
17713 const CANDIDATE: ConnectionId = ConnectionId::new(40);
17714
17715 struct Swap {
17716 registry: Arc<Registry>,
17717 forwarding: Arc<ForwardingTable>,
17718 handler: ControlHandler,
17719 incumbent_ctx: RouteCtx,
17720 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17721 candidate_ctx: RouteCtx,
17722 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17723 }
17724
17725 async fn swap_with_incumbent() -> Swap {
17726 let registry = Arc::new(Registry::default());
17727 let forwarding = Arc::new(ForwardingTable::default());
17728 let handler =
17729 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17730 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
17731 hello_via_sink(
17732 &handler,
17733 &incumbent_ctx,
17734 &mut incumbent_rx,
17735 hello_frame("aft", PROTOCOL_VERSION, 7),
17736 )
17737 .await;
17738 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
17739 Swap {
17740 registry,
17741 forwarding,
17742 handler,
17743 incumbent_ctx,
17744 incumbent_rx,
17745 candidate_ctx,
17746 candidate_rx,
17747 }
17748 }
17749
17750 fn register_candidate(swap: &Swap, ready: Option<bool>) {
17751 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
17752 candidate_manifest.ready = ready;
17753 let registration = swap
17754 .registry
17755 .register_candidate_with_control_ops(
17756 candidate_manifest,
17757 PROTOCOL_VERSION,
17758 CANDIDATE,
17759 module_baseline_control_ops(),
17760 )
17761 .unwrap();
17762 swap.forwarding
17763 .register_candidate_module_connection(
17764 CANDIDATE,
17765 "aft".to_string(),
17766 PROTOCOL_VERSION,
17767 manifest_concurrency(®istration.manifest),
17768 swap.candidate_ctx.egress.clone(),
17769 )
17770 .unwrap();
17771 }
17772
17773 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
17774 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
17775 swap.registry.promote_candidate("aft").unwrap().unwrap();
17776 cutover.incumbent.unwrap()
17777 }
17778
17779 #[tokio::test]
17788 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
17789 let mut swap = swap_with_incumbent().await;
17790 let hold = Arc::new(std::sync::Barrier::new(2));
17791 let handler = swap
17792 .handler
17793 .clone()
17794 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
17795 handler
17796 .supervisor
17797 .open_swap("aft", "candidate-nonce".to_string());
17798 let hello = {
17799 let handler = handler.clone();
17800 let candidate_ctx = swap.candidate_ctx.clone();
17801 std::thread::spawn(move || {
17802 handler
17803 .handle_hello(
17804 candidate_ctx.connection_id,
17805 Some(candidate_ctx.egress.clone()),
17806 hello_frame_with_nonce(
17807 "aft",
17808 PROTOCOL_VERSION,
17809 8,
17810 Some("candidate-nonce"),
17811 ),
17812 )
17813 .unwrap()
17814 })
17815 };
17816
17817 hold.wait();
17818 assert!(swap
17821 .registry
17822 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
17823 .unwrap()
17824 .is_some());
17825 assert!(swap
17826 .registry
17827 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17828 .unwrap()
17829 .is_none());
17830 assert!(swap
17831 .forwarding
17832 .module_endpoint_for_connection(CANDIDATE)
17833 .unwrap()
17834 .is_none());
17835
17836 hold.wait();
17837 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
17838 assert_eq!(
17839 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
17840 FrameType::HelloAck
17841 );
17842 let published = swap
17843 .registry
17844 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17845 .unwrap()
17846 .expect("candidate is published once its HELLO completes");
17847 assert_eq!(published.connection_id, CANDIDATE);
17848 assert!(swap
17849 .forwarding
17850 .module_endpoint_for_connection(CANDIDATE)
17851 .unwrap()
17852 .is_some());
17853 }
17854
17855 fn keyed_total(counters: &Value, key: &str) -> u64 {
17856 counters[key]
17857 .as_object()
17858 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
17859 .unwrap_or(0)
17860 }
17861
17862 #[tokio::test]
17863 async fn replacement_logs_old_end_and_new_admission_once() {
17864 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
17865 let mut swap = swap_with_incumbent().await;
17866 swap.handler
17867 .supervisor
17868 .open_swap("aft", "candidate-nonce".to_string());
17869 hello_via_sink(
17870 &swap.handler,
17871 &swap.candidate_ctx,
17872 &mut swap.candidate_rx,
17873 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
17874 )
17875 .await;
17876 cutover(&swap);
17877 swap.handler.cleanup_connection(INCUMBENT).unwrap();
17878
17879 let captured = crate::router::test_log::captured_logs(&logs);
17880 println!("captured replacement registry logs:\n{captured}");
17881 let new_admissions: Vec<_> = captured
17882 .lines()
17883 .filter(|line| {
17884 line.contains("connection_id=40")
17885 && line.contains("swap candidate registered; not routable until cutover")
17886 })
17887 .collect();
17888 assert_eq!(
17889 new_admissions.len(),
17890 1,
17891 "unexpected admission logs: {captured}"
17892 );
17893 assert!(new_admissions[0].contains("module_id=aft"));
17894 assert!(new_admissions[0].contains("connection_id=40"));
17895 assert!(
17896 new_admissions[0].contains("swap candidate registered; not routable until cutover")
17897 );
17898 assert!(new_admissions[0].contains("ready=true"));
17899
17900 let old_admissions: Vec<_> = captured
17901 .lines()
17902 .filter(|line| {
17903 line.contains("module registered module_id=aft ")
17904 && line.contains("connection_id=30")
17905 })
17906 .collect();
17907 assert_eq!(
17908 old_admissions.len(),
17909 1,
17910 "unexpected incumbent admission: {captured}"
17911 );
17912
17913 let old_ends: Vec<_> = captured
17914 .lines()
17915 .filter(|line| {
17916 line.contains("connection_id=30") && line.contains("module registration ended")
17917 })
17918 .collect();
17919 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
17920 assert!(old_ends[0].contains("module_id=aft"));
17921 assert!(old_ends[0].contains("connection_id=30"));
17922 assert!(old_ends[0].contains("reason=replaced"));
17923 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
17924 assert_eq!(
17925 captured
17926 .lines()
17927 .filter(|line| line.contains("module registration promoted"))
17928 .count(),
17929 1,
17930 "unexpected promotion logs: {captured}"
17931 );
17932 }
17933
17934 #[tokio::test]
17940 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
17941 let mut swap = swap_with_incumbent().await;
17942 let handler = swap.handler.clone();
17943
17944 let cotenant = ConnectionId::new(31);
17946 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
17947 let (cotenant_task, cotenant_bind) = relay_route_open(
17948 &handler,
17949 cotenant,
17950 &cotenant_ctx.egress,
17951 &mut swap.incumbent_rx,
17952 100,
17953 "aft",
17954 "swap-cotenant",
17955 )
17956 .await;
17957 handler
17958 .handle_control_frame(
17959 &swap.incumbent_ctx,
17960 route_bind_ack(cotenant_bind.header.corr),
17961 )
17962 .await
17963 .unwrap();
17964 assert!(cotenant_task.await.unwrap().is_empty());
17965 let (cotenant_channel, cotenant_epoch) =
17966 published_route(&cotenant_rx.recv().await.unwrap());
17967
17968 let caller = ConnectionId::new(32);
17970 let (caller_ctx, mut caller_rx) = route_ctx(caller);
17971 let (caller_task, caller_bind) = relay_route_open(
17972 &handler,
17973 caller,
17974 &caller_ctx.egress,
17975 &mut swap.incumbent_rx,
17976 101,
17977 "aft",
17978 "swap-caller",
17979 )
17980 .await;
17981 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
17982
17983 register_candidate(&swap, None);
17984 cutover(&swap);
17985
17986 let ack = handler
17988 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
17989 .await;
17990 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
17991 if module_loop_error.is_some() {
17992 handler.cleanup_connection(INCUMBENT).unwrap();
17995 }
17996
17997 assert!(
17999 cotenant_rx.try_recv().is_err(),
18000 "the co-tenant route on the incumbent was torn down by one late ack: \
18001 {module_loop_error:?}"
18002 );
18003 assert!(matches!(
18004 swap.forwarding
18005 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
18006 .unwrap(),
18007 DataRoute::Client(DataRouteState::Bound(_))
18008 ));
18009 assert_eq!(module_loop_error, None);
18010 assert!(swap
18011 .registry
18012 .get_module_by_connection(INCUMBENT)
18013 .unwrap()
18014 .is_some());
18015
18016 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
18018 .await
18019 .expect("the incumbent is told to drop the abandoned binding")
18020 .unwrap()
18021 .frame;
18022 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18023 assert_eq!(goodbye.header.channel, abandoned_channel);
18024 assert_eq!(goodbye.header.epoch, abandoned_epoch);
18025 assert!(swap.incumbent_rx.try_recv().is_err());
18026
18027 let response = caller_task.await.unwrap();
18029 assert_eq!(response.len(), 1);
18030 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
18031 assert!(caller_rx.try_recv().is_err());
18032
18033 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
18037 let counters = handler.counters().snapshot();
18038 assert_eq!(
18039 keyed_total(&counters, "route_open_accepted_by_principal"),
18040 1
18041 );
18042 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
18043 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
18044 }
18045
18046 #[tokio::test]
18050 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
18051 let mut swap = swap_with_incumbent().await;
18052 register_candidate(&swap, None);
18053 let incumbent = cutover(&swap);
18054 swap.forwarding
18055 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
18056 .unwrap()
18057 .expect("the incumbent is still registered");
18058
18059 let client = ConnectionId::new(33);
18060 let (client_ctx, mut client_rx) = route_ctx(client);
18061 let route_handler = swap.handler.clone();
18062 let open_ctx = RouteCtx {
18063 connection_id: client,
18064 egress: client_ctx.egress.clone(),
18065 };
18066 let mut route_task = tokio::spawn(async move {
18067 route_handler
18068 .handle_control_frame(
18069 &open_ctx,
18070 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
18071 )
18072 .await
18073 .unwrap()
18074 });
18075 let bind = tokio::select! {
18076 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
18077 response = &mut route_task => {
18078 let response = response.unwrap();
18079 panic!(
18080 "post-cutover route.open was refused instead of relayed to the candidate: {}",
18081 parse_error(&response[0])["code"]
18082 );
18083 }
18084 };
18085 swap.handler
18086 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
18087 .await
18088 .unwrap();
18089 assert!(route_task.await.unwrap().is_empty());
18090 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
18091 match swap
18092 .forwarding
18093 .lookup_data_route(client, channel, epoch)
18094 .unwrap()
18095 {
18096 DataRoute::Client(DataRouteState::Bound(route)) => {
18097 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
18098 }
18099 other => panic!("expected a bound route on the candidate, got {other:?}"),
18100 }
18101 assert!(swap.incumbent_rx.try_recv().is_err());
18102 }
18103
18104 #[tokio::test]
18109 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
18110 let swap = swap_with_incumbent().await;
18111 register_candidate(&swap, Some(false));
18112 let update = Frame::build(
18113 FrameType::Request,
18114 control_flags(),
18115 0,
18116 0,
18117 55,
18118 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
18119 provides: manifest("aft", PROTOCOL_VERSION).provides,
18120 capabilities: None,
18121 ready: Some(true),
18122 })
18123 .unwrap(),
18124 )
18125 .unwrap();
18126
18127 let replies = swap
18128 .handler
18129 .handle_control_frame(&swap.candidate_ctx, update)
18130 .await
18131 .unwrap();
18132
18133 assert_eq!(replies.len(), 1);
18134 assert_eq!(
18135 replies[0].header.ty,
18136 FrameType::Response,
18137 "candidate catalog.update was refused: {:?}",
18138 serde_json::from_slice::<Value>(&replies[0].body).ok()
18139 );
18140 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
18141 assert_eq!(
18142 swap.registry
18143 .get_module("aft")
18144 .unwrap()
18145 .unwrap()
18146 .connection_id,
18147 INCUMBENT
18148 );
18149 }
18150 }
18151
18152 mod swap_admission {
18156 use super::*;
18157
18158 const INCUMBENT_NONCE: &str = "incumbent-nonce";
18159 const CANDIDATE_NONCE: &str = "candidate-nonce";
18160
18161 fn handler_with_incumbent(
18162 module_id: &str,
18163 reserved: bool,
18164 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
18165 let registry = Arc::new(Registry::default());
18166 let supervisor = SupervisorHandle::new();
18167 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
18168 if reserved {
18169 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
18170 }
18171 let handler =
18172 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
18173 let incumbent = handler
18174 .handle_control(
18175 ConnectionId::new(1),
18176 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
18177 )
18178 .unwrap();
18179 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
18180 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
18181 (registry, supervisor, handler)
18182 }
18183
18184 #[test]
18190 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
18191 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
18192
18193 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
18194 let replies = handler
18195 .handle_control(
18196 ConnectionId::new(connection),
18197 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
18198 )
18199 .unwrap();
18200 assert_eq!(replies[0].header.ty, FrameType::Error);
18201 assert_eq!(
18202 parse_error(&replies[0])["code"],
18203 "swap_token_invalid",
18204 "nonce {nonce:?}"
18205 );
18206 }
18207 assert!(registry.get_candidate("aft").unwrap().is_none());
18208 assert_eq!(
18209 registry.get_module("aft").unwrap().unwrap().connection_id,
18210 ConnectionId::new(1)
18211 );
18212
18213 let admitted = handler
18216 .handle_control(
18217 ConnectionId::new(4),
18218 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
18219 )
18220 .unwrap();
18221 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
18222 assert_eq!(
18223 registry
18224 .get_candidate("aft")
18225 .unwrap()
18226 .unwrap()
18227 .connection_id,
18228 ConnectionId::new(4)
18229 );
18230 assert_eq!(
18231 registry.get_module("aft").unwrap().unwrap().connection_id,
18232 ConnectionId::new(1),
18233 "the candidate must not take the active slot"
18234 );
18235 let replayed = handler
18236 .handle_control(
18237 ConnectionId::new(5),
18238 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
18239 )
18240 .unwrap();
18241 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
18242
18243 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
18247 let squatter = handler
18248 .handle_control(
18249 ConnectionId::new(6),
18250 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
18251 )
18252 .unwrap();
18253 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
18254 assert!(
18255 registry.get_module("aft").unwrap().is_none(),
18256 "a squatter took the active slot of an id being swapped"
18257 );
18258 }
18259
18260 #[test]
18265 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
18266 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
18267
18268 let replies = handler
18269 .handle_control(
18270 ConnectionId::new(2),
18271 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
18272 )
18273 .unwrap();
18274
18275 assert_eq!(
18276 replies[0].header.ty,
18277 FrameType::HelloAck,
18278 "reserved candidate refused: {:?}",
18279 serde_json::from_slice::<Value>(&replies[0].body).ok()
18280 );
18281 assert_eq!(
18282 registry
18283 .get_candidate("vault")
18284 .unwrap()
18285 .unwrap()
18286 .connection_id,
18287 ConnectionId::new(2)
18288 );
18289 }
18290
18291 #[test]
18294 fn without_an_open_swap_the_ordinary_gates_decide() {
18295 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
18296 supervisor.close_swap("vault");
18297
18298 let candidate = handler
18299 .handle_control(
18300 ConnectionId::new(2),
18301 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
18302 )
18303 .unwrap();
18304 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
18305 let duplicate = handler
18306 .handle_control(
18307 ConnectionId::new(3),
18308 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
18309 )
18310 .unwrap();
18311 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
18312 assert!(registry.get_candidate("vault").unwrap().is_none());
18313 }
18314 }
18315
18316 mod scopes {
18320 mod idle_tests {
18321 use super::*;
18322 include!("scope_idle_tests.rs");
18323 }
18324 mod route_open_timings {
18325 use super::*;
18326 include!("route_open_timing_tests.rs");
18327 }
18328 mod route_open_refusal_log {
18329 use super::*;
18330 include!("route_open_refusal_log_tests.rs");
18331 }
18332 mod activity {
18333 include!("activity_scope_tests.rs");
18334 }
18335 mod delta {
18336 include!("scope_handler_delta_tests.rs");
18337 }
18338 mod call_modes {
18339 include!("scope_call_mode_tests.rs");
18340 }
18341 mod federated {
18342 include!("scope_federated_tests.rs");
18343 }
18344 mod authorize {
18345 include!("route_authorize_tests.rs");
18346 }
18347 #[tokio::test]
18348 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
18349 let rig = rig().await;
18350 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
18351 "s".into(),
18352 1,
18353 "tool".into(),
18354 ))
18355 .unwrap();
18356 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
18357 let (direct, _rx) = wide_ctx(999);
18358 for (ctx, expected) in [
18359 (&rig.owner, FrameType::Response),
18360 (&direct, FrameType::Error),
18361 ] {
18362 let frame =
18363 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
18364 .unwrap();
18365 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
18366 assert_eq!(replies[0].header.ty, expected);
18367 if expected == FrameType::Response {
18368 let reply: subc_control::ScopeServedByReply =
18369 serde_json::from_slice(&replies[0].body).unwrap();
18370 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
18371 } else {
18372 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
18373 }
18374 }
18375 let reply = call(
18376 &rig.handler,
18377 &rig.owner,
18378 &ModuleControlRequestFromModule::ScopeDescribe {
18379 owner: Principal::Reserved {
18380 module_id: OWNER.into(),
18381 },
18382 scope_ref: "s".into(),
18383 },
18384 )
18385 .await;
18386 assert!(matches!(
18387 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
18388 ModuleControlResponseToModule::ScopeDescribe { .. }
18389 ));
18390 }
18391
18392 use subc_protocol::scope::{
18393 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
18394 ScopeStatus,
18395 };
18396
18397 use super::*;
18398
18399 const OWNER: &str = "prefrontal-core";
18400
18401 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
18402 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
18403 }
18404
18405 async fn call(
18406 handler: &ControlHandler,
18407 ctx: &RouteCtx,
18408 request: &ModuleControlRequestFromModule,
18409 ) -> Frame {
18410 let body = serde_json::to_vec(request).unwrap();
18411 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
18412 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
18413 assert_eq!(replies.len(), 1, "{replies:?}");
18414 replies.pop().unwrap()
18415 }
18416
18417 async fn sync(
18418 handler: &ControlHandler,
18419 ctx: &RouteCtx,
18420 generation: u64,
18421 scopes: Vec<ScopeRecord>,
18422 ) -> Result<ModuleControlResponseToModule, String> {
18423 let reply = call(
18424 handler,
18425 ctx,
18426 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
18427 )
18428 .await;
18429 match reply.header.ty {
18430 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
18431 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
18432 }
18433 }
18434
18435 async fn describe(
18436 handler: &ControlHandler,
18437 ctx: &RouteCtx,
18438 owner: &str,
18439 scope_ref: &str,
18440 ) -> ModuleControlResponseToModule {
18441 let reply = call(
18442 handler,
18443 ctx,
18444 &ModuleControlRequestFromModule::ScopeDescribe {
18445 owner: Principal::Reserved {
18446 module_id: owner.to_string(),
18447 },
18448 scope_ref: scope_ref.to_string(),
18449 },
18450 )
18451 .await;
18452 assert_eq!(
18453 reply.header.ty,
18454 FrameType::Response,
18455 "{:?}",
18456 parse_error(&reply)
18457 );
18458 serde_json::from_slice(&reply.body).unwrap()
18459 }
18460
18461 #[tokio::test]
18462 async fn scope_describe_never_echoes_bind_proof_or_hash() {
18463 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18464 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18465 let mut rig = rig().await;
18466 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18467 .await;
18468 let _bound = rig
18469 .bound(
18470 Some(AFT),
18471 PLEXUS,
18472 Some(
18473 rig.selector("s", Some(1))
18474 .with_bind_proof(Some(PROOF.into())),
18475 ),
18476 )
18477 .await;
18478 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
18479 let output = serde_json::to_string(&reply).unwrap();
18480 assert!(output.contains("\"status\":\"live\""));
18481 assert!(!output.contains(PROOF));
18482 assert!(!output.contains(HASH));
18483 }
18484
18485 async fn module(
18487 handler: &ControlHandler,
18488 connection: u64,
18489 module_id: &str,
18490 nonce: Option<&str>,
18491 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
18492 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
18493 hello_via_sink(
18494 handler,
18495 &ctx,
18496 &mut rx,
18497 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
18498 )
18499 .await;
18500 (ctx, rx)
18501 }
18502
18503 #[tokio::test]
18506 async fn a_client_connection_cannot_sync_or_describe() {
18507 let handler = ControlHandler::new(Arc::new(Registry::default()));
18508 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
18509 for request in [
18510 ModuleControlRequestFromModule::ScopeSync {
18511 generation: 1,
18512 scopes: vec![head("s", 1)],
18513 },
18514 ModuleControlRequestFromModule::ScopeDescribe {
18515 owner: Principal::Direct,
18516 scope_ref: "s".to_string(),
18517 },
18518 ] {
18519 let reply = call(&handler, &ctx, &request).await;
18520 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
18521 }
18522 assert!(
18523 !handler
18524 .scopes
18525 .read()
18526 .unwrap()
18527 .describe(
18528 &Principal::Reserved {
18529 module_id: OWNER.to_string()
18530 },
18531 "s"
18532 )
18533 .owner_synced
18534 );
18535 }
18536
18537 #[tokio::test]
18540 async fn a_module_without_a_supervised_launch_cannot_sync() {
18541 let handler = ControlHandler::new(Arc::new(Registry::default()));
18542 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
18543 assert_eq!(
18544 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
18545 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18546 );
18547 }
18548
18549 #[tokio::test]
18550 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
18551 let supervisor = SupervisorHandle::new();
18552 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18553 let handler = ControlHandler::new(Arc::new(Registry::default()))
18554 .with_supervisor(supervisor.clone());
18555 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
18556 sync(&handler, &incumbent, 1, vec![head("s", 1)])
18557 .await
18558 .expect("the current launch syncs");
18559
18560 supervisor.open_swap(OWNER, "n2".to_string());
18563 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
18564 assert_eq!(
18565 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
18566 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18567 );
18568 sync(&handler, &incumbent, 2, vec![head("s", 1)])
18569 .await
18570 .expect("the serving owner syncs during the swap");
18571
18572 supervisor.close_swap(OWNER);
18575 assert_eq!(
18576 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
18577 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18578 );
18579 sync(&handler, &incumbent, 3, vec![head("s", 1)])
18580 .await
18581 .expect("the serving owner syncs after the rollback");
18582 handler.cleanup_connection(candidate.connection_id).unwrap();
18583
18584 supervisor.open_swap(OWNER, "n3".to_string());
18589 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
18590 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
18591 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
18592 .await
18593 .expect("the promoted launch takes authority");
18594 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18595 panic!("unexpected reply {reply:?}");
18596 };
18597 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
18598 assert_eq!(
18599 sync(&handler, &incumbent, 4, Vec::new()).await,
18600 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18601 );
18602 }
18603
18604 #[tokio::test]
18607 async fn closing_the_authority_connection_frees_sync_authority() {
18608 let supervisor = SupervisorHandle::new();
18609 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18610 let handler = ControlHandler::new(Arc::new(Registry::default()))
18611 .with_supervisor(supervisor.clone());
18612 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
18613 sync(&handler, &first, 10, vec![head("s", 1)])
18614 .await
18615 .unwrap();
18616 handler.cleanup_connection(first.connection_id).unwrap();
18617
18618 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
18619 sync(&handler, &second, 1, vec![head("s", 1)])
18620 .await
18621 .expect("the next connection takes the released authority");
18622 }
18623
18624 #[tokio::test]
18625 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
18626 let supervisor = SupervisorHandle::new();
18627 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18628 let handler =
18629 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
18630 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
18631 sync(&handler, &first, 10, vec![head("s", 1)])
18632 .await
18633 .unwrap();
18634 handler
18635 .handle_control_frame(
18636 &first,
18637 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
18638 )
18639 .await
18640 .unwrap();
18641 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
18642 sync(&handler, &second, 1, vec![head("s", 1)])
18643 .await
18644 .expect("GOODBYE releases authority even if the old socket remains open");
18645 }
18646
18647 #[tokio::test]
18648 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
18649 let registry = Arc::new(Registry::default());
18650 let supervisor_handle = SupervisorHandle::new();
18651 let supervisor =
18652 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
18653 .with_handle(supervisor_handle.clone())
18654 .with_daemon_incarnation("incarnation-7".to_string());
18655 supervisor
18658 .supervise_configured(
18659 ModuleSpec {
18660 carry_containment: false,
18661 module_id: OWNER.to_string(),
18662 program: PathBuf::from("/nonexistent/prefrontal-core"),
18663 args: Vec::new(),
18664 env: Vec::new(),
18665 reserved: false,
18666 reserved_prefixes: Vec::new(),
18667 protocol: ModuleProtocol::Subc,
18668 overlap: Default::default(),
18669 },
18670 false,
18671 )
18672 .unwrap();
18673 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
18674 let handler =
18675 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
18676 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
18677
18678 let ModuleControlResponseToModule::ScopeDescribe {
18680 status,
18681 daemon_incarnation,
18682 owner_synced,
18683 owner_configured,
18684 scope,
18685 ..
18686 } = describe(&handler, &reader, OWNER, "s").await
18687 else {
18688 panic!("not a describe reply");
18689 };
18690 assert_eq!(status, ScopeStatus::NotLive);
18691 assert_eq!(daemon_incarnation, "incarnation-7");
18692 assert!(!owner_synced);
18693 assert!(owner_configured);
18694 assert!(scope.is_none());
18695
18696 let ModuleControlResponseToModule::ScopeDescribe {
18699 status,
18700 owner_configured,
18701 ..
18702 } = describe(&handler, &reader, "ghost", "s").await
18703 else {
18704 panic!("not a describe reply");
18705 };
18706 assert_eq!(status, ScopeStatus::NotLive);
18707 assert!(!owner_configured);
18708
18709 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
18711 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
18712 let ModuleControlResponseToModule::ScopeDescribe {
18713 status,
18714 scope_epoch,
18715 owner_synced,
18716 scope,
18717 ..
18718 } = describe(&handler, &reader, OWNER, "s").await
18719 else {
18720 panic!("not a describe reply");
18721 };
18722 assert_eq!(status, ScopeStatus::Live);
18723 assert_eq!(scope_epoch, Some(4));
18724 assert!(owner_synced);
18725 let stamp = scope.expect("a live scope carries its stamp");
18726 assert!(
18727 stamp.owner_authorized,
18728 "prefrontal-core is the default authority"
18729 );
18730 assert_eq!(stamp.kind, ScopeKind::Head);
18731 }
18732
18733 #[tokio::test]
18734 async fn scope_authority_owners_decides_owner_authorized() {
18735 let supervisor = SupervisorHandle::new();
18736 supervisor.set_spawn_nonce("broca", "b1".to_string());
18737 let handler = ControlHandler::new(Arc::new(Registry::default()))
18738 .with_supervisor(supervisor)
18739 .with_scope_authority_owners(vec!["broca".to_string()]);
18740 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
18741 let mut gated = head("s", 1);
18742 gated.attributes.agent_id = Some("agent".to_string());
18743 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
18744 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
18745 describe(&handler, &broca, "broca", "s").await
18746 else {
18747 panic!("not a describe reply");
18748 };
18749 assert!(scope.unwrap().owner_authorized);
18750 }
18751
18752 #[tokio::test]
18756 async fn scope_ops_and_the_scopes_capability_are_advertised() {
18757 let handler = ControlHandler::new(Arc::new(Registry::default()));
18758 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
18759 let ack = hello_via_sink(
18760 &handler,
18761 &ctx,
18762 &mut rx,
18763 hello_frame("m", PROTOCOL_VERSION, 1),
18764 )
18765 .await;
18766 let ack = parse_ack(&ack);
18767 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
18768 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
18769 }
18770 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
18771
18772 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
18773 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
18774 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
18775 let reply = handler
18776 .handle_control_frame(&client, frame)
18777 .await
18778 .unwrap()
18779 .pop()
18780 .unwrap();
18781 let ClientControlResponse::ServerDescribe { capabilities, .. } =
18782 serde_json::from_slice(&reply.body).unwrap()
18783 else {
18784 panic!("not a server.describe reply");
18785 };
18786 assert!(
18787 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
18788 "{capabilities:?}"
18789 );
18790 }
18791
18792 const PLEXUS: &str = "plexus";
18795 const OTHER: &str = "other";
18796 const AFT: &str = "aft";
18797 const BROCA: &str = "broca";
18798 const MAGIC: &str = "magic-context";
18799
18800 fn nonce(module_id: &str) -> String {
18801 format!("nonce-{module_id}")
18802 }
18803
18804 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
18805 let (tx, rx) = mpsc::channel(64);
18806 (
18807 RouteCtx {
18808 connection_id: ConnectionId::new(connection),
18809 egress: FrameSink::new(tx),
18810 },
18811 rx,
18812 )
18813 }
18814
18815 struct Rig {
18819 handler: ControlHandler,
18820 forwarding: Arc<ForwardingTable>,
18821 owner: RouteCtx,
18822 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18823 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
18824 generation: u64,
18825 next_connection: u64,
18826 _supervisor: Supervisor,
18827 }
18828
18829 async fn rig() -> Rig {
18830 rig_with_flow_support(true).await
18831 }
18832
18833 async fn rig_with_flow_support(flow_support: bool) -> Rig {
18834 rig_with_scope_capabilities(flow_support, false).await
18835 }
18836
18837 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
18838 let mut provides = Vec::new();
18839 if flow_support {
18840 provides.push("flow-scopes/v1");
18841 }
18842 if run_support {
18843 provides.push("agent-run-scopes/v1");
18844 }
18845 rig_with_target_capabilities(&provides).await
18846 }
18847
18848 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
18849 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
18850 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18851 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18854 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
18855 Frame::build(
18856 FrameType::Hello,
18857 control_flags(),
18858 0,
18859 0,
18860 connection,
18861 serde_json::to_vec(&body).unwrap(),
18862 )
18863 .unwrap()
18864 }
18865
18866 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
18867 let registry = Arc::new(Registry::default());
18868 let forwarding = Arc::new(ForwardingTable::default());
18869 let supervisor_handle = SupervisorHandle::new();
18870 let supervisor =
18871 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
18872 .with_handle(supervisor_handle.clone());
18873 supervisor
18874 .supervise_configured(
18875 ModuleSpec {
18876 carry_containment: false,
18877 module_id: OWNER.to_string(),
18878 program: PathBuf::from("/nonexistent/prefrontal-core"),
18879 args: Vec::new(),
18880 env: Vec::new(),
18881 reserved: false,
18882 reserved_prefixes: Vec::new(),
18883 protocol: ModuleProtocol::Subc,
18884 overlap: Default::default(),
18885 },
18886 false,
18887 )
18888 .unwrap();
18889 for module_id in [OWNER, AFT, BROCA, MAGIC] {
18890 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
18891 }
18892 let handler =
18893 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18894 .with_supervisor(supervisor_handle);
18895 let (owner, mut owner_rx) = wide_ctx(1);
18896 hello_via_sink(
18897 &handler,
18898 &owner,
18899 &mut owner_rx,
18900 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
18901 )
18902 .await;
18903 let mut modules = BTreeMap::new();
18904 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
18905 let (ctx, mut rx) = wide_ctx(connection);
18906 let hello = hello_with_capabilities(module_id, connection, provides);
18907 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
18908 modules.insert(module_id.to_string(), (ctx, rx));
18909 }
18910 Rig {
18911 handler,
18912 forwarding,
18913 owner,
18914 _owner_rx: owner_rx,
18915 modules,
18916 generation: 0,
18917 next_connection: 100,
18918 _supervisor: supervisor,
18919 }
18920 }
18921
18922 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
18923 ScopeCarrier::new(Principal::Reserved {
18924 module_id: module_id.to_string(),
18925 })
18926 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
18927 }
18928
18929 fn session(scope_epoch: u64) -> ScopeRecord {
18932 let mut record = head("s", scope_epoch);
18933 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
18934 record.attributes.agent_id = Some("agent-1".to_string());
18935 record.attributes.delegates = true;
18936 record
18937 }
18938
18939 impl Rig {
18940 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
18941 self.generation += 1;
18942 sync(&self.handler, &self.owner, self.generation, scopes)
18943 .await
18944 .expect("the owner's sync is accepted");
18945 }
18946
18947 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18948 ScopeSelector::new(
18949 Principal::Reserved {
18950 module_id: OWNER.to_string(),
18951 },
18952 scope_ref.to_string(),
18953 scope_epoch,
18954 )
18955 }
18956
18957 fn open_frame(
18958 &mut self,
18959 opener: Option<&str>,
18960 target: &str,
18961 scope: Option<ScopeSelector>,
18962 ) -> (
18963 RouteCtx,
18964 mpsc::Receiver<crate::router::OutboundFrame>,
18965 Frame,
18966 ) {
18967 self.next_connection += 1;
18968 let (ctx, rx) = wide_ctx(self.next_connection);
18969 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
18970 busy_refusals: None,
18971 target: RouteTarget::ToolProvider {
18972 module_id: target.to_string(),
18973 },
18974 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
18975 consumer_identity: opener.map(|module_id| ConsumerIdentity {
18976 module_id: module_id.to_string(),
18977 launch_nonce: nonce(module_id),
18978 }),
18979 consumer_capabilities: None,
18980 role_versions: None,
18981 admission_facts: None,
18982 scope,
18983 })
18984 .unwrap();
18985 let frame = Frame::build(
18986 FrameType::Request,
18987 control_flags(),
18988 0,
18989 0,
18990 self.next_connection,
18991 body,
18992 )
18993 .unwrap();
18994 (ctx, rx, frame)
18995 }
18996
18997 async fn refused(
18999 &mut self,
19000 opener: Option<&str>,
19001 target: &str,
19002 scope: Option<ScopeSelector>,
19003 ) -> String {
19004 self.refusal_body(opener, target, scope).await["code"]
19005 .as_str()
19006 .unwrap()
19007 .to_string()
19008 }
19009
19010 async fn refusal_body(
19011 &mut self,
19012 opener: Option<&str>,
19013 target: &str,
19014 scope: Option<ScopeSelector>,
19015 ) -> Value {
19016 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
19017 let replies = tokio::time::timeout(
19018 Duration::from_secs(2),
19019 self.handler.handle_control_frame(&ctx, frame),
19020 )
19021 .await
19022 .expect("the open must be refused before waiting for a bind ack")
19023 .unwrap();
19024 assert_eq!(replies.len(), 1, "{replies:?}");
19025 assert_eq!(replies[0].header.ty, FrameType::Error);
19026 let (_, module_rx) = self.modules.get_mut(target).unwrap();
19027 assert!(
19028 module_rx.try_recv().is_err(),
19029 "a refused open relays nothing"
19030 );
19031 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
19032 parse_error(&replies[0])
19033 }
19034
19035 async fn relayed(
19037 &mut self,
19038 opener: Option<&str>,
19039 target: &str,
19040 scope: Option<ScopeSelector>,
19041 ) -> Relayed {
19042 self.relayed_with_legacy_root(opener, target, scope, None)
19043 .await
19044 }
19045
19046 async fn relayed_with_legacy_root(
19047 &mut self,
19048 opener: Option<&str>,
19049 target: &str,
19050 scope: Option<ScopeSelector>,
19051 legacy_root: Option<&std::path::Path>,
19052 ) -> Relayed {
19053 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
19054 if let Some(root) = legacy_root {
19055 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
19056 body["identity"]["project_root"] = serde_json::json!(root);
19057 frame.body = serde_json::to_vec(&body).unwrap();
19058 }
19059 let handler = self.handler.clone();
19060 let task_ctx = ctx.clone();
19061 let task = tokio::spawn(async move {
19062 handler
19063 .handle_control_frame(&task_ctx, frame)
19064 .await
19065 .unwrap()
19066 });
19067 let (_, module_rx) = self.modules.get_mut(target).unwrap();
19068 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
19069 .await
19070 .expect("the target receives the relayed route.bind")
19071 .unwrap()
19072 .frame;
19073 Relayed {
19074 target: target.to_string(),
19075 client: ctx,
19076 client_rx: rx,
19077 task,
19078 bind,
19079 }
19080 }
19081
19082 async fn ack(&self, relayed: &Relayed) {
19083 let (module, _) = &self.modules[&relayed.target];
19084 self.handler
19085 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
19086 .await
19087 .unwrap();
19088 }
19089
19090 async fn bound(
19092 &mut self,
19093 opener: Option<&str>,
19094 target: &str,
19095 scope: Option<ScopeSelector>,
19096 ) -> Bound {
19097 let relayed = self.relayed(opener, target, scope).await;
19098 self.commit_relayed(relayed).await
19099 }
19100
19101 async fn bound_with_legacy_root(
19102 &mut self,
19103 opener: Option<&str>,
19104 target: &str,
19105 scope: Option<ScopeSelector>,
19106 root: &std::path::Path,
19107 ) -> Bound {
19108 let relayed = self
19109 .relayed_with_legacy_root(opener, target, scope, Some(root))
19110 .await;
19111 self.commit_relayed(relayed).await
19112 }
19113
19114 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
19115 self.ack(&relayed).await;
19116 let Relayed {
19117 target,
19118 client,
19119 mut client_rx,
19120 task,
19121 bind,
19122 } = relayed;
19123 assert!(
19124 task.await.unwrap().is_empty(),
19125 "the open is answered by commit"
19126 );
19127 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
19128 Bound {
19129 target,
19130 client,
19131 client_rx,
19132 channel,
19133 epoch,
19134 bind,
19135 }
19136 }
19137
19138 fn live(&self, route: &Bound) -> bool {
19139 matches!(
19140 self.forwarding
19141 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
19142 .unwrap(),
19143 DataRoute::Client(DataRouteState::Bound(_))
19144 )
19145 }
19146 }
19147
19148 struct Relayed {
19149 target: String,
19150 client: RouteCtx,
19151 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19152 task: tokio::task::JoinHandle<Vec<Frame>>,
19153 bind: Frame,
19154 }
19155
19156 struct Bound {
19157 target: String,
19158 client: RouteCtx,
19159 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19160 channel: u16,
19161 epoch: u32,
19162 bind: Frame,
19163 }
19164
19165 impl Bound {
19166 fn closed_reason(&mut self) -> RouteCloseReason {
19169 let mut reason = None;
19170 let mut goodbye = false;
19171 while let Ok(outbound) = self.client_rx.try_recv() {
19172 let frame = outbound.frame;
19173 match frame.header.ty {
19174 FrameType::Goodbye => {
19175 assert_eq!(
19176 (frame.header.channel, frame.header.epoch),
19177 (self.channel, self.epoch)
19178 );
19179 goodbye = true;
19180 }
19181 FrameType::Push => {
19182 let ClientControlPush::RouteClosed {
19183 reason: r,
19184 module_id,
19185 ..
19186 } = serde_json::from_slice(&frame.body).unwrap()
19187 else {
19188 panic!("unexpected push");
19189 };
19190 assert_eq!(module_id, self.target);
19191 reason = Some(r);
19192 }
19193 other => panic!("unexpected frame {other:?}"),
19194 }
19195 }
19196 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
19197 reason.expect("the client is told why the route closed")
19198 }
19199
19200 fn untouched(&mut self) -> bool {
19201 self.client_rx.try_recv().is_err()
19202 }
19203
19204 fn stamp(&self) -> Option<ScopeStamp> {
19205 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
19206 ModuleControlRequest::RouteBind { scope, .. } => scope,
19207 other => panic!("expected a route.bind, got {other:?}"),
19208 }
19209 }
19210 }
19211
19212 #[tokio::test]
19213 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
19214 ) {
19215 let mut rig = rig().await;
19216 let mut record = session(1);
19217 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
19218 record.child_owners = vec![Principal::Reserved {
19219 module_id: MAGIC.to_string(),
19220 }];
19221 rig.sync(vec![record]).await;
19222 let scope = || Some(rig_selector("s", Some(1)));
19223
19224 rig.bound(Some(OWNER), PLEXUS, scope()).await;
19227 rig.bound(Some(AFT), OTHER, scope()).await;
19228 rig.bound(Some(BROCA), PLEXUS, scope()).await;
19229
19230 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
19234 assert_eq!(
19235 rig.refused(opener, target, scope()).await,
19236 error_codes::SCOPE_NOT_CARRIER,
19237 "{opener:?} -> {target}"
19238 );
19239 }
19240 }
19241
19242 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
19243 ScopeSelector::new(
19244 Principal::Reserved {
19245 module_id: OWNER.to_string(),
19246 },
19247 scope_ref.to_string(),
19248 scope_epoch,
19249 )
19250 }
19251
19252 #[tokio::test]
19253 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
19254 let mut rig = rig().await;
19255 rig.sync(vec![session(1)]).await;
19256 for opener in [OWNER, AFT] {
19257 assert_eq!(
19258 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
19259 .await,
19260 error_codes::SCOPE_EPOCH_REQUIRED,
19261 "{opener}"
19262 );
19263 }
19264 }
19265
19266 #[tokio::test]
19267 async fn admission_separates_not_synced_not_live_and_ended() {
19268 let mut rig = rig().await;
19269 let code = rig
19271 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19272 .await;
19273 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
19274 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
19275
19276 let ghost = ScopeSelector::new(
19278 Principal::Reserved {
19279 module_id: "ghost".to_string(),
19280 },
19281 "s".to_string(),
19282 Some(1),
19283 );
19284 assert_eq!(
19285 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
19286 error_codes::SCOPE_NOT_LIVE
19287 );
19288
19289 rig.sync(vec![session(2)]).await;
19290 assert_eq!(
19291 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
19292 .await,
19293 error_codes::SCOPE_NOT_LIVE
19294 );
19295 for epoch in [1, 3] {
19296 assert_eq!(
19297 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
19298 .await,
19299 error_codes::SCOPE_ENDED,
19300 "epoch {epoch}"
19301 );
19302 }
19303 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
19305 .await;
19306 }
19307
19308 #[tokio::test]
19309 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
19310 let mut rig = rig().await;
19311 rig.sync(vec![session(1)]).await;
19312 let route = rig
19313 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19314 .await;
19315 let stamp = route.stamp().expect("a scoped bind carries the stamp");
19316 assert_eq!(stamp.scope_ref, "s");
19317 assert_eq!(stamp.scope_epoch, 1);
19318 assert_eq!(stamp.kind, ScopeKind::Head);
19319 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
19320 assert!(stamp.attributes.delegates);
19321 assert!(stamp.owner_authorized);
19322 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19323 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
19324
19325 let (broca, mut broca_rx) = wide_ctx(50);
19328 hello_via_sink(
19329 &rig.handler,
19330 &broca,
19331 &mut broca_rx,
19332 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
19333 )
19334 .await;
19335 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
19336 .await
19337 .unwrap();
19338 let own = ScopeSelector::new(
19339 Principal::Reserved {
19340 module_id: BROCA.to_string(),
19341 },
19342 "b".to_string(),
19343 Some(1),
19344 );
19345 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
19346 assert!(!route.stamp().unwrap().owner_authorized);
19347 }
19348
19349 #[tokio::test]
19350 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
19351 let mut rig = rig().await;
19352 let mut record = head("s", 1);
19353 record.carriers = vec![carrier(AFT, None)];
19354 let flow_id = "Flow:run-7/step_2!~";
19355 record.attributes.flow_id = Some(flow_id.to_string());
19356 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
19357 .await
19358 .unwrap();
19359 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
19360 panic!("not a sync reply");
19361 };
19362 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
19363 let route = rig
19364 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19365 .await;
19366 assert!(rig.live(&route), "the stamped bind committed");
19367 let stamp = route.stamp().expect("a flow scope carries a stamp");
19368 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
19369 assert_eq!(stamp.attributes.agent_id, None);
19370 assert!(!stamp.attributes.delegates);
19371 assert!(stamp.owner_authorized);
19372 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19373 assert_eq!(unscoped.stamp(), None);
19374 }
19375
19376 #[tokio::test]
19377 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
19378 let mut rig = rig_with_flow_support(false).await;
19379 let mut record = session(1);
19380 record.attributes.flow_id = Some("flow:7".to_string());
19381 rig.sync(vec![record]).await;
19382 for opener in [OWNER, AFT] {
19383 let body = rig
19384 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
19385 .await;
19386 assert_eq!(body["code"], "target_flow_unsupported");
19387 let message = body["message"].as_str().unwrap();
19388 for required in [PLEXUS, "flow-scopes/v1"] {
19389 assert!(message.contains(required), "{message}");
19390 }
19391 }
19392 }
19393
19394 #[tokio::test]
19395 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
19396 let mut rig = rig_with_flow_support(true).await;
19397 let mut record = session(1);
19398 record.attributes.flow_id = Some("flow:7".to_string());
19399 rig.sync(vec![record]).await;
19400 for opener in [OWNER, AFT] {
19401 let route = rig
19402 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
19403 .await;
19404 assert!(rig.live(&route));
19405 assert_eq!(
19406 route.stamp().unwrap().attributes.flow_id.as_deref(),
19407 Some("flow:7")
19408 );
19409 }
19410 }
19411
19412 #[tokio::test]
19413 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
19414 use std::future::Future;
19415
19416 let mut rig = rig_with_flow_support(true).await;
19417 let mut record = session(1);
19418 record.attributes.flow_id = Some("flow:7".to_string());
19419 rig.sync(vec![record]).await;
19420 let (client, mut client_rx, frame) =
19421 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
19422 for _ in 0..64 {
19425 client.egress.try_send(route_bind_ack(1)).unwrap();
19426 }
19427 let handler = rig.handler.clone();
19428 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
19429 std::future::poll_fn(|cx| {
19430 assert!(open.as_mut().poll(cx).is_pending());
19431 std::task::Poll::Ready(())
19432 })
19433 .await;
19434 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19435
19436 let old_connection = rig.modules[PLEXUS].0.connection_id;
19437 rig.handler.cleanup_connection(old_connection).unwrap();
19438 let (replacement, mut replacement_rx) = wide_ctx(200);
19439 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
19440 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
19441 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
19442 let hello = Frame::build(
19443 FrameType::Hello,
19444 control_flags(),
19445 0,
19446 0,
19447 200,
19448 serde_json::to_vec(&body).unwrap(),
19449 )
19450 .unwrap();
19451 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
19452
19453 client_rx.try_recv().unwrap();
19454 let replies = tokio::time::timeout(Duration::from_secs(2), open)
19455 .await
19456 .expect("the replacement is refused without waiting for a bind ack")
19457 .unwrap();
19458 assert_eq!(replies.len(), 1);
19459 let body = parse_error(&replies[0]);
19460 assert_eq!(body["code"], "target_flow_unsupported");
19461 for required in [PLEXUS, "flow-scopes/v1"] {
19462 assert!(body["message"].as_str().unwrap().contains(required));
19463 }
19464 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
19465 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
19466 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19467 assert!(rig
19468 .handler
19469 .registry
19470 .get_module_by_connection(replacement.connection_id)
19471 .unwrap()
19472 .is_some());
19473 }
19474
19475 #[tokio::test]
19476 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
19477 use std::future::Future;
19478 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
19479 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19480 const NEXT_HASH: &str =
19481 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
19482 let mut rig = rig().await;
19483 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
19484 record.carriers.push(ScopeCarrier::new(Principal::Direct));
19485 rig.sync(vec![record.clone()]).await;
19486 let (client, mut rx, frame) = rig.open_frame(
19487 None,
19488 PLEXUS,
19489 Some(
19490 rig.selector("s", Some(1))
19491 .with_bind_proof(Some(PROOF.into())),
19492 ),
19493 );
19494 for _ in 0..64 {
19497 client.egress.try_send(route_bind_ack(1)).unwrap();
19498 }
19499 let handler = rig.handler.clone();
19500 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
19501 std::future::poll_fn(|cx| {
19502 assert!(open.as_mut().poll(cx).is_pending());
19503 std::task::Poll::Ready(())
19504 })
19505 .await;
19506 record.bind_proof_sha256 = Some(NEXT_HASH.into());
19507 rig.sync(vec![record]).await;
19508 rx.try_recv().unwrap();
19509 let replies = tokio::time::timeout(Duration::from_secs(2), open)
19510 .await
19511 .unwrap()
19512 .unwrap();
19513 assert_eq!(
19514 parse_error(&replies[0])["code"],
19515 error_codes::SCOPE_BIND_PROOF_MISMATCH
19516 );
19517 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
19518 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
19519 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19520 }
19521
19522 #[tokio::test]
19523 async fn scope_bind_proof_reserved_openers_ignore_proof() {
19524 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19525 let mut rig = rig().await;
19526 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
19527 .await;
19528 for opener in [OWNER, AFT] {
19529 for proof in [None, Some("malformed+proof=")] {
19530 let route = rig
19531 .bound(
19532 Some(opener),
19533 PLEXUS,
19534 Some(
19535 rig.selector("s", Some(1))
19536 .with_bind_proof(proof.map(str::to_string)),
19537 ),
19538 )
19539 .await;
19540 assert!(rig.live(&route));
19541 }
19542 }
19543 }
19544
19545 #[tokio::test]
19546 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
19547 use std::future::Future;
19548 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19549 const NEXT_HASH: &str =
19550 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
19551 let mut rig = rig().await;
19552 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
19553 .await;
19554 let (client, mut rx, frame) =
19555 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
19556 for _ in 0..64 {
19557 client.egress.try_send(route_bind_ack(1)).unwrap();
19558 }
19559 let handler = rig.handler.clone();
19560 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
19561 std::future::poll_fn(|cx| {
19562 assert!(open.as_mut().poll(cx).is_pending());
19563 std::task::Poll::Ready(())
19564 })
19565 .await;
19566 rig.sync(vec![
19567 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
19568 ])
19569 .await;
19570 rx.try_recv().unwrap();
19571 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19572 let bind = std::future::poll_fn(|cx| {
19573 assert!(open.as_mut().poll(cx).is_pending());
19574 module_rx.poll_recv(cx)
19575 })
19576 .await
19577 .unwrap()
19578 .frame;
19579 handler
19580 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
19581 .await
19582 .unwrap();
19583 let replies = tokio::time::timeout(Duration::from_secs(2), open)
19584 .await
19585 .unwrap()
19586 .unwrap();
19587 assert_eq!(replies.len(), 1);
19588 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
19591 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19592 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
19593 let route = rig
19594 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
19595 .await;
19596 assert!(rig.live(&route));
19597 }
19598
19599 #[tokio::test]
19600 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
19601 {
19602 let mut rig = rig_with_flow_support(false).await;
19603 rig.sync(vec![session(1)]).await;
19604 let route = rig
19605 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19606 .await;
19607 assert!(rig.live(&route));
19608 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
19609 let mut record = session(1);
19610 record.attributes.flow_id = Some("flow:7".to_string());
19611 rig.sync(vec![record]).await;
19612 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
19613 assert!(rig.live(&unscoped));
19614 assert_eq!(unscoped.stamp(), None);
19615 }
19616
19617 #[tokio::test]
19618 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
19619 let mut rig = rig().await;
19620 let mut record = session(1);
19621 record.attributes.flow_id = Some("flow:7".to_string());
19622 rig.sync(vec![record.clone()]).await;
19623 let before = rig.forwarding.published_scope_tag(OWNER, "s");
19624 let mut owner_route = rig
19625 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19626 .await;
19627 let mut carrier_route = rig
19628 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
19629 .await;
19630 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19631 rig.sync(vec![record.clone()]).await;
19632 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19633 assert!(rig.live(&owner_route) && owner_route.untouched());
19634 assert!(rig.live(&carrier_route) && carrier_route.untouched());
19635
19636 record.attributes.flow_id = Some("flow:8".to_string());
19637 rig.sync(vec![record]).await;
19638 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
19639 let before = before.unwrap();
19640 assert_eq!(after.scope_epoch, before.scope_epoch);
19641 assert!(after.version > before.version);
19642 for route in [&mut owner_route, &mut carrier_route] {
19643 assert!(!rig.live(route));
19644 assert_eq!(
19645 route.closed_reason(),
19646 RouteCloseReason::ScopeDelegationChanged
19647 );
19648 }
19649 assert!(rig.live(&unscoped) && unscoped.untouched());
19650 for target in [PLEXUS, OTHER] {
19653 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
19654 let goodbye = module_rx
19655 .try_recv()
19656 .expect("the provider sees the drain")
19657 .frame;
19658 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19659 assert!(module_rx.try_recv().is_err());
19660 }
19661 let rebound = rig
19662 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19663 .await;
19664 assert_eq!(
19665 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
19666 Some("flow:8")
19667 );
19668 }
19669
19670 #[tokio::test]
19675 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
19676 let mut rig = rig().await;
19677 rig.sync(vec![session(1)]).await;
19678 let mut cotenant = rig
19679 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19680 .await;
19681
19682 let mut changed = session(1);
19683 changed.child_owners.push(Principal::Reserved {
19684 module_id: MAGIC.to_string(),
19685 });
19686 let mut ended = None;
19687 for (code, next) in [
19688 (error_codes::SCOPE_CHANGED, vec![changed]),
19689 (error_codes::SCOPE_ENDED, Vec::new()),
19690 ] {
19691 let relayed = rig
19692 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19693 .await;
19694 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
19695 ended = Some(next.is_empty());
19696 rig.sync(next).await;
19697 rig.ack(&relayed).await;
19698 let replies = relayed.task.await.unwrap();
19699 assert_eq!(replies.len(), 1, "{replies:?}");
19700 assert_eq!(parse_error(&replies[0])["code"], code);
19701 assert_eq!(
19702 subc_protocol::error_codes::is_retryable_route_open(code),
19703 code == error_codes::SCOPE_CHANGED
19704 );
19705 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19706 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19708 let mut goodbyes = Vec::new();
19711 while let Ok(outbound) = plexus_rx.try_recv() {
19712 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
19713 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
19714 }
19715 assert!(
19716 goodbyes.contains(&(bind_channel, bind_epoch)),
19717 "{goodbyes:?}"
19718 );
19719 assert!(rig
19720 .handler
19721 .registry
19722 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
19723 .unwrap()
19724 .is_some());
19725 }
19726 assert_eq!(ended, Some(true));
19727 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
19730 }
19731
19732 #[tokio::test]
19735 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
19736 struct Case {
19737 name: &'static str,
19738 change: fn(&mut ScopeRecord),
19739 closed: [Option<RouteCloseReason>; 4],
19742 }
19743 use RouteCloseReason::*;
19744 let cases = [
19745 Case {
19746 name: "a carrier entry removed",
19747 change: |r| {
19748 r.carriers.retain(|c| {
19749 c.principal
19750 != Principal::Reserved {
19751 module_id: AFT.to_string(),
19752 }
19753 })
19754 },
19755 closed: [None, Some(ScopeCarrierRemoved), None, None],
19756 },
19757 Case {
19758 name: "a target removed from a carrier",
19759 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
19760 closed: [None, None, None, Some(ScopeCarrierRemoved)],
19761 },
19762 Case {
19763 name: "a bare carrier narrowed to targets",
19764 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
19765 closed: [None, Some(ScopeCarrierRemoved), None, None],
19766 },
19767 Case {
19768 name: "delegates turned off",
19769 change: |r| r.attributes.delegates = false,
19770 closed: [Some(ScopeDelegationChanged); 4],
19771 },
19772 Case {
19773 name: "agent_id changed",
19774 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
19775 closed: [Some(ScopeDelegationChanged); 4],
19776 },
19777 Case {
19778 name: "a carrier added, child owners changed, the record re-sent",
19779 change: |r| {
19780 r.carriers.push(carrier(MAGIC, None));
19781 r.child_owners.push(Principal::Reserved {
19782 module_id: MAGIC.to_string(),
19783 });
19784 },
19785 closed: [None; 4],
19786 },
19787 Case {
19788 name: "a target added",
19789 change: |r| {
19790 r.carriers[1]
19791 .targets
19792 .as_mut()
19793 .unwrap()
19794 .push("third".to_string())
19795 },
19796 closed: [None; 4],
19797 },
19798 Case {
19799 name: "delegates turned on",
19800 change: |r| r.attributes.delegates = true,
19801 closed: [None; 4],
19802 },
19803 ];
19804 for case in cases {
19805 let mut rig = rig().await;
19806 rig.sync(vec![session(1)]).await;
19807 let scope = || Some(rig_selector("s", Some(1)));
19808 let mut routes = [
19809 rig.bound(Some(OWNER), PLEXUS, scope()).await,
19810 rig.bound(Some(AFT), PLEXUS, scope()).await,
19811 rig.bound(Some(BROCA), PLEXUS, scope()).await,
19812 rig.bound(Some(BROCA), OTHER, scope()).await,
19813 ];
19814 let mut record = session(1);
19815 (case.change)(&mut record);
19816 rig.sync(vec![record]).await;
19817 for (index, expected) in case.closed.iter().enumerate() {
19818 let route = &mut routes[index];
19819 match expected {
19820 Some(reason) => {
19821 assert!(!rig.live(route), "{}: route {index} still live", case.name);
19822 assert_eq!(
19823 route.closed_reason(),
19824 *reason,
19825 "{}: route {index}",
19826 case.name
19827 );
19828 }
19829 None => {
19830 assert!(rig.live(route), "{}: route {index} closed", case.name);
19831 assert!(
19832 route.untouched(),
19833 "{}: route {index} was told something",
19834 case.name
19835 );
19836 }
19837 }
19838 }
19839 }
19840 }
19841
19842 #[tokio::test]
19843 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
19844 for next in [Vec::new(), vec![session(2)]] {
19846 let mut rig = rig().await;
19847 rig.sync(vec![session(1)]).await;
19848 let mut route = rig
19849 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19850 .await;
19851 rig.sync(next).await;
19852 assert!(!rig.live(&route));
19853 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
19854 }
19855
19856 let mut rig = rig().await;
19859 let mut child = session(1);
19860 child.scope_ref = "child".to_string();
19861 child.kind = ScopeKind::Worker;
19862 child.parent = Some(ScopeParent::new(
19863 Principal::Reserved {
19864 module_id: OWNER.to_string(),
19865 },
19866 "s".to_string(),
19867 1,
19868 ));
19869 rig.sync(vec![session(1), child.clone()]).await;
19870 let mut child_route = rig
19871 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
19872 .await;
19873 assert_eq!(
19874 child_route.stamp().unwrap().parent_state,
19875 Some(ParentState::Linked)
19876 );
19877 rig.sync(vec![child]).await;
19878 assert!(!rig.live(&child_route));
19879 assert_eq!(
19880 child_route.closed_reason(),
19881 RouteCloseReason::ScopeParentEnded
19882 );
19883 }
19884
19885 #[tokio::test]
19886 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
19887 ) {
19888 let mut rig = rig().await;
19889 rig.sync(vec![session(1)]).await;
19890 let mut route = rig
19891 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19892 .await;
19893 let before = rig.forwarding.published_scope_tag(OWNER, "s");
19894 rig.sync(vec![session(1)]).await;
19895 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19896 assert!(rig.live(&route) && route.untouched());
19897
19898 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
19902 .forwarding
19903 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
19904 .unwrap()
19905 else {
19906 panic!("the route is bound");
19907 };
19908 binding.flow.acquire_tagged(9, false).await.unwrap();
19909 let mut widened = session(1);
19910 widened.carriers.push(carrier(MAGIC, None));
19911 rig.sync(vec![widened]).await;
19912 assert!(rig.live(&route) && route.untouched());
19913 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19914 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
19915 assert_eq!(binding.flow.in_flight(), 1);
19918 binding
19919 .flow
19920 .acquire_tagged(10, false)
19921 .await
19922 .expect("the flow is still open");
19923 }
19924
19925 #[tokio::test]
19928 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
19929 let mut rig = rig().await;
19930 rig.sync(vec![session(1)]).await;
19931 let mut on_incumbent = rig
19932 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19933 .await;
19934
19935 let (candidate, _candidate_rx) = wide_ctx(9);
19938 let registration = rig
19939 .handler
19940 .registry
19941 .register_candidate_with_control_ops(
19942 manifest(PLEXUS, PROTOCOL_VERSION),
19943 PROTOCOL_VERSION,
19944 candidate.connection_id,
19945 module_baseline_control_ops(),
19946 )
19947 .unwrap();
19948 rig.forwarding
19949 .register_candidate_module_connection(
19950 candidate.connection_id,
19951 PLEXUS.to_string(),
19952 PROTOCOL_VERSION,
19953 manifest_concurrency(®istration.manifest),
19954 candidate.egress.clone(),
19955 )
19956 .unwrap();
19957 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
19958 rig.handler
19959 .registry
19960 .promote_candidate(PLEXUS)
19961 .unwrap()
19962 .unwrap();
19963 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
19964
19965 rig.sync(Vec::new()).await;
19966 assert!(!rig.live(&on_incumbent));
19967 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
19968 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19969 let goodbye = incumbent_rx
19970 .try_recv()
19971 .expect("the superseded endpoint is told")
19972 .frame;
19973 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19974 }
19975 }
19976}
19977
19978#[cfg(test)]
19979mod concurrency_default_exposure_tests {
19980 use super::*;
19981
19982 fn hello_body(role_json: &str) -> Vec<u8> {
19983 format!(
19984 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
19985 )
19986 .into_bytes()
19987 }
19988
19989 fn manifest_from(body: &[u8]) -> ModuleManifest {
19990 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
19991 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
19992 .expect("manifest parses")
19993 }
19994
19995 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
19996
19997 #[test]
19998 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
19999 let body = hello_body(&format!(
20000 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
20001 ));
20002 let manifest = manifest_from(&body);
20003 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
20006 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
20007 }
20008
20009 #[test]
20010 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
20011 let body = hello_body(&format!(
20012 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
20013 ));
20014 let manifest = manifest_from(&body);
20015 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
20016 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
20017 }
20018
20019 #[test]
20020 fn non_management_roles_are_never_reported() {
20021 let body = hello_body(
20022 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
20023 );
20024 let manifest = manifest_from(&body);
20025 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
20026 }
20027}