1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_flow_operations, validate_hello_rich_ui_descriptor,
26 validate_hello_role_declarations, validate_hello_self_signal_declarations,
27 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
28 ProviderRole,
29 },
30 scope::{
31 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
32 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
33 SCOPE_SYNC_OP,
34 },
35 session::{
36 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
37 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
38 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
39 ROLE_VERSIONS_FIELD,
40 },
41 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
42 Priority, RouteTarget, PROTOCOL_VERSION,
43};
44use tokio::time::{timeout_at, Instant};
45use tracing::{debug, info, warn};
46
47use crate::{
48 caller_identity::CallerIdentity,
49 capability_requirements::{
50 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
51 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
52 RuntimeModule,
53 },
54 child_roster::DaemonShutdownFlag,
55 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
56 forwarding::{
57 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
58 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
59 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
60 },
61 observability::{
62 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
63 },
64 privileged_prompt::{prompt_text, PrivilegedOperation},
65 provenance::{
66 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
67 },
68 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
69 router::{RouteCtx, RouterError},
70 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
71 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
72 stderr_tail::{CaptureState, TailEntry},
73 supervise::{
74 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
75 SupervisorHandle, SwapHelloAdmission,
76 },
77 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
78};
79
80pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
86
87const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
88const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
89const CAP_PING_PONG: &str = "ping_pong_v1";
90const CAP_SESSION_ATTACH: &str = "session_attach_v1";
91const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
92
93const SUBC_CONTROL_OPS: &[&str] = &[
94 ops::ACTIVITY_RECORD,
95 ops::ACTIVITY_QUERY,
96 ops::ACTIVITY_SUBSCRIBE,
97 ops::SERVER_DESCRIBE,
98 ops::CATALOG_LIST,
99 ops::ROUTE_OPEN,
100 ops::ROUTE_POLL,
101 ops::ROUTE_CLOSING,
102 ops::ROUTE_CLOSED,
103 ops::SUPERVISOR_LIST,
104 ops::SUPERVISOR_RESTART,
105 ops::SUPERVISOR_SWAP,
106 ops::SUPERVISOR_RELOAD,
107 ops::SUPERVISOR_RESCAN,
108 ops::SUPERVISOR_RELEASE_RESERVED,
109 ops::SUPERVISOR_SET_ENABLED,
110 ops::SUPERVISOR_HEALTH_PROBE,
111 ops::SUPERVISOR_HEALTH,
112 ops::SUPERVISOR_STDERR_TAIL,
113 ops::SUPERVISOR_TERMINALS,
114 ops::SUPERVISOR_ROUTES,
115 ops::SUPERVISOR_PROVENANCE,
116 ops::SUPERVISOR_SPAWN_SNAPSHOT,
117 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
118];
119
120const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
127
128mod rescan_checked;
129use rescan_checked::{RescanMode, RescanPlan};
130
131const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
132 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
133 "supervisor.live_roots",
134 SCOPE_SYNC_OP,
135 SCOPE_APPLY_OP,
136 SCOPE_DESCRIBE_OP,
137 "operator.confirm",
138];
139
140const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
144
145const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
146
147pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
155
156pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
171
172pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
190
191const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
192const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
193
194fn reload_verdict(
195 configured: &Path,
196 spawned_from: Option<&Path>,
197 image: subc_control::RunningImageAgreement,
198) -> PendingReloadVerdict {
199 let path = match spawned_from {
200 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
201 ReloadPathAgreement::Match
202 }
203 Some(spawned_from) => ReloadPathAgreement::Mismatch {
204 configured: configured.to_path_buf(),
205 spawned_from: spawned_from.to_path_buf(),
206 },
207 None => ReloadPathAgreement::Unavailable {
208 reason: if matches!(
209 image,
210 subc_control::RunningImageAgreement::Unavailable {
211 reason: subc_control::RunningImageUnavailableReason::NotRunning
212 }
213 ) {
214 ReloadPathUnavailableReason::NotRunning
215 } else {
216 ReloadPathUnavailableReason::SpawnedPathUnavailable
217 },
218 },
219 };
220 PendingReloadVerdict {
221 path,
222 image,
223 spec: None,
224 }
225}
226
227fn launch_spec_agreement(
228 configured: &crate::supervise::ModuleSpec,
229 running: bool,
230 spawned: Option<&crate::supervise::LaunchSpecDigest>,
231) -> subc_control::LaunchSpecAgreement {
232 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
233 if !running {
234 return Agreement::Unavailable {
235 reason: Reason::NotRunning,
236 };
237 }
238 match spawned {
239 None => Agreement::Unavailable {
240 reason: Reason::SpawnedSpecUnavailable,
241 },
242 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
243 Agreement::Match
244 }
245 Some(_) => Agreement::Mismatch,
246 }
247}
248
249fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
250 if configured == spawned_from {
251 return true;
252 }
253 #[cfg(windows)]
254 {
255 match (
261 std::fs::canonicalize(configured),
262 std::fs::canonicalize(spawned_from),
263 ) {
264 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
265 _ => false,
266 }
267 }
268 #[cfg(not(windows))]
269 {
270 false
271 }
272}
273
274#[derive(Clone)]
275struct DaemonProvenanceFacts {
276 build: DaemonBuildProvenance,
277 pid: Option<u32>,
278 started_at_ms: Option<u64>,
279 start_clock: Option<crate::clock::StartClock>,
280 executable_path: Option<PathBuf>,
281 executable_identity: Option<SpawnedFileIdentity>,
282 process_start_time: Option<u64>,
283 probe: ExecutableIdentityProbe,
284}
285
286impl Default for DaemonProvenanceFacts {
287 fn default() -> Self {
288 Self {
289 build: DaemonBuildProvenance {
290 build_git_sha: None,
291 build_lock_digest: None,
292 },
293 pid: None,
294 started_at_ms: None,
295 start_clock: None,
296 executable_path: None,
297 executable_identity: None,
298 process_start_time: None,
299 probe: ExecutableIdentityProbe::default(),
300 }
301 }
302}
303
304#[derive(Debug, Clone)]
305struct SupervisorRescanContext {
306 supervisor: Supervisor,
307 config_path: PathBuf,
308 configured_port: Option<u16>,
309 storage_config: Option<crate::daemon_config::StorageConfig>,
310 admission_facts_carrier_module_id: Option<String>,
311 admission_facts_targets: Option<Vec<String>>,
312 scope_authority_owners: Vec<String>,
313 privileged_operator_prompt: PrivilegedOperatorPrompt,
314 generation: Arc<std::sync::atomic::AtomicU64>,
320}
321
322const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
343 "reloading",
344 "supervisor_not_live",
345 "registration_not_active",
346 "no_forwarding_connection",
347 "relay_send_failed",
348];
349
350#[derive(Clone)]
352pub struct ControlHandler {
353 registry: Arc<Registry>,
354 activity: Option<crate::activity::ActivityLedger>,
355 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
356 forwarding: Arc<ForwardingTable>,
357 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
358 supervisor: SupervisorHandle,
359 subc_capabilities: Arc<[String]>,
360 route_bind_relay_timeout: Duration,
364 route_bind_relay_timeouts: BTreeMap<String, Duration>,
368 route_bind_breakers: RouteBindBreakers,
371 route_bind_concurrency: RouteBindConcurrency,
374 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
379 route_bind_breaker_threshold: u32,
381 route_bind_breaker_cooldown: Duration,
383 health_probe_timeout: Duration,
384 storage_config: Option<crate::daemon_config::StorageConfig>,
387 machine_id: Option<crate::machine_id::MachineId>,
391 admission_facts_carrier_module_id: Option<String>,
392 admission_facts_targets: Option<Vec<String>>,
393 scopes: Arc<RwLock<ScopeTable>>,
397 scope_deadline_added: Arc<tokio::sync::Notify>,
401 scope_authority_owners: Vec<String>,
404 privileged_operator_prompt: PrivilegedOperatorPrompt,
406 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
409 #[cfg(test)]
410 caller_lookup: Option<TestCallerLookup>,
411 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
414 rescan: Option<SupervisorRescanContext>,
415 connected_clients: ConnectedClients,
416 counters: DaemonCounters,
417 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
418 daemon_shutdown: DaemonShutdownFlag,
423 daemon_provenance: DaemonProvenanceFacts,
424 #[cfg(test)]
425 control_dispatch_delay: Option<Duration>,
426 #[cfg(test)]
427 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
428 #[cfg(test)]
432 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
433 #[cfg(test)]
439 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
440}
441
442impl fmt::Debug for ControlHandler {
443 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
444 f.debug_struct("ControlHandler")
445 .field("registry", &self.registry)
446 .field("forwarding", &self.forwarding)
447 .field("process_liveness", &self.process_liveness.is_some())
448 .field("supervisor", &self.supervisor)
449 .field("subc_capabilities", &self.subc_capabilities)
450 .finish()
451 }
452}
453
454struct RouteOpenRequest {
455 target: RouteTarget,
456 identity: BindIdentity,
457 consumer_identity: Option<ConsumerIdentity>,
458 consumer_capabilities: Option<Vec<String>>,
459 role_versions: Option<BTreeMap<String, String>>,
460 admission_facts: Option<serde_json::Value>,
461 scope: Option<ScopeSelector>,
462}
463
464#[derive(Clone, Copy)]
465enum RouteOpenPhase {
466 Admission,
467 ProjectRoot,
468 EgressReservation,
469 BindSend,
470 BindAckWait,
471}
472
473struct RouteOpenTimings {
477 module_id: String,
478 principal: String,
479 admission: Duration,
480 project_root: Duration,
481 egress_reservation: Duration,
482 bind_send: Duration,
483 bind_ack_wait: Duration,
484 phase: Option<RouteOpenPhase>,
485 phase_started_at: Instant,
486}
487
488impl RouteOpenTimings {
489 fn new() -> Self {
490 Self {
491 module_id: String::new(),
492 principal: "unresolved".to_string(),
494 admission: Duration::ZERO,
495 project_root: Duration::ZERO,
496 egress_reservation: Duration::ZERO,
497 bind_send: Duration::ZERO,
498 bind_ack_wait: Duration::ZERO,
499 phase: Some(RouteOpenPhase::Admission),
500 phase_started_at: Instant::now(),
501 }
502 }
503
504 fn finish(&mut self) {
505 let elapsed = self.phase_started_at.elapsed();
506 match self.phase.take() {
507 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
508 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
509 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
510 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
511 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
512 None => {}
513 }
514 }
515
516 fn enter(&mut self, phase: RouteOpenPhase) {
517 self.finish();
518 self.phase = Some(phase);
519 self.phase_started_at = Instant::now();
520 }
521}
522
523struct RouteBindReservationGuard {
524 forwarding: Arc<ForwardingTable>,
525 endpoint: ModuleEndpointId,
526 relay_corr: u64,
527 armed: bool,
528}
529
530struct ModuleControlRpcGuard {
531 forwarding: Arc<ForwardingTable>,
532 endpoint: ModuleEndpointId,
533 corr: u64,
534 armed: bool,
535}
536
537impl ModuleControlRpcGuard {
538 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
539 Self {
540 forwarding,
541 endpoint,
542 corr,
543 armed: true,
544 }
545 }
546
547 fn disarm(&mut self) {
548 self.armed = false;
549 }
550}
551
552impl Drop for ModuleControlRpcGuard {
553 fn drop(&mut self) {
554 if self.armed {
555 let _ = self
556 .forwarding
557 .cancel_module_control_rpc(self.endpoint, self.corr);
558 }
559 }
560}
561
562impl RouteBindReservationGuard {
563 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
564 Self {
565 forwarding,
566 endpoint,
567 relay_corr,
568 armed: true,
569 }
570 }
571
572 fn release_and_disarm(&mut self) {
573 if !self.armed {
574 return;
575 }
576 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
577 self.endpoint,
578 self.relay_corr,
579 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
580 ) {
581 send_goodbye_target_best_effort(
582 &self.forwarding.counters(),
583 &target,
584 "canceled route.bind",
585 );
586 }
587 self.armed = false;
588 }
589
590 fn disarm(&mut self) {
591 self.armed = false;
592 }
593}
594
595impl Drop for RouteBindReservationGuard {
596 fn drop(&mut self) {
597 self.release_and_disarm();
598 }
599}
600
601#[derive(Debug, Clone, Default)]
631pub(crate) struct RouteBindBreakers {
632 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
633}
634
635#[derive(Debug, Clone, Default)]
636pub(crate) struct RouteBindConcurrency {
637 modules: Arc<Mutex<HashMap<String, usize>>>,
638}
639
640struct RouteBindConcurrencyGuard {
641 concurrency: RouteBindConcurrency,
642 module_id: String,
643}
644
645impl RouteBindConcurrency {
646 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
649 let mut modules = self
650 .modules
651 .lock()
652 .expect("route.bind concurrency mutex poisoned");
653 let in_flight = modules.entry(module_id.to_string()).or_default();
654 if *in_flight >= limit {
655 return Err(*in_flight);
656 }
657 *in_flight += 1;
658 Ok(RouteBindConcurrencyGuard {
659 concurrency: self.clone(),
660 module_id: module_id.to_string(),
661 })
662 }
663}
664
665impl Drop for RouteBindConcurrencyGuard {
666 fn drop(&mut self) {
667 let mut modules = self
668 .concurrency
669 .modules
670 .lock()
671 .expect("route.bind concurrency mutex poisoned");
672 let remove = {
673 let in_flight = modules
674 .get_mut(&self.module_id)
675 .expect("admitted route.bind has a concurrency entry");
676 *in_flight -= 1;
677 *in_flight == 0
678 };
679 if remove {
680 modules.remove(&self.module_id);
681 }
682 }
683}
684
685#[derive(Debug, Default)]
686struct ModuleBreakerState {
687 consecutive_timeouts: u32,
689 cooldown_until: Option<Instant>,
692 probe_in_flight: Option<Arc<()>>,
697}
698
699enum RouteBindAdmission<'a> {
701 Admitted {
702 guard: RouteBindBreakerGuard<'a>,
703 probe: bool,
706 },
707 Refused {
708 consecutive_timeouts: u32,
709 retry_in: Duration,
713 probe_in_flight: bool,
714 },
715}
716
717struct RouteBindBreakerGuard<'a> {
724 breakers: RouteBindBreakers,
725 module_id: &'a str,
726 probe_token: Option<Arc<()>>,
727 settled: bool,
728}
729
730impl RouteBindBreakerGuard<'_> {
731 fn record_accepted(&mut self) -> bool {
735 self.settled = true;
736 self.breakers.record_accepted(self.module_id)
737 }
738
739 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
742 self.settled = true;
743 self.breakers.record_timeout(
744 self.module_id,
745 self.probe_token.as_ref(),
746 threshold,
747 cooldown,
748 )
749 }
750
751 fn record_inconclusive(&mut self) {
760 self.settled = true;
761 self.breakers
762 .record_inconclusive(self.module_id, self.probe_token.as_ref());
763 }
764}
765
766impl Drop for RouteBindBreakerGuard<'_> {
767 fn drop(&mut self) {
768 if !self.settled {
769 self.breakers
770 .record_inconclusive(self.module_id, self.probe_token.as_ref());
771 }
772 }
773}
774
775struct BreakerOpened {
779 consecutive_timeouts: u32,
780 reopened_after_probe: bool,
783}
784
785impl RouteBindBreakers {
786 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
787 self.modules
788 .lock()
789 .expect("route.bind breaker mutex poisoned")
790 }
791
792 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
795 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
796 probe: probe_token.is_some(),
797 guard: RouteBindBreakerGuard {
798 breakers: self.clone(),
799 module_id,
800 probe_token,
801 settled: false,
802 },
803 };
804
805 let mut modules = self.lock();
806 let Some(state) = modules.get_mut(module_id) else {
807 return admitted(None);
808 };
809 let Some(cooldown_until) = state.cooldown_until else {
810 return admitted(None);
811 };
812 if state.probe_in_flight.is_some() {
813 return RouteBindAdmission::Refused {
814 consecutive_timeouts: state.consecutive_timeouts,
815 retry_in: Duration::ZERO,
816 probe_in_flight: true,
817 };
818 }
819 let now = Instant::now();
820 if now < cooldown_until {
821 return RouteBindAdmission::Refused {
822 consecutive_timeouts: state.consecutive_timeouts,
823 retry_in: cooldown_until - now,
824 probe_in_flight: false,
825 };
826 }
827 let token = Arc::new(());
828 state.probe_in_flight = Some(Arc::clone(&token));
829 admitted(Some(token))
830 }
831
832 fn record_accepted(&self, module_id: &str) -> bool {
833 self.lock()
834 .remove(module_id)
835 .is_some_and(|state| state.cooldown_until.is_some())
836 }
837
838 fn record_timeout(
839 &self,
840 module_id: &str,
841 probe_token: Option<&Arc<()>>,
842 threshold: u32,
843 cooldown: Duration,
844 ) -> Option<BreakerOpened> {
845 let mut modules = self.lock();
846 let state = modules.entry(module_id.to_string()).or_default();
847 let was_open = state.cooldown_until.is_some();
848 let was_probe = Self::owns_probe(state, probe_token);
849 if was_probe {
850 state.probe_in_flight = None;
851 }
852 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
853 if state.consecutive_timeouts < threshold {
854 return None;
855 }
856 state.cooldown_until = Some(Instant::now() + cooldown);
857 Some(BreakerOpened {
858 consecutive_timeouts: state.consecutive_timeouts,
859 reopened_after_probe: was_open && was_probe,
860 })
861 }
862
863 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
864 state
868 .probe_in_flight
869 .as_ref()
870 .zip(token)
871 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
872 }
873
874 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
875 if let Some(state) = self.lock().get_mut(module_id) {
876 if Self::owns_probe(state, probe_token) {
877 state.probe_in_flight = None;
878 }
879 }
880 }
881
882 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
897 self.lock()
898 .remove(module_id)
899 .map(|state| state.consecutive_timeouts)
900 .filter(|discarded| *discarded > 0)
901 }
902
903 fn open_snapshot(&self) -> Option<serde_json::Value> {
910 let now = Instant::now();
911 let modules = self.lock();
912 let open = modules
913 .iter()
914 .filter_map(|(module_id, state)| {
915 let cooldown_until = state.cooldown_until?;
916 Some((
917 module_id.clone(),
918 serde_json::json!({
919 "consecutive_timeouts": state.consecutive_timeouts,
920 "cooldown_remaining_ms":
921 cooldown_until.saturating_duration_since(now).as_millis() as u64,
922 "probe_in_flight": state.probe_in_flight.is_some(),
923 }),
924 ))
925 })
926 .collect::<serde_json::Map<String, serde_json::Value>>();
927 (!open.is_empty()).then_some(serde_json::Value::Object(open))
928 }
929}
930
931impl ControlHandler {
932 pub fn new(registry: Arc<Registry>) -> Self {
933 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
934 }
935
936 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
937 let counters = forwarding.counters();
938 let route_bind_breakers = forwarding.route_bind_breakers();
942 let route_bind_concurrency = forwarding.route_bind_concurrency();
943 let route_outages = forwarding.route_outages();
944 Self {
945 registry,
946 activity: None,
947 activity_streams: Arc::new(Mutex::new(HashMap::new())),
948 forwarding,
949 process_liveness: None,
950 supervisor: SupervisorHandle::new(),
951 subc_capabilities: Arc::from([
952 CAP_MANIFEST_REGISTRATION.to_string(),
953 CAP_CHANNEL_LIFECYCLE.to_string(),
954 CAP_PING_PONG.to_string(),
955 CAP_SESSION_ATTACH.to_string(),
956 CAP_ADMISSION_FACTS_RELAY.to_string(),
957 CAP_SCOPES_V1.to_string(),
958 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
959 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
960 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
961 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
962 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
963 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
964 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
965 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
966 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
967 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
968 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
969 ]),
970 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
971 route_bind_relay_timeouts: BTreeMap::new(),
972 route_bind_breakers,
973 route_bind_concurrency,
974 route_outages,
975 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
976 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
977 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
978 storage_config: None,
979 machine_id: None,
980 admission_facts_carrier_module_id: None,
981 admission_facts_targets: None,
982 scopes: Arc::new(RwLock::new(ScopeTable::new(
983 crate::daemon_config::default_scope_authority_owners(),
984 ))),
985 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
986 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
987 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
988 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
989 #[cfg(test)]
990 caller_lookup: None,
991 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
992 rescan: None,
993 connected_clients: ConnectedClients::new(),
994 counters,
995 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
996 daemon_shutdown: DaemonShutdownFlag::default(),
997 daemon_provenance: DaemonProvenanceFacts::default(),
998 #[cfg(test)]
999 control_dispatch_delay: None,
1000 #[cfg(test)]
1001 provenance_probe_override: None,
1002 #[cfg(test)]
1003 route_bind_relay_sent_hold: None,
1004 #[cfg(test)]
1005 hello_registry_to_forwarding_hold: None,
1006 }
1007 }
1008
1009 pub fn with_storage_config(
1012 mut self,
1013 storage_config: Option<crate::daemon_config::StorageConfig>,
1014 ) -> Self {
1015 self.storage_config = storage_config;
1016 self
1017 }
1018
1019 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1022 self.activity = Some(crate::activity::ActivityLedger::open(
1023 run_dir.join("activity.jsonl"),
1024 incarnation,
1025 ));
1026 self.supervisor.set_activity(self.activity.clone());
1027 self
1028 }
1029
1030 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1033 self.machine_id = machine_id;
1034 self
1035 }
1036
1037 pub fn with_admission_facts_config(
1041 mut self,
1042 carrier_module_id: Option<String>,
1043 targets: Option<Vec<String>>,
1044 ) -> Self {
1045 self.admission_facts_carrier_module_id = carrier_module_id;
1046 self.admission_facts_targets = targets;
1047 self
1048 }
1049
1050 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1054 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1055 self.scope_authority_owners = owners;
1056 self
1057 }
1058
1059 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1061 self.privileged_operator_prompt = mode;
1062 self
1063 }
1064
1065 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1068 self.route_bind_relay_timeout = timeout;
1069 self
1070 }
1071
1072 pub fn with_route_bind_relay_timeouts(
1078 mut self,
1079 timeouts: impl IntoIterator<Item = (String, Duration)>,
1080 ) -> Self {
1081 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1082 self
1083 }
1084
1085 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1091 self.route_bind_relay_timeouts
1092 .get(module_id)
1093 .copied()
1094 .unwrap_or(self.route_bind_relay_timeout)
1095 }
1096
1097 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1105 self.route_bind_breaker_threshold = threshold.max(1);
1106 self.route_bind_breaker_cooldown = cooldown;
1107 self
1108 }
1109
1110 #[cfg(test)]
1111 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1112 self.health_probe_timeout = timeout;
1113 self
1114 }
1115
1116 #[cfg(test)]
1117 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1118 self.control_dispatch_delay = Some(delay);
1119 self
1120 }
1121
1122 #[cfg(test)]
1126 pub(crate) fn with_route_bind_relay_sent_hold(
1127 mut self,
1128 hold: Arc<tokio::sync::Barrier>,
1129 ) -> Self {
1130 self.route_bind_relay_sent_hold = Some(hold);
1131 self
1132 }
1133
1134 #[cfg(test)]
1139 pub(crate) fn with_hello_registry_to_forwarding_hold(
1140 mut self,
1141 hold: Arc<std::sync::Barrier>,
1142 ) -> Self {
1143 self.hello_registry_to_forwarding_hold = Some(hold);
1144 self
1145 }
1146
1147 #[cfg(test)]
1148 fn hold_between_registry_and_forwarding(&self) {
1149 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1150 hold.wait();
1151 hold.wait();
1152 }
1153 }
1154
1155 pub fn with_process_liveness(
1156 mut self,
1157 process_liveness: Arc<dyn ModuleProcessLiveness>,
1158 ) -> Self {
1159 self.process_liveness = Some(process_liveness);
1160 self
1161 }
1162
1163 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1164 self.supervisor = supervisor;
1165 self.supervisor.set_activity(self.activity.clone());
1166 self
1167 }
1168
1169 pub fn with_daemon_provenance(
1170 mut self,
1171 pid: u32,
1172 started_at_ms: u64,
1173 executable_path: Option<PathBuf>,
1174 build_git_sha: Option<String>,
1175 build_lock_digest: Option<String>,
1176 ) -> Self {
1177 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1178 let process_start_time = process_start_time(pid);
1179 self.daemon_provenance = DaemonProvenanceFacts {
1180 build: DaemonBuildProvenance {
1181 build_git_sha,
1182 build_lock_digest,
1183 },
1184 pid: Some(pid),
1185 started_at_ms: Some(started_at_ms),
1186 start_clock: None,
1187 executable_path,
1188 executable_identity,
1189 process_start_time,
1190 probe: ExecutableIdentityProbe::default(),
1191 };
1192 self
1193 }
1194
1195 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1196 self.daemon_provenance.start_clock = Some(clock);
1197 self
1198 }
1199
1200 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1202 self.scopes
1203 .write()
1204 .unwrap_or_else(|poisoned| poisoned.into_inner())
1205 .set_wall_clock(Arc::new(clock));
1206 self
1207 }
1208
1209 #[cfg(test)]
1210 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1211 self.provenance_probe_override = Some(result);
1212 self
1213 }
1214
1215 pub fn with_capability_config(
1219 self,
1220 modules: impl IntoIterator<Item = (String, bool)>,
1221 reserved_capabilities: BTreeMap<String, String>,
1222 ) -> Self {
1223 self.capability_evaluator
1224 .configure(modules, reserved_capabilities);
1225 self
1226 }
1227
1228 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1232 self.daemon_shutdown = flag;
1233 self
1234 }
1235
1236 pub fn with_supervisor_rescan(
1237 mut self,
1238 supervisor: Supervisor,
1239 config_path: impl Into<PathBuf>,
1240 configured_port: Option<u16>,
1241 ) -> Self {
1242 self.rescan = Some(SupervisorRescanContext {
1243 supervisor,
1244 config_path: config_path.into(),
1245 configured_port,
1246 storage_config: self.storage_config.clone(),
1247 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1248 admission_facts_targets: self.admission_facts_targets.clone(),
1249 scope_authority_owners: self.scope_authority_owners.clone(),
1250 privileged_operator_prompt: self.privileged_operator_prompt,
1251 generation: Arc::default(),
1252 });
1253 self
1254 }
1255
1256 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1257 self.connected_clients = connected_clients;
1258 self
1259 }
1260
1261 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1262 Arc::clone(&self.forwarding)
1263 }
1264
1265 pub(crate) fn counters(&self) -> DaemonCounters {
1266 self.counters.clone()
1267 }
1268
1269 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1272 tokio::spawn(async move {
1273 loop {
1274 self.capability_evaluator
1275 .wait_for_change_or_deadline()
1276 .await;
1277 self.refresh_capability_requirements();
1278 }
1279 });
1280 }
1281
1282 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1288 tokio::spawn(async move {
1289 loop {
1290 let has_deadlines = self
1291 .scopes
1292 .read()
1293 .unwrap_or_else(|poisoned| poisoned.into_inner())
1294 .has_deadlines();
1295 if !has_deadlines {
1296 self.scope_deadline_added.notified().await;
1297 continue;
1298 }
1299 tokio::time::sleep(Duration::from_secs(1)).await;
1300 if let Err(error) = self.sweep_expired_scopes() {
1301 warn!(%error, "scope expiry sweep failed");
1302 }
1303 }
1304 });
1305 }
1306
1307 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1308 let mut table = self
1309 .scopes
1310 .write()
1311 .unwrap_or_else(|poisoned| poisoned.into_inner());
1312 let swept = table.sweep_expired();
1313 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1314 drop(table);
1315 self.close_scope_drained_routes(drained);
1316 Ok(())
1317 }
1318
1319 fn publish_scope_changes(
1320 &self,
1321 changes: &[crate::scopes::ScopeTagChange],
1322 expired: &[crate::scopes::ScopeExpired],
1323 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1324 if changes.is_empty() {
1325 return Ok(Vec::new());
1326 }
1327 let drained = self
1328 .forwarding
1329 .publish_scope_changes(changes)
1330 .map_err(RouterError::Forwarding)?;
1331 let mut counts = HashMap::new();
1332 for route in &drained {
1333 *counts
1334 .entry((
1335 route.scope.owner.as_str(),
1336 route.scope.scope_ref.as_str(),
1337 route.scope.tag.scope_epoch,
1338 ))
1339 .or_insert(0usize) += 1;
1340 }
1341 for scope in expired {
1342 let routes_closed = counts
1343 .get(&(
1344 scope.owner.as_str(),
1345 scope.scope_ref.as_str(),
1346 scope.scope_epoch,
1347 ))
1348 .copied()
1349 .unwrap_or(0);
1350 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1351 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1352 routes_closed, "scope expired");
1353 }
1354 Ok(drained)
1355 }
1356
1357 fn runtime_capability_snapshot(
1358 &self,
1359 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1360 let runtime = self
1361 .supervisor
1362 .list()
1363 .into_iter()
1364 .map(|module| {
1365 let status = module.status().map_err(|err| {
1366 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1367 })?;
1368 Ok(RuntimeModule {
1369 module_id: status.module_id,
1370 state: status.state,
1371 enabled: status.enabled,
1372 })
1373 })
1374 .collect::<Result<Vec<_>, RouterError>>()?;
1375 let registrations = self.registry.list_registered_modules().map_err(|err| {
1381 RouterError::backend(
1382 0,
1383 0,
1384 format!("failed to list capability registrations: {err}"),
1385 )
1386 })?;
1387 let registrations = registrations
1388 .into_iter()
1389 .map(|registration| RegisteredModule {
1390 module_id: registration.manifest.module_id,
1391 module_version: registration.manifest.module_version,
1392 capabilities: registration.manifest.capabilities,
1393 })
1394 .collect();
1395 Ok((runtime, registrations))
1396 }
1397
1398 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1405 let cached_registration = RegisteredModule {
1406 module_id: registration.manifest.module_id.clone(),
1407 module_version: registration.manifest.module_version.clone(),
1408 capabilities: registration.manifest.capabilities.clone(),
1409 };
1410 if self.capability_evaluator.record_hello(&cached_registration) {
1411 warn!(
1412 module_id = %cached_registration.module_id,
1413 "capability claims drifted from the cached manifest"
1414 );
1415 }
1416 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1417 self.enforce_capability_denies();
1418 }
1419 self.refresh_capability_requirements();
1420 }
1421
1422 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1426 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1427 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1428 self.supervisor.set_swap_promotion_observer(observer);
1429 }
1430
1431 pub fn refresh_capability_requirements(&self) {
1432 match self.runtime_capability_snapshot() {
1433 Ok((runtime, registrations)) => {
1434 log_requirement_events(
1435 self.capability_evaluator
1436 .evaluate_now(&runtime, ®istrations),
1437 &self.daemon_shutdown,
1438 );
1439 }
1440 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1441 }
1442 }
1443
1444 fn enforce_capability_denies(&self) {
1448 let registrations = match self.registry.list_registered_modules() {
1449 Ok(snapshot) => snapshot,
1450 Err(err) => {
1451 warn!(error = %err, "failed to read registrations for capability deny census");
1452 return;
1453 }
1454 };
1455 let manifests = registrations
1456 .into_iter()
1457 .map(|registration| {
1458 (
1459 registration.manifest.module_id.clone(),
1460 registration.manifest,
1461 )
1462 })
1463 .collect::<BTreeMap<_, _>>();
1464 let census = match self.forwarding.route_census(None) {
1465 Ok(census) => census,
1466 Err(err) => {
1467 warn!(error = %err, "failed to read route census for capability deny enforcement");
1468 return;
1469 }
1470 };
1471
1472 for (target_module_id, routes) in census {
1473 let Some(target_manifest) = manifests.get(&target_module_id) else {
1474 continue;
1475 };
1476 let mut closed_routes = Vec::new();
1477 let mut goodbyes = Vec::new();
1478 for route in routes {
1479 let Principal::Reserved {
1480 module_id: opening_module_id,
1481 } = &route.principal
1482 else {
1483 continue;
1484 };
1485 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1486 continue;
1487 };
1488 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1489 continue;
1490 };
1491
1492 match self.forwarding.release_client_route(
1493 route.goodbye_target.connection_id,
1494 route.goodbye_target.channel,
1495 route.goodbye_target.epoch,
1496 ) {
1497 Ok(RouteRelease::Removed(module_goodbye)) => {
1498 warn!(
1499 opening_module_id,
1500 target_module_id,
1501 capability,
1502 "force-closing route because an attested capability deny edge now matches"
1503 );
1504 goodbyes.push(module_goodbye);
1505 goodbyes.push(route.goodbye_target.clone());
1506 closed_routes.push(route);
1507 }
1508 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1509 Err(err) => warn!(
1510 opening_module_id,
1511 target_module_id,
1512 capability,
1513 error = %err,
1514 "failed to force-close capability-denied route"
1515 ),
1516 }
1517 }
1518
1519 if closed_routes.is_empty() {
1520 continue;
1521 }
1522 send_route_control_pushes(
1523 &self.forwarding,
1524 closed_routes,
1525 ClientControlPush::RouteClosed {
1526 module_id: target_module_id,
1527 channels: Vec::new(),
1528 reason: RouteCloseReason::CapabilityDenied,
1529 drained: false,
1530 abandoned: 0,
1531 excluded_subscriptions: 0,
1532 terminal: Some(false),
1533 },
1534 );
1535 self.emit_route_goodbyes(goodbyes);
1538 }
1539 }
1540
1541 fn not_ready_reason(
1547 &self,
1548 registration: &crate::registry::ModuleRegistration,
1549 ) -> Option<NotReadyReason> {
1550 if !registration.ready {
1551 return Some(NotReadyReason {
1552 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1553 capability: None,
1554 });
1555 }
1556 self.first_unprovided_required_capability(registration)
1557 .map(|capability| NotReadyReason {
1558 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1559 capability: Some(capability),
1560 })
1561 }
1562
1563 fn first_unprovided_required_capability(
1581 &self,
1582 registration: &crate::registry::ModuleRegistration,
1583 ) -> Option<String> {
1584 let required = registration
1585 .manifest
1586 .capabilities
1587 .iter()
1588 .flat_map(|declarations| declarations.requires.iter())
1589 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1590 .map(|requirement| requirement.capability.as_str())
1591 .collect::<BTreeSet<_>>();
1592 if required.is_empty() {
1593 return None;
1594 }
1595 let module_id = registration.manifest.module_id.as_str();
1596 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1597 if required
1598 .iter()
1599 .any(|capability| verdict(capability).is_none())
1600 {
1601 self.refresh_capability_requirements();
1602 }
1603 required
1604 .into_iter()
1605 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1606 .map(str::to_string)
1607 }
1608
1609 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1610 self.capability_evaluator
1611 .statuses()
1612 .into_iter()
1613 .map(capability_requirement_status)
1614 .collect()
1615 }
1616
1617 fn deregister_connection(
1624 &self,
1625 connection_id: ConnectionId,
1626 reason: RegistrationEndReason,
1627 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1628 self.registry
1629 .deregister_connection_with_reason(connection_id, reason)
1630 }
1631
1632 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1633 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1634 return None;
1635 }
1636 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1637 parse_client_control_request(&frame.body)
1638 else {
1639 return None;
1640 };
1641 Some(target_module_id(&target).to_string())
1642 }
1643
1644 pub(crate) fn route_open_capacity_refusal(
1645 &self,
1646 ctx: &RouteCtx,
1647 frame: &Frame,
1648 target_module_id: &str,
1649 in_flight: usize,
1650 limit: usize,
1651 ) -> Result<Frame, RouterError> {
1652 let caller = self.route_open_caller_from_frame(frame);
1655 self.route_open_admission_refusal_frame(
1656 ctx,
1657 &caller,
1658 frame,
1659 target_module_id,
1660 "open_admission_full",
1661 (in_flight, limit),
1662 format!(
1663 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1664 ),
1665 )
1666 }
1667
1668 fn route_open_target_capacity_refusal(
1669 &self,
1670 ctx: &RouteCtx,
1671 caller: &RouteOpenCaller,
1672 frame: &Frame,
1673 target_module_id: &str,
1674 in_flight: usize,
1675 ) -> Result<Frame, RouterError> {
1676 self.route_open_admission_refusal_frame(
1677 ctx,
1678 caller,
1679 frame,
1680 target_module_id,
1681 "target_binds_full",
1682 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1683 format!(
1684 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1685 ),
1686 )
1687 }
1688
1689 #[allow(clippy::too_many_arguments)]
1697 fn route_open_admission_refusal_frame(
1698 &self,
1699 ctx: &RouteCtx,
1700 caller: &RouteOpenCaller,
1701 frame: &Frame,
1702 target_module_id: &str,
1703 reason: &'static str,
1704 (in_flight, limit): (usize, usize),
1705 message: impl Into<String>,
1706 ) -> Result<Frame, RouterError> {
1707 let code = error_codes::TARGET_UNAVAILABLE;
1708 self.counters.increment_route_open_refused(code);
1709 info!(
1710 target: "control",
1711 code,
1712 reason,
1713 module_id = ?target_module_id,
1714 connection_id = ctx.connection_id.get(),
1715 principal = %caller.principal,
1716 scope = %caller.scope,
1717 harness = %caller.harness,
1718 in_flight,
1719 limit,
1720 "route.open refused"
1721 );
1722 control_error_frame(frame, code, message.into())
1723 }
1724
1725 #[cfg(test)]
1731 pub fn handle_control(
1732 &self,
1733 connection_id: ConnectionId,
1734 frame: Frame,
1735 ) -> Result<Vec<Frame>, RouterError> {
1736 match frame.header.ty {
1737 FrameType::Ping => Ok(vec![pong(&frame)?]),
1738 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1739 FrameType::Goodbye => self.handle_goodbye(connection_id),
1740 ty => Ok(vec![control_error_frame(
1741 &frame,
1742 "unsupported_control_frame",
1743 format!("unsupported channel-0 frame {ty:?}"),
1744 )?]),
1745 }
1746 }
1747
1748 pub async fn handle_control_frame(
1749 &self,
1750 ctx: &RouteCtx,
1751 frame: Frame,
1752 ) -> Result<Vec<Frame>, RouterError> {
1753 self.handle_control_frame_timed(ctx, frame, None).await
1754 }
1755
1756 pub(crate) fn record_caller_endpoints(
1757 &self,
1758 connection: ConnectionId,
1759 endpoints: Option<(SocketAddr, SocketAddr)>,
1760 ) {
1761 let mut callers = self
1762 .caller_endpoints
1763 .lock()
1764 .unwrap_or_else(|p| p.into_inner());
1765 callers.remove(&connection);
1766 if let Some(endpoints) = endpoints {
1767 callers.insert(connection, endpoints);
1768 }
1769 }
1770
1771 #[cfg(test)]
1772 pub(crate) fn with_caller_lookup_for_test(
1773 mut self,
1774 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1775 ) -> Self {
1776 self.caller_lookup = Some(Arc::new(lookup));
1777 self
1778 }
1779
1780 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1781 let endpoints = self
1782 .caller_endpoints
1783 .lock()
1784 .unwrap_or_else(|p| p.into_inner())
1785 .get(&connection)
1786 .copied();
1787 #[cfg(test)]
1788 let lookup = self.caller_lookup.clone();
1789 tokio::task::spawn_blocking(move || {
1790 #[cfg(test)]
1791 if let Some(lookup) = lookup {
1792 return lookup(endpoints);
1793 }
1794 endpoints
1795 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1796 .unwrap_or_default()
1797 })
1798 .await
1799 .unwrap_or_default()
1800 }
1801
1802 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1803 frame.header.channel == 0
1804 && frame.header.ty == FrameType::Request
1805 && self
1806 .forwarding
1807 .module_endpoint_for_connection(ctx.connection_id)
1808 .is_ok_and(|endpoint| endpoint.is_none())
1809 && parse_client_control_request(&frame.body)
1810 .ok()
1811 .is_some_and(|request| privileged_operation(&request).is_some())
1812 }
1813
1814 pub(crate) async fn handle_control_frame_timed(
1815 &self,
1816 ctx: &RouteCtx,
1817 frame: Frame,
1818 dispatch_started_at: Option<StdInstant>,
1819 ) -> Result<Vec<Frame>, RouterError> {
1820 match frame.header.ty {
1821 FrameType::Ping => Ok(vec![pong(&frame)?]),
1822 FrameType::Hello => {
1823 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1824 }
1825 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1826 FrameType::Cancel => {
1827 if self
1830 .forwarding
1831 .operator_confirms()
1832 .cancel(ctx.connection_id, frame.header.corr)
1833 || self
1834 .supervisor
1835 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1836 || self.cancel_activity_subscription(ctx, &frame)?
1837 {
1838 Ok(Vec::new())
1839 } else {
1840 Ok(vec![control_error_frame(
1841 &frame,
1842 "unknown_subscription",
1843 "no subscription has this correlation id",
1844 )?])
1845 }
1846 }
1847 FrameType::Request => {
1848 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1851 if op
1852 .as_ref()
1853 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1854 {
1855 return self.handle_activity_record(ctx, frame).await;
1856 }
1857 if op.as_ref().is_some_and(|probe| {
1858 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1859 }) {
1860 return self.handle_activity_read(ctx, frame).await;
1861 }
1862 if op
1863 .as_ref()
1864 .is_some_and(|probe| probe.op == "operator.confirm")
1865 {
1866 return self.handle_operator_confirm(ctx, frame);
1867 }
1868 if op
1869 .as_ref()
1870 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1871 {
1872 return self.handle_scope_served_by(ctx.connection_id, frame);
1873 }
1874 if self
1875 .forwarding
1876 .module_endpoint_for_connection(ctx.connection_id)
1877 .map_err(RouterError::Forwarding)?
1878 .is_some()
1879 {
1880 if !is_known_module_request_op(&frame.body) {
1881 return Ok(vec![control_error_frame(
1882 &frame,
1883 "unsupported_control_frame",
1884 "module-originated channel-0 REQUEST is not supported",
1885 )?]);
1886 }
1887 let request = match parse_module_control_request_from_module(&frame.body) {
1888 Ok(request) => request,
1889 Err((err, ControlRequestBodyError::UnknownOp)) => {
1890 return Ok(vec![control_error_frame(
1891 &frame,
1892 "unsupported_control_frame",
1893 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1894 )?])
1895 }
1896 Err((err, ControlRequestBodyError::InvalidBody)) => {
1897 return Ok(vec![control_error_frame(
1898 &frame,
1899 "invalid_control_body",
1900 format!("malformed module control body: {err}"),
1901 )?])
1902 }
1903 };
1904 let op = module_control_request_op(&request);
1905 let corr = frame.header.corr;
1906 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1907 let result =
1908 self.handle_module_control_request(ctx.connection_id, frame, request);
1909 log_slow_control_dispatch(
1910 dispatch_started_at,
1911 op,
1912 ctx.connection_id,
1913 corr,
1914 None,
1915 );
1916 return result;
1917 }
1918
1919 if is_known_module_request_op(&frame.body) {
1920 return Ok(vec![control_error_frame(
1921 &frame,
1922 "not_registered",
1923 "catalog.update requires an active module registration owned by this connection",
1924 )?]);
1925 }
1926
1927 let request = match parse_client_control_request(&frame.body) {
1928 Ok(request) => request,
1929 Err((err, ControlRequestBodyError::UnknownOp)) => {
1930 return Ok(vec![control_error_frame(
1931 &frame,
1932 "unknown_control_op",
1933 format!("unknown client control op: {err}"),
1934 )?])
1935 }
1936 Err((err, ControlRequestBodyError::InvalidBody)) => {
1937 return Ok(vec![control_error_frame(
1938 &frame,
1939 "invalid_control_body",
1940 format!("malformed client control body: {err}"),
1941 )?])
1942 }
1943 };
1944 let op = client_control_request_op(&request);
1945 let corr = frame.header.corr;
1946 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1947 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1948 #[cfg(test)]
1949 if let Some(delay) = self.control_dispatch_delay {
1950 tokio::time::sleep(delay).await;
1951 }
1952 let result = self
1953 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1954 .await;
1955 if let Some(timings) = route_open_timings.as_mut() {
1956 timings.finish();
1957 }
1958 log_slow_control_dispatch(
1959 dispatch_started_at,
1960 op,
1961 ctx.connection_id,
1962 corr,
1963 route_open_timings.as_ref(),
1964 );
1965 result
1966 }
1967 FrameType::Push => {
1968 let Some(endpoint) = self
1969 .forwarding
1970 .module_endpoint_for_connection(ctx.connection_id)
1971 .map_err(RouterError::Forwarding)?
1972 else {
1973 return Ok(vec![control_error_frame(
1974 &frame,
1975 "unsupported_control_frame",
1976 "client-originated channel-0 PUSH is not supported",
1977 )?]);
1978 };
1979 self.handle_status_update(endpoint, frame)
1980 }
1981 FrameType::Response | FrameType::Error
1982 if self
1983 .forwarding
1984 .module_endpoint_for_connection(ctx.connection_id)
1985 .map_err(RouterError::Forwarding)?
1986 .is_some() =>
1987 {
1988 self.handle_module_relay_response(ctx.connection_id, frame)
1989 }
1990 ty => Ok(vec![control_error_frame(
1991 &frame,
1992 "unsupported_control_frame",
1993 format!("unsupported channel-0 frame {ty:?}"),
1994 )?]),
1995 }
1996 }
1997
1998 pub fn cleanup_connection(
1999 &self,
2000 connection_id: ConnectionId,
2001 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2002 self.caller_endpoints
2003 .lock()
2004 .unwrap_or_else(|p| p.into_inner())
2005 .remove(&connection_id);
2006 self.forwarding
2007 .operator_confirms()
2008 .connection_closed(connection_id);
2009 self.cleanup_connection_with_end_reason(
2010 connection_id,
2011 RegistrationEndReason::ConnectionClosed,
2012 )
2013 }
2014
2015 fn cleanup_connection_with_end_reason(
2016 &self,
2017 connection_id: ConnectionId,
2018 requested_reason: RegistrationEndReason,
2019 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2020 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2021 self.registry
2022 .get_module_by_connection(connection_id)?
2023 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2024 .and_then(|module| module.registration_end_reason().ok().flatten())
2025 .unwrap_or(requested_reason)
2026 } else {
2027 requested_reason
2028 };
2029 let crash_closed = self
2030 .registry
2031 .get_module_by_connection(connection_id)?
2032 .and_then(|registration| {
2033 self.forwarding
2034 .module_endpoint_for_connection(connection_id)
2035 .ok()
2036 .flatten()
2037 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2038 .map(|routes| (registration.manifest.module_id, routes))
2039 });
2040 let crash_closed = crash_closed.map(|(module_id, routes)| {
2041 let terminal = match self.supervisor.get(&module_id) {
2042 None => false,
2043 Some(module) => match module.will_recover_after_connection_loss() {
2044 Ok(will_recover) => !will_recover,
2045 Err(err) => {
2046 warn!(
2047 %module_id,
2048 error = %err,
2049 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2050 );
2051 false
2052 }
2053 },
2054 };
2055 let (reason, drained) = match self
2059 .forwarding
2060 .daemon_drain_result_for_connection(connection_id)
2061 {
2062 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2063 Ok(None) => (RouteCloseReason::Crash, false),
2064 Err(err) => {
2065 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2066 (RouteCloseReason::Crash, false)
2067 }
2068 };
2069 (module_id, routes, reason, drained, terminal)
2070 });
2071 let registrations = self.deregister_connection(connection_id, end_reason);
2072 let cleanup = if crash_closed.is_some() {
2073 self.forwarding.cleanup_connection_counted(connection_id)
2074 } else {
2075 self.forwarding
2081 .cleanup_connection(connection_id)
2082 .map(|released| crate::forwarding::ConnectionCleanup {
2083 released,
2084 abandoned_relays: 0,
2085 })
2086 };
2087 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2092 let abandoned = cleanup
2093 .as_ref()
2094 .map_or(0, |cleanup| cleanup.abandoned_relays);
2095 send_route_control_pushes(
2096 &self.forwarding,
2097 routes,
2098 ClientControlPush::RouteClosed {
2099 module_id,
2100 channels: Vec::new(),
2101 reason,
2102 drained,
2103 abandoned,
2104 excluded_subscriptions: 0,
2105 terminal: Some(terminal),
2106 },
2107 );
2108 }
2109 if let Ok(cleanup) = cleanup {
2110 self.emit_route_goodbyes(cleanup.released);
2111 }
2112 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2116 crate::supervise::notify_registration_release();
2117 self.capability_evaluator.wake_deadline_loop();
2118 self.refresh_capability_requirements();
2119 }
2120 self.supervisor.remove_spawn_subscribers(connection_id);
2121 self.activity_streams
2122 .lock()
2123 .unwrap_or_else(|p| p.into_inner())
2124 .retain(|(id, _), task| {
2125 if *id == connection_id {
2126 task.abort();
2127 false
2128 } else {
2129 true
2130 }
2131 });
2132 self.hello_launch_nonces
2135 .lock()
2136 .unwrap_or_else(|poisoned| poisoned.into_inner())
2137 .forget(connection_id);
2138 self.scopes
2139 .write()
2140 .unwrap_or_else(|poisoned| poisoned.into_inner())
2141 .release_connection(connection_id);
2142 registrations
2143 }
2144
2145 pub(crate) fn handle_route_goodbye(
2146 &self,
2147 connection_id: ConnectionId,
2148 route_channel: u16,
2149 route_epoch: u32,
2150 ) -> Result<bool, RouterError> {
2151 debug!(
2152 connection_id = connection_id.get(),
2153 route_channel, route_epoch, "handling route GOODBYE"
2154 );
2155 let RouteRelease::Removed(released_route) = self
2156 .forwarding
2157 .release_client_route(connection_id, route_channel, route_epoch)
2158 .map_err(RouterError::Forwarding)?
2159 else {
2160 return Ok(false);
2161 };
2162 self.emit_route_goodbyes(vec![released_route]);
2163 Ok(true)
2164 }
2165
2166 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2167 for released in released_routes {
2168 let frame = match Frame::build_with_version(
2169 released.negotiated_ver,
2170 FrameType::Goodbye,
2171 control_flags(),
2172 released.channel,
2173 released.epoch,
2174 0,
2175 Vec::new(),
2176 ) {
2177 Ok(frame) => frame,
2178 Err(err) => {
2179 warn!(
2180 route_channel = released.channel,
2181 error = %err,
2182 "failed to build route GOODBYE frame"
2183 );
2184 continue;
2185 }
2186 };
2187 if !released.close_on_delivery_failure() {
2188 crate::forwarding::send_module_route_goodbye(
2189 &self.counters,
2190 &released.sink,
2191 frame,
2192 released.module_id.as_deref(),
2193 "client route released",
2194 );
2195 continue;
2196 }
2197 if let Err(err) = released.sink.try_send(frame) {
2198 warn!(
2199 target_connection_id = released.connection_id.get(),
2200 route_channel = released.channel,
2201 error = %err,
2202 "route GOODBYE was not delivered to client; closing target connection"
2203 );
2204 if self
2205 .forwarding
2206 .escalate_client_delivery_failure(
2207 released.connection_id,
2208 released.channel,
2209 released.epoch,
2210 CloseReason::new(
2211 "route_goodbye_delivery_failed",
2212 format!(
2213 "failed to enqueue route GOODBYE for channel {}: {err}",
2214 released.channel
2215 ),
2216 ),
2217 crate::forwarding::UndeliveredFrame {
2218 module_id: released.module_id.as_deref(),
2219 sink: &released.sink,
2220 },
2221 )
2222 .unwrap_or(false)
2223 {
2224 self.counters.increment_goodbye_relay_client_failed();
2225 }
2226 }
2227 }
2228 }
2229
2230 fn send_abandoned_route_bind_goodbye(
2242 &self,
2243 module_sink: &crate::FrameSink,
2244 negotiated_ver: u8,
2245 module_channel: u16,
2246 module_epoch: u32,
2247 ) {
2248 let frame = match Frame::build_with_version(
2249 negotiated_ver,
2250 FrameType::Goodbye,
2251 control_flags(),
2252 module_channel,
2253 module_epoch,
2254 0,
2255 Vec::new(),
2256 ) {
2257 Ok(frame) => frame,
2258 Err(err) => {
2259 warn!(
2260 route_channel = module_channel,
2261 error = %err,
2262 "failed to build GOODBYE for abandoned route.bind"
2263 );
2264 return;
2265 }
2266 };
2267 crate::forwarding::send_module_route_goodbye(
2268 &self.counters,
2269 module_sink,
2270 frame,
2271 None,
2272 "abandoned route.bind",
2273 );
2274 }
2275
2276 fn handle_hello(
2277 &self,
2278 connection_id: ConnectionId,
2279 sink: Option<crate::FrameSink>,
2280 frame: Frame,
2281 ) -> Result<Vec<Frame>, RouterError> {
2282 debug!(
2283 connection_id = connection_id.get(),
2284 corr = frame.header.corr,
2285 "handling HELLO"
2286 );
2287 if self
2291 .registry
2292 .get_module_by_connection(connection_id)
2293 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2294 .is_some()
2295 {
2296 return Ok(vec![control_error_frame(
2297 &frame,
2298 "invalid_hello",
2299 "connection is already registered as a module",
2300 )?]);
2301 }
2302 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2303 Ok(value) => value,
2304 Err(err) => {
2305 return Ok(vec![control_error_frame(
2306 &frame,
2307 "invalid_hello",
2308 format!("malformed HELLO body: {err}"),
2309 )?])
2310 }
2311 };
2312 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2313 return Ok(vec![control_error_frame(
2314 &frame,
2315 "invalid_capability_grammar",
2316 err.to_string(),
2317 )?]);
2318 }
2319 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2320 return Ok(vec![control_error_frame(
2321 &frame,
2322 "invalid_manifest",
2323 err.to_string(),
2324 )?]);
2325 }
2326 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2327 return Ok(vec![control_error_frame(
2328 &frame,
2329 "invalid_event_declaration",
2330 err.to_string(),
2331 )?]);
2332 }
2333 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2334 return Ok(vec![control_error_frame(
2335 &frame,
2336 "invalid_manifest",
2337 err.to_string(),
2338 )?]);
2339 }
2340 if let Some(provenance) = hello_value
2341 .get("manifest")
2342 .and_then(|manifest| manifest.get("provenance"))
2343 {
2344 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2345 return Ok(vec![control_error_frame(
2346 &frame,
2347 "invalid_manifest",
2348 format!("malformed manifest provenance: {err}"),
2349 )?]);
2350 }
2351 }
2352 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2353 let module_id = hello_value["manifest"]["module_id"]
2354 .as_str()
2355 .unwrap_or("<unknown>");
2356 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2357 }
2358 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2359 let module_id = hello_value["manifest"]["module_id"]
2360 .as_str()
2361 .unwrap_or("<unknown>");
2362 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2363 }
2364 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2365 Ok(hello) => hello,
2366 Err(err) => {
2367 return Ok(vec![control_error_frame(
2368 &frame,
2369 "invalid_hello",
2370 format!("malformed HELLO body: {err}"),
2371 )?])
2372 }
2373 };
2374
2375 if hello.protocol_ver != hello.manifest.protocol_ver {
2376 return Ok(vec![control_error_frame(
2377 &frame,
2378 "invalid_manifest",
2379 format!(
2380 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2381 hello.protocol_ver, hello.manifest.protocol_ver
2382 ),
2383 )?]);
2384 }
2385
2386 if hello.manifest.module_id.trim().is_empty() {
2387 return Ok(vec![control_error_frame(
2388 &frame,
2389 "invalid_manifest",
2390 "manifest module_id must not be empty",
2391 )?]);
2392 }
2393
2394 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2395 Ok(negotiated_ver) => negotiated_ver,
2396 Err(message) => {
2397 return Ok(vec![control_error_frame(
2398 &frame,
2399 "version_unsupported",
2400 message,
2401 )?])
2402 }
2403 };
2404
2405 let swap_admission = self
2413 .supervisor
2414 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2415 if swap_admission == SwapHelloAdmission::Refused {
2416 warn!(
2417 module_id = %hello.manifest.module_id,
2418 connection_id = connection_id.get(),
2419 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2420 );
2421 return Ok(vec![control_error_frame(
2422 &frame,
2423 "swap_token_invalid",
2424 format!(
2425 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2426 hello.manifest.module_id
2427 ),
2428 )?]);
2429 }
2430 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2431
2432 let nonce_verified = if swap_candidate {
2437 true
2438 } else {
2439 match self.supervisor.reserved_hello_verification(
2440 &hello.manifest.module_id,
2441 hello.launch_nonce.as_deref(),
2442 ) {
2443 Ok(verified) => verified,
2444 Err(rejection) => {
2445 let message = match rejection {
2446 ReservedHelloRejection::Exact { module_id } => format!(
2447 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2448 ),
2449 ReservedHelloRejection::Prefix {
2450 prefix,
2451 owner_module_id,
2452 } => format!(
2453 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2454 hello.manifest.module_id
2455 ),
2456 };
2457 return Ok(vec![control_error_frame(
2458 &frame,
2459 "reserved_module",
2460 message,
2461 )?]);
2462 }
2463 }
2464 };
2465
2466 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2467 &hello.manifest.module_id,
2468 hello.manifest.capabilities.as_ref(),
2469 );
2470 if let Some(refusal) = reserved_capability_refusals.first() {
2471 let capability = refusal.capability.clone();
2472 let bound_module = refusal.claimants[0].clone();
2473 log_duplicate_claim_events(reserved_capability_refusals);
2474 return Ok(vec![control_error_frame(
2475 &frame,
2476 "reserved_capability",
2477 format!(
2478 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2479 capability, bound_module, hello.manifest.module_id
2480 ),
2481 )?]);
2482 }
2483
2484 if self
2487 .forwarding
2488 .connection_has_client_routes(connection_id)
2489 .map_err(RouterError::Forwarding)?
2490 {
2491 return Ok(vec![control_error_frame(
2492 &frame,
2493 "invalid_hello",
2494 "connection has open client routes and cannot also register as a module",
2495 )?]);
2496 }
2497
2498 self.hello_launch_nonces
2503 .lock()
2504 .unwrap_or_else(|poisoned| poisoned.into_inner())
2505 .record(connection_id, hello.launch_nonce.as_deref());
2506 let control_ops = effective_module_control_ops(hello.control_ops);
2507 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2510 if swap_candidate {
2511 return self.register_swap_candidate(
2512 connection_id,
2513 sink,
2514 &frame,
2515 hello.manifest,
2516 negotiated_ver,
2517 control_ops,
2518 hello_ack,
2519 );
2520 }
2521 let registration = match self
2528 .registry
2529 .register_unpublished_verified_with_control_ops(
2530 hello.manifest,
2531 negotiated_ver,
2532 connection_id,
2533 control_ops,
2534 nonce_verified,
2535 ) {
2536 Ok(registration) => registration,
2537 Err(RegistryError::DuplicateModuleId { module_id }) => {
2538 return Ok(vec![control_error_frame(
2539 &frame,
2540 "duplicate_module_id",
2541 format!(
2542 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2543 ),
2544 )?])
2545 }
2546 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2547 return Ok(vec![control_error_frame(
2548 &frame,
2549 "invalid_module_id",
2550 err.to_string(),
2551 )?])
2552 }
2553 Err(err) => {
2554 return Ok(vec![control_error_frame(
2555 &frame,
2556 "registry_error",
2557 err.to_string(),
2558 )?])
2559 }
2560 };
2561 #[cfg(test)]
2562 self.hold_between_registry_and_forwarding();
2563
2564 let reply = if let Some(sink) = sink {
2565 let concurrency = manifest_concurrency(®istration.manifest);
2581 if let Err(err) = self.forwarding.register_module_connection_acked(
2582 connection_id,
2583 registration.manifest.module_id.clone(),
2584 negotiated_ver,
2585 concurrency,
2586 sink,
2587 hello_ack,
2588 ) {
2589 if matches!(
2594 self.deregister_connection(
2595 connection_id,
2596 RegistrationEndReason::RegistrationFailed,
2597 ),
2598 Ok(r) if !r.is_empty()
2599 ) {
2600 crate::supervise::notify_registration_release();
2601 }
2602 return Ok(vec![control_error_frame(
2603 &frame,
2604 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2605 "invalid_hello"
2606 } else {
2607 forwarding_error_code(&err)
2608 },
2609 err.to_string(),
2610 )?]);
2611 }
2612 Vec::new()
2613 } else {
2614 vec![hello_ack]
2617 };
2618
2619 if !self
2624 .registry
2625 .publish_registration(connection_id)
2626 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2627 {
2628 debug!(
2629 module_id = %registration.manifest.module_id,
2630 connection_id = connection_id.get(),
2631 "registration was removed before it could be published"
2632 );
2633 }
2634
2635 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2644 info!(
2645 module_id = %registration.manifest.module_id,
2646 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2647 );
2648 }
2649
2650 self.apply_registration_capabilities(®istration);
2651
2652 self.supervisor
2653 .record_activity_registration(®istration.manifest.module_id);
2654
2655 info!(
2656 module_id = %registration.manifest.module_id,
2657 module_version = %registration.manifest.module_version,
2658 negotiated_ver,
2659 routable_provider = manifest_provides_routable_role(®istration.manifest),
2660 connection_id = connection_id.get(),
2661 "module registered"
2662 );
2663
2664 Ok(reply)
2665 }
2666
2667 #[allow(clippy::too_many_arguments)]
2684 fn register_swap_candidate(
2685 &self,
2686 connection_id: ConnectionId,
2687 sink: Option<crate::FrameSink>,
2688 frame: &Frame,
2689 manifest: ModuleManifest,
2690 negotiated_ver: u8,
2691 control_ops: Vec<String>,
2692 hello_ack: Frame,
2693 ) -> Result<Vec<Frame>, RouterError> {
2694 let module_id = manifest.module_id.clone();
2695 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2696 manifest,
2697 negotiated_ver,
2698 connection_id,
2699 control_ops,
2700 ) {
2701 Ok(registration) => registration,
2702 Err(RegistryError::DuplicateModuleId { module_id }) => {
2703 return Ok(vec![control_error_frame(
2704 frame,
2705 "duplicate_module_id",
2706 format!(
2707 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2708 ),
2709 )?])
2710 }
2711 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2712 return Ok(vec![control_error_frame(
2713 frame,
2714 "invalid_module_id",
2715 err.to_string(),
2716 )?])
2717 }
2718 Err(err) => {
2719 return Ok(vec![control_error_frame(
2720 frame,
2721 "registry_error",
2722 err.to_string(),
2723 )?])
2724 }
2725 };
2726 #[cfg(test)]
2727 self.hold_between_registry_and_forwarding();
2728 let reply = if let Some(sink) = sink {
2729 let concurrency = manifest_concurrency(®istration.manifest);
2733 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2734 connection_id,
2735 module_id.clone(),
2736 negotiated_ver,
2737 concurrency,
2738 sink,
2739 hello_ack,
2740 ) {
2741 if matches!(
2742 self.deregister_connection(
2743 connection_id,
2744 RegistrationEndReason::RegistrationFailed,
2745 ),
2746 Ok(r) if !r.is_empty()
2747 ) {
2748 crate::supervise::notify_registration_release();
2749 }
2750 return Ok(vec![control_error_frame(
2751 frame,
2752 forwarding_error_code(&err),
2753 err.to_string(),
2754 )?]);
2755 }
2756 Vec::new()
2757 } else {
2758 vec![hello_ack]
2759 };
2760 if !self
2761 .registry
2762 .publish_registration(connection_id)
2763 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2764 {
2765 debug!(
2766 module_id = %module_id,
2767 connection_id = connection_id.get(),
2768 "swap candidate registration was removed before it could be published"
2769 );
2770 }
2771 self.supervisor.mark_swap_candidate_admitted(&module_id);
2772 info!(
2773 module_id = %module_id,
2774 module_version = %registration.manifest.module_version,
2775 negotiated_ver,
2776 ready = registration.ready,
2777 connection_id = connection_id.get(),
2778 "swap candidate registered; not routable until cutover"
2779 );
2780 Ok(reply)
2781 }
2782
2783 fn build_hello_ack(
2784 &self,
2785 frame: &Frame,
2786 negotiated_ver: u8,
2787 module_id: &str,
2788 ) -> Result<Frame, RouterError> {
2789 let ack = ModuleHelloAckBody {
2790 negotiated_ver,
2791 subc_ops: module_subc_ops(),
2792 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2793 storage: self
2794 .storage_config
2795 .as_ref()
2796 .map(|cfg| cfg.descriptor_for(module_id)),
2797 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2798 };
2799 let body = serde_json::to_vec(&ack).map_err(|err| {
2800 RouterError::backend(
2801 0,
2802 frame.header.corr,
2803 format!("failed to encode HELLO_ACK: {err}"),
2804 )
2805 })?;
2806
2807 Frame::build_with_version(
2808 negotiated_ver,
2809 FrameType::HelloAck,
2810 control_flags(),
2811 0,
2812 0,
2813 frame.header.corr,
2814 body,
2815 )
2816 .map_err(RouterError::FrameBuild)
2817 }
2818
2819 async fn handle_client_control_request(
2820 &self,
2821 ctx: &RouteCtx,
2822 frame: Frame,
2823 request: ClientControlRequest,
2824 route_open_timings: &mut Option<RouteOpenTimings>,
2825 ) -> Result<Vec<Frame>, RouterError> {
2826 if let Some(operation) = privileged_operation(&request) {
2827 let caller = self.resolve_caller(ctx.connection_id).await;
2828 let op = client_control_request_op(&request);
2829 let text = prompt_text(operation, &caller);
2830 match self.privileged_operator_prompt {
2831 PrivilegedOperatorPrompt::Log => {
2832 info!(target: "subc_daemon::privileged_prompt", operation = op,
2833 target = ?operation, caller = %caller.description(), summary = %text,
2834 "privileged request would prompt the operator");
2835 }
2836 PrivilegedOperatorPrompt::Enforce => {
2837 let key =
2838 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2839 let outcome = self
2840 .forwarding
2841 .operator_confirms()
2842 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2843 .await;
2844 if outcome.code != "confirmed" {
2845 return Ok(vec![outcome.frame(&frame)]);
2846 }
2847 }
2848 }
2849 }
2850 match request {
2851 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2852 ClientControlRequest::CatalogList { module_id } => {
2853 self.handle_catalog_list(frame, module_id)
2854 }
2855 ClientControlRequest::RouteOpen {
2856 target,
2857 identity,
2858 consumer_identity,
2859 consumer_capabilities,
2860 role_versions,
2861 admission_facts,
2862 scope,
2863 } => {
2864 self.handle_route_open(
2865 ctx,
2866 frame,
2867 RouteOpenRequest {
2868 target,
2869 identity,
2870 consumer_identity,
2871 consumer_capabilities,
2872 role_versions,
2873 admission_facts,
2874 scope,
2875 },
2876 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2877 )
2878 .await
2879 }
2880 ClientControlRequest::RoutePoll {
2881 route_channel,
2882 route_epoch,
2883 kind,
2884 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2885 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2886 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2887 self.handle_supervisor_spawn_snapshot(frame)
2888 }
2889 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2890 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2891 }
2892 ClientControlRequest::SupervisorRestart {
2893 module_id,
2894 drain_timeout_ms,
2895 } => {
2896 self.log_supervisor_request_received(
2897 ctx,
2898 frame.header.corr,
2899 ops::SUPERVISOR_RESTART,
2900 Some(&module_id),
2901 None,
2902 )?;
2903 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2904 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2905 .await
2906 }
2907 ClientControlRequest::SupervisorSwap {
2908 module_id,
2909 ready_timeout_ms,
2910 } => {
2911 self.log_supervisor_request_received(
2912 ctx,
2913 frame.header.corr,
2914 ops::SUPERVISOR_SWAP,
2915 Some(&module_id),
2916 None,
2917 )?;
2918 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2919 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2920 .await
2921 }
2922 ClientControlRequest::SupervisorReload { module_id } => {
2923 self.log_supervisor_request_received(
2924 ctx,
2925 frame.header.corr,
2926 ops::SUPERVISOR_RELOAD,
2927 Some(&module_id),
2928 None,
2929 )?;
2930 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2931 self.handle_supervisor_reload(frame, module_id, requester)
2932 .await
2933 }
2934 ClientControlRequest::SupervisorRescan { preview } => {
2935 if !preview {
2936 self.log_supervisor_request_received(
2937 ctx,
2938 frame.header.corr,
2939 ops::SUPERVISOR_RESCAN,
2940 None,
2941 None,
2942 )?;
2943 }
2944 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2945 self.handle_supervisor_rescan(
2946 frame,
2947 if preview {
2948 RescanMode::Preview
2949 } else {
2950 RescanMode::Execute
2951 },
2952 requester,
2953 )
2954 .await
2955 }
2956 ClientControlRequest::SupervisorRescanChecked {
2957 preview,
2958 candidate,
2959 expect,
2960 } => {
2961 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2962 .await
2963 }
2964 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2965 self.log_supervisor_request_received(
2966 ctx,
2967 frame.header.corr,
2968 ops::SUPERVISOR_RELEASE_RESERVED,
2969 Some(&module_id),
2970 None,
2971 )?;
2972 self.handle_supervisor_release_reserved(frame, module_id)
2973 .await
2974 }
2975 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2976 self.log_supervisor_request_received(
2977 ctx,
2978 frame.header.corr,
2979 ops::SUPERVISOR_SET_ENABLED,
2980 Some(&module_id),
2981 Some(enabled),
2982 )?;
2983 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2984 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2985 .await
2986 }
2987 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2988 self.handle_supervisor_health_probe(frame, module_id).await
2989 }
2990 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2991 ClientControlRequest::SupervisorRoutes { module_id } => {
2992 self.handle_supervisor_routes(frame, module_id)
2993 }
2994 ClientControlRequest::SupervisorProvenance { module_id } => {
2995 self.handle_supervisor_provenance(frame, module_id).await
2996 }
2997 ClientControlRequest::SupervisorStderrTail {
2998 module_id,
2999 max_lines,
3000 max_bytes,
3001 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3002 ClientControlRequest::SupervisorTerminals { module_id } => {
3003 self.handle_supervisor_terminals(frame, module_id).await
3004 }
3005 }
3006 }
3007
3008 fn log_supervisor_request_received(
3009 &self,
3010 ctx: &RouteCtx,
3011 corr: u64,
3012 op: &'static str,
3013 module_id: Option<&str>,
3014 enabled: Option<bool>,
3015 ) -> Result<(), RouterError> {
3016 let caller = self.supervisor_requester(ctx, corr)?;
3017 let caller = principal_label(&caller);
3018 info!(
3021 target: "control",
3022 op,
3023 module_id,
3024 enabled,
3025 connection_id = ctx.connection_id.get(),
3026 caller = %caller,
3027 "supervisor request received"
3028 );
3029 Ok(())
3030 }
3031
3032 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3033 Ok(self
3034 .registry
3035 .get_module_by_connection(ctx.connection_id)
3036 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3037 .map(|registration| Principal::Reserved {
3038 module_id: registration.manifest.module_id,
3039 })
3040 .unwrap_or(Principal::Direct))
3041 }
3042
3043 fn handle_module_control_request(
3044 &self,
3045 connection_id: ConnectionId,
3046 frame: Frame,
3047 request: ModuleControlRequestFromModule,
3048 ) -> Result<Vec<Frame>, RouterError> {
3049 match request {
3050 ModuleControlRequestFromModule::CatalogUpdate {
3051 provides,
3052 capabilities,
3053 ready,
3054 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3055 ModuleControlRequestFromModule::LiveRoots {} => {
3056 let registered = self
3057 .registry
3058 .get_module_by_connection(connection_id)
3059 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3060 let Some(registration) = registered else {
3061 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3062 };
3063 let response = self
3064 .forwarding
3065 .live_roots(®istration.manifest.module_id)
3066 .map_err(RouterError::Forwarding)?;
3067 Ok(vec![control_response_body_frame(
3068 &frame,
3069 &response,
3070 "ModuleControlResponseToModule::LiveRoots",
3071 )?])
3072 }
3073 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3074 self.handle_scope_sync(connection_id, frame, generation, scopes)
3075 }
3076 ModuleControlRequestFromModule::ScopeApply {
3077 generation,
3078 upsert,
3079 end,
3080 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3081 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3082 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3083 }
3084 }
3085 }
3086
3087 fn handle_scope_served_by(
3088 &self,
3089 connection_id: ConnectionId,
3090 frame: Frame,
3091 ) -> Result<Vec<Frame>, RouterError> {
3092 let registered = self
3093 .registry
3094 .get_module_by_connection(connection_id)
3095 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3096 let Some(registered) = registered else {
3097 return Ok(vec![control_error_frame(
3098 &frame,
3099 "not_registered",
3100 "scope.served_by requires an active module registration owned by this connection",
3101 )?]);
3102 };
3103 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3104 {
3105 Ok(request) => request,
3106 Err(err) => {
3107 return Ok(vec![control_error_frame(
3108 &frame,
3109 "invalid_control_body",
3110 format!("malformed scope.served_by body: {err}"),
3111 )?])
3112 }
3113 };
3114 let answer = self
3115 .scopes
3116 .read()
3117 .unwrap_or_else(|poisoned| poisoned.into_inner())
3118 .served_by(
3119 ®istered.manifest.module_id,
3120 &request.scope_ref,
3121 request.scope_epoch,
3122 &request.module_id,
3123 );
3124 let reply = subc_control::ScopeServedByReply::new(
3125 answer,
3126 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3127 );
3128 Ok(vec![control_response_body_frame(
3129 &frame,
3130 &reply,
3131 "ScopeServedByReply",
3132 )?])
3133 }
3134
3135 fn handle_operator_confirm(
3136 &self,
3137 ctx: &RouteCtx,
3138 frame: Frame,
3139 ) -> Result<Vec<Frame>, RouterError> {
3140 use crate::operator_confirm::{audit, Outcome};
3141 let registration = self
3142 .registry
3143 .get_module_by_connection(ctx.connection_id)
3144 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3145 let Some(registration) = registration else {
3146 let outcome = Outcome::refusal("not_registered");
3147 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3148 return Ok(vec![outcome.frame(&frame)]);
3149 };
3150 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3151 Ok(request) => request,
3152 Err(_) => {
3153 let outcome = Outcome::refusal("invalid_control_body");
3154 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3155 return Ok(vec![outcome.frame(&frame)]);
3156 }
3157 };
3158 let module_id = registration.manifest.module_id;
3159 let nonce = self
3163 .hello_launch_nonces
3164 .lock()
3165 .unwrap_or_else(|p| p.into_inner())
3166 .nonce(ctx.connection_id)
3167 .map(str::to_owned);
3168 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3169 self.supervisor
3170 .spawned_consumer_authorized(&module_id, nonce)
3171 });
3172 let confirms = self.forwarding.operator_confirms();
3173 self.forwarding
3174 .with_operator_route(
3175 ctx.connection_id,
3176 request.route_channel,
3177 request.route_epoch,
3178 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3179 )
3180 .map_err(RouterError::Forwarding)
3181 }
3182
3183 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3186 let mut scopes = ScopeRecords::from(scopes);
3187 for (index, record) in scopes.records.iter_mut().enumerate() {
3188 if let Some(refusal) = Self::canonicalize_workdir(record) {
3189 scopes.refusals.insert(index, refusal);
3190 }
3191 }
3192 scopes
3193 }
3194
3195 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3196 let workdir = record.workdir.as_mut()?;
3197 let path = std::path::Path::new(workdir);
3198 if !path.is_absolute() {
3199 return Some((
3200 error_codes::INVALID_WORKDIR,
3201 "workdir must be absolute".to_string(),
3202 ));
3203 }
3204 match ProjectRootId::from_path_allowing_missing(path) {
3205 Ok(root) => match root.as_path().to_str() {
3206 Some(canonical) => {
3207 *workdir = canonical.to_string();
3208 None
3209 }
3210 None => Some((
3211 error_codes::INVALID_WORKDIR,
3212 "canonical workdir is not UTF-8".to_string(),
3213 )),
3214 },
3215 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3216 }
3217 }
3218
3219 fn handle_scope_sync(
3223 &self,
3224 connection_id: ConnectionId,
3225 frame: Frame,
3226 generation: u64,
3227 scopes: Vec<ScopeRecord>,
3228 ) -> Result<Vec<Frame>, RouterError> {
3229 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3230 }
3231
3232 fn handle_scope_change(
3233 &self,
3234 connection_id: ConnectionId,
3235 frame: Frame,
3236 generation: u64,
3237 scopes: Vec<ScopeRecord>,
3238 end: Option<Vec<ScopeEnd>>,
3239 ) -> Result<Vec<Frame>, RouterError> {
3240 let is_apply = end.is_some();
3241 let Some(registration) = self
3242 .registry
3243 .get_module_by_connection(connection_id)
3244 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3245 else {
3246 return Ok(vec![control_error_frame(
3247 &frame,
3248 "not_registered",
3249 "scope.sync and scope.apply require an active module registration owned by this connection",
3250 )?]);
3251 };
3252 let owner = registration.manifest.module_id;
3253 let scopes = Self::prepare_scope_records(scopes);
3254 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3255 let is_current_launch = |connection: ConnectionId| {
3256 self.hello_launch_nonces
3257 .lock()
3258 .unwrap_or_else(|poisoned| poisoned.into_inner())
3259 .presented(connection, current_nonce.as_deref())
3260 };
3261 let mut table = self
3266 .scopes
3267 .write()
3268 .unwrap_or_else(|poisoned| poisoned.into_inner());
3269 let outcome = match end {
3270 Some(end) => table.apply(
3271 &owner,
3272 connection_id,
3273 is_current_launch,
3274 generation,
3275 scopes,
3276 end,
3277 ),
3278 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3279 };
3280 let drained = match &outcome {
3281 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3282 Err(_) => Vec::new(),
3283 };
3284 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3285 drop(table);
3286 if has_deadlines {
3287 self.scope_deadline_added.notify_one();
3288 }
3289 match outcome {
3290 Ok(applied) => {
3291 let counts = ScopeOutcomeCounts::of(&applied.results);
3292 info!(
3293 owner = %owner,
3294 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3295 generation,
3296 records = applied.results.len(),
3297 created = counts.created,
3298 replaced = counts.replaced,
3299 updated = counts.updated,
3300 unchanged = counts.unchanged,
3301 refused = counts.refused,
3302 ended = applied.ended.len(),
3303 tag_changes = applied.tag_changes.len(),
3304 routes_closed = drained.len(),
3305 "scope change accepted"
3306 );
3307 for refused in applied
3313 .results
3314 .iter()
3315 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3316 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3317 {
3318 warn!(
3319 owner = %owner,
3320 generation,
3321 scope_ref = %refused.scope_ref,
3322 scope_epoch = refused.scope_epoch,
3323 code = refused.code.as_deref().unwrap_or(""),
3324 "scope record refused"
3325 );
3326 }
3327 self.close_scope_drained_routes(drained);
3328 let response = if is_apply {
3329 ModuleControlResponseToModule::ScopeApply {
3330 generation,
3331 results: applied.results,
3332 end_results: applied.end_results,
3333 ended: applied.ended,
3334 }
3335 } else {
3336 ModuleControlResponseToModule::ScopeSync {
3337 generation,
3338 results: applied.results,
3339 ended: applied.ended,
3340 }
3341 };
3342 Ok(vec![control_response_body_frame(
3343 &frame,
3344 &response,
3345 "scope change response",
3346 )?])
3347 }
3348 Err(refusal) => {
3349 info!(
3350 owner = %owner,
3351 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3352 generation,
3353 code = refusal.code,
3354 "scope change refused"
3355 );
3356 Ok(vec![control_error_frame(
3357 &frame,
3358 refusal.code,
3359 refusal.message,
3360 )?])
3361 }
3362 }
3363 }
3364
3365 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3371 if drained.is_empty() {
3372 return;
3373 }
3374 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3375 BTreeMap::new();
3376 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3377 for route in drained {
3378 warn!(
3379 module_id = %route.module_id,
3380 reason = ?route.reason,
3381 client_connection_id = route.client.connection_id.get(),
3382 route_channel = route.client.channel,
3383 "closing route because its scope changed"
3384 );
3385 pushes
3386 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3387 .or_insert_with(|| (route.reason, Vec::new()))
3388 .1
3389 .push(EndpointRoute {
3390 goodbye_target: route.client.clone(),
3391 principal: Principal::Unverified,
3392 bound_at: Instant::now(),
3393 draining: false,
3394 drain_reason: None,
3395 });
3396 goodbyes.push(route.module);
3397 goodbyes.push(route.client);
3398 }
3399 for ((module_id, _), (reason, routes)) in pushes {
3400 send_route_control_pushes(
3401 &self.forwarding,
3402 routes,
3403 ClientControlPush::RouteClosed {
3404 module_id,
3405 channels: Vec::new(),
3406 reason,
3407 drained: false,
3408 abandoned: 0,
3409 excluded_subscriptions: 0,
3410 terminal: Some(false),
3411 },
3412 );
3413 }
3414 self.emit_route_goodbyes(goodbyes);
3415 }
3416
3417 fn handle_scope_describe(
3420 &self,
3421 connection_id: ConnectionId,
3422 frame: Frame,
3423 owner: Principal,
3424 scope_ref: String,
3425 ) -> Result<Vec<Frame>, RouterError> {
3426 let registered = self
3427 .registry
3428 .get_module_by_connection(connection_id)
3429 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3430 let Some(registered) = registered else {
3431 return Ok(vec![control_error_frame(
3432 &frame,
3433 "not_registered",
3434 "scope.describe requires an active module registration owned by this connection",
3435 )?]);
3436 };
3437 let mut description = self
3438 .scopes
3439 .read()
3440 .unwrap_or_else(|poisoned| poisoned.into_inner())
3441 .describe(&owner, &scope_ref);
3442 if !registered
3445 .manifest
3446 .capabilities
3447 .as_ref()
3448 .is_some_and(|caps| {
3449 caps.provides
3450 .iter()
3451 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3452 })
3453 {
3454 if let Some(stamp) = description.stamp.as_mut() {
3455 stamp.attributes.on_behalf_of_agent = None;
3456 }
3457 }
3458 if !registered
3459 .manifest
3460 .capabilities
3461 .as_ref()
3462 .is_some_and(|caps| {
3463 caps.provides
3464 .iter()
3465 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3466 })
3467 {
3468 if let Some(stamp) = description.stamp.as_mut() {
3469 stamp.attributes.flow_version = None;
3470 stamp.attributes.flow_code_hash = None;
3471 stamp.attributes.flow_install_id = None;
3472 }
3473 }
3474 let owner_configured = match &owner {
3475 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3480 _ => false,
3481 };
3482 let response = ModuleControlResponseToModule::ScopeDescribe {
3483 status: description.status,
3484 scope_epoch: description.scope_epoch,
3485 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3486 owner_synced: description.owner_synced,
3487 owner_configured,
3488 scope: description.stamp,
3489 };
3490 Ok(vec![control_response_body_frame(
3491 &frame,
3492 &response,
3493 "ModuleControlResponseToModule::ScopeDescribe",
3494 )?])
3495 }
3496
3497 fn handle_catalog_update(
3498 &self,
3499 connection_id: ConnectionId,
3500 frame: Frame,
3501 provides: Vec<ProviderRole>,
3502 capabilities: Option<CapabilityDeclarations>,
3503 ready: Option<bool>,
3504 ) -> Result<Vec<Frame>, RouterError> {
3505 self.refresh_capability_requirements();
3506 let Some(registration) = self
3507 .registry
3508 .get_module_by_connection(connection_id)
3509 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3510 else {
3511 return Ok(vec![control_error_frame(
3512 &frame,
3513 "not_registered",
3514 "catalog.update requires an active module registration owned by this connection",
3515 )?]);
3516 };
3517
3518 if let Some(message) =
3519 catalog_update_frozen_field_message(®istration.manifest, &provides)
3520 {
3521 return Ok(vec![control_error_frame(
3522 &frame,
3523 "catalog_update_frozen_field",
3524 message,
3525 )?]);
3526 }
3527
3528 let mut candidate = registration.manifest.clone();
3529 candidate.provides = provides.clone();
3530 candidate.capabilities = capabilities
3531 .clone()
3532 .or_else(|| registration.manifest.capabilities.clone());
3533 if let Some(rich_ui) = &candidate.rich_ui {
3534 if let Err(err) = rich_ui.validate() {
3535 return Ok(vec![control_error_frame(
3536 &frame,
3537 "invalid_manifest",
3538 err.to_string(),
3539 )?]);
3540 }
3541 }
3542 if let Err(err) = candidate.validate_capability_grammar() {
3543 return Ok(vec![control_error_frame(
3544 &frame,
3545 "invalid_capability_grammar",
3546 err.to_string(),
3547 )?]);
3548 }
3549
3550 let mut conflicts = self
3553 .capability_evaluator
3554 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3555 if let Some(conflict) = conflicts.first() {
3556 let message = format!(
3557 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3558 conflict.capability, conflict.claimants[0], candidate.module_id
3559 );
3560 for conflict in &mut conflicts {
3561 conflict.source = DuplicateClaimSource::CatalogUpdate;
3562 }
3563 log_duplicate_claim_events(conflicts);
3564 return Ok(vec![control_error_frame(
3565 &frame,
3566 "reserved_capability",
3567 message,
3568 )?]);
3569 }
3570
3571 let updated = self
3572 .registry
3573 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3574 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3575 if updated.is_none() {
3576 return Ok(vec![control_error_frame(
3577 &frame,
3578 "not_registered",
3579 "catalog.update requires an active module registration owned by this connection",
3580 )?]);
3581 }
3582 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3583 log_duplicate_claim_events(
3584 self.capability_evaluator
3585 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3586 );
3587 }
3588 if capability_census_trigger(
3589 registration.manifest.capabilities.as_ref(),
3590 updated
3591 .as_ref()
3592 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3593 ) {
3594 self.enforce_capability_denies();
3595 }
3596 self.refresh_capability_requirements();
3597
3598 let response = ModuleControlResponseToModule::CatalogUpdate {};
3599 control_response_body_frame(
3600 &frame,
3601 &response,
3602 "ModuleControlResponseToModule::CatalogUpdate",
3603 )
3604 .map(|frame| vec![frame])
3605 }
3606
3607 pub(crate) fn with_activity_ledger_config(
3608 mut self,
3609 run_dir: &Path,
3610 incarnation: String,
3611 config: crate::activity::ActivityConfig,
3612 ) -> Self {
3613 self.activity = Some(crate::activity::ActivityLedger::open_config(
3614 run_dir.join("activity.jsonl"),
3615 incarnation,
3616 config,
3617 ));
3618 self.supervisor.set_activity(self.activity.clone());
3619 self
3620 }
3621
3622 pub(crate) async fn configure_activity(
3623 &self,
3624 config: crate::activity::ActivityConfig,
3625 ) -> Result<(), &'static str> {
3626 match &self.activity {
3627 Some(ledger) => ledger.configure(config).await,
3628 None => Ok(()),
3629 }
3630 }
3631
3632 fn cancel_activity_subscription(
3633 &self,
3634 ctx: &RouteCtx,
3635 frame: &Frame,
3636 ) -> Result<bool, RouterError> {
3637 let task = self
3638 .activity_streams
3639 .lock()
3640 .unwrap_or_else(|p| p.into_inner())
3641 .remove(&(ctx.connection_id, frame.header.corr));
3642 let Some(task) = task else {
3643 return Ok(false);
3644 };
3645 task.abort();
3646 let end = Frame::build_with_version(
3647 response_version(frame),
3648 FrameType::StreamEnd,
3649 control_flags(),
3650 0,
3651 0,
3652 frame.header.corr,
3653 Vec::new(),
3654 )
3655 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3656 let sink = ctx.egress.clone();
3657 tokio::spawn(async move {
3658 let _ = sink.send(end).await;
3659 });
3660 Ok(true)
3661 }
3662
3663 async fn handle_activity_read(
3664 &self,
3665 ctx: &RouteCtx,
3666 frame: Frame,
3667 ) -> Result<Vec<Frame>, RouterError> {
3668 use subc_control::{
3669 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3670 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3671 };
3672 #[derive(Deserialize)]
3673 struct Identity {
3674 #[serde(default)]
3675 consumer_identity: Option<ConsumerIdentity>,
3676 }
3677 let principal = match self
3681 .forwarding
3682 .module_id_for_connection(ctx.connection_id)
3683 .map_err(RouterError::Forwarding)?
3684 {
3685 Some(module_id) => Ok(Principal::Reserved { module_id }),
3686 None => {
3687 let identity: Identity = match serde_json::from_slice(&frame.body) {
3688 Ok(identity) => identity,
3689 Err(_) => {
3690 return Ok(vec![control_error_frame(
3691 &frame,
3692 ACTIVITY_MALFORMED,
3693 "malformed activity read",
3694 )?])
3695 }
3696 };
3697 self.route_open_principal(&frame, identity.consumer_identity)?
3698 }
3699 };
3700 let admitted = matches!(principal, Ok(Principal::Direct))
3701 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3702 if !admitted {
3703 return Ok(vec![control_error_frame(
3704 &frame,
3705 ACTIVITY_READ_FORBIDDEN,
3706 "activity reads require an operator or trusted mirror principal",
3707 )?]);
3708 }
3709 let Some(ledger) = &self.activity else {
3710 return Ok(vec![control_error_frame(
3711 &frame,
3712 ACTIVITY_UNAVAILABLE,
3713 "activity ledger unavailable",
3714 )?]);
3715 };
3716 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3717 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3718 {
3719 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3720 Ok(request) => ledger.query(request).await,
3721 Err(_) => Err(ACTIVITY_MALFORMED),
3722 };
3723 return match result {
3724 Ok(reply) => Ok(vec![control_response_body_frame(
3725 &frame,
3726 &reply,
3727 ops::ACTIVITY_QUERY,
3728 )?]),
3729 Err(code) => Ok(vec![control_error_frame(
3730 &frame,
3731 code,
3732 "activity query refused",
3733 )?]),
3734 };
3735 }
3736 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3737 Ok(request) => request,
3738 Err(_) => {
3739 return Ok(vec![control_error_frame(
3740 &frame,
3741 ACTIVITY_MALFORMED,
3742 "malformed activity subscription",
3743 )?])
3744 }
3745 };
3746 let mut subscription = match ledger.subscribe(request.since).await {
3747 Ok(subscription) => subscription,
3748 Err(code) => {
3749 return Ok(vec![control_error_frame(
3750 &frame,
3751 code,
3752 "activity subscription refused",
3753 )?])
3754 }
3755 };
3756 let key = (ctx.connection_id, frame.header.corr);
3757 let sink = ctx.egress.clone();
3758 let mut streams = self
3759 .activity_streams
3760 .lock()
3761 .unwrap_or_else(|p| p.into_inner());
3762 streams.retain(|_, task| !task.is_finished());
3763 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3764 return Ok(vec![control_error_frame(
3765 &frame,
3766 "duplicate_subscription",
3767 "activity subscription correlation id already in use",
3768 )?]);
3769 }
3770 let task = tokio::spawn(async move {
3771 loop {
3772 let entry = if let Some(entry) = subscription.replay.pop_front() {
3773 Some(entry)
3774 } else {
3775 subscription.receiver.recv().await
3776 };
3777 let Some(entry) = entry else {
3778 break;
3779 };
3780 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3781 return;
3782 };
3783 let Ok(next) = Frame::build_with_version(
3784 response_version(&frame),
3785 FrameType::StreamData,
3786 control_flags(),
3787 0,
3788 0,
3789 frame.header.corr,
3790 body,
3791 ) else {
3792 return;
3793 };
3794 if sink.send(next).await.is_err() {
3795 return;
3796 }
3797 }
3798 if let Some(code) = subscription.terminal_code() {
3801 if let Ok(end) =
3802 control_error_frame(&frame, code, "activity subscriber fell behind")
3803 {
3804 let _ = sink.send(end).await;
3805 }
3806 }
3807 });
3808 streams.insert(key, task.abort_handle());
3809 Ok(Vec::new())
3810 }
3811
3812 async fn handle_activity_record(
3813 &self,
3814 ctx: &RouteCtx,
3815 frame: Frame,
3816 ) -> Result<Vec<Frame>, RouterError> {
3817 use subc_control::{
3818 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3819 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3820 };
3821 let result = async {
3822 let module = self
3823 .forwarding
3824 .module_id_for_connection(ctx.connection_id)
3825 .ok()
3826 .flatten()
3827 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3828 let request: ActivityRecordRequest =
3829 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3830 crate::activity::validate(&module, &request)?;
3831 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3832 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3833 let at_ms = std::time::SystemTime::now()
3834 .duration_since(std::time::UNIX_EPOCH)
3835 .unwrap_or_default()
3836 .as_millis() as u64;
3837 ledger.record(module, request, actor, at_ms).await
3838 }
3839 .await;
3840 match result {
3841 Ok(entry) => Ok(vec![control_response_body_frame(
3842 &frame,
3843 &ActivityRecordResponse { seq: entry.seq },
3844 "activity.record",
3845 )?]),
3846 Err(code) => Ok(vec![control_error_frame(
3847 &frame,
3848 code,
3849 "activity recording refused",
3850 )?]),
3851 }
3852 }
3853
3854 fn activity_actor(
3855 &self,
3856 connection: ConnectionId,
3857 module: &str,
3858 on: Option<&subc_control::ActivityOn>,
3859 ) -> Result<subc_control::ActivityActor, &'static str> {
3860 use crate::forwarding::{DataRoute, DataRouteState};
3861 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3862 let principal = |id: &str| Principal::Reserved {
3863 module_id: id.to_owned(),
3864 };
3865 let mut actor = ActivityActor {
3866 principal: principal(module),
3867 scope_owner: None,
3868 agent_id: None,
3869 run_id: None,
3870 flow_id: None,
3871 project_root: None,
3872 workdir: None,
3873 project_id: None,
3874 };
3875 let scope = match on {
3876 None => return Ok(actor),
3877 Some(ActivityOn::Scope(scope)) => {
3878 let selector = subc_protocol::scope::ScopeSelector::new(
3879 principal(module),
3880 scope.scope_ref.clone(),
3881 Some(scope.epoch),
3882 );
3883 self.scopes
3884 .read()
3885 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3886 .attribute(&principal(module), module, &selector, true)
3887 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3888 }
3889 Some(ActivityOn::Route(route)) => {
3890 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3891 .forwarding
3892 .lookup_data_route(connection, route.channel, route.epoch)
3893 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3894 else {
3895 return Err(ACTIVITY_ACTOR_UNKNOWN);
3896 };
3897 if binding.module_id != module {
3898 return Err(ACTIVITY_ACTOR_UNKNOWN);
3899 }
3900 actor.principal = binding.principal.clone();
3901 actor.project_root = binding
3902 .project_root
3903 .as_ref()
3904 .map(|root| root.as_path().to_path_buf());
3905 let Some(bound) = &binding.scope else {
3906 return Ok(actor);
3907 };
3908 let owner = principal(&bound.owner);
3909 let selector = subc_protocol::scope::ScopeSelector::new(
3910 owner.clone(),
3911 bound.scope_ref.clone(),
3912 Some(bound.tag.scope_epoch),
3913 );
3914 let admitted = self
3915 .scopes
3916 .read()
3917 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3918 .attribute(&owner, module, &selector, true)
3919 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3920 if admitted.tag != bound.tag {
3921 return Err(ACTIVITY_ACTOR_UNKNOWN);
3922 }
3923 admitted
3924 }
3925 };
3926 actor.scope_owner = Some(scope.stamp.owner);
3927 actor.agent_id = scope.stamp.attributes.agent_id;
3928 actor.run_id = scope.stamp.attributes.run_id;
3929 actor.flow_id = scope.stamp.attributes.flow_id;
3930 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3931 actor.project_id = scope.stamp.project_id;
3932 Ok(actor)
3933 }
3934
3935 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3936 self.refresh_capability_requirements();
3937 let mut counters = self.counters.snapshot();
3943 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3944 obj.insert(
3945 "activity_unparseable_lines".into(),
3946 ledger.read_errors().into(),
3947 );
3948 obj.insert(
3949 "activity_lifecycle_append_failures".into(),
3950 ledger.lifecycle_failures().into(),
3951 );
3952 obj.insert(
3953 "activity_started_digest_skips".into(),
3954 ledger.lifecycle_digest_skips().into(),
3955 );
3956 }
3957 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3958 self.forwarding.client_route_concentration(),
3959 counters.as_object_mut(),
3960 ) {
3961 obj.insert(
3962 "client_connections_with_routes".into(),
3963 connections_with_routes.into(),
3964 );
3965 obj.insert("max_routes_on_one_connection".into(), max.into());
3966 }
3967 if let (Some(open_breakers), Some(obj)) = (
3973 self.route_bind_breakers.open_snapshot(),
3974 counters.as_object_mut(),
3975 ) {
3976 obj.insert("route_bind_breakers_open".into(), open_breakers);
3977 }
3978 let response = ClientControlResponse::ServerDescribe {
3979 protocol_ver: PROTOCOL_VERSION,
3980 subc_ops: subc_ops(),
3981 capabilities: self.describe_capabilities(),
3982 connected_clients: self.connected_clients.count(),
3983 counters: Some(counters),
3984 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3985 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3986 capability_requirements: self.capability_requirement_statuses(),
3987 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3988 config_path: self
3989 .rescan
3990 .as_ref()
3991 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3992 };
3993 Ok(vec![control_response_body_frame(
3994 &frame,
3995 &response,
3996 "ClientControlResponse::ServerDescribe",
3997 )?])
3998 }
3999
4000 fn handle_catalog_list(
4001 &self,
4002 frame: Frame,
4003 module_id: Option<String>,
4004 ) -> Result<Vec<Frame>, RouterError> {
4005 let (generation, modules) = self
4006 .registry
4007 .list_modules_with_nonce_verification()
4008 .map_err(|err| {
4009 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4010 })?;
4011 let entries = modules
4012 .into_iter()
4013 .filter(|(registration, _)| {
4014 module_id
4015 .as_deref()
4016 .map(|wanted| registration.manifest.module_id == wanted)
4017 .unwrap_or(true)
4018 })
4019 .map(|(registration, nonce_verified)| {
4020 let not_ready = self.not_ready_reason(®istration);
4021 let roles = registration.manifest.provides;
4022 CatalogEntry::new(
4023 registration.manifest.module_id,
4024 roles,
4025 registration.control_ops,
4026 )
4027 .with_ready(not_ready.is_none())
4028 .with_not_ready(not_ready)
4029 .with_module_version(Some(registration.manifest.module_version))
4030 .with_capabilities(registration.manifest.capabilities)
4031 .with_self_signals(registration.manifest.self_signals)
4032 .with_events(registration.manifest.events)
4033 .with_operation_delivery(registration.manifest.operation_delivery)
4034 .with_operation_payload(registration.manifest.operation_payload)
4035 .with_flow_operations(registration.manifest.flow_operations)
4036 .with_rich_ui(registration.manifest.rich_ui)
4037 .with_role_declarations(registration.manifest.role_declarations)
4038 .with_nonce_verified(nonce_verified)
4039 })
4040 .collect();
4041 let response = ClientControlResponse::CatalogList {
4042 generation,
4043 modules: entries,
4044 subc_ops: subc_ops(),
4045 };
4046 Ok(vec![control_response_body_frame(
4047 &frame,
4048 &response,
4049 "ClientControlResponse::CatalogList",
4050 )?])
4051 }
4052
4053 fn attest_route_open_consumer(
4058 &self,
4059 consumer_identity: Option<&ConsumerIdentity>,
4060 ) -> Option<Principal> {
4061 let Some(consumer_identity) = consumer_identity else {
4062 return Some(Principal::Direct);
4063 };
4064 self.supervisor
4065 .spawned_consumer_authorized(
4066 &consumer_identity.module_id,
4067 &consumer_identity.launch_nonce,
4068 )
4069 .then(|| Principal::Reserved {
4070 module_id: consumer_identity.module_id.clone(),
4071 })
4072 }
4073
4074 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4078 let Ok(ClientControlRequest::RouteOpen {
4079 identity,
4080 consumer_identity,
4081 scope,
4082 ..
4083 }) = parse_client_control_request(&frame.body)
4084 else {
4085 return RouteOpenCaller::unknown();
4086 };
4087 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4088 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4089 }
4090
4091 fn route_open_principal(
4092 &self,
4093 frame: &Frame,
4094 consumer_identity: Option<ConsumerIdentity>,
4095 ) -> Result<Result<Principal, Frame>, RouterError> {
4096 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4097 self.route_open_principal_attested(frame, consumer_identity, attested)
4098 }
4099
4100 fn route_open_principal_attested(
4104 &self,
4105 frame: &Frame,
4106 consumer_identity: Option<ConsumerIdentity>,
4107 attested: Option<Principal>,
4108 ) -> Result<Result<Principal, Frame>, RouterError> {
4109 if let Some(principal) = attested {
4110 return Ok(Ok(principal));
4111 }
4112 let Some(consumer_identity) = consumer_identity else {
4113 return Ok(Ok(Principal::Direct));
4114 };
4115
4116 Ok(Err(control_error_frame(
4117 frame,
4118 "bad_consumer_identity",
4119 format!(
4120 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4121 consumer_identity.module_id
4122 ),
4123 )?))
4124 }
4125
4126 #[allow(clippy::too_many_arguments)]
4133 fn route_open_refusal_frame(
4134 &self,
4135 ctx: &RouteCtx,
4136 caller: &RouteOpenCaller,
4137 frame: &Frame,
4138 module_id: &str,
4139 reason: &'static str,
4140 code: &'static str,
4141 message: impl Into<String>,
4142 ) -> Result<Frame, RouterError> {
4143 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4144 control_error_frame(frame, code, message.into())
4145 }
4146
4147 #[allow(clippy::too_many_arguments)]
4165 fn route_open_breaker_refusal_frame(
4166 &self,
4167 ctx: &RouteCtx,
4168 caller: &RouteOpenCaller,
4169 frame: &Frame,
4170 module_id: &str,
4171 consecutive_timeouts: u32,
4172 retry_in: Duration,
4173 probe_in_flight: bool,
4174 ) -> Result<Frame, RouterError> {
4175 self.counters
4176 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4177 debug!(
4178 target: "control",
4179 code = "module_timeout",
4180 module_id = ?module_id,
4181 connection_id = ctx.connection_id.get(),
4182 principal = %caller.principal,
4183 scope = %caller.scope,
4184 harness = %caller.harness,
4185 consecutive_timeouts,
4186 retry_in_ms = retry_in.as_millis() as u64,
4187 probe_in_flight,
4188 "route.open refused by open bind-relay breaker"
4189 );
4190 let detail = if probe_in_flight {
4197 "one test route is already being tried; retry once it settles".to_string()
4198 } else {
4199 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4200 };
4201 control_error_frame(
4202 frame,
4203 "module_timeout",
4204 format!(
4205 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4206 timed out in a row); {detail}; its established routes are unaffected"
4207 ),
4208 )
4209 }
4210
4211 fn observe_route_open_refusal(
4224 &self,
4225 ctx: &RouteCtx,
4226 caller: &RouteOpenCaller,
4227 module_id: &str,
4228 reason: &'static str,
4229 code: &'static str,
4230 ) {
4231 self.counters.increment_route_open_refused(code);
4232 info!(
4233 target: "control",
4234 code,
4235 reason,
4236 module_id = ?module_id,
4237 connection_id = ctx.connection_id.get(),
4238 principal = %caller.principal,
4239 scope = %caller.scope,
4240 harness = %caller.harness,
4241 "route.open refused"
4242 );
4243 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4244 self.route_outages.record_not_serving(module_id, reason);
4245 }
4246 }
4247
4248 fn observe_route_open_accept(
4293 &self,
4294 ctx: &RouteCtx,
4295 module_id: &str,
4296 principal: &str,
4297 scope: &str,
4298 ) {
4299 self.route_outages.record_accepted(module_id);
4300 self.counters.increment_route_open_accepted(principal);
4301 info!(
4302 target: "control",
4303 principal,
4304 module_id,
4305 connection_id = ctx.connection_id.get(),
4306 scope = %scope,
4307 "route.open accepted"
4308 );
4309 }
4310
4311 fn supervised_absent_route_open_refusal_frame(
4312 &self,
4313 ctx: &RouteCtx,
4314 caller: &RouteOpenCaller,
4315 frame: &Frame,
4316 module_id: &str,
4317 code: &'static str,
4318 status: &crate::supervise::ModuleStatus,
4319 ) -> Result<Frame, RouterError> {
4320 self.counters.increment_route_open_refused(code);
4321 info!(
4322 target: "control",
4323 code,
4324 reason = "supervised_not_registered",
4325 module_id = ?module_id,
4326 connection_id = ctx.connection_id.get(),
4327 principal = %caller.principal,
4328 scope = %caller.scope,
4329 harness = %caller.harness,
4330 state = %status.state,
4331 enabled = status.enabled,
4332 live = status.live,
4333 "route.open refused"
4334 );
4335 self.route_outages
4339 .record_not_serving(module_id, "supervised_not_registered");
4340 let message = if let Some(pid) = status.waiting_for_previous_pid {
4341 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4342 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4343 })
4344 } else {
4345 format!(
4346 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4347 status.state, status.enabled, status.live
4348 )
4349 };
4350 control_error_frame(frame, code, message)
4351 }
4352
4353 async fn handle_route_open(
4354 &self,
4355 ctx: &RouteCtx,
4356 frame: Frame,
4357 request: RouteOpenRequest,
4358 timings: &mut RouteOpenTimings,
4359 ) -> Result<Vec<Frame>, RouterError> {
4360 let RouteOpenRequest {
4361 target,
4362 mut identity,
4363 consumer_identity,
4364 consumer_capabilities,
4365 role_versions,
4366 admission_facts,
4367 scope,
4368 } = request;
4369 let target_module_id = target_module_id(&target).to_string();
4370 timings.module_id.clone_from(&target_module_id);
4371 if self
4372 .registry
4373 .get_module_by_connection(ctx.connection_id)
4374 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4375 .is_some()
4376 {
4377 return Ok(vec![control_error_frame(
4378 &frame,
4379 "invalid_request",
4380 "module connections cannot open client routes",
4381 )?]);
4382 }
4383 debug!(
4384 connection_id = ctx.connection_id.get(),
4385 corr = frame.header.corr,
4386 module_id = %target_module_id,
4387 "handling route.open"
4388 );
4389
4390 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4395 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4396
4397 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4403 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4404 self.observe_route_open_refusal(
4405 ctx,
4406 &caller,
4407 &target_module_id,
4408 "invalid_role_versions",
4409 error_codes::INVALID_REQUEST,
4410 );
4411 return Ok(vec![control_error_body_frame(
4412 &frame,
4413 ErrorBody {
4414 code: error_codes::INVALID_REQUEST.to_string(),
4415 message: error.to_string(),
4416 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4417 },
4418 )?]);
4419 }
4420
4421 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4422 if status.waiting_for_previous_pid.is_some() {
4423 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4424 ctx,
4425 &caller,
4426 &frame,
4427 &target_module_id,
4428 "module_warming",
4429 &status,
4430 )?]);
4431 }
4432 }
4433
4434 let Some(registration) = self
4453 .registry
4454 .get_module(&target_module_id)
4455 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4456 else {
4457 if let Some((status, warming)) =
4458 self.supervisor_status(&target_module_id, frame.header.corr)?
4459 {
4460 if status.protocol == ModuleProtocol::None {
4467 return Ok(vec![self.route_open_refusal_frame(
4468 ctx,
4469 &caller,
4470 &frame,
4471 &target_module_id,
4472 "protocol_none",
4473 error_codes::MODULE_NO_PROTOCOL,
4474 format!(
4475 "module_id '{target_module_id}' is declared protocol: none; \
4476 it speaks no subc wire and serves no routes"
4477 ),
4478 )?]);
4479 }
4480 let code = if warming {
4481 "module_warming"
4482 } else {
4483 "target_unavailable"
4484 };
4485 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4486 ctx,
4487 &caller,
4488 &frame,
4489 &target_module_id,
4490 code,
4491 &status,
4492 )?]);
4493 }
4494 if let Some(removed_ago_ms) =
4495 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4496 {
4497 return Ok(vec![self.route_open_refusal_frame(
4498 ctx,
4499 &caller,
4500 &frame,
4501 &target_module_id,
4502 "removed",
4503 error_codes::MODULE_REMOVED,
4504 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4505 )?]);
4506 }
4507 return Ok(vec![self.route_open_refusal_frame(
4508 ctx,
4509 &caller,
4510 &frame,
4511 &target_module_id,
4512 "not_registered",
4513 error_codes::UNKNOWN_MODULE,
4514 format!("module_id '{target_module_id}' is not registered"),
4515 )?]);
4516 };
4517
4518 if !registration.ready {
4522 self.counters
4523 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4524 info!(
4525 target: "control",
4526 code = error_codes::MODULE_WARMING,
4527 module_id = ?target_module_id,
4528 connection_id = ctx.connection_id.get(),
4529 principal = %caller.principal,
4530 scope = %caller.scope,
4531 harness = %caller.harness,
4532 reason = "declared_not_ready",
4533 "route.open refused"
4534 );
4535 self.route_outages
4538 .record_not_serving(&target_module_id, "declared_not_ready");
4539 return Ok(vec![control_error_body_frame(
4540 &frame,
4541 ErrorBody {
4542 code: error_codes::MODULE_WARMING.to_string(),
4543 message: format!(
4544 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4545 ),
4546 detail: Some(serde_json::json!({
4547 "reason": "declared_not_ready"
4548 })),
4549 },
4550 )?]);
4551 }
4552
4553 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4572 self.counters
4573 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4574 info!(
4575 target: "control",
4576 code = error_codes::MODULE_WARMING,
4577 module_id = ?target_module_id,
4578 connection_id = ctx.connection_id.get(),
4579 principal = %caller.principal,
4580 scope = %caller.scope,
4581 harness = %caller.harness,
4582 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4583 capability = %capability,
4584 "route.open refused"
4585 );
4586 return Ok(vec![control_error_body_frame(
4587 &frame,
4588 ErrorBody {
4589 code: error_codes::MODULE_WARMING.to_string(),
4590 message: format!(
4591 "module_id '{target_module_id}' requires capability '{capability}', \
4592 which no registered module provides; retry"
4593 ),
4594 detail: Some(serde_json::json!({
4595 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4596 "capability": capability,
4597 })),
4598 },
4599 )?]);
4600 }
4601
4602 if !target_has_required_role(&target, ®istration.manifest.provides) {
4603 return Ok(vec![self.route_open_refusal_frame(
4604 ctx,
4605 &caller,
4606 &frame,
4607 &target_module_id,
4608 "role_not_provided",
4609 "target_unavailable",
4610 format!("module_id '{target_module_id}' does not provide the requested target"),
4611 )?]);
4612 }
4613
4614 if registration.state != ChannelState::Active {
4615 return Ok(vec![self.route_open_refusal_frame(
4616 ctx,
4617 &caller,
4618 &frame,
4619 &target_module_id,
4620 "registration_not_active",
4621 "target_unavailable",
4622 format!("module_id '{target_module_id}' is not active"),
4623 )?]);
4624 }
4625
4626 if self
4627 .forwarding
4628 .module_is_draining(&target_module_id)
4629 .map_err(RouterError::Forwarding)?
4630 {
4631 return Ok(vec![self.route_open_refusal_frame(
4632 ctx,
4633 &caller,
4634 &frame,
4635 &target_module_id,
4636 "reloading",
4637 "module_reloading",
4638 format!("module_id '{target_module_id}' is reloading"),
4639 )?]);
4640 }
4641
4642 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4643 process_liveness.process_live(&target_module_id) == Some(false)
4644 }) {
4645 if process_liveness.process_replacing(&target_module_id) {
4652 return Ok(vec![self.route_open_refusal_frame(
4653 ctx,
4654 &caller,
4655 &frame,
4656 &target_module_id,
4657 "reloading",
4658 "module_reloading",
4659 format!("module_id '{target_module_id}' is reloading"),
4660 )?]);
4661 }
4662 return Ok(vec![self.route_open_refusal_frame(
4663 ctx,
4664 &caller,
4665 &frame,
4666 &target_module_id,
4667 "supervisor_not_live",
4668 "target_unavailable",
4669 format!("module_id '{target_module_id}' is not live"),
4670 )?]);
4671 }
4672
4673 if !self
4674 .forwarding
4675 .has_live_module_connection(&target_module_id)
4676 .map_err(RouterError::Forwarding)?
4677 {
4678 return Ok(vec![self.route_open_refusal_frame(
4679 ctx,
4680 &caller,
4681 &frame,
4682 &target_module_id,
4683 "no_forwarding_connection",
4684 "target_unavailable",
4685 format!("module_id '{target_module_id}' has no live forwarding connection"),
4686 )?]);
4687 }
4688
4689 if let Some(error) =
4690 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4691 {
4692 self.observe_route_open_refusal(
4693 ctx,
4694 &caller,
4695 &target_module_id,
4696 "op_not_allowed",
4697 "op_not_allowed",
4698 );
4699 return Ok(vec![error]);
4700 }
4701
4702 let principal =
4703 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4704 Ok(principal) => principal,
4705 Err(error) => {
4706 self.observe_route_open_refusal(
4707 ctx,
4708 &caller,
4709 &target_module_id,
4710 "bad_consumer_identity",
4711 "bad_consumer_identity",
4712 );
4713 return Ok(vec![error]);
4714 }
4715 };
4716 timings.principal = principal_label(&principal);
4717
4718 if let Principal::Reserved {
4722 module_id: opening_module_id,
4723 } = &principal
4724 {
4725 if let Some(opening_registration) = self
4726 .registry
4727 .get_module(opening_module_id)
4728 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4729 {
4730 if let Some(capability) =
4731 denied_capability(&opening_registration.manifest, ®istration.manifest)
4732 {
4733 warn!(
4734 opening_module_id,
4735 target_module_id,
4736 capability,
4737 "refusing route.open because an attested capability deny edge matches"
4738 );
4739 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4740 &frame,
4741 &target_module_id,
4742 "capability_deny_edge",
4743 "capability_forbidden",
4744 format!(
4745 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4746 ),
4747 )?]);
4748 }
4749 }
4750 }
4751
4752 if admission_facts.is_some() {
4753 let carrier_matches = matches!(
4754 &principal,
4755 Principal::Reserved { module_id }
4756 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4757 );
4758 if !carrier_matches {
4759 return Ok(vec![self.route_open_refusal_frame(
4760 ctx,
4761 &caller,
4762 &frame,
4763 &target_module_id,
4764 "admission_facts_carrier_not_permitted",
4765 "admission_facts_not_permitted",
4766 "admission facts may only be carried by the configured reserved module",
4767 )?]);
4768 }
4769
4770 let target_allowed = self
4771 .admission_facts_targets
4772 .as_ref()
4773 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4774 if !target_allowed {
4775 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4776 &frame,
4777 &target_module_id,
4778 "admission_facts_target_not_listed",
4779 "admission_facts_target_not_allowed",
4780 format!(
4781 "admission facts are not permitted for target module_id '{target_module_id}'"
4782 ),
4783 )?]);
4784 }
4785
4786 }
4790
4791 let (bound_scope, mut scope_stamp) = match &scope {
4797 None => (None, None),
4798 Some(selector) => {
4799 let owner_configured = match &selector.owner {
4800 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4806 _ => false,
4807 };
4808 let mut scopes = self
4809 .scopes
4810 .write()
4811 .unwrap_or_else(|poisoned| poisoned.into_inner());
4812 let admitted = scopes
4813 .admit(&principal, &target_module_id, selector, owner_configured)
4814 .and_then(|admission| {
4815 crate::scopes::check_target_flow_support(
4816 &admission.stamp,
4817 &target_module_id,
4818 registration.manifest.capabilities.as_ref(),
4819 )?;
4820 crate::scopes::check_target_agent_run_support(
4821 &admission.stamp,
4822 &target_module_id,
4823 registration.manifest.capabilities.as_ref(),
4824 )?;
4825 let bound = scopes.reserve_bind(
4826 &admission,
4827 &principal,
4828 ctx.connection_id,
4829 selector.hold_presence,
4830 );
4831 Ok((admission, bound))
4832 });
4833 match admitted {
4834 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4835 Err(refusal) => {
4836 return Ok(vec![self.route_open_refusal_frame(
4837 ctx,
4838 &caller,
4839 &frame,
4840 &target_module_id,
4841 refusal.code,
4842 refusal.code,
4843 refusal.message,
4844 )?]);
4845 }
4846 }
4847 }
4848 };
4849 if let Some(stamp) = &scope_stamp {
4852 caller.scope = route_open_scope_label(stamp);
4853 }
4854
4855 timings.enter(RouteOpenPhase::ProjectRoot);
4877 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4878 Ok(project_root) => project_root,
4879 Err(err) => {
4880 return Ok(vec![control_error_frame(
4881 &frame,
4882 "invalid_project_root",
4883 err.to_string(),
4884 )?])
4885 }
4886 };
4887 timings.enter(RouteOpenPhase::Admission);
4888 identity.project_root = project_root.as_path().to_path_buf();
4889
4890 let _concurrency_guard = match self
4901 .route_bind_concurrency
4902 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4903 {
4904 Ok(guard) => guard,
4905 Err(in_flight) => {
4906 return Ok(vec![self.route_open_target_capacity_refusal(
4907 ctx,
4908 &caller,
4909 &frame,
4910 &target_module_id,
4911 in_flight,
4912 )?]);
4913 }
4914 };
4915
4916 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4919 RouteBindAdmission::Admitted { guard, probe } => {
4920 if probe {
4921 info!(
4922 module_id = %target_module_id,
4923 connection_id = ctx.connection_id.get(),
4924 "route.bind breaker half-open: admitting one probe"
4925 );
4926 }
4927 guard
4928 }
4929 RouteBindAdmission::Refused {
4930 consecutive_timeouts,
4931 retry_in,
4932 probe_in_flight,
4933 } => {
4934 return Ok(vec![self.route_open_breaker_refusal_frame(
4935 ctx,
4936 &caller,
4937 &frame,
4938 &target_module_id,
4939 consecutive_timeouts,
4940 retry_in,
4941 probe_in_flight,
4942 )?]);
4943 }
4944 };
4945
4946 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4951 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4952 timings.enter(RouteOpenPhase::EgressReservation);
4953 let pending = match self
4954 .forwarding
4955 .begin_route_bind_relay_for(
4956 ctx.connection_id,
4957 ctx.egress.clone(),
4958 response_version(&frame),
4959 frame.header.corr,
4960 &target_module_id,
4961 principal.clone(),
4962 bound_scope,
4963 Some(project_root),
4964 relay_deadline,
4965 )
4966 .await
4967 {
4968 Ok(pending) => pending,
4969 Err(err) => {
4970 return Ok(vec![self.route_open_refusal_frame(
4971 ctx,
4972 &caller,
4973 &frame,
4974 &target_module_id,
4975 "relay_reservation_failed",
4976 forwarding_error_code(&err),
4977 err.to_string(),
4978 )?])
4979 }
4980 };
4981 timings.enter(RouteOpenPhase::Admission);
4982 let crate::forwarding::PendingRouteBindRelay {
4983 endpoint,
4984 module_sink,
4985 negotiated_ver,
4986 client_channel,
4987 client_epoch,
4988 module_channel,
4989 module_epoch,
4990 corr: relay_corr,
4991 receiver,
4992 } = pending;
4993 let mut reservation =
4994 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4995
4996 let relay_registration = self
5001 .registry
5002 .get_module_by_connection(endpoint.connection_id)
5003 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5004 let relay_capabilities = relay_registration
5005 .as_ref()
5006 .and_then(|registration| registration.manifest.capabilities.as_ref());
5007 if let Some(selector) = &scope {
5008 if let Err(refusal) = self
5011 .scopes
5012 .read()
5013 .unwrap_or_else(|poisoned| poisoned.into_inner())
5014 .check_bind_proof(&principal, selector)
5015 {
5016 reservation.release_and_disarm();
5017 return Ok(vec![self.route_open_refusal_frame(
5018 ctx,
5019 &caller,
5020 &frame,
5021 &target_module_id,
5022 refusal.code,
5023 refusal.code,
5024 refusal.message,
5025 )?]);
5026 }
5027 }
5028 if let Some(stamp) = scope_stamp
5029 .as_ref()
5030 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5031 {
5032 if let Err(refusal) = crate::scopes::check_target_flow_support(
5033 stamp,
5034 &target_module_id,
5035 relay_capabilities,
5036 )
5037 .and_then(|()| {
5038 crate::scopes::check_target_agent_run_support(
5039 stamp,
5040 &target_module_id,
5041 relay_capabilities,
5042 )
5043 }) {
5044 reservation.release_and_disarm();
5045 return Ok(vec![self.route_open_refusal_frame(
5046 ctx,
5047 &caller,
5048 &frame,
5049 &target_module_id,
5050 refusal.code,
5051 refusal.code,
5052 refusal.message,
5053 )?]);
5054 }
5055 }
5056
5057 if !relay_capabilities.is_some_and(|caps| {
5060 caps.provides
5061 .iter()
5062 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5063 }) {
5064 if let Some(stamp) = scope_stamp.as_mut() {
5065 stamp.workdir = None;
5066 stamp.project_id = None;
5067 }
5068 }
5069 let worker_run = scope_stamp
5074 .as_ref()
5075 .is_some_and(crate::scopes::is_worker_run)
5076 && relay_capabilities.is_some_and(|caps| {
5077 caps.provides
5078 .iter()
5079 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5080 });
5081 if !worker_run
5082 && !relay_capabilities.is_some_and(|caps| {
5083 caps.provides
5084 .iter()
5085 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5086 })
5087 {
5088 if let Some(stamp) = scope_stamp.as_mut() {
5089 stamp.attributes.on_behalf_of_agent = None;
5090 }
5091 }
5092 if !relay_capabilities.is_some_and(|caps| {
5093 caps.provides
5094 .iter()
5095 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5096 }) {
5097 if let Some(stamp) = scope_stamp.as_mut() {
5098 stamp.attributes.flow_version = None;
5099 stamp.attributes.flow_code_hash = None;
5100 stamp.attributes.flow_install_id = None;
5101 }
5102 }
5103 let call_mode = if worker_run
5104 || relay_capabilities
5105 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5106 {
5107 Some(derive_call_mode(scope_stamp.as_ref()))
5108 } else {
5109 if let Some(stamp) = scope_stamp.as_mut() {
5110 stamp.expires_at_ms = None;
5111 }
5112 None
5113 };
5114
5115 debug!(
5116 connection_id = ctx.connection_id.get(),
5117 client_channel,
5118 client_epoch,
5119 module_channel,
5120 module_epoch,
5121 "reserved route handle pair"
5122 );
5123 let route_principal_label = principal_label(&principal);
5126 identity.build_class = None;
5130 let relay = ModuleControlRequest::RouteBind {
5131 route_channel: module_channel,
5132 epoch: module_epoch,
5133 target,
5134 identity,
5135 principal: Some(principal),
5136 consumer_capabilities,
5137 role_versions,
5138 admission_facts,
5139 scope: scope_stamp,
5140 call_mode,
5141 };
5142 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5143 RouterError::backend(
5144 0,
5145 frame.header.corr,
5146 format!("failed to encode route.bind request: {err}"),
5147 )
5148 })?;
5149 let relay_frame = Frame::build_with_version(
5150 negotiated_ver,
5151 FrameType::Request,
5152 control_flags(),
5153 0,
5154 0,
5155 relay_corr,
5156 relay_body,
5157 )
5158 .map_err(RouterError::FrameBuild)?;
5159
5160 timings.enter(RouteOpenPhase::BindSend);
5161 let sent = module_sink.send(relay_frame).await;
5162 timings.enter(RouteOpenPhase::Admission);
5163 if let Err(err) = sent {
5164 reservation.release_and_disarm();
5165 return Ok(vec![self.route_open_refusal_frame(
5166 ctx,
5167 &caller,
5168 &frame,
5169 &target_module_id,
5170 "relay_send_failed",
5171 "target_unavailable",
5172 err.to_string(),
5173 )?]);
5174 }
5175
5176 #[cfg(test)]
5177 if let Some(hold) = &self.route_bind_relay_sent_hold {
5178 hold.wait().await;
5182 hold.wait().await;
5183 }
5184
5185 if self
5190 .forwarding
5191 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5192 .map_err(RouterError::Forwarding)?
5193 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5194 {
5195 self.send_abandoned_route_bind_goodbye(
5196 &module_sink,
5197 negotiated_ver,
5198 module_channel,
5199 module_epoch,
5200 );
5201 }
5202
5203 timings.enter(RouteOpenPhase::BindAckWait);
5204 let outcome = timeout_at(relay_deadline, receiver).await;
5205 timings.enter(RouteOpenPhase::Admission);
5206 match outcome {
5207 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5208 reservation.disarm();
5209 if breaker.record_accepted() {
5210 info!(
5211 module_id = %target_module_id,
5212 "route.bind breaker closed: the probe was accepted"
5213 );
5214 }
5215 self.observe_route_open_accept(
5216 ctx,
5217 &target_module_id,
5218 &route_principal_label,
5219 &caller.scope,
5220 );
5221 Ok(Vec::new())
5222 }
5223 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5224 reservation.release_and_disarm();
5225 breaker.record_inconclusive();
5229 let scope_code = match body.code.as_str() {
5233 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5234 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5235 _ => None,
5236 };
5237 if let Some(code) = scope_code {
5238 self.observe_route_open_refusal(
5239 ctx,
5240 &caller,
5241 &target_module_id,
5242 "scope_changed_before_commit",
5243 code,
5244 );
5245 return Ok(vec![control_error_body_frame(&frame, body)?]);
5246 }
5247 self.counters
5248 .increment_route_open_refused("module_rejected");
5249 info!(
5250 target: "control",
5251 code = "module_rejected",
5252 module_code = ?body.code,
5253 module_id = ?target_module_id,
5254 connection_id = ctx.connection_id.get(),
5255 principal = %caller.principal,
5256 scope = %caller.scope,
5257 harness = %caller.harness,
5258 "route.open refused"
5259 );
5260 Ok(vec![control_error_body_frame(&frame, body)?])
5261 }
5262 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5263 reservation.release_and_disarm();
5264 breaker.record_inconclusive();
5265 tracing::warn!(
5272 module_id = %target_module_id,
5273 "route.bind relay abandoned: {message}"
5274 );
5275 Ok(vec![self.route_open_refusal_frame(
5276 ctx,
5277 &caller,
5278 &frame,
5279 &target_module_id,
5280 "relay_abandoned",
5281 "target_unavailable",
5282 message,
5283 )?])
5284 }
5285 Ok(Err(_)) => {
5286 reservation.release_and_disarm();
5287 breaker.record_inconclusive();
5288 Ok(vec![self.route_open_refusal_frame(
5289 ctx,
5290 &caller,
5291 &frame,
5292 &target_module_id,
5293 "relay_waiter_canceled",
5294 "target_unavailable",
5295 "route.bind relay waiter was canceled before the module responded",
5296 )?])
5297 }
5298 Err(_) => {
5299 reservation.release_and_disarm();
5300 if let Some(opened) = breaker.record_timeout(
5304 self.route_bind_breaker_threshold,
5305 self.route_bind_breaker_cooldown,
5306 ) {
5307 warn!(
5308 module_id = %target_module_id,
5309 consecutive_timeouts = opened.consecutive_timeouts,
5310 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5311 reopened_after_probe = opened.reopened_after_probe,
5312 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5313 );
5314 }
5315 tracing::warn!(
5323 module_id = %target_module_id,
5324 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5325 "route.bind relay timed out: module did not ack within budget"
5326 );
5327 Ok(vec![self.route_open_refusal_frame(
5328 ctx,
5329 &caller,
5330 &frame,
5331 &target_module_id,
5332 "relay_timed_out",
5333 "module_timeout",
5334 format!(
5335 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5336 route_bind_relay_timeout
5337 ),
5338 )?])
5339 }
5340 }
5341 }
5342
5343 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5344 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5345 snapshot: self.supervisor.spawn_snapshot(),
5346 };
5347 Ok(vec![control_response_body_frame(
5348 &frame,
5349 &response,
5350 "ClientControlResponse::SupervisorSpawnSnapshot",
5351 )?])
5352 }
5353
5354 fn handle_supervisor_spawn_subscribe(
5355 &self,
5356 ctx: &RouteCtx,
5357 frame: Frame,
5358 since: Option<SpawnCursor>,
5359 ) -> Result<Vec<Frame>, RouterError> {
5360 match self.supervisor.subscribe_spawns(
5361 ctx.connection_id,
5362 frame.header.corr,
5363 response_version(&frame),
5364 since,
5365 ctx.egress.clone(),
5366 ) {
5367 Ok(()) => Ok(Vec::new()),
5368 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5369 Ok(vec![control_error_body_frame(
5370 &frame,
5371 ErrorBody {
5372 code: "spawn_cursor_incarnation_mismatch".to_string(),
5373 message: "spawn cursor belongs to a different daemon incarnation"
5374 .to_string(),
5375 detail: Some(serde_json::json!({
5376 "current_daemon_incarnation": current
5377 })),
5378 },
5379 )?])
5380 }
5381 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5382 &frame,
5383 ErrorBody {
5384 code: "spawn_cursor_too_old".to_string(),
5385 message: "spawn cursor predates the retained event ring".to_string(),
5386 detail: Some(serde_json::json!({
5387 "oldest_retained_cursor": oldest
5388 })),
5389 },
5390 )?]),
5391 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5392 0,
5393 frame.header.corr,
5394 format!("failed to open supervisor spawn subscription: {error}"),
5395 )),
5396 }
5397 }
5398
5399 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5400 let generation = self
5401 .registry
5402 .generation()
5403 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5404 let mut modules = Vec::new();
5405 for module in self.supervisor.list() {
5406 let status = module.status_for_control("list").map_err(|err| {
5407 RouterError::backend(
5408 0,
5409 frame.header.corr,
5410 format!("failed to read supervisor status: {err}"),
5411 )
5412 })?;
5413 let (configured, _) = module.configuration().map_err(|err| {
5414 RouterError::backend(
5415 0,
5416 frame.header.corr,
5417 format!("failed to read module configuration: {err}"),
5418 )
5419 })?;
5420 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5421 RouterError::backend(
5422 0,
5423 frame.header.corr,
5424 format!("failed to read retry time: {err}"),
5425 )
5426 })?;
5427 let image = module.running_image_agreement().await;
5429 let resources = Some(module.child_resource_usage());
5432 let mut verdict =
5433 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5434 verdict.spec = Some(launch_spec_agreement(
5435 &configured,
5436 status.process_alive,
5437 status.launch_spec_digest.as_ref(),
5438 ));
5439 let pending_reload = Some(verdict);
5440 let configured_enabled = self
5441 .capability_evaluator
5442 .configured_enabled(&status.module_id);
5443 modules.push(
5446 SupervisorEntry::new(
5447 status.module_id,
5448 status.state.to_string(),
5449 status.enabled,
5450 status.live,
5451 status.health.status,
5452 )
5453 .with_configured_enabled(configured_enabled)
5454 .with_next_retry_at_ms(next_retry_at_ms)
5455 .with_launch_nonce_env(Some(!cfg!(unix)))
5456 .with_launch_nonce_source(status.launch_nonce_source)
5457 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5458 .with_protocol(status.protocol)
5459 .with_pending_reload(pending_reload)
5460 .with_last_probe_ms(status.health.last_probe_ms)
5461 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5462 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5463 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5464 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5465 .with_restart_count(Some(status.restart_count))
5466 .with_max_restarts(Some(status.max_restarts))
5467 .with_lifetime_restarts(Some(status.lifetime_restarts))
5468 .with_spawn_generation(Some(status.spawn_generation))
5469 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5470 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5471 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5472 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5473 .with_resources(resources),
5474 );
5475 }
5476 let response = ClientControlResponse::SupervisorList {
5477 generation,
5478 modules,
5479 };
5480 Ok(vec![control_response_body_frame(
5481 &frame,
5482 &response,
5483 "ClientControlResponse::SupervisorList",
5484 )?])
5485 }
5486
5487 fn handle_supervisor_stderr_tail(
5488 &self,
5489 frame: Frame,
5490 module_id: String,
5491 max_lines: Option<u32>,
5492 max_bytes: Option<u32>,
5493 ) -> Result<Vec<Frame>, RouterError> {
5494 let Some(module) = self.supervisor.get(&module_id) else {
5495 return Ok(vec![control_error_frame(
5496 &frame,
5497 "unknown_module",
5498 format!("module_id '{module_id}' is not supervised"),
5499 )?]);
5500 };
5501
5502 let snapshot = module.stderr_tail(
5503 max_lines.map(|value| value as usize),
5504 max_bytes.map(|value| value as usize),
5505 );
5506
5507 let response = ClientControlResponse::SupervisorStderrTail {
5508 module_id,
5509 tail: StderrTail {
5510 capture: match snapshot.capture {
5511 CaptureState::Captured => StderrCaptureState::Captured,
5512 CaptureState::Incomplete { reason } => {
5513 StderrCaptureState::Incomplete { reason }
5514 }
5515 CaptureState::NotCaptured { reason } => {
5516 StderrCaptureState::NotCaptured { reason }
5517 }
5518 },
5519 entries: snapshot
5520 .entries
5521 .into_iter()
5522 .map(|entry| match entry {
5523 TailEntry::Line {
5524 text,
5525 truncated,
5526 at_ms,
5527 } => StderrTailEntry::Line {
5528 text,
5529 truncated,
5530 at_ms,
5531 },
5532 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5533 })
5534 .collect(),
5535 dropped_lines: snapshot.dropped_lines,
5536 },
5537 };
5538 Ok(vec![control_response_body_frame(
5539 &frame,
5540 &response,
5541 "ClientControlResponse::SupervisorStderrTail",
5542 )?])
5543 }
5544
5545 async fn handle_supervisor_terminals(
5546 &self,
5547 frame: Frame,
5548 module_id: String,
5549 ) -> Result<Vec<Frame>, RouterError> {
5550 let Some(module) = self.supervisor.get(&module_id) else {
5551 return Ok(vec![control_error_frame(
5552 &frame,
5553 "unknown_module",
5554 format!("module_id '{module_id}' is not supervised"),
5555 )?]);
5556 };
5557
5558 let terminals = module
5561 .read_durable_terminal_history()
5562 .await
5563 .map_err(|error| {
5564 RouterError::backend(
5565 0,
5566 frame.header.corr,
5567 format!("failed to read terminal history: {error}"),
5568 )
5569 })?;
5570 let response = ClientControlResponse::SupervisorTerminals {
5571 module_id,
5572 terminals,
5573 };
5574 Ok(vec![control_response_body_frame(
5575 &frame,
5576 &response,
5577 "ClientControlResponse::SupervisorTerminals",
5578 )?])
5579 }
5580
5581 fn handle_supervisor_routes(
5582 &self,
5583 frame: Frame,
5584 module_id: Option<String>,
5585 ) -> Result<Vec<Frame>, RouterError> {
5586 let modules = self
5587 .forwarding
5588 .route_census(module_id.as_deref())
5589 .map_err(RouterError::Forwarding)?
5590 .into_iter()
5591 .map(|(module_id, routes)| SupervisorRouteModule {
5592 module_id,
5593 routes: routes
5594 .into_iter()
5595 .map(|route| SupervisorRoute {
5596 consumer: match route.principal {
5597 Principal::Reserved { module_id } => {
5598 SupervisorRouteConsumer::Reserved { module_id }
5599 }
5600 Principal::Direct | Principal::Unverified => {
5601 SupervisorRouteConsumer::Direct {
5602 connection_id: route.goodbye_target.connection_id.get(),
5603 }
5604 }
5605 },
5606 age_ms: Instant::now()
5607 .saturating_duration_since(route.bound_at)
5608 .as_millis()
5609 .try_into()
5610 .unwrap_or(u64::MAX),
5611 draining: route.draining,
5612 drain_reason: route.drain_reason,
5613 })
5614 .collect(),
5615 })
5616 .collect();
5617 let response = ClientControlResponse::SupervisorRoutes { modules };
5618 Ok(vec![control_response_body_frame(
5619 &frame,
5620 &response,
5621 "ClientControlResponse::SupervisorRoutes",
5622 )?])
5623 }
5624
5625 async fn handle_supervisor_provenance(
5626 &self,
5627 frame: Frame,
5628 module_id: Option<String>,
5629 ) -> Result<Vec<Frame>, RouterError> {
5630 let mut selected = if let Some(module_id) = module_id {
5631 let Some(module) = self.supervisor.get(&module_id) else {
5632 return Ok(vec![control_error_frame(
5633 &frame,
5634 "unknown_module",
5635 format!("module_id '{module_id}' is not supervised"),
5636 )?]);
5637 };
5638 vec![module]
5639 } else {
5640 self.supervisor.list()
5641 };
5642
5643 let mut modules = Vec::with_capacity(selected.len());
5644 for module in selected.drain(..) {
5645 let (status, observed_image) = module
5646 .status_and_running_image_agreement()
5647 .await
5648 .map_err(|err| {
5649 RouterError::backend(
5650 0,
5651 frame.header.corr,
5652 format!("failed to read supervisor status: {err}"),
5653 )
5654 })?;
5655 let module_declared = self
5656 .registry
5657 .get_module(&status.module_id)
5658 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5659 .and_then(|registration| registration.manifest.provenance)
5660 .map(|build| ModuleDeclaredProvenance::Reported { build })
5661 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5662 #[cfg(test)]
5663 let running_image = match &self.provenance_probe_override {
5664 Some(result) => result.clone(),
5665 None => observed_image,
5666 };
5667 #[cfg(not(test))]
5668 let running_image = observed_image;
5669 modules.push(SupervisorModuleProvenance {
5670 module_id: status.module_id,
5671 module_declared,
5672 daemon_observed: SupervisorObservedProcess {
5673 launch_nonce_source: status.launch_nonce_source,
5674 pid: status.pid,
5675 spawned_at_ms: status.spawned_at_ms,
5676 spawned_from: status.spawned_from,
5677 running_image,
5678 },
5679 });
5680 }
5681 let daemon = SupervisorDaemonProvenance {
5682 daemon_build: self.daemon_provenance.build.clone(),
5683 daemon_observed: DaemonObservedProcess {
5684 pid: self.daemon_provenance.pid,
5685 started_at_ms: self
5686 .daemon_provenance
5687 .start_clock
5688 .map(|clock| clock.started_at_ms())
5689 .or(self.daemon_provenance.started_at_ms),
5690 running_image: self
5691 .daemon_provenance
5692 .probe
5693 .observe(
5694 self.daemon_provenance.pid,
5695 self.daemon_provenance.executable_path.as_deref(),
5696 self.daemon_provenance.executable_identity.clone(),
5697 self.daemon_provenance.process_start_time,
5698 )
5699 .await,
5700 },
5701 };
5702 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5703 Ok(vec![control_response_body_frame(
5704 &frame,
5705 &response,
5706 "ClientControlResponse::SupervisorProvenance",
5707 )?])
5708 }
5709
5710 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5711 self.refresh_capability_requirements();
5712 let generation = self
5713 .registry
5714 .generation()
5715 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5716 let modules = self
5717 .supervisor
5718 .list()
5719 .into_iter()
5720 .map(|module| {
5721 let status = module.status_for_control("health").map_err(|err| {
5722 RouterError::backend(
5723 0,
5724 frame.header.corr,
5725 format!("failed to read supervisor health: {err}"),
5726 )
5727 })?;
5728 let module_id = status.module_id;
5729 let capability_detail = self
5730 .capability_evaluator
5731 .required_problem_detail(&module_id);
5732 Ok(SupervisorHealthEntry {
5733 module_id,
5734 status: status.health.status,
5735 detail: append_capability_problem_detail(
5736 status.health.detail,
5737 capability_detail,
5738 ),
5739 metrics: status.health.metrics,
5740 consecutive_failures: status.health.consecutive_failures,
5741 late_answer_count: status.health.late_answer_count,
5742 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5743 last_action: status.health.last_action,
5744 last_action_ms: status.health.last_action_ms,
5745 last_probe_ms: status.health.last_probe_ms,
5746 })
5747 })
5748 .collect::<Result<Vec<_>, RouterError>>()?;
5749 let response = ClientControlResponse::SupervisorHealth {
5750 generation,
5751 modules,
5752 };
5753 Ok(vec![control_response_body_frame(
5754 &frame,
5755 &response,
5756 "ClientControlResponse::SupervisorHealth",
5757 )?])
5758 }
5759
5760 async fn handle_supervisor_restart(
5761 &self,
5762 frame: Frame,
5763 module_id: String,
5764 drain_timeout_ms: Option<u64>,
5765 requester: Principal,
5766 ) -> Result<Vec<Frame>, RouterError> {
5767 let operation_lock = self.supervisor.operation_lock();
5768 let _operation_guard = operation_lock.lock().await;
5769 let Some(module) = self.supervisor.get(&module_id) else {
5770 return Ok(vec![control_error_frame(
5771 &frame,
5772 "unknown_module",
5773 format!("module_id '{module_id}' is not supervised"),
5774 )?]);
5775 };
5776
5777 self.route_outages.mark_operator_action(&module_id);
5778 if let Err(err) = module
5779 .restart_requested(drain_timeout_ms, Some(requester))
5780 .await
5781 {
5782 self.route_outages
5783 .operator_action_ended_unrefused(&module_id);
5784 let (code, message) = match err {
5785 crate::supervise::SuperviseError::Disabled { .. } => {
5786 ("module_disabled", err.to_string())
5787 }
5788 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5789 ("swap_in_progress", err.to_string())
5790 }
5791 _ => (
5792 "target_unavailable",
5793 format!("failed to restart module_id '{module_id}': {err}"),
5794 ),
5795 };
5796 return Ok(vec![control_error_frame(&frame, code, message)?]);
5797 }
5798
5799 let response = ClientControlResponse::SupervisorAck {
5800 module_id,
5801 applied: true,
5802 };
5803 Ok(vec![control_response_body_frame(
5804 &frame,
5805 &response,
5806 "ClientControlResponse::SupervisorAck",
5807 )?])
5808 }
5809
5810 async fn handle_supervisor_swap(
5814 &self,
5815 frame: Frame,
5816 module_id: String,
5817 ready_timeout_ms: Option<u64>,
5818 requester: Principal,
5819 ) -> Result<Vec<Frame>, RouterError> {
5820 let module = {
5827 let operation_lock = self.supervisor.operation_lock();
5828 let _operation_guard = operation_lock.lock().await;
5829 self.supervisor.get(&module_id)
5830 };
5831 let Some(module) = module else {
5832 return Ok(vec![control_error_frame(
5833 &frame,
5834 "unknown_module",
5835 format!("module_id '{module_id}' is not supervised"),
5836 )?]);
5837 };
5838
5839 self.route_outages.mark_operator_action(&module_id);
5840 if let Err(err) = module
5841 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5842 .await
5843 {
5844 self.route_outages
5845 .operator_action_ended_unrefused(&module_id);
5846 use crate::supervise::SuperviseError;
5847 let message = err.to_string();
5848 let error = match err {
5849 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5850 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5851 code: "swap_refused".to_string(),
5852 message,
5853 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5854 },
5855 SuperviseError::SwapFailed {
5856 arm,
5857 candidate_exit,
5858 ..
5859 } => ErrorBody {
5860 code: "swap_failed".to_string(),
5861 message,
5862 detail: Some(serde_json::json!({
5863 "arm": arm.as_str(),
5864 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5865 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5866 })),
5867 },
5868 _ => ErrorBody::new(
5869 "target_unavailable",
5870 format!("failed to swap module_id '{module_id}': {message}"),
5871 ),
5872 };
5873 return Ok(vec![control_error_body_frame(&frame, error)?]);
5874 }
5875 self.route_outages
5879 .operator_action_ended_unrefused(&module_id);
5880
5881 let response = ClientControlResponse::SupervisorAck {
5882 module_id,
5883 applied: true,
5884 };
5885 Ok(vec![control_response_body_frame(
5886 &frame,
5887 &response,
5888 "ClientControlResponse::SupervisorAck",
5889 )?])
5890 }
5891
5892 async fn handle_supervisor_reload(
5893 &self,
5894 frame: Frame,
5895 module_id: String,
5896 requester: Principal,
5897 ) -> Result<Vec<Frame>, RouterError> {
5898 let operation_lock = self.supervisor.operation_lock();
5899 let _operation_guard = operation_lock.lock().await;
5900 let Some(module) = self.supervisor.get(&module_id) else {
5901 return Ok(vec![control_error_frame(
5902 &frame,
5903 "unknown_module",
5904 format!("module_id '{module_id}' is not supervised"),
5905 )?]);
5906 };
5907
5908 self.route_outages.mark_operator_action(&module_id);
5909 if let Err(err) = module.reload_requested(Some(requester)).await {
5910 self.route_outages
5911 .operator_action_ended_unrefused(&module_id);
5912 let (code, message) = match err {
5913 crate::supervise::SuperviseError::Disabled { .. } => {
5914 ("module_disabled", err.to_string())
5915 }
5916 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5917 ("swap_in_progress", err.to_string())
5918 }
5919 _ => (
5920 "reload_failed",
5921 format!("failed to reload module_id '{module_id}': {err}"),
5922 ),
5923 };
5924 return Ok(vec![control_error_frame(&frame, code, message)?]);
5925 }
5926
5927 let response = ClientControlResponse::SupervisorAck {
5928 module_id,
5929 applied: true,
5930 };
5931 Ok(vec![control_response_body_frame(
5932 &frame,
5933 &response,
5934 "ClientControlResponse::SupervisorAck",
5935 )?])
5936 }
5937
5938 async fn handle_supervisor_rescan(
5942 &self,
5943 frame: Frame,
5944 mode: RescanMode,
5945 requester: Principal,
5946 ) -> Result<Vec<Frame>, RouterError> {
5947 let preview = mode.previews();
5948 let Some(context) = self.rescan.clone() else {
5949 return Ok(vec![control_error_frame(
5950 &frame,
5951 "rescan_unavailable",
5952 "the daemon was not started with a reloadable config path".to_string(),
5953 )?]);
5954 };
5955
5956 let operation_lock = self.supervisor.operation_lock();
5957 let _operation_guard = operation_lock.lock().await;
5958 let loaded = match self.load_rescan_config(&mode, &context) {
5962 Ok(config) => config,
5963 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5964 };
5965 let Some((config, activity_config)) = loaded else {
5973 return Ok(vec![control_error_frame(
5974 &frame,
5975 "invalid_daemon_config",
5976 format!(
5977 "daemon config not found at {}; refusing to rescan (an absent config would \
5978 retire every supervised module)",
5979 context.config_path.display()
5980 ),
5981 )?]);
5982 };
5983 let (
5984 configured_port,
5985 storage_config,
5986 admission_facts_carrier_module_id,
5987 admission_facts_targets,
5988 scope_authority_owners,
5989 privileged_operator_prompt,
5990 modules,
5991 reserved_capabilities,
5992 ) = (
5993 config.port,
5994 config.storage,
5995 config.admission_facts_carrier_module_id,
5996 config.admission_facts_targets,
5997 config.scope_authority_owners,
5998 config.privileged_operator_prompt,
5999 config.modules,
6000 config.reserved_capabilities,
6001 );
6002
6003 let mut restart_required = Vec::new();
6012 for section in RestartRequiredSection::ALL {
6013 let changed = match section {
6014 RestartRequiredSection::Port => configured_port != context.configured_port,
6015 RestartRequiredSection::Storage => storage_config != context.storage_config,
6016 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6017 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6018 }
6019 RestartRequiredSection::AdmissionFactsTargets => {
6020 admission_facts_targets != context.admission_facts_targets
6021 }
6022 RestartRequiredSection::ScopeAuthorityOwners => {
6023 scope_authority_owners != context.scope_authority_owners
6024 }
6025 RestartRequiredSection::PrivilegedOperatorPrompt => {
6026 privileged_operator_prompt != context.privileged_operator_prompt
6027 }
6028 };
6029 if changed {
6030 restart_required.push(section.label().to_string());
6031 }
6032 }
6033 if !restart_required.is_empty() {
6034 warn!(
6035 config_path = %context.config_path.display(),
6036 sections = %restart_required.join(", "),
6037 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6038 );
6039 }
6040
6041 for configured in &modules {
6042 if let Err(err) = validate_spec(&configured.module_spec()) {
6043 return Ok(vec![control_error_frame(
6044 &frame,
6045 "invalid_daemon_config",
6046 format!("supervisor rescan rejected daemon config: {err}"),
6047 )?]);
6048 }
6049 }
6050
6051 let configured_capabilities = modules
6052 .iter()
6053 .map(|module| (module.module_id.clone(), module.enabled))
6054 .collect::<Vec<_>>();
6055 let preview_capability_warnings = if preview {
6056 let (_, registrations) = self.runtime_capability_snapshot()?;
6057 let current_modules = self
6058 .supervisor
6059 .list()
6060 .into_iter()
6061 .map(|module| module.module_id().to_string())
6062 .collect::<BTreeSet<_>>();
6063 let resulting_modules = configured_capabilities.clone();
6064 let removed = current_modules
6065 .into_iter()
6066 .filter(|module_id| {
6067 !resulting_modules
6068 .iter()
6069 .any(|(configured_id, _)| configured_id == module_id)
6070 })
6071 .collect::<Vec<_>>();
6072 let disabled = match &mode {
6075 RescanMode::Candidate(_) => resulting_modules
6076 .iter()
6077 .filter(|(_, enabled)| !enabled)
6078 .map(|(module_id, _)| module_id.clone())
6079 .collect::<Vec<_>>(),
6080 _ => Vec::new(),
6081 };
6082 self.capability_evaluator.preview_warnings(
6083 resulting_modules,
6084 &removed,
6085 &disabled,
6086 ®istrations,
6087 )
6088 } else {
6089 Vec::new()
6090 };
6091 let reconciled = match &mode {
6092 RescanMode::Candidate(candidate) => self
6093 .preview_candidate(&context, modules, candidate)
6094 .map(|result| (result, Vec::new())),
6095 _ => {
6096 self.reconcile_supervised_modules(
6097 &context.supervisor,
6098 modules,
6099 preview,
6100 requester.clone(),
6101 )
6102 .await
6103 }
6104 };
6105 let (result, changed) = match reconciled {
6106 Ok(result) => result,
6107 Err(message) => {
6108 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6109 }
6110 };
6111 if !preview {
6112 if let Some(ledger) = &self.activity {
6113 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6114 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6115 }
6116 if let Err(code) = self.configure_activity(activity_config).await {
6117 warn!(
6118 code,
6119 "activity retention could not be applied during rescan"
6120 );
6121 }
6122 self.capability_evaluator
6123 .configure(configured_capabilities, reserved_capabilities);
6124 self.capability_evaluator.wake_deadline_loop();
6125 self.refresh_capability_requirements();
6126 }
6127 let mut result = result;
6128 result.restart_required = restart_required;
6129 result.capability_warnings = preview_capability_warnings;
6130 let response = ClientControlResponse::SupervisorRescan { result };
6131 Ok(vec![control_response_body_frame(
6132 &frame,
6133 &response,
6134 "ClientControlResponse::SupervisorRescan",
6135 )?])
6136 }
6137
6138 async fn handle_supervisor_release_reserved(
6139 &self,
6140 frame: Frame,
6141 module_id: String,
6142 ) -> Result<Vec<Frame>, RouterError> {
6143 let Some(context) = self.rescan.clone() else {
6144 return Ok(vec![control_error_frame(
6145 &frame,
6146 "release_unavailable",
6147 "reserved-id release requires a daemon started with a reloadable config path",
6148 )?]);
6149 };
6150 let operation_lock = self.supervisor.operation_lock();
6151 let _operation_guard = operation_lock.lock().await;
6152 let loaded = match crate::daemon_config::load(&context.config_path) {
6153 Ok(Some(config)) => config,
6154 Ok(None) => {
6155 return Ok(vec![control_error_frame(
6156 &frame,
6157 "invalid_daemon_config",
6158 format!(
6159 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6160 context.config_path.display()
6161 ),
6162 )?])
6163 }
6164 Err(err) => {
6165 return Ok(vec![control_error_frame(
6166 &frame,
6167 "invalid_daemon_config",
6168 format!("unable to verify reserved-id release against daemon config: {err}"),
6169 )?])
6170 }
6171 };
6172 if loaded
6173 .modules
6174 .iter()
6175 .any(|configured| configured.module_id == module_id)
6176 {
6177 return Ok(vec![control_error_frame(
6178 &frame,
6179 "reserved_module_configured",
6180 format!(
6181 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6182 ),
6183 )?]);
6184 }
6185 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6186 return Ok(vec![control_error_frame(
6187 &frame,
6188 "reserved_gate_not_retained",
6189 format!(
6190 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6191 ),
6192 )?]);
6193 }
6194
6195 let response = ClientControlResponse::SupervisorAck {
6196 module_id,
6197 applied: true,
6198 };
6199 Ok(vec![control_response_body_frame(
6200 &frame,
6201 &response,
6202 "ClientControlResponse::SupervisorAck",
6203 )?])
6204 }
6205
6206 fn plan_rescan(
6210 &self,
6211 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6212 ) -> Result<RescanPlan, String> {
6213 let mut current = BTreeMap::new();
6214 for module in self.supervisor.list() {
6215 let (spec, health) = module.configuration().map_err(|err| {
6216 format!(
6217 "failed to read configuration for module_id '{}': {err}",
6218 module.module_id()
6219 )
6220 })?;
6221 let enabled = module
6222 .status()
6223 .map_err(|err| {
6224 format!(
6225 "failed to read status for module_id '{}': {err}",
6226 module.module_id()
6227 )
6228 })?
6229 .enabled;
6230 current.insert(
6231 module.module_id().to_string(),
6232 (module, spec, health, enabled),
6233 );
6234 }
6235 let configured = configured_modules
6236 .into_iter()
6237 .map(|module| (module.module_id.clone(), module))
6238 .collect::<BTreeMap<_, _>>();
6239
6240 let added = configured
6241 .keys()
6242 .filter(|module_id| !current.contains_key(*module_id))
6243 .cloned()
6244 .collect::<Vec<_>>();
6245 let removed = current
6246 .keys()
6247 .filter(|module_id| !configured.contains_key(*module_id))
6248 .cloned()
6249 .collect::<Vec<_>>();
6250 let mut changed_pending_reload = Vec::new();
6251 let mut configuration_changes = BTreeSet::new();
6252 let mut enabled_changes = BTreeSet::new();
6253 let mut unchanged = 0_u32;
6254
6255 for (module_id, configured_module) in &configured {
6256 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6257 else {
6258 continue;
6259 };
6260 let launch_changed = *current_spec != configured_module.module_spec();
6264 let configuration_changed =
6265 launch_changed || *current_health != configured_module.health;
6266 let enabled_changed = *current_enabled != configured_module.enabled;
6267 if configuration_changed {
6268 configuration_changes.insert(module_id.clone());
6269 }
6270 if launch_changed {
6271 changed_pending_reload.push(module_id.clone());
6272 }
6273 if enabled_changed {
6274 enabled_changes.insert(module_id.clone());
6275 }
6276 if !configuration_changed && !enabled_changed {
6277 unchanged = unchanged.saturating_add(1);
6278 }
6279 }
6280 Ok(RescanPlan {
6281 current,
6282 configured,
6283 added,
6284 removed,
6285 changed_pending_reload,
6286 configuration_changes,
6287 enabled_changes,
6288 unchanged,
6289 })
6290 }
6291
6292 async fn reconcile_supervised_modules(
6301 &self,
6302 supervisor: &Supervisor,
6303 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6304 preview: bool,
6305 requester: Principal,
6306 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6307 let plan = self.plan_rescan(configured_modules)?;
6308
6309 if preview {
6315 return Ok((plan.preview_result(), Vec::new()));
6316 }
6317 let changed = plan
6318 .configuration_changes
6319 .union(&plan.enabled_changes)
6320 .cloned()
6321 .collect::<Vec<_>>();
6322 let RescanPlan {
6323 current,
6324 configured,
6325 added,
6326 removed,
6327 changed_pending_reload,
6328 configuration_changes,
6329 enabled_changes,
6330 unchanged,
6331 } = plan;
6332 if !(added.is_empty()
6337 && removed.is_empty()
6338 && configuration_changes.is_empty()
6339 && enabled_changes.is_empty())
6340 {
6341 if let Some(context) = &self.rescan {
6342 context
6343 .generation
6344 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6345 }
6346 }
6347
6348 for module_id in &removed {
6349 let module = ¤t
6350 .get(module_id)
6351 .expect("removed module came from current supervisor state")
6352 .0;
6353 module
6354 .retire_requested(Some(requester.clone()))
6355 .await
6356 .map_err(|err| {
6357 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6358 })?;
6359 self.supervisor.record_rescan_removal(module_id);
6393 self.supervisor.retire(module_id);
6394 self.route_outages.forget(module_id);
6395 }
6396
6397 for module_id in configured.keys() {
6398 let Some((module, _, _, _)) = current.get(module_id) else {
6399 continue;
6400 };
6401 let configured_module = configured
6402 .get(module_id)
6403 .expect("configured module id came from configured map");
6404 if configuration_changes.contains(module_id) {
6405 module
6406 .update_configuration(
6407 configured_module.module_spec(),
6408 configured_module.health.clone(),
6409 configured_module.drain_timeout_ms,
6410 )
6411 .await
6412 .map_err(|err| {
6413 format!(
6414 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6415 )
6416 })?;
6417 }
6418 if enabled_changes.contains(module_id) {
6419 self.route_outages.mark_operator_action(module_id);
6422 module
6423 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6424 .await
6425 .map_err(|err| {
6426 self.route_outages.operator_action_ended_unrefused(module_id);
6427 format!(
6428 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6429 configured_module.enabled
6430 )
6431 })?;
6432 }
6433 }
6434
6435 for module_id in &added {
6436 let configured_module = configured
6437 .get(module_id)
6438 .expect("added module id came from configured map");
6439 supervisor
6440 .supervise_configured_requested(
6441 configured_module.module_spec(),
6442 configured_module.enabled,
6443 configured_module.health.clone(),
6444 configured_module.drain_timeout_ms,
6445 configured_module.restart,
6446 configured_module.register_timeout,
6447 Some(requester.clone()),
6448 )
6449 .map_err(|err| {
6450 format!("failed to add module_id '{module_id}' during rescan: {err}")
6451 })?;
6452 }
6453
6454 Ok((
6455 SupervisorRescanResult {
6456 added,
6457 removed,
6458 changed_pending_reload,
6459 enabled_changes: enabled_changes.iter().cloned().collect(),
6460 unchanged,
6461 preview: false,
6462 restart_required: Vec::new(),
6465 capability_warnings: Vec::new(),
6466 module_actions: Vec::new(),
6467 earlier_pending: Vec::new(),
6468 expect: None,
6469 },
6470 changed,
6471 ))
6472 }
6473
6474 async fn handle_supervisor_set_enabled(
6475 &self,
6476 frame: Frame,
6477 module_id: String,
6478 enabled: bool,
6479 requester: Principal,
6480 ) -> Result<Vec<Frame>, RouterError> {
6481 let operation_lock = self.supervisor.operation_lock();
6482 let _operation_guard = operation_lock.lock().await;
6483 let Some(module) = self.supervisor.get(&module_id) else {
6484 return Ok(vec![control_error_frame(
6485 &frame,
6486 "unknown_module",
6487 format!("module_id '{module_id}' is not supervised"),
6488 )?]);
6489 };
6490
6491 self.route_outages.mark_operator_action(&module_id);
6494 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6495 Ok(applied) => applied,
6496 Err(err) => {
6497 self.route_outages
6498 .operator_action_ended_unrefused(&module_id);
6499 return Ok(vec![control_error_frame(
6500 &frame,
6501 "target_unavailable",
6502 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6503 )?]);
6504 }
6505 };
6506 if !applied {
6507 self.route_outages
6510 .operator_action_ended_unrefused(&module_id);
6511 }
6512
6513 self.capability_evaluator.wake_deadline_loop();
6514 self.refresh_capability_requirements();
6515 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6516 Ok(vec![control_response_body_frame(
6517 &frame,
6518 &response,
6519 "ClientControlResponse::SupervisorAck",
6520 )?])
6521 }
6522
6523 async fn handle_supervisor_health_probe(
6524 &self,
6525 frame: Frame,
6526 module_id: String,
6527 ) -> Result<Vec<Frame>, RouterError> {
6528 self.refresh_capability_requirements();
6529 let Some(registration) = self
6530 .registry
6531 .get_module(&module_id)
6532 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6533 else {
6534 return Ok(vec![control_error_frame(
6535 &frame,
6536 "unknown_module",
6537 format!("module_id '{module_id}' is not registered"),
6538 )?]);
6539 };
6540
6541 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6554 {
6555 return Ok(vec![control_error_frame(
6556 &frame,
6557 "health_not_advertised",
6558 format!("module_id '{module_id}' did not advertise health.check"),
6559 )?]);
6560 }
6561
6562 let deadline = Instant::now() + self.health_probe_timeout;
6563 let pending = match self.forwarding.begin_module_control_rpc_for(
6564 &module_id,
6565 MODULE_CONTROL_OP_HEALTH_CHECK,
6566 deadline,
6567 ) {
6568 Ok(pending) => pending,
6569 Err(err) => {
6570 return Ok(vec![control_error_frame(
6571 &frame,
6572 forwarding_error_code(&err),
6573 err.to_string(),
6574 )?])
6575 }
6576 };
6577
6578 let PendingModuleControlRpc {
6579 endpoint,
6580 module_sink,
6581 negotiated_ver,
6582 corr: probe_corr,
6583 receiver,
6584 } = pending;
6585 let mut guard =
6586 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6587 let probe_body =
6588 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6589 RouterError::backend(
6590 0,
6591 frame.header.corr,
6592 format!("failed to encode health.check request: {err}"),
6593 )
6594 })?;
6595 let probe_frame = Frame::build_with_version(
6596 negotiated_ver,
6597 FrameType::Request,
6598 control_flags(),
6599 0,
6600 0,
6601 probe_corr,
6602 probe_body,
6603 )
6604 .map_err(RouterError::FrameBuild)?;
6605
6606 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6609 Ok(Ok(())) => {}
6610 Ok(Err(err)) => {
6611 return Ok(vec![control_error_frame(
6612 &frame,
6613 "target_unavailable",
6614 err.to_string(),
6615 )?]);
6616 }
6617 Err(_) => {
6618 return Ok(vec![control_error_frame(
6619 &frame,
6620 "module_timeout",
6621 format!(
6622 "module_id '{module_id}' did not answer health.check within {:?}",
6623 self.health_probe_timeout
6624 ),
6625 )?]);
6626 }
6627 }
6628
6629 match timeout_at(deadline, receiver).await {
6630 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6631 guard.disarm();
6632 let Some(report) = response.health_report() else {
6633 return Ok(vec![control_error_frame(
6634 &frame,
6635 "invalid_control_body",
6636 "health.check RPC returned a non-health response",
6637 )?]);
6638 };
6639 let HealthReport {
6644 status,
6645 detail,
6646 metrics,
6647 } = report;
6648 let capability_detail = self
6649 .capability_evaluator
6650 .required_problem_detail(&module_id);
6651 let response = ClientControlResponse::SupervisorHealthProbe {
6652 module_id,
6653 status,
6654 detail: append_capability_problem_detail(detail, capability_detail),
6655 metrics,
6656 };
6657 Ok(vec![control_response_body_frame(
6658 &frame,
6659 &response,
6660 "ClientControlResponse::SupervisorHealthProbe",
6661 )?])
6662 }
6663 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6664 guard.disarm();
6665 Ok(vec![control_error_body_frame(&frame, body)?])
6666 }
6667 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6668 guard.disarm();
6669 Ok(vec![control_error_frame(
6670 &frame,
6671 "target_unavailable",
6672 message,
6673 )?])
6674 }
6675 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6676 guard.disarm();
6677 Ok(vec![control_error_frame(
6678 &frame,
6679 "invalid_control_body",
6680 message,
6681 )?])
6682 }
6683 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6684 guard.disarm();
6685 Ok(vec![control_error_frame(
6686 &frame,
6687 "invalid_control_body",
6688 format!("expected module-control op '{expected}', got '{actual}'"),
6689 )?])
6690 }
6691 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6692 guard.disarm();
6693 Ok(vec![control_error_frame(
6694 &frame,
6695 "module_timeout",
6696 format!(
6697 "module_id '{module_id}' answered health.check after {:?}",
6698 self.health_probe_timeout
6699 ),
6700 )?])
6701 }
6702 Ok(Err(_)) => Ok(vec![control_error_frame(
6703 &frame,
6704 "target_unavailable",
6705 "health.check waiter was canceled before the module responded",
6706 )?]),
6707 Err(_) => Ok(vec![control_error_frame(
6708 &frame,
6709 "module_timeout",
6710 format!(
6711 "module_id '{module_id}' did not answer health.check within {:?}",
6712 self.health_probe_timeout
6713 ),
6714 )?]),
6715 }
6716 }
6717
6718 fn supervisor_status(
6719 &self,
6720 module_id: &str,
6721 corr: u64,
6722 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6723 self.supervisor
6724 .get(module_id)
6725 .map(|module| {
6726 let warming = module.is_warming_for_control("status").map_err(|err| {
6727 RouterError::backend(
6728 0,
6729 corr,
6730 format!(
6731 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6732 ),
6733 )
6734 })?;
6735 module.status_for_control("status").map_err(|err| {
6736 RouterError::backend(
6737 0,
6738 corr,
6739 format!(
6740 "failed to read supervisor status for module_id '{module_id}': {err}"
6741 ),
6742 )
6743 }).map(|status| (status, warming))
6744 })
6745 .transpose()
6746 }
6747
6748 fn guard_module_control_op(
6749 &self,
6750 frame: &Frame,
6751 module_id: &str,
6752 op: &str,
6753 ) -> Result<Option<Frame>, RouterError> {
6754 if self.module_grants_op(module_id, op, frame.header.corr)? {
6755 return Ok(None);
6756 }
6757
6758 Ok(Some(control_error_frame(
6759 frame,
6760 "op_not_allowed",
6761 format!("module_id '{module_id}' did not grant control op '{op}'"),
6762 )?))
6763 }
6764
6765 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6766 let Some(registration) = self
6767 .registry
6768 .get_module(module_id)
6769 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6770 else {
6771 return Ok(false);
6772 };
6773 Ok(module_registration_grants_op(®istration.control_ops, op))
6774 }
6775
6776 fn handle_status_update(
6777 &self,
6778 endpoint: ModuleEndpointId,
6779 frame: Frame,
6780 ) -> Result<Vec<Frame>, RouterError> {
6781 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6782 Ok(update) => update,
6783 Err(err) => {
6784 if is_known_module_push_op(&frame.body) {
6789 return Ok(vec![control_error_frame(
6790 &frame,
6791 "invalid_control_body",
6792 format!("malformed module control push body: {err}"),
6793 )?]);
6794 }
6795 return Ok(Vec::new());
6796 }
6797 };
6798
6799 match update {
6800 ModuleControlPush::RouteStatus {
6801 route_channel,
6802 route_epoch,
6803 status,
6804 } => {
6805 self.forwarding
6806 .cache_status(endpoint, route_channel, route_epoch, status)
6807 .map_err(RouterError::Forwarding)?;
6808 }
6809 }
6810 Ok(Vec::new())
6811 }
6812
6813 fn handle_route_poll(
6814 &self,
6815 ctx: &RouteCtx,
6816 frame: Frame,
6817 route_channel: u16,
6818 route_epoch: u32,
6819 kind: PollKind,
6820 ) -> Result<Vec<Frame>, RouterError> {
6821 let snapshot = self
6822 .forwarding
6823 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6824 .map_err(RouterError::Forwarding)?;
6825 let response = match (kind, snapshot) {
6826 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6827 ClientControlResponse::RoutePoll {
6828 route_channel,
6829 route_epoch,
6830 status,
6831 live: None,
6832 }
6833 }
6834 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6835 route_channel,
6836 route_epoch,
6837 status: None,
6838 live: None,
6839 },
6840 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6841 let live = self
6867 .process_liveness
6868 .as_ref()
6869 .and_then(|source| source.process_live(&module_id))
6870 .unwrap_or(true);
6871 ClientControlResponse::RoutePoll {
6872 route_channel,
6873 route_epoch,
6874 status: None,
6875 live: Some(live),
6876 }
6877 }
6878 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6879 route_channel,
6880 route_epoch,
6881 status: None,
6882 live: Some(false),
6883 },
6884 };
6885
6886 Ok(vec![control_response_body_frame(
6887 &frame,
6888 &response,
6889 "ClientControlResponse::RoutePoll",
6890 )?])
6891 }
6892
6893 pub(crate) fn observe_module_control_completion(
6894 &self,
6895 completion: ModuleControlRpcCompletion,
6896 ) -> bool {
6897 match completion {
6898 ModuleControlRpcCompletion::Unknown => false,
6899 ModuleControlRpcCompletion::Settled => true,
6900 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6901 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6902 info!(
6903 module_id = %module_id,
6904 latency_ms,
6905 "late health.check answer proves the module is alive"
6906 );
6907 match self
6908 .supervisor
6909 .record_late_health_answer(&module_id, latency_ms)
6910 {
6911 Ok(true) => {}
6912 Ok(false) => debug!(
6913 module_id = %module_id,
6914 latency_ms,
6915 "late health.check answer has no active supervisor snapshot"
6916 ),
6917 Err(err) => warn!(
6918 module_id = %module_id,
6919 latency_ms,
6920 error = %err,
6921 "failed to record late health.check answer"
6922 ),
6923 }
6924 true
6925 }
6926 }
6927 }
6928
6929 fn refuse_to_end_module_connection_for_a_client(
6950 &self,
6951 module_connection_id: ConnectionId,
6952 corr: u64,
6953 err: ForwardingError,
6954 ) -> Result<(), RouterError> {
6955 if let ForwardingError::ConnectionClosing { connection_id } = err {
6956 if connection_id != module_connection_id {
6957 warn!(
6958 module_connection_id = module_connection_id.get(),
6959 client_connection_id = connection_id.get(),
6960 corr,
6961 "dropping a route.bind response for a closing client; the module connection keeps serving"
6962 );
6963 return Ok(());
6964 }
6965 }
6966 Err(RouterError::Forwarding(err))
6967 }
6968
6969 fn handle_module_relay_response(
6970 &self,
6971 connection_id: ConnectionId,
6972 frame: Frame,
6973 ) -> Result<Vec<Frame>, RouterError> {
6974 let mut secondary_error = None;
6975 let outcome = match frame.header.ty {
6976 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6977 Ok(probe) if probe.op == "route.bind" => {
6978 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6979 Ok(ModuleControlResponse::RouteBindAck {}) => {
6980 RouteBindRelayOutcome::Accepted
6981 }
6982 Ok(other) => {
6983 let message =
6984 format!("route.bind response carried unexpected body: {other:?}");
6985 secondary_error = Some(control_error_frame(
6986 &frame,
6987 "invalid_control_body",
6988 message.clone(),
6989 )?);
6990 RouteBindRelayOutcome::ModuleGone(message)
6991 }
6992 Err(err) => {
6993 let message = format!("malformed route.bind response body: {err}");
6994 secondary_error = Some(control_error_frame(
6995 &frame,
6996 "invalid_control_body",
6997 message.clone(),
6998 )?);
6999 RouteBindRelayOutcome::ModuleGone(message)
7000 }
7001 }
7002 }
7003 Ok(probe) => {
7004 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7005 {
7006 Ok(response) => ModuleControlRpcOutcome::Response(response),
7007 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7008 "malformed {} response body: {err}",
7009 probe.op
7010 )),
7011 };
7012 let completion = self
7013 .forwarding
7014 .complete_module_control_rpc(
7015 connection_id,
7016 frame.header.corr,
7017 Some(&probe.op),
7018 outcome,
7019 )
7020 .map_err(RouterError::Forwarding)?;
7021 if !self.observe_module_control_completion(completion) {
7022 debug!(
7023 connection_id = connection_id.get(),
7024 corr = frame.header.corr,
7025 op = %probe.op,
7026 "dropping late or unknown module-control RPC response"
7027 );
7028 }
7029 return Ok(Vec::new());
7030 }
7031 Err(err) => {
7032 if let Some(expected_op) = self
7033 .forwarding
7034 .pending_module_control_op(connection_id, frame.header.corr)
7035 .map_err(RouterError::Forwarding)?
7036 {
7037 let completion = self
7038 .forwarding
7039 .complete_module_control_rpc(
7040 connection_id,
7041 frame.header.corr,
7042 None,
7043 ModuleControlRpcOutcome::MalformedResponse(format!(
7044 "malformed {expected_op} response body: {err}"
7045 )),
7046 )
7047 .map_err(RouterError::Forwarding)?;
7048 if !self.observe_module_control_completion(completion) {
7049 debug!(
7050 connection_id = connection_id.get(),
7051 corr = frame.header.corr,
7052 "dropping late malformed module-control RPC response"
7053 );
7054 }
7055 return Ok(Vec::new());
7056 }
7057 let message = format!("malformed route.bind response body: {err}");
7058 secondary_error = Some(control_error_frame(
7059 &frame,
7060 "invalid_control_body",
7061 message.clone(),
7062 )?);
7063 RouteBindRelayOutcome::ModuleGone(message)
7064 }
7065 },
7066 FrameType::Error => {
7067 if self
7068 .forwarding
7069 .pending_module_control_op(connection_id, frame.header.corr)
7070 .map_err(RouterError::Forwarding)?
7071 .is_some()
7072 {
7073 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7074 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7075 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7076 "malformed module-control ERROR body: {err}"
7077 )),
7078 };
7079 let completion = self
7080 .forwarding
7081 .complete_module_control_rpc(
7082 connection_id,
7083 frame.header.corr,
7084 None,
7085 outcome,
7086 )
7087 .map_err(RouterError::Forwarding)?;
7088 if !self.observe_module_control_completion(completion) {
7089 debug!(
7090 connection_id = connection_id.get(),
7091 corr = frame.header.corr,
7092 "dropping late or unknown module-control RPC error"
7093 );
7094 }
7095 return Ok(Vec::new());
7096 }
7097 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7098 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7099 Err(err) => {
7100 let message = format!("malformed route.bind ERROR body: {err}");
7101 secondary_error = Some(control_error_frame(
7102 &frame,
7103 "invalid_control_body",
7104 message.clone(),
7105 )?);
7106 RouteBindRelayOutcome::ModuleGone(message)
7107 }
7108 }
7109 }
7110 ty => {
7111 return Ok(vec![control_error_frame(
7112 &frame,
7113 "unsupported_control_frame",
7114 format!("unsupported module channel-0 frame {ty:?}"),
7115 )?])
7116 }
7117 };
7118
7119 let settled =
7120 self.forwarding
7121 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7122 let completion = match settled {
7123 Ok(completion) => completion,
7124 Err(err) => {
7125 self.refuse_to_end_module_connection_for_a_client(
7126 connection_id,
7127 frame.header.corr,
7128 err,
7129 )?;
7130 return Ok(secondary_error.into_iter().collect());
7131 }
7132 };
7133 if let Some(target) = completion.abandoned.as_ref() {
7134 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7135 }
7136 if !completion.settled {
7137 debug!(
7138 connection_id = connection_id.get(),
7139 corr = frame.header.corr,
7140 frame_type = ?frame.header.ty,
7141 "dropping late or unknown route.bind relay response"
7142 );
7143 }
7144 Ok(secondary_error.into_iter().collect())
7145 }
7146
7147 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7148 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7149 self.cleanup_connection_with_end_reason(
7153 connection_id,
7154 RegistrationEndReason::ExplicitGoodbye,
7155 )
7156 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7157 Ok(Vec::new())
7158 }
7159}
7160
7161impl Default for ControlHandler {
7162 fn default() -> Self {
7163 Self::new(Arc::new(Registry::default()))
7164 }
7165}
7166
7167impl crate::supervise::SwapPromotionObserver for ControlHandler {
7168 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7169 self.apply_registration_capabilities(registration);
7170 }
7171}
7172
7173fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7174 CapabilityRequirementStatus {
7175 consumer: status.consumer,
7176 capability: status.capability,
7177 need: match status.need {
7178 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7179 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7180 },
7181 verdict: status.verdict.as_str().to_string(),
7182 episode_seq: status.episode_seq,
7183 config_satisfiable: status.config_satisfiable,
7184 runtime_available: status.runtime_available,
7185 detail: status.detail,
7186 }
7187}
7188
7189fn append_capability_problem_detail(
7190 detail: Option<String>,
7191 capability_detail: Option<String>,
7192) -> Option<String> {
7193 match (detail, capability_detail) {
7194 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7195 (Some(detail), None) => Some(detail),
7196 (None, Some(capability_detail)) => Some(capability_detail),
7197 (None, None) => None,
7198 }
7199}
7200
7201fn subc_ops() -> Vec<String> {
7202 SUBC_CONTROL_OPS
7203 .iter()
7204 .map(|op| (*op).to_string())
7205 .collect()
7206}
7207
7208fn module_subc_ops() -> Vec<String> {
7209 SUBC_CONTROL_OPS
7210 .iter()
7211 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7212 .map(|op| (*op).to_string())
7213 .collect()
7214}
7215
7216#[cfg(test)]
7217fn module_baseline_control_ops() -> Vec<String> {
7218 MODULE_BASELINE_CONTROL_OPS
7219 .iter()
7220 .map(|op| (*op).to_string())
7221 .collect()
7222}
7223
7224fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7225 let mut seen = HashSet::new();
7226 let mut effective = Vec::new();
7227 for op in MODULE_BASELINE_CONTROL_OPS {
7228 if seen.insert((*op).to_string()) {
7229 effective.push((*op).to_string());
7230 }
7231 }
7232 for op in declared.unwrap_or_default() {
7233 if seen.insert(op.clone()) {
7234 effective.push(op);
7235 }
7236 }
7237 effective
7238}
7239
7240fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7241 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7242}
7243
7244fn target_module_id(target: &RouteTarget) -> &str {
7245 match target {
7246 RouteTarget::ToolProvider { module_id }
7247 | RouteTarget::ManagementSurface { module_id }
7248 | RouteTarget::InternalService { module_id, .. } => module_id,
7249 }
7250}
7251
7252fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7253 roles.iter().any(|role| match (target, role) {
7254 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7255 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7256 (
7257 RouteTarget::InternalService { service_id, .. },
7258 ProviderRole::InternalService {
7259 service_id: provided,
7260 ..
7261 },
7262 ) => service_id == provided,
7263 _ => false,
7264 })
7265}
7266
7267fn is_routable_role(role: &ProviderRole) -> bool {
7268 matches!(
7269 role,
7270 ProviderRole::ToolProvider { .. }
7271 | ProviderRole::ManagementSurface { .. }
7272 | ProviderRole::InternalService { .. }
7273 )
7274}
7275
7276#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7277enum ControlRequestBodyError {
7278 UnknownOp,
7279 InvalidBody,
7280}
7281
7282#[derive(Debug, Deserialize)]
7283struct ControlOpProbe {
7284 op: String,
7285}
7286
7287const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7290
7291fn is_known_module_push_op(body: &[u8]) -> bool {
7292 serde_json::from_slice::<ControlOpProbe>(body)
7293 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7294 .unwrap_or(false)
7295}
7296
7297fn is_known_module_request_op(body: &[u8]) -> bool {
7298 serde_json::from_slice::<ControlOpProbe>(body)
7299 .map(|probe| is_module_to_subc_op(&probe.op))
7300 .unwrap_or(false)
7301}
7302
7303fn is_module_to_subc_op(op: &str) -> bool {
7304 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7305}
7306
7307fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7308 debug!(
7309 op = %op,
7310 connection_id = connection_id.get(),
7311 corr,
7312 "control dispatch"
7313 );
7314}
7315
7316fn log_slow_control_dispatch(
7317 dispatch_started_at: Option<StdInstant>,
7318 op: &'static str,
7319 connection_id: ConnectionId,
7320 corr: u64,
7321 route_open_timings: Option<&RouteOpenTimings>,
7322) {
7323 let Some(dispatch_started_at) = dispatch_started_at else {
7324 return;
7325 };
7326 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7328 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7329 if let Some(timings) = route_open_timings {
7330 warn!(
7331 op = %op,
7332 connection_id = connection_id.get(),
7333 corr,
7334 elapsed_ms = elapsed.as_millis() as u64,
7335 module_id = ?timings.module_id,
7336 principal = %timings.principal,
7337 admission_ms = timings.admission.as_millis() as u64,
7338 project_root_ms = timings.project_root.as_millis() as u64,
7339 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7340 bind_send_ms = timings.bind_send.as_millis() as u64,
7341 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7342 "slow control dispatch"
7343 );
7344 return;
7345 }
7346 warn!(
7347 op = %op,
7348 connection_id = connection_id.get(),
7349 corr,
7350 elapsed_ms = elapsed.as_millis() as u64,
7351 "slow control dispatch"
7352 );
7353 }
7354}
7355
7356#[cfg(test)]
7358type TestCallerLookup =
7359 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7360
7361fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7363 match request {
7364 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7365 Some(PrivilegedOperation::Restart { module_id })
7366 }
7367 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7368 Some(PrivilegedOperation::Swap { module_id })
7369 }
7370 ClientControlRequest::SupervisorReload { module_id } => {
7371 Some(PrivilegedOperation::Reload { module_id })
7372 }
7373 ClientControlRequest::SupervisorRescan { preview } => {
7374 if *preview {
7375 None
7376 } else {
7377 Some(PrivilegedOperation::Rescan)
7378 }
7379 }
7380 ClientControlRequest::SupervisorRescanChecked {
7385 preview,
7386 candidate,
7387 expect,
7388 } => (!*preview && candidate.is_none() && expect.is_some())
7389 .then_some(PrivilegedOperation::Rescan),
7390 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7391 Some(PrivilegedOperation::SetEnabled {
7392 module_id,
7393 enabled: *enabled,
7394 })
7395 }
7396 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7397 Some(PrivilegedOperation::ReleaseReserved { module_id })
7398 }
7399 ClientControlRequest::ServerDescribe {}
7400 | ClientControlRequest::CatalogList { .. }
7401 | ClientControlRequest::RouteOpen { .. }
7402 | ClientControlRequest::RoutePoll { .. }
7403 | ClientControlRequest::SupervisorList {}
7404 | ClientControlRequest::SupervisorHealth {}
7405 | ClientControlRequest::SupervisorHealthProbe { .. }
7406 | ClientControlRequest::SupervisorRoutes { .. }
7407 | ClientControlRequest::SupervisorProvenance { .. }
7408 | ClientControlRequest::SupervisorStderrTail { .. }
7409 | ClientControlRequest::SupervisorTerminals { .. }
7410 | ClientControlRequest::SupervisorSpawnSnapshot {}
7411 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7412 }
7413}
7414
7415fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7416 match request {
7417 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7418 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7419 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7420 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7421 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7422 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7423 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7424 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7425 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7426 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7427 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7428 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7429 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7430 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7431 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7432 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7433 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7434 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7435 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7436 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7437 }
7438}
7439
7440struct RouteOpenCaller {
7448 principal: String,
7452 scope: String,
7455 harness: String,
7458}
7459
7460impl RouteOpenCaller {
7461 fn new(
7462 principal: Option<&Principal>,
7463 scope: Option<&ScopeSelector>,
7464 identity: &BindIdentity,
7465 ) -> Self {
7466 Self {
7467 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7468 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7469 harness: bounded_log_text(&identity.harness),
7470 }
7471 }
7472
7473 fn unknown() -> Self {
7476 Self {
7477 principal: "unknown".to_string(),
7478 scope: "unknown".to_string(),
7479 harness: "unknown".to_string(),
7480 }
7481 }
7482}
7483
7484const MAX_LOGGED_TEXT_CHARS: usize = 64;
7487
7488fn bounded_log_text(value: &str) -> String {
7492 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7493 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7494 bounded.push('…');
7495 }
7496 format!("{bounded:?}")
7497}
7498
7499fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7505 use subc_protocol::scope::ScopeKind;
7506 let kind = match stamp.kind {
7507 ScopeKind::Head => "head",
7508 ScopeKind::Worker => "worker",
7509 ScopeKind::Ephemeral => "ephemeral",
7510 };
7511 format!(
7512 "{kind}/{}/{}@{}",
7513 principal_label(&stamp.owner),
7514 bounded_log_text(&stamp.scope_ref),
7515 stamp.scope_epoch
7516 )
7517}
7518
7519fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7526 let epoch = selector
7527 .scope_epoch
7528 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7529 format!(
7530 "requested/{}/{}@{epoch}",
7531 bounded_log_text(&principal_label(&selector.owner)),
7532 bounded_log_text(&selector.scope_ref),
7533 )
7534}
7535
7536fn principal_label(principal: &Principal) -> String {
7537 match principal {
7538 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7539 Principal::Direct => "direct".to_string(),
7540 other => format!("{other:?}"),
7541 }
7542}
7543
7544fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7545 match request {
7546 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7547 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7548 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7549 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7550 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7551 }
7552}
7553
7554fn parse_client_control_request(
7555 body: &[u8],
7556) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7557 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7558 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7559 Ok(probe)
7560 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7561 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7562 {
7563 ControlRequestBodyError::InvalidBody
7564 }
7565 Ok(_) => ControlRequestBodyError::UnknownOp,
7566 Err(_) => ControlRequestBodyError::InvalidBody,
7567 };
7568 (err, classification)
7569 })
7570}
7571
7572fn parse_module_control_request_from_module(
7573 body: &[u8],
7574) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7575 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7576 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7577 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7578 Ok(_) => ControlRequestBodyError::UnknownOp,
7579 Err(_) => ControlRequestBodyError::InvalidBody,
7580 };
7581 (err, classification)
7582 })
7583}
7584
7585#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7586enum ProviderRoleKind {
7587 ToolProvider,
7588 PipelineStage,
7589 ManagementSurface,
7590 InternalService,
7591}
7592
7593fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7594 match role {
7595 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7596 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7597 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7598 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7599 }
7600}
7601
7602fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7603 roles.iter().map(provider_role_kind).collect()
7604}
7605
7606const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7609
7610#[derive(Debug, Default, PartialEq, Eq)]
7612struct ScopeOutcomeCounts {
7613 created: usize,
7614 replaced: usize,
7615 updated: usize,
7616 unchanged: usize,
7617 refused: usize,
7618}
7619
7620impl ScopeOutcomeCounts {
7621 fn of(results: &[ScopeRecordResult]) -> Self {
7622 let mut counts = Self::default();
7623 for result in results {
7624 let slot = match result.outcome {
7625 ScopeRecordOutcome::Created => &mut counts.created,
7626 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7627 ScopeRecordOutcome::Updated => &mut counts.updated,
7628 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7629 ScopeRecordOutcome::Refused => &mut counts.refused,
7630 };
7631 *slot += 1;
7632 }
7633 counts
7634 }
7635}
7636
7637#[cfg(test)]
7638mod scope_outcome_count_tests {
7639 use super::*;
7640
7641 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7642 ScopeRecordResult {
7643 scope_ref: "r".to_string(),
7644 scope_epoch: 1,
7645 outcome,
7646 code: None,
7647 message: None,
7648 version: None,
7649 parent_state: None,
7650 }
7651 }
7652
7653 #[test]
7656 fn every_outcome_is_counted_in_its_own_field() {
7657 let results = [
7658 result(ScopeRecordOutcome::Created),
7659 result(ScopeRecordOutcome::Created),
7660 result(ScopeRecordOutcome::Replaced),
7661 result(ScopeRecordOutcome::Updated),
7662 result(ScopeRecordOutcome::Unchanged),
7663 result(ScopeRecordOutcome::Refused),
7664 result(ScopeRecordOutcome::Refused),
7665 result(ScopeRecordOutcome::Refused),
7666 ];
7667 assert_eq!(
7668 ScopeOutcomeCounts::of(&results),
7669 ScopeOutcomeCounts {
7670 created: 2,
7671 replaced: 1,
7672 updated: 1,
7673 unchanged: 1,
7674 refused: 3,
7675 }
7676 );
7677 }
7678}
7679
7680fn capability_census_trigger(
7684 old: Option<&CapabilityDeclarations>,
7685 new: Option<&CapabilityDeclarations>,
7686) -> bool {
7687 let old_provides = old
7688 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7689 .unwrap_or_default();
7690 let old_denies = old
7691 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7692 .unwrap_or_default();
7693 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7694 provides: Vec::new(),
7695 requires: Vec::new(),
7696 must_never_reach: Vec::new(),
7697 });
7698
7699 new.provides
7700 .iter()
7701 .any(|capability| !old_provides.contains(capability))
7702 || new
7703 .must_never_reach
7704 .iter()
7705 .any(|capability| !old_denies.contains(capability))
7706}
7707
7708fn denied_capability<'a>(
7711 opening_manifest: &'a ModuleManifest,
7712 target_manifest: &ModuleManifest,
7713) -> Option<&'a str> {
7714 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7715 let target_capabilities = target_manifest.capabilities.as_ref()?;
7716 opening_capabilities
7717 .must_never_reach
7718 .iter()
7719 .find(|denied| {
7720 target_capabilities
7721 .provides
7722 .iter()
7723 .any(|provided| provided == *denied)
7724 })
7725 .map(String::as_str)
7726}
7727
7728fn catalog_update_frozen_field_message(
7729 registered: &ModuleManifest,
7730 provides: &[ProviderRole],
7731) -> Option<String> {
7732 let old_has_provides = !registered.provides.is_empty();
7733 let new_has_provides = !provides.is_empty();
7734 if old_has_provides != new_has_provides {
7735 return Some(format!(
7736 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7737 registered.module_id
7738 ));
7739 }
7740
7741 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7742 return Some(format!(
7743 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7744 registered.module_id
7745 ));
7746 }
7747
7748 let registered_concurrency = manifest_concurrency(registered);
7749 let mut candidate = registered.clone();
7750 candidate.provides = provides.to_vec();
7751 let candidate_concurrency = manifest_concurrency(&candidate);
7752 if candidate_concurrency != registered_concurrency {
7753 return Some(format!(
7754 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7755 registered.module_id, registered_concurrency, candidate_concurrency
7756 ));
7757 }
7758
7759 None
7762}
7763
7764fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7765 manifest.provides.iter().any(is_routable_role)
7766}
7767
7768fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7774 manifest
7775 .provides
7776 .iter()
7777 .find_map(|provider| match provider {
7778 ProviderRole::ToolProvider { concurrency, .. }
7779 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7780 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7781 })
7782 .unwrap_or(Concurrency::ModuleManaged)
7783}
7784
7785fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7792 let has_management_surface = manifest
7793 .provides
7794 .iter()
7795 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7796 if !has_management_surface {
7797 return false;
7798 }
7799 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7800 return false;
7801 };
7802 let Some(provides) = raw
7803 .get("manifest")
7804 .and_then(|manifest| manifest.get("provides"))
7805 .and_then(serde_json::Value::as_array)
7806 else {
7807 return false;
7808 };
7809 provides.iter().any(|role| {
7814 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7815 && role.get("concurrency").is_none()
7816 })
7817}
7818
7819fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7820 if peer_version != PROTOCOL_VERSION {
7821 return Err(format!(
7822 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7823 ));
7824 }
7825 Ok(PROTOCOL_VERSION)
7826}
7827
7828fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7829 Frame::build_with_version(
7830 response_version(frame),
7831 FrameType::Pong,
7832 frame.header.flags,
7833 0,
7834 0,
7835 frame.header.corr,
7836 Vec::new(),
7837 )
7838 .map_err(RouterError::FrameBuild)
7839}
7840
7841pub(crate) fn control_error_frame(
7842 frame: &Frame,
7843 code: &'static str,
7844 message: impl Into<String>,
7845) -> Result<Frame, RouterError> {
7846 control_error_body_frame(
7847 frame,
7848 ErrorBody {
7849 code: code.to_string(),
7850 message: message.into(),
7851 detail: None,
7852 },
7853 )
7854}
7855
7856fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7857 let body = serde_json::to_vec(&error).map_err(|err| {
7858 RouterError::backend(
7859 0,
7860 frame.header.corr,
7861 format!("failed to encode control ERROR: {err}"),
7862 )
7863 })?;
7864
7865 Frame::build_with_version(
7866 response_version(frame),
7867 FrameType::Error,
7868 control_flags(),
7869 0,
7870 0,
7871 frame.header.corr,
7872 body,
7873 )
7874 .map_err(RouterError::FrameBuild)
7875}
7876
7877fn control_response_body_frame<T: Serialize>(
7878 frame: &Frame,
7879 reply: &T,
7880 label: &'static str,
7881) -> Result<Frame, RouterError> {
7882 let body = serde_json::to_vec(reply).map_err(|err| {
7883 RouterError::backend(
7884 0,
7885 frame.header.corr,
7886 format!("failed to encode {label}: {err}"),
7887 )
7888 })?;
7889
7890 Frame::build_with_version(
7891 response_version(frame),
7892 FrameType::Response,
7893 control_flags(),
7894 0,
7895 0,
7896 frame.header.corr,
7897 body,
7898 )
7899 .map_err(RouterError::FrameBuild)
7900}
7901
7902fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7932 match err {
7933 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7934 ForwardingError::NoModuleConnection => "target_unavailable",
7935 ForwardingError::ModuleReloading { .. } => "module_reloading",
7936 ForwardingError::ClientRouteChannelExhausted { .. }
7937 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7938 ForwardingError::StaleModuleEndpoint
7939 | ForwardingError::UnknownReservation { .. }
7940 | ForwardingError::ConnectionClosing { .. }
7941 | ForwardingError::ClientEgressClosed { .. }
7942 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7943 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7946 ForwardingError::RelayCorrelationExhausted
7947 | ForwardingError::RouteOpenBuild(_)
7948 | ForwardingError::Poisoned => "forwarding_error",
7949 }
7950}
7951
7952fn response_version(frame: &Frame) -> u8 {
7953 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7954 frame.header.ver
7955 } else {
7956 PROTOCOL_VERSION
7957 }
7958}
7959
7960fn control_flags() -> Flags {
7961 Flags::new(false, Priority::Passive, false)
7962}
7963
7964fn send_goodbye_target_best_effort(
7969 counters: &DaemonCounters,
7970 target: &GoodbyeTarget,
7971 context: &'static str,
7972) {
7973 let Ok(frame) = Frame::build_with_version(
7974 target.negotiated_ver,
7975 FrameType::Goodbye,
7976 control_flags(),
7977 target.channel,
7978 target.epoch,
7979 0,
7980 Vec::new(),
7981 ) else {
7982 return;
7983 };
7984 crate::forwarding::send_module_route_goodbye(
7985 counters,
7986 &target.sink,
7987 frame,
7988 target.module_id.as_deref(),
7989 context,
7990 );
7991}
7992
7993pub(crate) fn send_route_control_pushes(
7994 forwarding: &ForwardingTable,
7995 routes: Vec<EndpointRoute>,
7996 push: ClientControlPush,
7997) {
7998 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7999 for route in routes {
8000 let target = route.goodbye_target;
8001 if let Some((existing, channels)) = targets
8002 .iter_mut()
8003 .find(|(existing, _)| existing.connection_id == target.connection_id)
8004 {
8005 debug_assert_eq!(
8006 existing.negotiated_ver, target.negotiated_ver,
8007 "one connection cannot negotiate multiple frame versions"
8008 );
8009 if !channels.contains(&target.channel) {
8010 channels.push(target.channel);
8011 }
8012 continue;
8013 }
8014 let channel = target.channel;
8015 targets.push((target, vec![channel]));
8016 }
8017 for (target, mut channels) in targets {
8018 channels.sort_unstable();
8019 let mut push = push.clone();
8020 match &mut push {
8021 ClientControlPush::RouteClosing {
8022 channels: covered, ..
8023 }
8024 | ClientControlPush::RouteClosed {
8025 channels: covered, ..
8026 } => *covered = channels,
8027 }
8028 let body = match serde_json::to_vec(&push) {
8029 Ok(body) => body,
8030 Err(err) => {
8031 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8032 continue;
8033 }
8034 };
8035 let frame = match Frame::build_with_version(
8036 target.negotiated_ver,
8037 FrameType::Push,
8038 control_flags(),
8039 0,
8040 0,
8041 0,
8042 body.clone(),
8043 ) {
8044 Ok(frame) => frame,
8045 Err(err) => {
8046 warn!(
8047 route_channel = target.channel,
8048 error = %err,
8049 "failed to build route lifecycle control PUSH frame"
8050 );
8051 continue;
8052 }
8053 };
8054 if let Err(err) = target.sink.try_send(frame) {
8055 if target.close_on_delivery_failure() {
8056 warn!(
8057 target_connection_id = target.connection_id.get(),
8058 route_channel = target.channel,
8059 error = %err,
8060 "route lifecycle control PUSH was not delivered to client; closing target connection"
8061 );
8062 let _ = forwarding.escalate_client_delivery_failure(
8063 target.connection_id,
8064 target.channel,
8065 target.epoch,
8066 CloseReason::new(
8067 "route_lifecycle_push_delivery_failed",
8068 format!(
8069 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8070 target.channel
8071 ),
8072 ),
8073 crate::forwarding::UndeliveredFrame {
8074 module_id: target.module_id.as_deref(),
8075 sink: &target.sink,
8076 },
8077 );
8078 }
8079 }
8080 }
8081}
8082
8083#[cfg(test)]
8084mod tests {
8085 mod activity_read {
8086 include!("activity_read_tests.rs");
8087 }
8088
8089 mod lifecycle_activity {
8090 include!("control_lifecycle_tests.rs");
8091 }
8092 #[cfg(unix)]
8093 #[tokio::test]
8094 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8095 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8096 let path = dir.join("subc.jsonc");
8097 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8098 "program":"/bin/sleep","args":["60"],"protocol":"none",
8099 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8100 }}}).to_string()).unwrap();
8101 let mut configured = crate::daemon_config::load(&path)
8102 .unwrap()
8103 .unwrap()
8104 .modules
8105 .pop()
8106 .unwrap();
8107 let registry = std::sync::Arc::new(crate::Registry::default());
8108 let handle = crate::SupervisorHandle::new();
8109 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8110 .with_handle(handle.clone());
8111 let module = supervisor
8112 .supervise_configured_with_health(
8113 configured.module_spec(),
8114 true,
8115 configured.health.clone(),
8116 None,
8117 configured.restart,
8118 )
8119 .unwrap();
8120 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8121 let before = module.status().unwrap().pid;
8122 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8123 configured.health.cadence = std::time::Duration::from_secs(3600);
8124 let health_only = handler
8125 .reconcile_supervised_modules(
8126 &supervisor,
8127 vec![configured.clone()],
8128 false,
8129 Principal::Direct,
8130 )
8131 .await
8132 .unwrap();
8133 assert!(
8134 health_only.0.changed_pending_reload.is_empty(),
8135 "health policy is already applied live"
8136 );
8137 assert_eq!(module.status().unwrap().pid, before);
8138 assert_eq!(
8139 module.configuration().unwrap().1.http,
8140 configured.health.http
8141 );
8142 configured.args = vec!["61".into()];
8143 let launch = handler
8144 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8145 .await
8146 .unwrap();
8147 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8148 assert_eq!(
8149 module.status().unwrap().pid,
8150 before,
8151 "a launch edit is stored until reload"
8152 );
8153 module.drain().await.unwrap();
8154 }
8155 use cortexkit_test_support::ScratchDir;
8156 use std::{
8157 collections::BTreeMap,
8158 fmt,
8159 path::PathBuf,
8160 sync::{Arc, Mutex},
8161 time::Duration,
8162 };
8163
8164 use serde_json::{json, Value};
8165 use subc_protocol::{
8166 manifest::{
8167 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8168 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8169 },
8170 session::HealthStatus,
8171 FrameType,
8172 };
8173
8174 use super::*;
8175 use crate::{
8176 forwarding::{DataRoute, DataRouteState},
8177 registry::ChannelState,
8178 router::FrameSink,
8179 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8180 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8181 RouteCtx, Router,
8182 };
8183 use tokio::{
8184 sync::mpsc,
8185 time::{sleep, Instant},
8186 };
8187 use tracing::{
8188 field::{Field, Visit},
8189 Event, Subscriber,
8190 };
8191 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8192
8193 fn fake_aft_stub_path() -> PathBuf {
8212 let mut path = std::env::current_exe().expect("current_exe available in tests");
8213 path.pop(); path.pop(); path.push(if cfg!(windows) {
8216 "fake-aft-stub.exe"
8217 } else {
8218 "fake-aft-stub"
8219 });
8220 assert!(
8221 path.exists(),
8222 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8223 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8224 path.display()
8225 );
8226 path
8227 }
8228
8229 fn client_retries(code: &str) -> bool {
8234 subc_protocol::error_codes::is_retryable_route_open(code)
8235 }
8236
8237 #[test]
8248 fn retryability_of_forwarding_codes_matches_the_failure() {
8249 let transient = [
8252 ForwardingError::NoModuleConnection,
8253 ForwardingError::ModuleReloading {
8254 module_id: "m".into(),
8255 },
8256 ForwardingError::StaleModuleEndpoint,
8257 ForwardingError::UnknownReservation {
8258 client_channel: 1,
8259 module_channel: 1,
8260 },
8261 ForwardingError::ConnectionClosing {
8262 connection_id: ConnectionId::new(1),
8263 },
8264 ForwardingError::ClientEgressClosed {
8265 connection_id: ConnectionId::new(1),
8266 },
8267 ForwardingError::ModuleEgressUnavailable {
8268 connection_id: ConnectionId::new(1),
8269 },
8270 ];
8271 for err in transient {
8272 let code = forwarding_error_code(&err);
8273 assert!(
8274 client_retries(code),
8275 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8276 );
8277 }
8278
8279 let permanent = [
8284 ForwardingError::ConnectionRoleConflict {
8285 connection_id: ConnectionId::new(1),
8286 },
8287 ForwardingError::ClientRouteChannelExhausted {
8288 connection_id: ConnectionId::new(1),
8289 },
8290 ForwardingError::ModuleRouteChannelExhausted {
8291 endpoint: ModuleEndpointId {
8292 connection_id: ConnectionId::new(1),
8293 generation: 1,
8294 },
8295 },
8296 ForwardingError::RelayCorrelationExhausted,
8297 ForwardingError::RouteOpenBuild("x".into()),
8298 ForwardingError::Poisoned,
8299 ];
8300 for err in permanent {
8301 let code = forwarding_error_code(&err);
8302 assert!(
8303 !client_retries(code),
8304 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8305 );
8306 }
8307 }
8308
8309 #[tokio::test]
8337 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8338 let handler = ControlHandler::default();
8339 let frame =
8340 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8341
8342 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8346 assert_eq!(
8347 stamped,
8348 Principal::Direct,
8349 "a caller that proved nothing must not be stamped as a supervised module"
8350 );
8351
8352 let forged = handler
8357 .route_open_principal(
8358 &frame,
8359 Some(ConsumerIdentity {
8360 module_id: "aft".to_string(),
8361 launch_nonce: "not-a-real-nonce".to_string(),
8362 }),
8363 )
8364 .unwrap();
8365 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8366 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8367 }
8368
8369 #[test]
8386 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8387 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8388 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8389 let ClientControlRequest::RouteOpen {
8390 consumer_identity, ..
8391 } = parsed
8392 else {
8393 panic!("route.open body must parse as RouteOpen");
8394 };
8395 assert_eq!(
8396 consumer_identity,
8397 Some(ConsumerIdentity {
8398 module_id: "aft".to_string(),
8399 launch_nonce: "n".to_string(),
8400 }),
8401 "the wire field name must reach the value route_open_principal reads"
8402 );
8403 }
8404
8405 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8406 ModuleManifest::builder(module_id, "0.1.0")
8407 .protocol_ver(protocol_ver)
8408 .provides(vec![ProviderRole::ToolProvider {
8409 tools: vec![Tool {
8410 name: "read".to_string(),
8411 description: None,
8412 execution_mode: ExecutionMode::Pure,
8413 schema: json!({"type": "object"}),
8414 }],
8415 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8416 concurrency: Concurrency::ModuleManaged,
8417 emits_push: true,
8418 sub_supervises: true,
8419 }])
8420 .build()
8421 }
8422
8423 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8424 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8425 }
8426
8427 fn hello_frame_with_control_ops(
8428 module_id: &str,
8429 protocol_ver: u8,
8430 corr: u64,
8431 control_ops: Option<Vec<String>>,
8432 ) -> Frame {
8433 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8434 }
8435
8436 fn hello_frame_with_nonce(
8437 module_id: &str,
8438 protocol_ver: u8,
8439 corr: u64,
8440 launch_nonce: Option<&str>,
8441 ) -> Frame {
8442 hello_frame_full(
8443 module_id,
8444 protocol_ver,
8445 corr,
8446 None,
8447 launch_nonce.map(ToOwned::to_owned),
8448 )
8449 }
8450
8451 fn hello_frame_full(
8452 module_id: &str,
8453 protocol_ver: u8,
8454 corr: u64,
8455 control_ops: Option<Vec<String>>,
8456 launch_nonce: Option<String>,
8457 ) -> Frame {
8458 let body = serde_json::to_vec(&ModuleHelloBody {
8459 manifest: manifest(module_id, protocol_ver),
8460 protocol_ver,
8461 control_ops,
8462 launch_nonce,
8463 })
8464 .unwrap();
8465 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8466 }
8467
8468 fn hello_frame_with_rich_ui(
8469 module_id: &str,
8470 protocol_ver: u8,
8471 corr: u64,
8472 launch_nonce: Option<&str>,
8473 rich_ui: Value,
8474 ) -> Frame {
8475 let mut body = serde_json::to_value(ModuleHelloBody {
8476 manifest: manifest(module_id, protocol_ver),
8477 protocol_ver,
8478 control_ops: None,
8479 launch_nonce: launch_nonce.map(str::to_string),
8480 })
8481 .unwrap();
8482 body["manifest"]["rich_ui"] = rich_ui;
8483 Frame::build(
8484 FrameType::Hello,
8485 control_flags(),
8486 0,
8487 0,
8488 corr,
8489 serde_json::to_vec(&body).unwrap(),
8490 )
8491 .unwrap()
8492 }
8493
8494 fn non_routable_hello_frame_with_control_ops(
8495 module_id: &str,
8496 corr: u64,
8497 control_ops: Option<Vec<String>>,
8498 ) -> Frame {
8499 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8500 manifest.provides.clear();
8501 let body = serde_json::to_vec(&ModuleHelloBody {
8502 manifest,
8503 protocol_ver: PROTOCOL_VERSION,
8504 control_ops,
8505 launch_nonce: None,
8506 })
8507 .unwrap();
8508 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8509 }
8510
8511 fn capability_grammar_hello_frame(
8512 capabilities: Value,
8513 runtime_computed: Option<Value>,
8514 corr: u64,
8515 ) -> Frame {
8516 let mut body = serde_json::to_value(ModuleHelloBody {
8517 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8518 protocol_ver: PROTOCOL_VERSION,
8519 control_ops: None,
8520 launch_nonce: None,
8521 })
8522 .expect("HELLO body serializes");
8523 body["manifest"]["capabilities"] = capabilities;
8524 if let Some(runtime_computed) = runtime_computed {
8525 body["runtime_computed"] = runtime_computed;
8526 }
8527 Frame::build(
8528 FrameType::Hello,
8529 control_flags(),
8530 0,
8531 0,
8532 corr,
8533 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8534 )
8535 .expect("HELLO frame builds")
8536 }
8537
8538 fn channel_request(channel: u16, corr: u64) -> Frame {
8539 Frame::build(
8540 FrameType::Request,
8541 Flags::new(true, Priority::Interactive, false),
8542 channel,
8543 0,
8544 corr,
8545 b"opaque".to_vec(),
8546 )
8547 .unwrap()
8548 }
8549
8550 fn route_ctx(
8551 connection_id: ConnectionId,
8552 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8553 let (tx, rx) = mpsc::channel(8);
8554 (
8555 RouteCtx {
8556 connection_id,
8557 egress: FrameSink::new(tx),
8558 },
8559 rx,
8560 )
8561 }
8562
8563 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8564 serde_json::from_slice(&frame.body).unwrap()
8565 }
8566
8567 async fn hello_via_sink(
8572 handler: &ControlHandler,
8573 ctx: &RouteCtx,
8574 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8575 hello: Frame,
8576 ) -> Frame {
8577 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8578 assert!(
8579 replies.is_empty(),
8580 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8581 );
8582 let ack = rx
8583 .try_recv()
8584 .expect("HELLO_ACK is queued on the module sink")
8585 .frame;
8586 assert_eq!(ack.header.ty, FrameType::HelloAck);
8587 ack
8588 }
8589
8590 fn parse_error(frame: &Frame) -> Value {
8591 serde_json::from_slice(&frame.body).unwrap()
8592 }
8593
8594 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8595 serde_json::from_slice(&frame.body).unwrap()
8596 }
8597
8598 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8599 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8600 route_channel,
8601 route_epoch: 0,
8602 kind,
8603 })
8604 .unwrap();
8605 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8606 }
8607
8608 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8609 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8610 module_id: module_id.to_string(),
8611 })
8612 .unwrap();
8613 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8614 }
8615
8616 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8617 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8618 }
8619
8620 fn route_open_frame_with_consumer_capabilities(
8621 corr: u64,
8622 module_id: &str,
8623 project_root: ScratchDir,
8624 consumer_capabilities: Option<Vec<String>>,
8625 ) -> Frame {
8626 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8627 target: RouteTarget::ToolProvider {
8628 module_id: module_id.to_string(),
8629 },
8630 identity: BindIdentity::new(
8631 project_root.path().to_path_buf(),
8632 "unit".to_string(),
8633 "session".to_string(),
8634 ),
8635 consumer_identity: None,
8636 consumer_capabilities,
8637 role_versions: None,
8638 admission_facts: None,
8639 scope: None,
8640 })
8641 .unwrap();
8642 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8643 }
8644
8645 fn route_open_frame_with_role_versions(
8646 corr: u64,
8647 module_id: &str,
8648 project_root: ScratchDir,
8649 role_versions: Option<BTreeMap<String, String>>,
8650 ) -> Frame {
8651 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8652 target: RouteTarget::ToolProvider {
8653 module_id: module_id.to_string(),
8654 },
8655 identity: BindIdentity::new(
8656 project_root.path().to_path_buf(),
8657 "unit".to_string(),
8658 format!("session-{corr}"),
8659 ),
8660 consumer_identity: None,
8661 consumer_capabilities: None,
8662 role_versions,
8663 admission_facts: None,
8664 scope: None,
8665 })
8666 .unwrap();
8667 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8668 }
8669
8670 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8671 entries
8672 .iter()
8673 .map(|(role, version)| (role.to_string(), version.to_string()))
8674 .collect()
8675 }
8676
8677 fn route_open_frame_with_admission_facts(
8678 corr: u64,
8679 module_id: &str,
8680 project_root: ScratchDir,
8681 consumer_identity: Option<subc_control::ConsumerIdentity>,
8682 facts: Option<Value>,
8683 ) -> Frame {
8684 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8685 target: RouteTarget::ToolProvider {
8686 module_id: module_id.to_string(),
8687 },
8688 identity: BindIdentity::new(
8689 project_root.path().to_path_buf(),
8690 "unit".to_string(),
8691 format!("session-{corr}"),
8692 ),
8693 consumer_identity,
8694 consumer_capabilities: None,
8695 role_versions: None,
8696 admission_facts: facts,
8697 scope: None,
8698 })
8699 .unwrap();
8700 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8701 }
8702
8703 #[derive(Clone, Default)]
8704 struct EventCapture {
8705 events: Arc<Mutex<Vec<CapturedEvent>>>,
8706 }
8707
8708 #[derive(Clone, Debug)]
8709 struct CapturedEvent {
8710 target: String,
8711 level: tracing::Level,
8712 fields: BTreeMap<String, String>,
8713 }
8714
8715 impl EventCapture {
8716 fn events(&self) -> Vec<CapturedEvent> {
8717 self.events.lock().unwrap().clone()
8718 }
8719 }
8720
8721 impl<S> Layer<S> for EventCapture
8722 where
8723 S: Subscriber,
8724 {
8725 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8726 let mut visitor = EventFieldVisitor::default();
8727 event.record(&mut visitor);
8728 self.events.lock().unwrap().push(CapturedEvent {
8729 target: event.metadata().target().to_string(),
8730 level: *event.metadata().level(),
8731 fields: visitor.fields,
8732 });
8733 }
8734 }
8735
8736 #[derive(Default)]
8737 struct EventFieldVisitor {
8738 fields: BTreeMap<String, String>,
8739 }
8740
8741 impl Visit for EventFieldVisitor {
8742 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8743 self.fields
8744 .insert(field.name().to_string(), format!("{value:?}"));
8745 }
8746 }
8747
8748 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8749 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8750 status,
8751 detail: Some("warming".to_string()),
8752 metrics: Some(json!({"queue_depth": 3})),
8753 })
8754 .unwrap();
8755 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8756 }
8757
8758 fn route_bind_ack(corr: u64) -> Frame {
8759 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8760 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8761 }
8762
8763 fn unique_project_root(label: &str) -> ScratchDir {
8764 ScratchDir::new(label)
8765 }
8766
8767 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8768 match parse_route_poll(frame) {
8769 ClientControlResponse::RoutePoll {
8770 status: None,
8771 live: Some(live),
8772 ..
8773 } => assert_eq!(live, expected_live),
8774 other => panic!("unexpected route.poll response: {other:?}"),
8775 }
8776 }
8777
8778 fn bind_liveness_route(
8779 registry: &Registry,
8780 forwarding: &ForwardingTable,
8781 module_id: &str,
8782 ) -> (RouteCtx, u16, u32) {
8783 let module_connection = ConnectionId::new(101);
8784 let client_connection = ConnectionId::new(202);
8785 let registration = registry
8786 .register_with_control_ops(
8787 manifest(module_id, PROTOCOL_VERSION),
8788 PROTOCOL_VERSION,
8789 module_connection,
8790 module_baseline_control_ops(),
8791 )
8792 .unwrap();
8793 let (module_tx, _module_rx) = mpsc::channel(8);
8794 let endpoint = forwarding
8795 .register_module_connection(
8796 module_connection,
8797 module_id.to_string(),
8798 PROTOCOL_VERSION,
8799 manifest_concurrency(®istration.manifest),
8800 FrameSink::new(module_tx),
8801 )
8802 .unwrap();
8803 let (client_ctx, _client_rx) = route_ctx(client_connection);
8804 let pending = forwarding
8805 .begin_route_bind_relay_for_test(
8806 client_connection,
8807 client_ctx.egress.clone(),
8808 1,
8809 module_id,
8810 )
8811 .unwrap();
8812 assert_eq!(pending.endpoint, endpoint);
8813 let route_channel = pending.client_channel;
8814 let route_epoch = pending.client_epoch;
8815 forwarding
8816 .complete_pending_relay(
8817 module_connection,
8818 pending.corr,
8819 RouteBindRelayOutcome::Accepted,
8820 )
8821 .unwrap();
8822 (client_ctx, route_channel, route_epoch)
8823 }
8824
8825 struct FakeProcessLiveness {
8826 live: Option<bool>,
8827 }
8828
8829 impl ModuleProcessLiveness for FakeProcessLiveness {
8830 fn process_live(&self, _module_id: &str) -> Option<bool> {
8831 self.live
8832 }
8833 }
8834
8835 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8836 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8837 {
8838 let registry = Arc::new(Registry::default());
8839 let supervisor_handle = SupervisorHandle::new();
8840 let supervisor = Supervisor::new_for_test(
8841 Arc::clone(®istry),
8842 RestartPolicy::new(1, Duration::from_millis(10)),
8843 )
8844 .with_handle(supervisor_handle.clone());
8845 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8846 let module = supervisor
8847 .spawn(ModuleSpec {
8848 module_id: "stderr-tail-wire".to_string(),
8849 program: fake_aft_stub_path(),
8850 args: Vec::new(),
8851 env: vec![
8852 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8853 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8854 ],
8855 reserved: false,
8856 reserved_prefixes: Vec::new(),
8857 protocol: ModuleProtocol::Subc,
8858 overlap: Default::default(),
8859 })
8860 .unwrap();
8861
8862 let deadline = Instant::now() + Duration::from_secs(5);
8863 loop {
8864 let tail = module.stderr_tail(None, None);
8865 if tail
8866 .entries
8867 .iter()
8868 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8869 && tail.entries.iter().any(|entry| {
8870 matches!(
8871 entry,
8872 TailEntry::Line {
8873 truncated: true,
8874 ..
8875 }
8876 )
8877 })
8878 {
8879 break;
8880 }
8881 assert!(
8882 Instant::now() < deadline,
8883 "module did not produce a truncated line and restart boundary: {tail:?}"
8884 );
8885 sleep(Duration::from_millis(10)).await;
8886 }
8887
8888 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8889 let request = ClientControlRequest::SupervisorStderrTail {
8890 module_id: "stderr-tail-wire".to_string(),
8891 max_lines: None,
8892 max_bytes: None,
8893 };
8894 let frame = Frame::build(
8895 FrameType::Request,
8896 control_flags(),
8897 0,
8898 0,
8899 1,
8900 serde_json::to_vec(&request).unwrap(),
8901 )
8902 .unwrap();
8903 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8904 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8905 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8906 serde_json::from_slice(&responses[0].body).unwrap()
8907 else {
8908 panic!("expected supervisor.stderr_tail response");
8909 };
8910
8911 assert!(
8912 tail.entries
8913 .iter()
8914 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8915 "the control response lost the restart boundary"
8916 );
8917 let Some(StderrTailEntry::Line {
8918 text,
8919 truncated,
8920 at_ms,
8921 }) = tail.entries.iter().find(|entry| {
8922 matches!(
8923 entry,
8924 StderrTailEntry::Line {
8925 truncated: true,
8926 ..
8927 }
8928 )
8929 })
8930 else {
8931 panic!("the control response lost the truncated line");
8932 };
8933 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8934 assert!(*truncated);
8935 assert!(
8936 at_ms.is_some(),
8937 "the control response lost the line's capture time"
8938 );
8939 }
8940
8941 #[tokio::test(flavor = "current_thread")]
8946 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8947 let dir = ScratchDir::new("terminals-off-worker");
8948 let journal_path = dir.join("terminals.jsonl");
8949 let registry = Arc::new(Registry::default());
8950 let supervisor_handle = SupervisorHandle::new();
8951 let supervisor =
8952 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8953 .with_handle(supervisor_handle.clone())
8954 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8955 let module = supervisor
8956 .spawn(ModuleSpec {
8957 module_id: "terminal-off-worker".to_string(),
8958 program: fake_aft_stub_path(),
8959 args: Vec::new(),
8960 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8961 reserved: false,
8962 reserved_prefixes: Vec::new(),
8963 protocol: ModuleProtocol::Subc,
8964 overlap: Default::default(),
8965 })
8966 .unwrap();
8967 let deadline = Instant::now() + Duration::from_secs(5);
8968 while module.terminal_history().entries.len() != 2 {
8969 assert!(Instant::now() < deadline, "module did not record two exits");
8970 sleep(Duration::from_millis(10)).await;
8971 }
8972
8973 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8974 let handler =
8975 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8976 let frame = Frame::build(
8977 FrameType::Request,
8978 control_flags(),
8979 0,
8980 0,
8981 1,
8982 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8983 module_id: "terminal-off-worker".to_string(),
8984 })
8985 .unwrap(),
8986 )
8987 .unwrap();
8988 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8989 let spawned_at = std::time::Instant::now();
8990 let read = tokio::spawn({
8991 let handler = Arc::clone(&handler);
8992 async move { handler.handle_control_frame(&ctx, frame).await }
8993 });
8994 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8997 .await
8998 .unwrap()
8999 .expect("the history read reached its pause");
9000 let elapsed = spawned_at.elapsed();
9001 assert!(
9002 elapsed < Duration::from_secs(2) && !read.is_finished(),
9003 "this task could not run while the history read was paused \
9004 (resumed after {elapsed:?}, read finished: {})",
9005 read.is_finished()
9006 );
9007
9008 drop(release);
9009 let responses = read.await.unwrap().unwrap();
9010 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9011 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9012 panic!("expected supervisor.terminals response");
9013 };
9014 assert_eq!(terminals.entries.len(), 2);
9015 assert_eq!(terminals.journal_skipped_lines, 0);
9016 assert_eq!(terminals.journal_read_errors, 0);
9017 }
9018
9019 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9020 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9021 let registry = Arc::new(Registry::default());
9022 let supervisor_handle = SupervisorHandle::new();
9023 let supervisor =
9024 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9025 .with_handle(supervisor_handle.clone());
9026 let module = supervisor
9027 .spawn(ModuleSpec {
9028 module_id: "terminal-golden".to_string(),
9029 program: fake_aft_stub_path(),
9030 args: Vec::new(),
9031 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9032 reserved: false,
9033 reserved_prefixes: Vec::new(),
9034 protocol: ModuleProtocol::Subc,
9035 overlap: Default::default(),
9036 })
9037 .unwrap();
9038
9039 let deadline = Instant::now() + Duration::from_secs(5);
9040 while module.terminal_history().entries.len() != 2 {
9041 assert!(
9042 Instant::now() < deadline,
9043 "module did not retain two terminal exits: {:?}",
9044 module.terminal_history()
9045 );
9046 sleep(Duration::from_millis(10)).await;
9047 }
9048
9049 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9050 let request = ClientControlRequest::SupervisorTerminals {
9051 module_id: "terminal-golden".to_string(),
9052 };
9053 let frame = Frame::build(
9054 FrameType::Request,
9055 control_flags(),
9056 0,
9057 0,
9058 1,
9059 serde_json::to_vec(&request).unwrap(),
9060 )
9061 .unwrap();
9062 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9063 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9064 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9065 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9066 panic!("expected supervisor.terminals response");
9067 };
9068 assert_eq!(terminals.entries.len(), 2);
9069 assert_eq!(terminals.dropped, 0);
9070
9071 let mut rendered = serde_json::to_value(response).unwrap();
9072 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9075 for (index, entry) in rendered["entries"]
9076 .as_array_mut()
9077 .expect("terminal response entries array")
9078 .iter_mut()
9079 .enumerate()
9080 {
9081 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9082 }
9083
9084 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9085 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9086 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9087 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9088 std::fs::write(&golden_path, &serialized).unwrap();
9089 }
9090 let expected: Value =
9091 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9092 assert_eq!(rendered, expected);
9093 }
9094
9095 #[test]
9096 fn hello_registers_manifest_and_returns_ack() {
9097 let registry = Arc::new(Registry::default());
9098 let handler = ControlHandler::new(Arc::clone(®istry));
9099 let conn = ConnectionId::new(1);
9100
9101 let responses = handler
9102 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9103 .unwrap();
9104
9105 assert_eq!(responses.len(), 1);
9106 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9107 assert_eq!(responses[0].header.channel, 0);
9108 assert_eq!(responses[0].header.corr, 7);
9109 let ack = parse_ack(&responses[0]);
9110 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9111 assert!(ack
9112 .subc_capabilities
9113 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9114 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9115 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9116 assert!(ack
9117 .subc_ops
9118 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9119 assert!(ack
9120 .subc_ops
9121 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9122
9123 let registration = registry.get_module("aft").unwrap().unwrap();
9124 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9125 assert_eq!(registration.state, ChannelState::Active);
9126 assert_eq!(registration.connection_id, conn);
9127 assert_eq!(registration.control_ops, module_baseline_control_ops());
9128 }
9129
9130 #[test]
9131 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9132 let invalid_identifiers = [
9133 ("case_change", "credentials-Provider/v1"),
9134 ("leading_zero", "credentials-provider/v01"),
9135 ("trailing_hyphen", "credentials-provider-/v1"),
9136 ("consecutive_hyphens", "credentials--provider/v1"),
9137 ("uppercase", "Credentials-provider/v1"),
9138 ("missing_v", "credentials-provider/1"),
9139 ("whitespace", "credentials provider/v1"),
9140 ("zero_version", "credentials-provider/v0"),
9141 ("out_of_range_version", "credentials-provider/v4294967296"),
9142 (
9143 "overlength_name",
9144 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9145 ),
9146 ];
9147 let mut cases = invalid_identifiers
9148 .into_iter()
9149 .map(|(name, identifier)| {
9150 (
9151 format!("identifier_{name}"),
9152 "capabilities.provides[0]".to_string(),
9153 identifier.to_string(),
9154 json!({ "provides": [identifier] }),
9155 None,
9156 )
9157 })
9158 .collect::<Vec<_>>();
9159 cases.extend([
9160 (
9161 "unknown_need".to_string(),
9162 "capabilities.requires[0].need".to_string(),
9163 "deferred".to_string(),
9164 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9165 None,
9166 ),
9167 (
9168 "duplicate_provides".to_string(),
9169 "capabilities.provides[1]".to_string(),
9170 "credentials-provider/v1".to_string(),
9171 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9172 None,
9173 ),
9174 (
9175 "duplicate_must_never_reach".to_string(),
9176 "capabilities.must_never_reach[1]".to_string(),
9177 "credentials-provider/v1".to_string(),
9178 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9179 None,
9180 ),
9181 (
9182 "duplicate_requires_same_need".to_string(),
9183 "capabilities.requires[1]".to_string(),
9184 "credentials-provider/v1".to_string(),
9185 json!({ "requires": [
9186 { "capability": "credentials-provider/v1", "need": "required" },
9187 { "capability": "credentials-provider/v1", "need": "required" }
9188 ] }),
9189 None,
9190 ),
9191 (
9192 "duplicate_requires_conflicting_need".to_string(),
9193 "capabilities.requires[1]".to_string(),
9194 "credentials-provider/v1".to_string(),
9195 json!({ "requires": [
9196 { "capability": "credentials-provider/v1", "need": "required" },
9197 { "capability": "credentials-provider/v1", "need": "optional" }
9198 ] }),
9199 None,
9200 ),
9201 (
9202 "capabilities_root_pointer".to_string(),
9203 "runtime_computed[0]".to_string(),
9204 "/capabilities".to_string(),
9205 json!({}),
9206 Some(json!(["/capabilities"])),
9207 ),
9208 (
9209 "capabilities_descendant_pointer".to_string(),
9210 "runtime_computed[0]".to_string(),
9211 "/capabilities/provides".to_string(),
9212 json!({}),
9213 Some(json!(["/capabilities/provides"])),
9214 ),
9215 (
9216 "malformed_pointer_without_leading_slash".to_string(),
9217 "runtime_computed[0]".to_string(),
9218 "capabilities".to_string(),
9219 json!({}),
9220 Some(json!(["capabilities"])),
9221 ),
9222 (
9223 "malformed_pointer_escape".to_string(),
9224 "runtime_computed[0]".to_string(),
9225 "/roles/~2/tools".to_string(),
9226 json!({}),
9227 Some(json!(["/roles/~2/tools"])),
9228 ),
9229 (
9230 "unknown_capabilities_field".to_string(),
9231 "capabilities.future".to_string(),
9232 "<array>".to_string(),
9233 json!({ "future": [] }),
9234 None,
9235 ),
9236 ]);
9237
9238 for (index, (name, field, value, capabilities, runtime_computed)) in
9239 cases.into_iter().enumerate()
9240 {
9241 let registry = Arc::new(Registry::default());
9242 let handler = ControlHandler::new(Arc::clone(®istry));
9243 let response = handler
9244 .handle_control(
9245 ConnectionId::new((index + 1) as u64),
9246 capability_grammar_hello_frame(
9247 capabilities,
9248 runtime_computed,
9249 index as u64 + 1,
9250 ),
9251 )
9252 .expect("invalid HELLO returns a refusal");
9253
9254 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9255 let error = parse_error(&response[0]);
9256 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9257 let message = error["message"]
9258 .as_str()
9259 .expect("error message is a string");
9260 assert!(
9261 message.contains(&field),
9262 "{name}: field missing from {message}"
9263 );
9264 assert!(
9265 message.contains(&value),
9266 "{name}: value missing from {message}"
9267 );
9268 assert_eq!(
9269 registry
9270 .active_registration_count()
9271 .expect("registry reads"),
9272 0,
9273 "{name}: refused HELLO must not create a catalog entry"
9274 );
9275 }
9276 }
9277
9278 #[test]
9279 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9280 let registry = Arc::new(Registry::default());
9281 let handler = ControlHandler::new(Arc::clone(®istry));
9282 let capabilities = json!({
9283 "provides": ["credentials-provider/v1"],
9284 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9285 "must_never_reach": ["federation-transport/v1"]
9286 });
9287 let response = handler
9288 .handle_control(
9289 ConnectionId::new(99),
9290 capability_grammar_hello_frame(
9291 capabilities.clone(),
9292 Some(json!(["/roles/0/tools"])),
9293 99,
9294 ),
9295 )
9296 .expect("valid HELLO registers");
9297 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9298
9299 let request = Frame::build(
9300 FrameType::Request,
9301 control_flags(),
9302 0,
9303 0,
9304 100,
9305 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9306 .expect("catalog request serializes"),
9307 )
9308 .expect("catalog request frame builds");
9309 let response = handler
9310 .handle_catalog_list(request, None)
9311 .expect("catalog list succeeds");
9312 let ClientControlResponse::CatalogList { modules, .. } =
9313 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9314 else {
9315 panic!("catalog request must return catalog.list");
9316 };
9317 assert_eq!(modules.len(), 1);
9318 assert_eq!(
9319 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9320 capabilities
9321 );
9322 }
9323
9324 #[test]
9325 fn catalog_list_mirrors_management_operation_description() {
9326 let registry = Arc::new(Registry::default());
9327 let handler = ControlHandler::new(Arc::clone(®istry));
9328 let description = "List managed records and return their identifiers and metadata.";
9329 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9330 manifest.provides = vec![ProviderRole::ManagementSurface {
9331 operations: vec![ManagementOperation {
9332 name: "records.list".to_string(),
9333 kind: ManagementOperationKind::Query,
9334 description: Some(description.to_string()),
9335 }],
9336 config_schema: json!({"type": "object"}),
9337 observability: vec![ObservabilitySurface {
9338 name: "records.stats".to_string(),
9339 kind: ObservabilityKind::Snapshot,
9340 }],
9341 identity_scope: vec![IdentityScope::Project],
9342 concurrency: Concurrency::ModuleManaged,
9343 }];
9344 registry
9345 .register_with_control_ops(
9346 manifest,
9347 PROTOCOL_VERSION,
9348 ConnectionId::new(99),
9349 Vec::new(),
9350 )
9351 .expect("described management manifest registers");
9352
9353 let request = Frame::build(
9354 FrameType::Request,
9355 control_flags(),
9356 0,
9357 0,
9358 100,
9359 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9360 .expect("catalog request serializes"),
9361 )
9362 .expect("catalog request frame builds");
9363 let response = handler
9364 .handle_catalog_list(request, None)
9365 .expect("catalog list succeeds");
9366 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9367 assert_eq!(
9368 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9369 "catalog.list must preserve the declared operation description verbatim"
9370 );
9371 }
9372
9373 #[test]
9374 fn catalog_list_mirrors_registered_operation_delivery() {
9375 let registry = Arc::new(Registry::default());
9376 let handler = ControlHandler::new(Arc::clone(®istry));
9377 let delivery =
9378 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9379 let mut body = serde_json::to_value(ModuleHelloBody {
9380 manifest: manifest("watch-management", PROTOCOL_VERSION),
9381 protocol_ver: PROTOCOL_VERSION,
9382 control_ops: None,
9383 launch_nonce: None,
9384 })
9385 .unwrap();
9386 body["manifest"]["provides"] = json!([{
9387 "role": "management_surface",
9388 "operations": [
9389 {"name": "status", "kind": "query"},
9390 {"name": "future", "kind": "query"},
9391 {"name": "reset", "kind": "mutate"}
9392 ],
9393 "config_schema": {}, "observability": [], "identity_scope": []
9394 }]);
9395 body["manifest"]["operation_delivery"] = delivery.clone();
9396 let hello = Frame::build(
9397 FrameType::Hello,
9398 control_flags(),
9399 0,
9400 0,
9401 99,
9402 serde_json::to_vec(&body).unwrap(),
9403 )
9404 .unwrap();
9405 let response = handler
9406 .handle_control(ConnectionId::new(99), hello)
9407 .unwrap();
9408 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9409
9410 let (_, registrations) = registry.list_modules().unwrap();
9411 let registered = ®istrations[0].manifest;
9412 assert!(registered.is_watch_operation("status"));
9413 assert!(!registered.is_watch_operation("future"));
9414 assert!(!registered.is_watch_operation("reset"));
9415 assert!(!registered.is_watch_operation("missing"));
9416
9417 let request = Frame::build(
9418 FrameType::Request,
9419 control_flags(),
9420 0,
9421 0,
9422 100,
9423 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9424 )
9425 .unwrap();
9426 let response = handler.handle_catalog_list(request, None).unwrap();
9427 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9428 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9429 let ClientControlResponse::CatalogList { modules, .. } =
9430 serde_json::from_slice(&response[0].body).unwrap()
9431 else {
9432 panic!("expected catalog.list");
9433 };
9434 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9435 }
9436
9437 #[test]
9438 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9439 let registry = Arc::new(Registry::default());
9440 let supervisor = SupervisorHandle::new();
9441 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9442 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9443 let descriptor = json!({
9444 "schema": "ckios.richui.manifest/1",
9445 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9446 "bytes": 4096,
9447 "chunks": 4
9448 });
9449 let connection = ConnectionId::new(201);
9450 let hello = handler
9451 .handle_control(
9452 connection,
9453 hello_frame_with_rich_ui(
9454 "vault",
9455 PROTOCOL_VERSION,
9456 201,
9457 Some("the-real-nonce"),
9458 descriptor.clone(),
9459 ),
9460 )
9461 .unwrap();
9462 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9463
9464 let request =
9465 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9466 let provides = registry
9467 .get_module_by_connection(connection)
9468 .unwrap()
9469 .unwrap()
9470 .manifest
9471 .provides;
9472 handler
9473 .handle_catalog_update(connection, request, provides, None, Some(false))
9474 .unwrap();
9475
9476 let request = Frame::build(
9477 FrameType::Request,
9478 control_flags(),
9479 0,
9480 0,
9481 203,
9482 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9483 )
9484 .unwrap();
9485 let response = handler.handle_catalog_list(request, None).unwrap();
9486 let ClientControlResponse::CatalogList { modules, .. } =
9487 serde_json::from_slice(&response[0].body).unwrap()
9488 else {
9489 panic!("expected catalog.list");
9490 };
9491 assert_eq!(
9492 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9493 descriptor
9494 );
9495 assert!(modules[0].nonce_verified);
9496 }
9497
9498 fn hello_with_role_declarations(declarations: Value) -> Frame {
9499 let mut body = serde_json::to_value(ModuleHelloBody {
9500 manifest: manifest("role-test", PROTOCOL_VERSION),
9501 protocol_ver: PROTOCOL_VERSION,
9502 control_ops: None,
9503 launch_nonce: None,
9504 })
9505 .unwrap();
9506 body["manifest"]["role_declarations"] = declarations;
9507 Frame::build(
9508 FrameType::Hello,
9509 control_flags(),
9510 0,
9511 0,
9512 301,
9513 serde_json::to_vec(&body).unwrap(),
9514 )
9515 .unwrap()
9516 }
9517
9518 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9519 let request = Frame::build(
9520 FrameType::Request,
9521 control_flags(),
9522 0,
9523 0,
9524 302,
9525 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9526 )
9527 .unwrap();
9528 let response = handler.handle_catalog_list(request, None).unwrap();
9529 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9530 let ClientControlResponse::CatalogList { mut modules, .. } =
9531 serde_json::from_slice(&response[0].body).unwrap()
9532 else {
9533 panic!("expected catalog.list");
9534 };
9535 (raw["modules"][0].clone(), modules.remove(0))
9536 }
9537
9538 #[test]
9539 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9540 let registry = Arc::new(Registry::default());
9541 let handler = ControlHandler::new(Arc::clone(®istry));
9542 let declarations = json!({
9543 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9544 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9545 });
9546 let connection = ConnectionId::new(301);
9547 let response = handler
9548 .handle_control(
9549 connection,
9550 hello_with_role_declarations(declarations.clone()),
9551 )
9552 .unwrap();
9553 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9554 let registered = registry
9555 .get_module_by_connection(connection)
9556 .unwrap()
9557 .unwrap()
9558 .manifest;
9559 assert_eq!(
9563 serde_json::to_value(®istered.role_declarations).unwrap(),
9564 declarations
9565 );
9566 let (raw, entry) = role_declarations_catalog(&handler);
9567 assert_eq!(raw["role_declarations"], declarations);
9568 assert_eq!(entry.role_declarations, registered.role_declarations);
9569
9570 let request =
9571 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9572 handler
9573 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9574 .unwrap();
9575 let (raw, entry) = role_declarations_catalog(&handler);
9576 assert_eq!(raw["role_declarations"], declarations);
9577 assert_eq!(entry.role_declarations, registered.role_declarations);
9578 }
9579
9580 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9581 let registry = Arc::new(Registry::default());
9582 let handler = ControlHandler::new(Arc::clone(®istry));
9583 let capture = EventCapture::default();
9584 let _subscriber =
9585 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9586 let response = handler
9587 .handle_control(
9588 ConnectionId::new(301),
9589 hello_with_role_declarations(declarations),
9590 )
9591 .unwrap();
9592 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9593 assert!(registry
9594 .get_module_by_connection(ConnectionId::new(301))
9595 .unwrap()
9596 .unwrap()
9597 .manifest
9598 .role_declarations
9599 .is_empty());
9600 let (raw, entry) = role_declarations_catalog(&handler);
9601 assert!(raw.get("role_declarations").is_none());
9602 assert!(entry.role_declarations.is_empty());
9603 let warnings: Vec<_> = capture
9604 .events()
9605 .into_iter()
9606 .filter(|event| {
9607 event.fields.get("message").map(String::as_str)
9608 == Some("dropping whole role_declarations map from HELLO")
9609 })
9610 .collect();
9611 assert_eq!(warnings.len(), 1);
9612 assert_eq!(warnings[0].level, tracing::Level::WARN);
9613 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9614 assert_eq!(warnings[0].fields["reason"], reason);
9615 }
9616
9617 #[test]
9618 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9619 for value in [
9620 json!(null),
9621 json!(false),
9622 json!(0),
9623 json!("opaque"),
9624 json!([]),
9625 ] {
9626 assert_hello_drops_role_declarations_with_warning(
9627 json!({"routing/v1": {}, "broken/v1": value}),
9628 "role_declarations values must be objects",
9629 );
9630 }
9631 assert_hello_drops_role_declarations_with_warning(
9632 json!([]),
9633 "role_declarations must be an object",
9634 );
9635 }
9636
9637 #[test]
9638 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9639 assert_hello_drops_role_declarations_with_warning(
9640 json!({"routing/v1": {}, "bad key": {}}),
9641 "role_declarations key must use capability-name grammar",
9642 );
9643 }
9644
9645 #[test]
9646 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9647 assert_hello_drops_role_declarations_with_warning(
9648 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9649 "role_declarations exceeds 16384 compact JSON bytes",
9650 );
9651 }
9652
9653 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
9654 let mut body = serde_json::to_value(ModuleHelloBody {
9655 manifest: manifest("flow-ops", PROTOCOL_VERSION),
9656 protocol_ver: PROTOCOL_VERSION,
9657 control_ops: None,
9658 launch_nonce: None,
9659 })
9660 .unwrap();
9661 body["manifest"]["provides"] = json!([{
9662 "role": "management_surface",
9663 "operations": [
9664 {"name": "diagnostics.findings", "kind": "query"},
9665 {"name": "reset", "kind": "mutate"}
9666 ],
9667 "config_schema": {}, "observability": [], "identity_scope": []
9668 }]);
9669 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
9670 if let Some(flow_operations) = flow_operations {
9671 body["manifest"]["flow_operations"] = flow_operations;
9672 }
9673 Frame::build(
9674 FrameType::Hello,
9675 control_flags(),
9676 0,
9677 0,
9678 401,
9679 serde_json::to_vec(&body).unwrap(),
9680 )
9681 .unwrap()
9682 }
9683
9684 #[test]
9685 fn catalog_list_mirrors_flow_operations_unchanged() {
9686 for (capability, flow_operations) in [
9689 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
9690 ("call-modes/v1", json!(["diagnostics.findings"])),
9691 ("flow-scopes/v1", json!([])),
9692 ] {
9693 let registry = Arc::new(Registry::default());
9694 let handler = ControlHandler::new(Arc::clone(®istry));
9695 let connection = ConnectionId::new(401);
9696 let response = handler
9697 .handle_control(
9698 connection,
9699 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
9700 )
9701 .unwrap();
9702 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9703 let registered = registry
9704 .get_module_by_connection(connection)
9705 .unwrap()
9706 .unwrap()
9707 .manifest;
9708 assert_eq!(
9709 serde_json::to_value(®istered.flow_operations).unwrap(),
9710 flow_operations
9711 );
9712 let (raw, entry) = role_declarations_catalog(&handler);
9713 assert_eq!(raw["flow_operations"], flow_operations);
9714 assert_eq!(entry.flow_operations, registered.flow_operations);
9715 }
9716
9717 let registry = Arc::new(Registry::default());
9718 let handler = ControlHandler::new(Arc::clone(®istry));
9719 let response = handler
9720 .handle_control(
9721 ConnectionId::new(401),
9722 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
9723 )
9724 .unwrap();
9725 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9726 let (raw, entry) = role_declarations_catalog(&handler);
9727 assert!(raw.get("flow_operations").is_none());
9728 assert_eq!(entry.flow_operations, None);
9729 }
9730
9731 fn assert_hello_drops_flow_operations_with_warning(
9732 capabilities: Value,
9733 flow_operations: Value,
9734 reason: &str,
9735 ) {
9736 let registry = Arc::new(Registry::default());
9737 let handler = ControlHandler::new(Arc::clone(®istry));
9738 let capture = EventCapture::default();
9739 let _subscriber =
9740 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9741 let response = handler
9742 .handle_control(
9743 ConnectionId::new(401),
9744 hello_with_flow_operations(capabilities, Some(flow_operations)),
9745 )
9746 .unwrap();
9747 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9748 let registered = registry
9749 .get_module_by_connection(ConnectionId::new(401))
9750 .unwrap()
9751 .expect("the module still registers")
9752 .manifest;
9753 assert_eq!(registered.module_id, "flow-ops");
9754 assert_eq!(registered.flow_operations, None);
9755 let (raw, entry) = role_declarations_catalog(&handler);
9756 assert_eq!(raw["module_id"], "flow-ops");
9757 assert!(raw.get("flow_operations").is_none());
9758 assert_eq!(entry.flow_operations, None);
9759 let warnings: Vec<_> = capture
9760 .events()
9761 .into_iter()
9762 .filter(|event| {
9763 event.fields.get("message").map(String::as_str)
9764 == Some("dropping flow_operations from HELLO")
9765 })
9766 .collect();
9767 assert_eq!(warnings.len(), 1);
9768 assert_eq!(warnings[0].level, tracing::Level::WARN);
9769 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
9770 assert_eq!(warnings[0].fields["reason"], reason);
9771 }
9772
9773 #[test]
9774 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
9775 assert_hello_drops_flow_operations_with_warning(
9776 json!(["flow-scopes/v1"]),
9777 json!(["diagnostics.findings", "missing"]),
9778 "flow_operations names 'missing', which is not a declared management operation",
9779 );
9780 }
9781
9782 #[test]
9783 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
9784 assert_hello_drops_flow_operations_with_warning(
9785 json!([]),
9786 json!(["diagnostics.findings"]),
9787 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
9788 );
9789 assert_hello_drops_flow_operations_with_warning(
9791 json!(["agent-run-scopes/v1"]),
9792 json!([]),
9793 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
9794 );
9795 }
9796
9797 #[test]
9798 fn hello_malformed_flow_operations_is_dropped_with_warning() {
9799 assert_hello_drops_flow_operations_with_warning(
9800 json!(["flow-scopes/v1"]),
9801 json!(["reset", "reset"]),
9802 "flow_operations names 'reset' more than once",
9803 );
9804 assert_hello_drops_flow_operations_with_warning(
9805 json!(["flow-scopes/v1"]),
9806 json!(vec!["reset"; 257]),
9807 "flow_operations lists 257 names, more than 256",
9808 );
9809 assert_hello_drops_flow_operations_with_warning(
9810 json!(["flow-scopes/v1"]),
9811 json!("reset"),
9812 "flow_operations must be an array of operation names",
9813 );
9814 }
9815
9816 #[test]
9817 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9818 let registry = Arc::new(Registry::default());
9819 let handler = ControlHandler::new(registry.clone());
9820 let descriptor = json!({
9821 "schema": "ckios.richui.manifest/1",
9822 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9823 "bytes": 128,
9824 "chunks": 2
9825 });
9826 let malformed = [
9827 ("schema", json!("")),
9828 ("schema", json!("a".repeat(65))),
9829 ("schema", json!("Bad schema")),
9830 (
9831 "manifest_digest",
9832 json!(format!("sha256:{}", "A".repeat(64))),
9833 ),
9834 ("manifest_digest", json!("sha256:abc")),
9835 ("bytes", json!(0)),
9836 ("bytes", json!(16_777_217)),
9837 ("bytes", json!("128")),
9838 ("chunks", json!(0)),
9839 ("chunks", json!(1025)),
9840 ("chunks", json!("2")),
9841 ];
9842 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9843 let mut invalid = descriptor.clone();
9844 invalid[field] = invalid_value;
9845 let response = handler
9846 .handle_control(
9847 ConnectionId::new(210 + index as u64),
9848 hello_frame_with_rich_ui(
9849 &format!("bad-rich-ui-{index}"),
9850 PROTOCOL_VERSION,
9851 210 + index as u64,
9852 None,
9853 invalid,
9854 ),
9855 )
9856 .unwrap();
9857 assert_eq!(response[0].header.ty, FrameType::Error);
9858 let error = parse_error(&response[0]);
9859 assert_eq!(error["code"], "invalid_manifest");
9860 assert!(error["message"]
9861 .as_str()
9862 .unwrap()
9863 .contains(&format!("rich_ui.{field}")));
9864 assert!(registry
9865 .get_module(&format!("bad-rich-ui-{index}"))
9866 .unwrap()
9867 .is_none());
9868 }
9869 }
9870
9871 #[test]
9872 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9873 let registry = Arc::new(Registry::default());
9874 let handler = ControlHandler::new(Arc::clone(®istry));
9875 let connection = ConnectionId::new(219);
9876 registry
9877 .register_with_control_ops(
9878 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9879 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9880 "bad schema",
9881 format!("sha256:{}", "a".repeat(64)),
9882 128,
9883 2,
9884 )))
9885 .build(),
9886 PROTOCOL_VERSION,
9887 connection,
9888 Vec::new(),
9889 )
9890 .unwrap();
9891 let request =
9892 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9893 let response = handler
9894 .handle_catalog_update(connection, request, Vec::new(), None, None)
9895 .unwrap();
9896 assert_eq!(response[0].header.ty, FrameType::Error);
9897 let error = parse_error(&response[0]);
9898 assert_eq!(error["code"], "invalid_manifest");
9899 assert!(error["message"]
9900 .as_str()
9901 .unwrap()
9902 .contains("rich_ui.schema"));
9903 }
9904
9905 #[test]
9906 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9907 let registry = Arc::new(Registry::default());
9908 let supervisor = SupervisorHandle::new();
9909 supervisor.open_swap("provider", "candidate-nonce".to_string());
9910 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9911 let connection = ConnectionId::new(218);
9912 let hello = handler
9913 .handle_control(
9914 connection,
9915 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9916 )
9917 .unwrap();
9918 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9919 assert!(registry.list_modules().unwrap().1.is_empty());
9920
9921 registry.promote_candidate("provider").unwrap().unwrap();
9922 let request = Frame::build(
9923 FrameType::Request,
9924 control_flags(),
9925 0,
9926 0,
9927 219,
9928 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9929 )
9930 .unwrap();
9931 let response = handler.handle_catalog_list(request, None).unwrap();
9932 let ClientControlResponse::CatalogList { modules, .. } =
9933 serde_json::from_slice(&response[0].body).unwrap()
9934 else {
9935 panic!("expected catalog.list");
9936 };
9937 assert_eq!(modules.len(), 1);
9938 assert!(modules[0].nonce_verified);
9939 }
9940
9941 #[test]
9942 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9943 let registry = Arc::new(Registry::default());
9944 let handler = ControlHandler::new(Arc::clone(®istry));
9945 let hello = Frame::build(
9946 FrameType::Hello,
9947 control_flags(),
9948 0,
9949 0,
9950 220,
9951 serde_json::to_vec(&ModuleHelloBody {
9952 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9953 protocol_ver: PROTOCOL_VERSION,
9954 control_ops: None,
9955 launch_nonce: Some("unverified".to_string()),
9956 })
9957 .unwrap(),
9958 )
9959 .unwrap();
9960 assert_eq!(
9961 handler
9962 .handle_control(ConnectionId::new(220), hello)
9963 .unwrap()[0]
9964 .header
9965 .ty,
9966 FrameType::HelloAck
9967 );
9968 let request = Frame::build(
9969 FrameType::Request,
9970 control_flags(),
9971 0,
9972 0,
9973 221,
9974 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9975 )
9976 .unwrap();
9977 let response = handler.handle_catalog_list(request, None).unwrap();
9978 let ClientControlResponse::CatalogList { modules, .. } =
9979 serde_json::from_slice(&response[0].body).unwrap()
9980 else {
9981 panic!("expected catalog.list");
9982 };
9983 assert_eq!(
9984 serde_json::to_vec(&modules[0]).unwrap(),
9985 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9986 );
9987 assert!(!modules[0].nonce_verified);
9988 }
9989
9990 #[test]
9991 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9992 let registry = Arc::new(Registry::default());
9993 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9994 [("vault".to_string(), true), ("squatter".to_string(), true)],
9995 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9996 );
9997 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9998 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9999 provides: vec!["credentials-provider/v1".to_string()],
10000 requires: Vec::new(),
10001 must_never_reach: Vec::new(),
10002 });
10003 let frame = Frame::build(
10004 FrameType::Hello,
10005 control_flags(),
10006 0,
10007 0,
10008 77,
10009 serde_json::to_vec(&ModuleHelloBody {
10010 manifest: squatter,
10011 protocol_ver: PROTOCOL_VERSION,
10012 control_ops: None,
10013 launch_nonce: None,
10014 })
10015 .expect("HELLO serializes"),
10016 )
10017 .expect("HELLO frame builds");
10018 let response = handler
10019 .handle_control(ConnectionId::new(77), frame)
10020 .expect("reserved claim receives a typed refusal");
10021 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10022 assert_eq!(
10023 registry
10024 .active_registration_count()
10025 .expect("registry reads"),
10026 0,
10027 "a reserved capability refusal must not leave a catalog entry"
10028 );
10029 }
10030
10031 #[test]
10032 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10033 for settlement in ["timeout", "inconclusive", "drop"] {
10034 let breakers = RouteBindBreakers::default();
10035 let RouteBindAdmission::Admitted {
10036 guard: mut old,
10037 probe: false,
10038 } = breakers.admit("prov")
10039 else {
10040 panic!("ordinary relay admitted")
10041 };
10042 let RouteBindAdmission::Admitted {
10043 guard: mut opener, ..
10044 } = breakers.admit("prov")
10045 else {
10046 panic!("second relay admitted")
10047 };
10048 assert!(
10049 !opener
10050 .record_timeout(1, Duration::ZERO)
10051 .unwrap()
10052 .reopened_after_probe
10053 );
10054 let RouteBindAdmission::Admitted {
10055 guard: mut probe,
10056 probe: true,
10057 } = breakers.admit("prov")
10058 else {
10059 panic!("one cooldown probe admitted")
10060 };
10061 match settlement {
10062 "timeout" => assert!(
10063 !old.record_timeout(1, Duration::ZERO)
10064 .unwrap()
10065 .reopened_after_probe
10066 ),
10067 "inconclusive" => old.record_inconclusive(),
10068 "drop" => drop(old),
10069 _ => unreachable!(),
10070 }
10071 assert!(
10072 matches!(
10073 breakers.admit("prov"),
10074 RouteBindAdmission::Refused {
10075 probe_in_flight: true,
10076 ..
10077 }
10078 ),
10079 "{settlement} of a pre-open relay cannot release the real probe"
10080 );
10081 assert!(
10082 probe
10083 .record_timeout(1, Duration::ZERO)
10084 .unwrap()
10085 .reopened_after_probe
10086 );
10087 assert!(matches!(
10088 breakers.admit("prov"),
10089 RouteBindAdmission::Admitted { probe: true, .. }
10090 ));
10091 }
10092 let breakers = RouteBindBreakers::default();
10093 let admit = || match breakers.admit("prov") {
10094 RouteBindAdmission::Admitted { guard, .. } => guard,
10095 _ => panic!("relay admitted"),
10096 };
10097 admit().record_timeout(1, Duration::ZERO);
10098 let mut old_probe = admit();
10099 breakers.reset_for_new_module_connection("prov");
10100 admit().record_timeout(1, Duration::ZERO);
10101 let _new_probe = admit();
10102 old_probe.record_inconclusive();
10103 assert!(matches!(
10104 breakers.admit("prov"),
10105 RouteBindAdmission::Refused {
10106 probe_in_flight: true,
10107 ..
10108 }
10109 ));
10110 }
10111
10112 #[tokio::test]
10113 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10114 for candidate in [false, true] {
10115 let registry = Arc::new(Registry::default());
10116 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10117 [("vault".to_string(), true), ("squatter".to_string(), true)],
10118 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10119 );
10120 let conn = ConnectionId::new(77);
10121 let (ctx, mut rx) = route_ctx(conn);
10122 let initial = capability_manifest("squatter", &[], &[]);
10123 if candidate {
10124 registry
10125 .register_candidate_with_control_ops(
10126 initial.clone(),
10127 PROTOCOL_VERSION,
10128 conn,
10129 module_baseline_control_ops(),
10130 )
10131 .unwrap();
10132 handler
10133 .forwarding
10134 .register_candidate_module_connection(
10135 conn,
10136 "squatter".to_string(),
10137 PROTOCOL_VERSION,
10138 manifest_concurrency(&initial),
10139 ctx.egress.clone(),
10140 )
10141 .unwrap();
10142 } else {
10143 hello_via_sink(
10144 &handler,
10145 &ctx,
10146 &mut rx,
10147 hello_frame_with_manifest(initial.clone(), 1),
10148 )
10149 .await;
10150 }
10151 let response = handler
10152 .handle_control_frame(
10153 &ctx,
10154 catalog_update_with_capabilities_frame(
10155 2,
10156 capability_manifest("squatter", &["credentials-provider/v1"], &[])
10157 .capabilities
10158 .unwrap(),
10159 ),
10160 )
10161 .await
10162 .unwrap();
10163 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10164 assert_eq!(
10165 registry
10166 .get_module_by_connection(conn)
10167 .unwrap()
10168 .unwrap()
10169 .manifest,
10170 initial
10171 );
10172 }
10173 }
10174
10175 #[test]
10176 fn server_describe_surfaces_required_capability_verdict_fields() {
10177 let registry = Arc::new(Registry::default());
10178 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10179 [
10180 ("consumer".to_string(), true),
10181 ("provider".to_string(), false),
10182 ],
10183 BTreeMap::new(),
10184 );
10185 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10186 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10187 provides: Vec::new(),
10188 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10189 capability: "credentials-provider/v1".to_string(),
10190 need: subc_protocol::manifest::CapabilityNeed::Required,
10191 }],
10192 must_never_reach: Vec::new(),
10193 });
10194 let hello = Frame::build(
10195 FrameType::Hello,
10196 control_flags(),
10197 0,
10198 0,
10199 78,
10200 serde_json::to_vec(&ModuleHelloBody {
10201 manifest: consumer,
10202 protocol_ver: PROTOCOL_VERSION,
10203 control_ops: None,
10204 launch_nonce: None,
10205 })
10206 .expect("HELLO serializes"),
10207 )
10208 .expect("HELLO frame builds");
10209 handler
10210 .handle_control(ConnectionId::new(78), hello)
10211 .expect("consumer registers");
10212 let describe = Frame::build(
10213 FrameType::Request,
10214 control_flags(),
10215 0,
10216 0,
10217 79,
10218 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10219 .expect("request serializes"),
10220 )
10221 .expect("describe frame builds");
10222 let response = handler
10223 .handle_server_describe(describe)
10224 .expect("server.describe succeeds");
10225 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10226 let requirement = &rendered["capability_requirements"][0];
10227 assert_eq!(requirement["consumer"], "consumer");
10228 assert_eq!(requirement["verdict"], "never_provided");
10229 assert_eq!(requirement["episode_seq"], 1);
10230 assert_eq!(requirement["config_satisfiable"], false);
10231 assert_eq!(requirement["runtime_available"], false);
10232 assert!(requirement["detail"]
10233 .as_str()
10234 .expect("detail string")
10235 .contains("credentials-provider/v1"));
10236 }
10237
10238 fn deregister_required_provider(
10244 daemon_shutdown_begun: bool,
10245 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10246 const CAPABILITY: &str = "project-identity/v1";
10247 let registry = Arc::new(Registry::default());
10248 let daemon_shutdown = DaemonShutdownFlag::default();
10249 let handler = ControlHandler::new(Arc::clone(®istry))
10250 .with_capability_config(
10251 [
10252 ("consumer".to_string(), true),
10253 ("provider".to_string(), true),
10254 ],
10255 BTreeMap::new(),
10256 )
10257 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10258 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10259 consumer.capabilities = Some(CapabilityDeclarations {
10260 provides: Vec::new(),
10261 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10262 capability: CAPABILITY.to_string(),
10263 need: subc_protocol::manifest::CapabilityNeed::Required,
10264 }],
10265 must_never_reach: Vec::new(),
10266 });
10267 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10268 let provider_conn = ConnectionId::new(182);
10269 for (conn, manifest, corr) in [
10270 (ConnectionId::new(181), consumer, 181),
10271 (provider_conn, provider, 182),
10272 ] {
10273 let hello = handler
10274 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10275 .expect("module registers");
10276 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10277 }
10278 assert_eq!(
10279 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10280 Some(CapabilityVerdict::Provided),
10281 "the registered provider must satisfy the requirement first"
10282 );
10283 if daemon_shutdown_begun {
10284 daemon_shutdown.set_for_test();
10285 }
10286
10287 let capture = EventCapture::default();
10288 let _subscriber =
10289 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10290 handler
10291 .cleanup_connection(provider_conn)
10292 .expect("provider deregisters");
10293 let events = capture
10294 .events()
10295 .into_iter()
10296 .filter(|event| {
10297 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10298 })
10299 .collect();
10300 (
10301 events,
10302 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10303 )
10304 }
10305
10306 #[test]
10307 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10308 let (events, verdict) = deregister_required_provider(true);
10309 assert_eq!(
10310 verdict,
10311 Some(CapabilityVerdict::NeverProvided),
10312 "shutdown must not change the verdict route.open refusals read"
10313 );
10314 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10315 let event = &events[0];
10316 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10317 assert_eq!(
10318 event.fields.get("cause").map(String::as_str),
10319 Some("\"daemon_shutdown\""),
10320 "{event:?}"
10321 );
10322 assert_eq!(
10323 event.fields.get("verdict").map(String::as_str),
10324 Some("\"never_provided\""),
10325 "{event:?}"
10326 );
10327 }
10328
10329 #[test]
10330 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10331 let (events, verdict) = deregister_required_provider(false);
10332 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10333 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10334 let event = &events[0];
10335 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10336 assert_eq!(event.fields.get("cause"), None, "{event:?}");
10337 assert_eq!(
10338 event.fields.get("verdict").map(String::as_str),
10339 Some("\"never_provided\""),
10340 "{event:?}"
10341 );
10342 }
10343
10344 #[test]
10345 fn catalog_list_omits_capabilities_for_legacy_manifest() {
10346 let registry = Arc::new(Registry::default());
10347 let handler = ControlHandler::new(Arc::clone(®istry));
10348 let hello = handler
10349 .handle_control(
10350 ConnectionId::new(101),
10351 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10352 )
10353 .expect("legacy HELLO registers");
10354 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10355
10356 let request = Frame::build(
10357 FrameType::Request,
10358 control_flags(),
10359 0,
10360 0,
10361 102,
10362 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10363 .expect("catalog request serializes"),
10364 )
10365 .expect("catalog request frame builds");
10366 let response = handler
10367 .handle_catalog_list(request, None)
10368 .expect("catalog list succeeds");
10369 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10370 assert!(
10371 body["modules"][0].get("capabilities").is_none(),
10372 "legacy manifest must retain an absent capabilities field on catalog.list"
10373 );
10374 }
10375
10376 #[test]
10377 fn hello_ack_omits_storage_when_no_storage_config() {
10378 let registry = Arc::new(Registry::default());
10379 let handler = ControlHandler::new(Arc::clone(®istry));
10380 let responses = handler
10381 .handle_control(
10382 ConnectionId::new(1),
10383 hello_frame("aft", PROTOCOL_VERSION, 7),
10384 )
10385 .unwrap();
10386 let ack = parse_ack(&responses[0]);
10387 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10388 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10389 }
10390
10391 #[tokio::test]
10392 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10393 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10394 let registry = Arc::new(Registry::default());
10395 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
10396 let responses = handler
10397 .handle_control(
10398 ConnectionId::new(1),
10399 hello_frame("aft", PROTOCOL_VERSION, 7),
10400 )
10401 .unwrap();
10402 let ack = parse_ack(&responses[0]);
10403 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10404
10405 let described = handler
10406 .handle_control_frame(
10407 &route_ctx(ConnectionId::new(2)).0,
10408 Frame::build(
10409 FrameType::Request,
10410 control_flags(),
10411 0,
10412 0,
10413 9,
10414 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10415 )
10416 .unwrap(),
10417 )
10418 .await
10419 .unwrap();
10420 let ClientControlResponse::ServerDescribe { machine_id, .. } =
10421 serde_json::from_slice(&described[0].body).unwrap()
10422 else {
10423 panic!("server.describe answered with another shape");
10424 };
10425 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10426 }
10427
10428 #[test]
10429 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10430 let registry = Arc::new(Registry::default());
10433 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
10434 crate::daemon_config::StorageConfig::Sqlite {
10435 data_home: std::path::PathBuf::from("/data"),
10436 },
10437 ));
10438
10439 let responses = handler
10440 .handle_control(
10441 ConnectionId::new(1),
10442 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10443 )
10444 .unwrap();
10445 let ack = parse_ack(&responses[0]);
10446 assert_eq!(
10447 ack.storage,
10448 Some(serde_json::json!({
10449 "module_id": "alfonso-routing",
10450 "storage_namespace": "default",
10451 "isolation": { "kind": "module" },
10452 "backend": {
10453 "backend": "sqlite",
10454 "path": "/data/cortexkit/alfonso-routing/store.db"
10455 }
10456 })),
10457 "the delivered descriptor is the module's own sqlite store path"
10458 );
10459 }
10460
10461 #[test]
10462 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10463 let registry = Arc::new(Registry::default());
10464 let handler = ControlHandler::new(Arc::clone(®istry));
10465 let conn = ConnectionId::new(1);
10466 let responses = handler
10467 .handle_control(
10468 conn,
10469 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10470 )
10471 .unwrap();
10472 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10473 let registration = registry.get_module("aft").unwrap().unwrap();
10474 assert_eq!(registration.control_ops, module_baseline_control_ops());
10475
10476 let frame =
10477 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10478 assert!(handler
10479 .guard_module_control_op(&frame, "aft", "route.bind")
10480 .unwrap()
10481 .is_none());
10482 let error = handler
10483 .guard_module_control_op(&frame, "aft", "test.synthetic")
10484 .unwrap()
10485 .expect("synthetic ungranted op should be rejected");
10486 assert_eq!(error.header.ty, FrameType::Error);
10487 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10488 }
10489
10490 #[test]
10491 fn hello_control_ops_some_adds_optional_grants() {
10492 let registry = Arc::new(Registry::default());
10493 let handler = ControlHandler::new(Arc::clone(®istry));
10494 handler
10495 .handle_control(
10496 ConnectionId::new(1),
10497 hello_frame_with_control_ops(
10498 "aft",
10499 PROTOCOL_VERSION,
10500 7,
10501 Some(vec![
10502 "future.synthetic".to_string(),
10503 "route.bind".to_string(),
10504 ]),
10505 ),
10506 )
10507 .unwrap();
10508 let registration = registry.get_module("aft").unwrap().unwrap();
10509 assert_eq!(
10510 registration.control_ops,
10511 vec![
10512 "route.bind".to_string(),
10513 "route.status".to_string(),
10514 "future.synthetic".to_string(),
10515 ]
10516 );
10517 let frame =
10518 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10519 assert!(handler
10520 .guard_module_control_op(&frame, "aft", "future.synthetic")
10521 .unwrap()
10522 .is_none());
10523 }
10524
10525 #[tokio::test]
10526 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10527 let registry = Arc::new(Registry::default());
10528 let forwarding = Arc::new(ForwardingTable::default());
10529 let handler =
10530 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10531 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10532 hello_via_sink(
10533 &handler,
10534 &module_ctx,
10535 &mut module_rx,
10536 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10537 )
10538 .await;
10539
10540 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10541 let responses = handler
10542 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10543 .await
10544 .unwrap();
10545 assert_eq!(responses.len(), 1);
10546 assert_eq!(responses[0].header.ty, FrameType::Error);
10547 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10548 assert!(module_rx.try_recv().is_err());
10549 }
10550
10551 #[tokio::test]
10552 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10553 let registry = Arc::new(Registry::default());
10554 let forwarding = Arc::new(ForwardingTable::default());
10555 let handler =
10556 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10557 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10558 hello_via_sink(
10559 &handler,
10560 &module_ctx,
10561 &mut module_rx,
10562 hello_frame_with_control_ops(
10563 "aft",
10564 PROTOCOL_VERSION,
10565 7,
10566 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10567 ),
10568 )
10569 .await;
10570
10571 let project_root = unique_project_root("demux");
10572 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10573 let route_handler = handler.clone();
10574 let route_task = tokio::spawn(async move {
10575 route_handler
10576 .handle_control_frame(
10577 &route_client_ctx,
10578 route_open_frame(100, "aft", project_root),
10579 )
10580 .await
10581 .unwrap()
10582 });
10583 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10584 .await
10585 .unwrap()
10586 .unwrap();
10587 assert!(matches!(
10588 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10589 ModuleControlRequest::RouteBind { .. }
10590 ));
10591
10592 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10593 let health_handler = handler.clone();
10594 let health_task = tokio::spawn(async move {
10595 health_handler
10596 .handle_control_frame(
10597 &health_client_ctx,
10598 supervisor_health_probe_frame(101, "aft"),
10599 )
10600 .await
10601 .unwrap()
10602 });
10603 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10604 .await
10605 .unwrap()
10606 .unwrap();
10607 assert_eq!(
10608 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10609 ModuleControlRequest::HealthCheck {}
10610 );
10611
10612 handler
10613 .handle_control_frame(
10614 &module_ctx,
10615 health_response(health_frame.header.corr, HealthStatus::Degraded),
10616 )
10617 .await
10618 .unwrap();
10619 let health_response = health_task.await.unwrap();
10620 assert_eq!(health_response.len(), 1);
10621 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10622 ClientControlResponse::SupervisorHealthProbe {
10623 module_id,
10624 status,
10625 detail,
10626 metrics,
10627 } => {
10628 assert_eq!(module_id, "aft");
10629 assert_eq!(status, HealthStatus::Degraded);
10630 assert_eq!(detail.as_deref(), Some("warming"));
10631 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10632 }
10633 other => panic!("unexpected health response: {other:?}"),
10634 }
10635
10636 handler
10637 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10638 .await
10639 .unwrap();
10640 let route_response = route_task.await.unwrap();
10641 assert!(route_response.is_empty());
10642 let published = route_client_rx.recv().await.unwrap();
10643 assert!(matches!(
10644 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10645 ClientControlResponse::RouteOpen { .. }
10646 ));
10647 }
10648
10649 async fn relay_route_open(
10654 handler: &ControlHandler,
10655 client_connection: ConnectionId,
10656 client_egress: &FrameSink,
10657 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10658 corr: u64,
10659 module_id: &str,
10660 project_root_label: &str,
10661 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10662 let ctx = RouteCtx {
10663 connection_id: client_connection,
10664 egress: client_egress.clone(),
10665 };
10666 let handler = handler.clone();
10667 let project_root = unique_project_root(project_root_label);
10668 let module_id = module_id.to_string();
10669 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10670 let task = tokio::spawn(async move {
10671 let _guard = tracing::dispatcher::set_default(&dispatch);
10672 handler
10673 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10674 .await
10675 .unwrap()
10676 });
10677 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10678 .await
10679 .expect("module receives the relayed route.bind")
10680 .expect("module egress is open");
10681 (task, bind.frame)
10682 }
10683
10684 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10685 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10686 ModuleControlRequest::RouteBind {
10687 route_channel,
10688 epoch,
10689 ..
10690 } => (route_channel, epoch),
10691 other => panic!("expected a route.bind request, got {other:?}"),
10692 }
10693 }
10694
10695 fn published_route(frame: &Frame) -> (u16, u32) {
10696 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10697 ClientControlResponse::RouteOpen {
10698 route_channel,
10699 route_epoch,
10700 } => (route_channel, route_epoch),
10701 other => panic!("expected a route.open response, got {other:?}"),
10702 }
10703 }
10704
10705 #[tokio::test]
10725 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10726 let registry = Arc::new(Registry::default());
10727 let forwarding = Arc::new(ForwardingTable::default());
10728 let handler =
10729 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10730
10731 let module_connection = ConnectionId::new(30);
10732 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10733 hello_via_sink(
10734 &handler,
10735 &module_ctx,
10736 &mut module_rx,
10737 hello_frame("aft", PROTOCOL_VERSION, 7),
10738 )
10739 .await;
10740
10741 let dying_client = ConnectionId::new(31);
10742 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10743
10744 let (first_task, first_bind) = relay_route_open(
10747 &handler,
10748 dying_client,
10749 &dying_ctx.egress,
10750 &mut module_rx,
10751 100,
10752 "aft",
10753 "closing-first",
10754 )
10755 .await;
10756 handler
10757 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10758 .await
10759 .unwrap();
10760 assert!(first_task.await.unwrap().is_empty());
10761 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10762
10763 let (second_task, second_bind) = relay_route_open(
10765 &handler,
10766 dying_client,
10767 &dying_ctx.egress,
10768 &mut module_rx,
10769 101,
10770 "aft",
10771 "closing-second",
10772 )
10773 .await;
10774 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10775
10776 assert!(forwarding
10779 .escalate_client_delivery_failure(
10780 dying_client,
10781 first_channel,
10782 first_epoch,
10783 CloseReason::new(
10784 "module_to_client_delivery_failed",
10785 "client egress refused a module frame",
10786 ),
10787 crate::forwarding::UndeliveredFrame {
10788 module_id: None,
10789 sink: &dying_ctx.egress,
10790 },
10791 )
10792 .unwrap());
10793 assert!(!dying_ctx.egress.is_closed());
10794
10795 let ack = handler
10797 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10798 .await;
10799 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10800 if module_loop_error.is_some() {
10801 handler.cleanup_connection(module_connection).unwrap();
10805 }
10806 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10810 .await
10811 .ok()
10812 .flatten();
10813
10814 assert!(
10816 registry
10817 .get_module_by_connection(module_connection)
10818 .unwrap()
10819 .is_some(),
10820 "one client's closing connection ended the shared module connection: \
10821 {module_loop_error:?}"
10822 );
10823 let cotenant = ConnectionId::new(32);
10825 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10826 let (cotenant_task, cotenant_bind) = relay_route_open(
10827 &handler,
10828 cotenant,
10829 &cotenant_ctx.egress,
10830 &mut module_rx,
10831 102,
10832 "aft",
10833 "closing-cotenant",
10834 )
10835 .await;
10836 handler
10837 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10838 .await
10839 .unwrap();
10840 assert!(cotenant_task.await.unwrap().is_empty());
10841 let (cotenant_channel, cotenant_epoch) =
10842 published_route(&cotenant_rx.recv().await.unwrap());
10843 assert!(matches!(
10844 forwarding
10845 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10846 .unwrap(),
10847 DataRoute::Client(DataRouteState::Bound(_))
10848 ));
10849
10850 let goodbye = post_ack_module_frame
10853 .expect("module receives a GOODBYE for the abandoned route channel");
10854 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10855 assert_eq!(goodbye.header.channel, abandoned_channel);
10856 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10857
10858 assert!(dying_rx.try_recv().is_err());
10862 let second_response = second_task.await.unwrap();
10863 assert_eq!(second_response.len(), 1);
10864 assert_eq!(
10865 parse_error(&second_response[0])["code"],
10866 "target_unavailable"
10867 );
10868 }
10869
10870 async fn catalog_snapshot(
10873 handler: &ControlHandler,
10874 client_ctx: &RouteCtx,
10875 corr: u64,
10876 ) -> (u64, Vec<(String, bool)>) {
10877 let body =
10878 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10879 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10880 let reply = handler
10881 .handle_control_frame(client_ctx, request)
10882 .await
10883 .unwrap()
10884 .pop()
10885 .unwrap();
10886 let ClientControlResponse::CatalogList {
10887 generation,
10888 modules,
10889 ..
10890 } = serde_json::from_slice(&reply.body).unwrap()
10891 else {
10892 panic!("not a catalog.list reply");
10893 };
10894 (
10895 generation,
10896 modules
10897 .into_iter()
10898 .map(|entry| (entry.module_id, entry.ready))
10899 .collect(),
10900 )
10901 }
10902
10903 fn hello_held_before_forwarding() -> (
10909 ControlHandler,
10910 Arc<std::sync::Barrier>,
10911 std::thread::JoinHandle<Vec<Frame>>,
10912 RouteCtx,
10913 mpsc::Receiver<crate::router::OutboundFrame>,
10914 ) {
10915 let hold = Arc::new(std::sync::Barrier::new(2));
10916 let handler = ControlHandler::with_forwarding(
10917 Arc::new(Registry::default()),
10918 Arc::new(ForwardingTable::default()),
10919 )
10920 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10921 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10922 let hello = {
10923 let handler = handler.clone();
10924 let module_ctx = module_ctx.clone();
10925 std::thread::spawn(move || {
10926 handler
10927 .handle_hello(
10928 module_ctx.connection_id,
10929 Some(module_ctx.egress.clone()),
10930 hello_frame("aft", PROTOCOL_VERSION, 7),
10931 )
10932 .unwrap()
10933 })
10934 };
10935 (handler, hold, hello, module_ctx, module_rx)
10936 }
10937
10938 #[tokio::test]
10947 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10948 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10949 let client = ConnectionId::new(41);
10950 let (client_ctx, mut client_rx) = route_ctx(client);
10951
10952 hold.wait();
10953 assert!(handler.registry.get_module("aft").unwrap().is_some());
10955 assert!(!handler
10956 .forwarding
10957 .has_live_module_connection("aft")
10958 .unwrap());
10959 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10960 assert!(
10961 listed.is_empty(),
10962 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10963 );
10964 let refused = handler
10965 .handle_control_frame(
10966 &client_ctx,
10967 route_open_frame(301, "aft", unique_project_root("listed-window")),
10968 )
10969 .await
10970 .unwrap();
10971 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10972 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10973 let duplicate = handler
10974 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10975 .await
10976 .unwrap();
10977 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10978
10979 hold.wait();
10980 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10981 let ack = module_rx.try_recv().unwrap().frame;
10982 assert_eq!(ack.header.ty, FrameType::HelloAck);
10983 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10984 assert_eq!(listed, vec![("aft".to_string(), true)]);
10985
10986 let (open_task, bind) = relay_route_open(
10987 &handler,
10988 client,
10989 &client_ctx.egress,
10990 &mut module_rx,
10991 303,
10992 "aft",
10993 "listed-after",
10994 )
10995 .await;
10996 assert!(handler
10997 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10998 .await
10999 .unwrap()
11000 .is_empty());
11001 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11002 published_route(&client_rx.try_recv().unwrap().frame);
11003 }
11004
11005 #[tokio::test]
11012 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11013 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11014 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11015 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11016
11017 hold.wait();
11018 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11019 assert!(listed.is_empty());
11020 assert_eq!(
11021 during, before,
11022 "the generation moved while the module was not listed"
11023 );
11024
11025 hold.wait();
11026 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11027 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11028 assert_eq!(listed, vec![("aft".to_string(), true)]);
11029 assert_eq!(
11030 after,
11031 before + 1,
11032 "the module appeared without exactly one generation bump"
11033 );
11034 }
11035
11036 #[tokio::test]
11041 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11042 let registry = Arc::new(Registry::default());
11043 let handler = ControlHandler::with_forwarding(
11044 Arc::clone(®istry),
11045 Arc::new(ForwardingTable::default()),
11046 );
11047 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11048 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11049 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11050 drop(module_rx);
11051
11052 let replies = handler
11053 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11054 .await
11055 .unwrap();
11056 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11057 assert!(registry.get_module("aft").unwrap().is_none());
11058 assert!(registry
11059 .get_module_by_connection(module_ctx.connection_id)
11060 .unwrap()
11061 .is_none());
11062 assert_eq!(
11063 catalog_snapshot(&handler, &client_ctx, 301).await,
11064 (before, Vec::new())
11065 );
11066 }
11067
11068 async fn route_open_held_after_bind_send() -> (
11072 ControlHandler,
11073 Arc<tokio::sync::Barrier>,
11074 RouteCtx,
11075 mpsc::Receiver<crate::router::OutboundFrame>,
11076 ) {
11077 let hold = Arc::new(tokio::sync::Barrier::new(2));
11078 let handler = ControlHandler::with_forwarding(
11079 Arc::new(Registry::default()),
11080 Arc::new(ForwardingTable::default()),
11081 )
11082 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11083 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11084 hello_via_sink(
11085 &handler,
11086 &module_ctx,
11087 &mut module_rx,
11088 hello_frame("aft", PROTOCOL_VERSION, 7),
11089 )
11090 .await;
11091 (handler, hold, module_ctx, module_rx)
11092 }
11093
11094 #[tokio::test]
11101 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11102 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11103 let client = ConnectionId::new(41);
11104 let (client_ctx, mut client_rx) = route_ctx(client);
11105
11106 let (open_task, bind) = relay_route_open(
11107 &handler,
11108 client,
11109 &client_ctx.egress,
11110 &mut module_rx,
11111 200,
11112 "aft",
11113 "acked-before-mark",
11114 )
11115 .await;
11116 let (module_channel, module_epoch) = route_bind_channel(&bind);
11117 hold.wait().await;
11119
11120 assert!(handler
11122 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11123 .await
11124 .unwrap()
11125 .is_empty());
11126 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11127
11128 hold.wait().await;
11129 assert!(
11130 open_task.await.unwrap().is_empty(),
11131 "route.open answers accepted"
11132 );
11133
11134 match module_rx.try_recv() {
11137 Err(mpsc::error::TryRecvError::Empty) => {}
11138 Ok(outbound) => panic!(
11139 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11140 outbound.frame.header.ty,
11141 outbound.frame.header.channel,
11142 outbound.frame.header.epoch
11143 ),
11144 Err(err) => panic!("module egress closed: {err}"),
11145 }
11146
11147 let router = Router::with_control_handler(Arc::new(handler.clone()));
11150 let request = Frame::build(
11151 FrameType::Request,
11152 Flags::new(true, Priority::Interactive, false),
11153 client_channel,
11154 client_epoch,
11155 201,
11156 b"first request".to_vec(),
11157 )
11158 .unwrap();
11159 router
11160 .route_for_connection(&client_ctx, request)
11161 .await
11162 .unwrap();
11163 let delivered = module_rx.try_recv().unwrap().frame;
11164 assert_eq!(delivered.header.ty, FrameType::Request);
11165 assert_eq!(
11166 (delivered.header.channel, delivered.header.epoch),
11167 (module_channel, module_epoch)
11168 );
11169 assert_eq!(delivered.body, b"first request");
11170 }
11171
11172 #[tokio::test]
11178 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11179 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11180 let client = ConnectionId::new(42);
11181 let (client_ctx, _client_rx) = route_ctx(client);
11182
11183 let (open_task, bind) = relay_route_open(
11184 &handler,
11185 client,
11186 &client_ctx.egress,
11187 &mut module_rx,
11188 300,
11189 "aft",
11190 "released-before-mark",
11191 )
11192 .await;
11193 let (module_channel, module_epoch) = route_bind_channel(&bind);
11194 hold.wait().await;
11195
11196 handler.cleanup_connection(client).unwrap();
11197 assert!(
11198 module_rx.try_recv().is_err(),
11199 "client teardown must not GOODBYE a bind route.open is still queueing"
11200 );
11201
11202 hold.wait().await;
11203 let response = open_task.await.unwrap();
11204 assert_eq!(response.len(), 1);
11205 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11206
11207 let goodbye = module_rx
11208 .try_recv()
11209 .expect("module is told to drop the binding the daemon abandoned")
11210 .frame;
11211 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11212 assert_eq!(
11213 (goodbye.header.channel, goodbye.header.epoch),
11214 (module_channel, module_epoch)
11215 );
11216 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11217 }
11218
11219 #[tokio::test]
11224 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11225 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11226 let client = ConnectionId::new(43);
11227 let (client_ctx, client_rx) = route_ctx(client);
11228
11229 let (open_task, bind) = relay_route_open(
11230 &handler,
11231 client,
11232 &client_ctx.egress,
11233 &mut module_rx,
11234 400,
11235 "aft",
11236 "gone-client-before-mark",
11237 )
11238 .await;
11239 let (module_channel, module_epoch) = route_bind_channel(&bind);
11240 hold.wait().await;
11241
11242 drop(client_rx);
11243 assert!(client_ctx.egress.is_closed());
11244 handler
11245 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11246 .await
11247 .unwrap();
11248
11249 hold.wait().await;
11250 let response = open_task.await.unwrap();
11251 assert_eq!(response.len(), 1);
11252 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11253
11254 let goodbye = module_rx
11255 .try_recv()
11256 .expect("module is told to drop the binding it acked")
11257 .frame;
11258 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11259 assert_eq!(
11260 (goodbye.header.channel, goodbye.header.epoch),
11261 (module_channel, module_epoch)
11262 );
11263 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11264 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11265 }
11266
11267 #[test]
11274 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11275 let handler = ControlHandler::default();
11276 let module_connection = ConnectionId::new(30);
11277 let client_connection = ConnectionId::new(31);
11278
11279 handler
11280 .refuse_to_end_module_connection_for_a_client(
11281 module_connection,
11282 77,
11283 ForwardingError::ConnectionClosing {
11284 connection_id: client_connection,
11285 },
11286 )
11287 .expect("a closing client must never end the module connection");
11288
11289 assert!(matches!(
11290 handler.refuse_to_end_module_connection_for_a_client(
11291 module_connection,
11292 78,
11293 ForwardingError::ConnectionClosing {
11294 connection_id: module_connection,
11295 },
11296 ),
11297 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11298 connection_id
11299 })) if connection_id == module_connection
11300 ));
11301 assert!(matches!(
11302 handler.refuse_to_end_module_connection_for_a_client(
11303 module_connection,
11304 79,
11305 ForwardingError::Poisoned,
11306 ),
11307 Err(RouterError::Forwarding(ForwardingError::Poisoned))
11308 ));
11309 assert!(matches!(
11310 handler.refuse_to_end_module_connection_for_a_client(
11311 module_connection,
11312 80,
11313 ForwardingError::StaleModuleEndpoint,
11314 ),
11315 Err(RouterError::Forwarding(
11316 ForwardingError::StaleModuleEndpoint
11317 ))
11318 ));
11319 }
11320
11321 #[tokio::test]
11334 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11335 let registry = Arc::new(Registry::default());
11336 let forwarding = Arc::new(ForwardingTable::default());
11337 let supervisor = SupervisorHandle::new();
11338 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11339 let handler =
11340 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11341 .with_supervisor(supervisor);
11342
11343 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11344 hello_via_sink(
11345 &handler,
11346 &target_ctx,
11347 &mut target_rx,
11348 hello_frame("target", PROTOCOL_VERSION, 1),
11349 )
11350 .await;
11351
11352 let wrong_nonce = handler
11356 .handle_control_frame(
11357 &route_ctx(ConnectionId::new(91)).0,
11358 route_open_frame_with_admission_facts(
11359 20,
11360 "target",
11361 unique_project_root("admission-facts"),
11362 Some(subc_control::ConsumerIdentity {
11363 module_id: "fed".to_string(),
11364 launch_nonce: "not-the-real-nonce".to_string(),
11365 }),
11366 None,
11367 ),
11368 )
11369 .await
11370 .unwrap();
11371 assert_eq!(
11372 parse_error(&wrong_nonce[0])["code"],
11373 "bad_consumer_identity",
11374 "a mismatched launch nonce must be refused, not stamped Reserved"
11375 );
11376
11377 let never_spawned = handler
11381 .handle_control_frame(
11382 &route_ctx(ConnectionId::new(92)).0,
11383 route_open_frame_with_admission_facts(
11384 21,
11385 "target",
11386 unique_project_root("admission-facts"),
11387 Some(subc_control::ConsumerIdentity {
11388 module_id: "never-spawned".to_string(),
11389 launch_nonce: "any-nonce".to_string(),
11390 }),
11391 None,
11392 ),
11393 )
11394 .await
11395 .unwrap();
11396 assert_eq!(
11397 parse_error(&never_spawned[0])["code"],
11398 "bad_consumer_identity",
11399 "an unspawned module_id must be refused rather than accepted for lack of a record"
11400 );
11401 }
11402
11403 #[tokio::test]
11426 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11427 let registry = Arc::new(Registry::default());
11428 let forwarding = Arc::new(ForwardingTable::default());
11429 let supervisor = SupervisorHandle::new();
11430 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11431 let handler =
11432 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11433 .with_supervisor(supervisor);
11434
11435 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11436 hello_via_sink(
11437 &handler,
11438 &target_ctx,
11439 &mut target_rx,
11440 hello_frame("target", PROTOCOL_VERSION, 1),
11441 )
11442 .await;
11443
11444 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11445 let route_handler = handler.clone();
11446 let route_task = tokio::spawn(async move {
11447 route_handler
11448 .handle_control_frame(
11449 &client_ctx,
11450 route_open_frame_with_admission_facts(
11451 30,
11452 "target",
11453 unique_project_root("admission-facts"),
11454 Some(subc_control::ConsumerIdentity {
11455 module_id: "fed".to_string(),
11456 launch_nonce: "fed-nonce".to_string(),
11457 }),
11458 None,
11459 ),
11460 )
11461 .await
11462 .unwrap()
11463 });
11464
11465 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11472 .await
11473 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11474 .expect("module control channel closed before route.bind");
11475 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11476 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11477 panic!("expected route.bind")
11478 };
11479 assert_eq!(
11480 principal,
11481 Some(Principal::Reserved {
11482 module_id: "fed".to_string()
11483 }),
11484 "a correctly attested consumer must be stamped Reserved for its own id"
11485 );
11486
11487 handler
11488 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11489 .await
11490 .unwrap();
11491 assert!(route_task.await.unwrap().is_empty());
11492 assert!(
11493 matches!(
11494 serde_json::from_slice::<ClientControlResponse>(
11495 &client_rx.recv().await.unwrap().body
11496 )
11497 .unwrap(),
11498 ClientControlResponse::RouteOpen { .. }
11499 ),
11500 "the route must actually open, not merely avoid an error"
11501 );
11502 }
11503
11504 #[tokio::test(start_paused = true)]
11505 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11506 let registry = Arc::new(Registry::default());
11507 let forwarding = Arc::new(ForwardingTable::default());
11508 let supervisor = SupervisorHandle::new();
11509 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11510 let handler =
11511 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11512 .with_supervisor(supervisor);
11513
11514 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11515 hello_via_sink(
11516 &handler,
11517 &target_ctx,
11518 &mut target_rx,
11519 hello_frame("target", PROTOCOL_VERSION, 1),
11520 )
11521 .await;
11522
11523 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11524 let direct_handler = handler.clone();
11525 let direct_open = tokio::spawn(async move {
11526 direct_handler
11527 .handle_control_frame(
11528 &direct_ctx,
11529 route_open_frame(2, "target", unique_project_root("route-census-direct")),
11530 )
11531 .await
11532 .unwrap()
11533 });
11534 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11535 .await
11536 .expect("no direct route.bind within 5s")
11537 .expect("target control channel closed before direct route.bind");
11538 handler
11539 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11540 .await
11541 .unwrap();
11542 assert!(direct_open.await.unwrap().is_empty());
11543 let _ = direct_rx.recv().await.unwrap();
11544
11545 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11546 let reserved_handler = handler.clone();
11547 let reserved_open = tokio::spawn(async move {
11548 reserved_handler
11549 .handle_control_frame(
11550 &reserved_ctx,
11551 route_open_frame_with_admission_facts(
11552 3,
11553 "target",
11554 unique_project_root("admission-facts"),
11555 Some(ConsumerIdentity {
11556 module_id: "fed".to_string(),
11557 launch_nonce: "fed-nonce".to_string(),
11558 }),
11559 None,
11560 ),
11561 )
11562 .await
11563 .unwrap()
11564 });
11565 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11566 .await
11567 .expect("no reserved route.bind within 5s")
11568 .expect("target control channel closed before reserved route.bind");
11569 handler
11570 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11571 .await
11572 .unwrap();
11573 assert!(reserved_open.await.unwrap().is_empty());
11574 let _ = reserved_rx.recv().await.unwrap();
11575
11576 forwarding
11577 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11578 .unwrap();
11579 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11580 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11581 module_id: Some("target".to_string()),
11582 })
11583 .unwrap();
11584 let census_frame =
11585 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11586 let response = handler
11587 .handle_control_frame(&census_ctx, census_frame)
11588 .await
11589 .unwrap()
11590 .pop()
11591 .unwrap();
11592 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11593 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11594 assert!(matches!(
11595 decoded,
11596 ClientControlResponse::SupervisorRoutes { .. }
11597 ));
11598 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11599 assert_eq!(routes.len(), 2);
11600 assert!(routes.iter().all(|route| route["draining"] == true));
11601 assert!(
11604 routes.iter().all(|route| route["drain_reason"] == "reload"),
11605 "draining routes must name the drain's reason: {routes:?}"
11606 );
11607 assert!(routes.iter().any(|route| {
11608 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11609 }));
11610 assert!(routes.iter().any(|route| {
11611 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11612 }));
11613
11614 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11615 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11616 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11617 std::fs::write(
11618 &golden_path,
11619 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11620 )
11621 .unwrap();
11622 }
11623 let expected: Value =
11624 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11625 assert_eq!(actual, expected);
11626 }
11627
11628 async fn query_live_roots(
11629 handler: &ControlHandler,
11630 module_ctx: &RouteCtx,
11631 ) -> ModuleControlResponseToModule {
11632 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11633 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11634 let response = handler
11635 .handle_control_frame(module_ctx, frame)
11636 .await
11637 .unwrap()
11638 .pop()
11639 .unwrap();
11640 serde_json::from_slice(&response.body).unwrap()
11641 }
11642
11643 #[tokio::test(start_paused = true)]
11644 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11645 let registry = Arc::new(Registry::default());
11646 let forwarding = Arc::new(ForwardingTable::default());
11647 let handler = ControlHandler::with_forwarding(registry, forwarding);
11648 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11649 hello_via_sink(
11650 &handler,
11651 &target_ctx,
11652 &mut target_rx,
11653 hello_frame("target", PROTOCOL_VERSION, 1),
11654 )
11655 .await;
11656 let root = unique_project_root("live-roots-known");
11657 let path = ProjectRootId::from_path_allowing_missing(root.path())
11658 .unwrap()
11659 .as_path()
11660 .to_path_buf();
11661 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11662 let open_handler = handler.clone();
11663 let opened = tokio::spawn(async move {
11664 open_handler
11665 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11666 .await
11667 .unwrap()
11668 });
11669 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11670 .await
11671 .unwrap()
11672 .unwrap();
11673 handler
11674 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11675 .await
11676 .unwrap();
11677 assert!(opened.await.unwrap().is_empty());
11678 let _ = client_rx.recv().await.unwrap();
11679
11680 let root = unique_project_root("live-roots-pending");
11681 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11682 .unwrap()
11683 .as_path()
11684 .to_path_buf();
11685 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11686 let open_handler = handler.clone();
11687 let pending = tokio::spawn(async move {
11688 open_handler
11689 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11690 .await
11691 .unwrap()
11692 });
11693 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11694 .await
11695 .unwrap()
11696 .unwrap();
11697 let actual = query_live_roots(&handler, &target_ctx).await;
11698 let ModuleControlResponseToModule::LiveRoots {
11699 roots,
11700 unknown_root_bindings,
11701 total_bindings,
11702 } = actual
11703 else {
11704 panic!("expected live roots")
11705 };
11706 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11707 assert_eq!(unknown_root_bindings, 0);
11708 assert_eq!(
11709 roots.len(),
11710 2,
11711 "root-known arm must retain each canonical root"
11712 );
11713 assert_eq!(
11714 total_bindings,
11715 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11716 );
11717 let counts = roots
11718 .iter()
11719 .map(|root| (root.project_root.clone(), root.bound, root.pending))
11720 .collect::<Vec<_>>();
11721 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11722 expected.sort_by(|a, b| a.0.cmp(&b.0));
11723 assert_eq!(
11724 counts, expected,
11725 "roots must sort by path and count pending separately"
11726 );
11727 handler
11728 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11729 .await
11730 .unwrap();
11731 assert!(pending.await.unwrap().is_empty());
11732 }
11733
11734 #[tokio::test(start_paused = true)]
11735 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11736 let forwarding = Arc::new(ForwardingTable::default());
11737 let handler =
11738 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11739 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11740 hello_via_sink(
11741 &handler,
11742 &target_ctx,
11743 &mut target_rx,
11744 hello_frame("target", PROTOCOL_VERSION, 1),
11745 )
11746 .await;
11747 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11748 let pending = forwarding
11749 .begin_route_bind_relay_for_test(
11750 client_ctx.connection_id,
11751 client_ctx.egress.clone(),
11752 2,
11753 "target",
11754 )
11755 .unwrap();
11756 forwarding
11757 .complete_pending_relay(
11758 target_ctx.connection_id,
11759 pending.corr,
11760 RouteBindRelayOutcome::Accepted,
11761 )
11762 .unwrap();
11763 let actual = query_live_roots(&handler, &target_ctx).await;
11764 let ModuleControlResponseToModule::LiveRoots {
11765 roots,
11766 unknown_root_bindings,
11767 total_bindings,
11768 } = actual
11769 else {
11770 panic!("expected live roots")
11771 };
11772 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11773 assert_eq!(
11774 unknown_root_bindings, 1,
11775 "unknown-root arm must not read as no bindings"
11776 );
11777 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11778 assert_eq!(
11779 total_bindings,
11780 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11781 );
11782 }
11783
11784 #[tokio::test(start_paused = true)]
11792 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11793 let forwarding = Arc::new(ForwardingTable::default());
11794 let handler =
11795 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11796 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11797 let replies = handler
11798 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11799 .await
11800 .unwrap();
11801 let queued_by_hello = module_rx.len();
11802
11803 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11804 let open_handler = handler.clone();
11805 let open = tokio::spawn(async move {
11806 open_handler
11807 .handle_control_frame(
11808 &client_ctx,
11809 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11810 )
11811 .await
11812 .unwrap()
11813 });
11814 let mut spins = 0;
11816 while module_rx.len() == queued_by_hello {
11817 spins += 1;
11818 assert!(spins < 10_000, "route.open never queued a route.bind");
11819 tokio::task::yield_now().await;
11820 }
11821
11822 for reply in replies {
11824 module_ctx.egress.send(reply).await.unwrap();
11825 }
11826
11827 let first = module_rx.recv().await.unwrap().frame;
11828 assert_eq!(
11829 first.header.ty,
11830 FrameType::HelloAck,
11831 "the first frame a registering module reads must be its HELLO_ACK"
11832 );
11833 assert_eq!(first.header.corr, 7);
11834 let second = module_rx.recv().await.unwrap().frame;
11835 assert_eq!(second.header.ty, FrameType::Request);
11836 assert!(
11837 matches!(
11838 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11839 ModuleControlRequest::RouteBind { .. }
11840 ),
11841 "the route.bind follows the ack"
11842 );
11843 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11844
11845 handler
11846 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11847 .await
11848 .unwrap();
11849 assert!(open.await.unwrap().is_empty());
11850 let _ = client_rx.recv().await.unwrap();
11851 }
11852
11853 #[tokio::test(start_paused = true)]
11854 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11855 let handler = ControlHandler::with_forwarding(
11856 Arc::new(Registry::default()),
11857 Arc::new(ForwardingTable::default()),
11858 );
11859 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11860 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11861 hello_via_sink(
11862 &handler,
11863 &first_ctx,
11864 &mut first_rx,
11865 hello_frame("first", PROTOCOL_VERSION, 1),
11866 )
11867 .await;
11868 hello_via_sink(
11869 &handler,
11870 &second_ctx,
11871 &mut second_rx,
11872 hello_frame("second", PROTOCOL_VERSION, 2),
11873 )
11874 .await;
11875 let root = unique_project_root("second-only");
11876 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11877 let cloned = handler.clone();
11878 let open = tokio::spawn(async move {
11879 cloned
11880 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11881 .await
11882 .unwrap()
11883 });
11884 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11885 .await
11886 .unwrap()
11887 .unwrap();
11888 let first = query_live_roots(&handler, &first_ctx).await;
11889 let second = query_live_roots(&handler, &second_ctx).await;
11890 assert!(
11891 matches!(
11892 first,
11893 ModuleControlResponseToModule::LiveRoots {
11894 total_bindings: 0,
11895 ..
11896 }
11897 ),
11898 "cross-module scope must not expose another module's roots"
11899 );
11900 assert!(
11901 matches!(
11902 second,
11903 ModuleControlResponseToModule::LiveRoots {
11904 total_bindings: 1,
11905 ..
11906 }
11907 ),
11908 "second module must see its pending route"
11909 );
11910 handler
11911 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11912 .await
11913 .unwrap();
11914 assert!(open.await.unwrap().is_empty());
11915 }
11916
11917 #[tokio::test(start_paused = true)]
11918 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11919 let handler = ControlHandler::with_forwarding(
11920 Arc::new(Registry::default()),
11921 Arc::new(ForwardingTable::default()),
11922 );
11923 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11924 hello_via_sink(
11925 &handler,
11926 &target_ctx,
11927 &mut target_rx,
11928 hello_frame("target", PROTOCOL_VERSION, 1),
11929 )
11930 .await;
11931 let actual = query_live_roots(&handler, &target_ctx).await;
11932 let ModuleControlResponseToModule::LiveRoots {
11933 roots,
11934 unknown_root_bindings,
11935 total_bindings,
11936 } = actual
11937 else {
11938 panic!("expected live roots")
11939 };
11940 assert!(roots.is_empty());
11941 assert_eq!(unknown_root_bindings, 0);
11942 assert_eq!(total_bindings, 0);
11943 assert_eq!(
11944 total_bindings,
11945 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11946 );
11947 }
11948
11949 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11956 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11957 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11958 let text = std::fs::read_to_string(&path)
11959 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11960 let vectors: Vec<(String, Value)> = text
11961 .lines()
11962 .filter(|line| !line.trim().is_empty())
11963 .map(|line| {
11964 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11965 let id = entry["corpus_id"]
11966 .as_str()
11967 .expect("every vector carries a corpus_id")
11968 .to_string();
11969 (id, entry["package"].clone())
11970 })
11971 .collect();
11972 assert_eq!(
11976 vectors.len(),
11977 3,
11978 "vendored fed corpus changed size; re-sync from subc-federation"
11979 );
11980
11981 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11996 let richest = vectors
11997 .iter()
11998 .filter_map(|(_, package)| package.as_object())
11999 .map(|object| {
12000 object
12001 .keys()
12002 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12003 .count()
12004 })
12005 .max()
12006 .unwrap_or(0);
12007 assert!(
12008 richest >= 2,
12009 "vendored corpus no longer carries a package with unmodelled fields, \
12010 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12011 );
12012
12013 vectors
12014 }
12015
12016 #[tokio::test]
12024 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12025 for (corpus_id, package) in fed_admission_facts_vectors() {
12026 let registry = Arc::new(Registry::default());
12027 let forwarding = Arc::new(ForwardingTable::default());
12028 let supervisor = SupervisorHandle::new();
12029 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12030 let handler =
12031 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12032 .with_supervisor(supervisor)
12033 .with_admission_facts_config(
12034 Some("fed".to_string()),
12035 Some(vec!["target".to_string()]),
12036 );
12037
12038 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12039 hello_via_sink(
12040 &handler,
12041 &target_ctx,
12042 &mut target_rx,
12043 hello_frame("target", PROTOCOL_VERSION, 1),
12044 )
12045 .await;
12046
12047 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12048 let route_handler = handler.clone();
12049 let expected = package.clone();
12050 let route_task = tokio::spawn(async move {
12051 route_handler
12052 .handle_control_frame(
12053 &client_ctx,
12054 route_open_frame_with_admission_facts(
12055 20,
12056 "target",
12057 unique_project_root("admission-facts"),
12058 Some(subc_control::ConsumerIdentity {
12059 module_id: "fed".to_string(),
12060 launch_nonce: "fed-nonce".to_string(),
12061 }),
12062 Some(package),
12063 ),
12064 )
12065 .await
12066 .unwrap()
12067 });
12068
12069 let bind_frame = target_rx.recv().await.unwrap();
12070 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12071 let ModuleControlRequest::RouteBind {
12072 admission_facts, ..
12073 } = bind
12074 else {
12075 panic!("{corpus_id}: expected route.bind")
12076 };
12077 assert_eq!(
12078 admission_facts,
12079 Some(expected),
12080 "{corpus_id}: relay must not add, drop or reshape any field"
12081 );
12082
12083 handler
12084 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12085 .await
12086 .unwrap();
12087 route_task.await.unwrap();
12088 }
12089 }
12090
12091 #[tokio::test]
12092 async fn route_bind_never_relays_the_openers_build_class() {
12093 let registry = Arc::new(Registry::default());
12094 let forwarding = Arc::new(ForwardingTable::default());
12095 let handler =
12096 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12097 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12098 hello_via_sink(
12099 &handler,
12100 &target_ctx,
12101 &mut target_rx,
12102 hello_frame("target", PROTOCOL_VERSION, 1),
12103 )
12104 .await;
12105
12106 let root = unique_project_root("build-class");
12107 let mut identity = BindIdentity::new(
12108 root.path().to_path_buf(),
12109 "unit".to_string(),
12110 "s".to_string(),
12111 );
12112 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12113 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12114 target: RouteTarget::ToolProvider {
12115 module_id: "target".to_string(),
12116 },
12117 identity,
12118 consumer_identity: None,
12119 consumer_capabilities: None,
12120 role_versions: None,
12121 admission_facts: None,
12122 scope: None,
12123 })
12124 .unwrap();
12125 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12126
12127 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12128 let route_handler = handler.clone();
12129 let route_task = tokio::spawn(async move {
12130 route_handler
12131 .handle_control_frame(&client_ctx, open)
12132 .await
12133 .unwrap()
12134 });
12135 let bind_frame = target_rx.recv().await.unwrap();
12136 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12137 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12138 panic!("expected route.bind")
12139 };
12140 assert_eq!(
12141 identity.build_class, None,
12142 "route.bind carried the build class"
12143 );
12144 assert!(
12145 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12146 "the route.bind bytes still name build_class"
12147 );
12148 handler
12149 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12150 .await
12151 .unwrap();
12152 route_task.await.unwrap();
12153 }
12154
12155 #[tokio::test]
12156 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12157 let registry = Arc::new(Registry::default());
12158 let forwarding = Arc::new(ForwardingTable::default());
12159 let supervisor = SupervisorHandle::new();
12160 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12161 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12162 let handler =
12163 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12164 .with_supervisor(supervisor)
12165 .with_admission_facts_config(
12166 Some("fed".to_string()),
12167 Some(vec!["target".to_string()]),
12168 );
12169
12170 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12171 hello_via_sink(
12172 &handler,
12173 &target_ctx,
12174 &mut target_rx,
12175 hello_frame("target", PROTOCOL_VERSION, 1),
12176 )
12177 .await;
12178 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12179 hello_via_sink(
12180 &handler,
12181 &other_ctx,
12182 &mut other_rx,
12183 hello_frame("other", PROTOCOL_VERSION, 2),
12184 )
12185 .await;
12186
12187 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12188 let expected_facts = facts.clone();
12189 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12190 let route_handler = handler.clone();
12191 let route_task = tokio::spawn(async move {
12192 route_handler
12193 .handle_control_frame(
12194 &client_ctx,
12195 route_open_frame_with_admission_facts(
12196 10,
12197 "target",
12198 unique_project_root("admission-facts"),
12199 Some(subc_control::ConsumerIdentity {
12200 module_id: "fed".to_string(),
12201 launch_nonce: "fed-nonce".to_string(),
12202 }),
12203 Some(facts.clone()),
12204 ),
12205 )
12206 .await
12207 .unwrap()
12208 });
12209 let bind_frame = target_rx.recv().await.unwrap();
12210 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12211 let ModuleControlRequest::RouteBind {
12212 admission_facts, ..
12213 } = bind
12214 else {
12215 panic!("expected route.bind")
12216 };
12217 assert_eq!(admission_facts, Some(expected_facts));
12218 handler
12219 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12220 .await
12221 .unwrap();
12222 assert!(route_task.await.unwrap().is_empty());
12223 assert!(matches!(
12224 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12225 .unwrap(),
12226 ClientControlResponse::RouteOpen { .. }
12227 ));
12228
12229 let direct = handler
12230 .handle_control_frame(
12231 &route_ctx(ConnectionId::new(73)).0,
12232 route_open_frame_with_admission_facts(
12233 11,
12234 "target",
12235 unique_project_root("admission-facts"),
12236 None,
12237 Some(json!({"x": 1})),
12238 ),
12239 )
12240 .await
12241 .unwrap();
12242 assert_eq!(
12243 parse_error(&direct[0])["code"],
12244 "admission_facts_not_permitted"
12245 );
12246
12247 let different_reserved = handler
12248 .handle_control_frame(
12249 &route_ctx(ConnectionId::new(77)).0,
12250 route_open_frame_with_admission_facts(
12251 15,
12252 "target",
12253 unique_project_root("admission-facts"),
12254 Some(subc_control::ConsumerIdentity {
12255 module_id: "other".to_string(),
12256 launch_nonce: "other-nonce".to_string(),
12257 }),
12258 Some(json!({"x": 1})),
12259 ),
12260 )
12261 .await
12262 .unwrap();
12263 assert_eq!(
12264 parse_error(&different_reserved[0])["code"],
12265 "admission_facts_not_permitted"
12266 );
12267
12268 let other_target = handler
12269 .handle_control_frame(
12270 &route_ctx(ConnectionId::new(74)).0,
12271 route_open_frame_with_admission_facts(
12272 12,
12273 "other",
12274 unique_project_root("admission-facts"),
12275 Some(subc_control::ConsumerIdentity {
12276 module_id: "fed".to_string(),
12277 launch_nonce: "fed-nonce".to_string(),
12278 }),
12279 Some(json!({"x": 1})),
12280 ),
12281 )
12282 .await
12283 .unwrap();
12284 assert_eq!(
12285 parse_error(&other_target[0])["code"],
12286 "admission_facts_target_not_allowed"
12287 );
12288
12289 let nonexistent = handler
12290 .handle_control_frame(
12291 &route_ctx(ConnectionId::new(75)).0,
12292 route_open_frame_with_admission_facts(
12293 13,
12294 "missing",
12295 unique_project_root("admission-facts"),
12296 None,
12297 Some(json!({"x": 1})),
12298 ),
12299 )
12300 .await
12301 .unwrap();
12302 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12303
12304 let described = handler
12305 .handle_control_frame(
12306 &route_ctx(ConnectionId::new(76)).0,
12307 Frame::build(
12308 FrameType::Request,
12309 control_flags(),
12310 0,
12311 0,
12312 14,
12313 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12314 )
12315 .unwrap(),
12316 )
12317 .await
12318 .unwrap();
12319 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12320 serde_json::from_slice(&described[0].body).unwrap()
12321 else {
12322 panic!("expected server.describe response")
12323 };
12324 assert!(capabilities
12325 .iter()
12326 .any(|cap| cap == "admission_facts_relay_v1"));
12327 }
12328
12329 #[tokio::test]
12330 async fn admission_facts_without_configured_carrier_are_rejected() {
12331 let registry = Arc::new(Registry::default());
12332 let forwarding = Arc::new(ForwardingTable::default());
12333 let handler = ControlHandler::with_forwarding(registry, forwarding);
12334 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12335 hello_via_sink(
12336 &handler,
12337 &target_ctx,
12338 &mut target_rx,
12339 hello_frame("target", PROTOCOL_VERSION, 1),
12340 )
12341 .await;
12342
12343 let responses = handler
12344 .handle_control_frame(
12345 &route_ctx(ConnectionId::new(79)).0,
12346 route_open_frame_with_admission_facts(
12347 16,
12348 "target",
12349 unique_project_root("admission-facts"),
12350 None,
12351 Some(json!({"x": 1})),
12352 ),
12353 )
12354 .await
12355 .unwrap();
12356 assert_eq!(
12357 parse_error(&responses[0])["code"],
12358 "admission_facts_not_permitted"
12359 );
12360 }
12361
12362 #[tokio::test]
12363 async fn route_open_relays_consumer_capabilities_verbatim() {
12364 let registry = Arc::new(Registry::default());
12365 let forwarding = Arc::new(ForwardingTable::default());
12366 let handler =
12367 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12368 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12369 hello_via_sink(
12370 &handler,
12371 &module_ctx,
12372 &mut module_rx,
12373 hello_frame("aft", PROTOCOL_VERSION, 7),
12374 )
12375 .await;
12376
12377 let expected = vec!["elicitation".to_string(), "roots".to_string()];
12378 let expected_for_request = expected.clone();
12379 let project_root = unique_project_root("consumer-capabilities-present");
12380 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12381 let route_handler = handler.clone();
12382 let route_task = tokio::spawn(async move {
12383 route_handler
12384 .handle_control_frame(
12385 &client_ctx,
12386 route_open_frame_with_consumer_capabilities(
12387 401,
12388 "aft",
12389 project_root,
12390 Some(expected_for_request),
12391 ),
12392 )
12393 .await
12394 .unwrap()
12395 });
12396 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12397 .await
12398 .unwrap()
12399 .unwrap();
12400 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12401 let ModuleControlRequest::RouteBind {
12402 consumer_capabilities,
12403 ..
12404 } = bind
12405 else {
12406 panic!("expected route.bind request, got {bind:?}");
12407 };
12408 assert_eq!(consumer_capabilities, Some(expected.clone()));
12409
12410 handler
12411 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12412 .await
12413 .unwrap();
12414 let route_response = route_task.await.unwrap();
12415 assert!(route_response.is_empty());
12416 let published = client_rx.recv().await.unwrap();
12417 assert!(matches!(
12418 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12419 ClientControlResponse::RouteOpen { .. }
12420 ));
12421 }
12422
12423 #[tokio::test]
12424 async fn route_open_without_consumer_capabilities_relays_none() {
12425 let registry = Arc::new(Registry::default());
12426 let forwarding = Arc::new(ForwardingTable::default());
12427 let handler =
12428 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12429 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12430 hello_via_sink(
12431 &handler,
12432 &module_ctx,
12433 &mut module_rx,
12434 hello_frame("aft", PROTOCOL_VERSION, 7),
12435 )
12436 .await;
12437
12438 let project_root = unique_project_root("consumer-capabilities-absent");
12439 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12440 let route_handler = handler.clone();
12441 let route_task = tokio::spawn(async move {
12442 route_handler
12443 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12444 .await
12445 .unwrap()
12446 });
12447 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12448 .await
12449 .unwrap()
12450 .unwrap();
12451 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12452 let ModuleControlRequest::RouteBind {
12453 consumer_capabilities,
12454 ..
12455 } = bind
12456 else {
12457 panic!("expected route.bind request, got {bind:?}");
12458 };
12459 assert_eq!(consumer_capabilities, None);
12460
12461 handler
12462 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12463 .await
12464 .unwrap();
12465 let route_response = route_task.await.unwrap();
12466 assert!(route_response.is_empty());
12467 let published = client_rx.recv().await.unwrap();
12468 assert!(matches!(
12469 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12470 ClientControlResponse::RouteOpen { .. }
12471 ));
12472 }
12473
12474 async fn bind_role_versions_for(
12478 sent: Option<BTreeMap<String, String>>,
12479 connection: u64,
12480 ) -> Option<BTreeMap<String, String>> {
12481 let registry = Arc::new(Registry::default());
12482 let forwarding = Arc::new(ForwardingTable::default());
12483 let handler =
12484 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12485 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12486 hello_via_sink(
12487 &handler,
12488 &module_ctx,
12489 &mut module_rx,
12490 hello_frame("aft", PROTOCOL_VERSION, 7),
12491 )
12492 .await;
12493 let project_root = unique_project_root("role-versions");
12494 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12495 let route_handler = handler.clone();
12496 let route_task = tokio::spawn(async move {
12497 route_handler
12498 .handle_control_frame(
12499 &client_ctx,
12500 route_open_frame_with_role_versions(403, "aft", project_root, sent),
12501 )
12502 .await
12503 .unwrap()
12504 });
12505 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12506 .await
12507 .expect("a well-formed route.open reaches the module as a bind")
12508 .unwrap();
12509 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12510 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12511 panic!("expected route.bind request, got {bind:?}");
12512 };
12513 handler
12514 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12515 .await
12516 .unwrap();
12517 assert!(route_task.await.unwrap().is_empty());
12518 let published = client_rx.recv().await.unwrap();
12519 assert!(matches!(
12520 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12521 ClientControlResponse::RouteOpen { .. }
12522 ));
12523 role_versions
12524 }
12525
12526 #[tokio::test]
12527 async fn route_open_relays_role_versions_verbatim() {
12528 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12529 assert_eq!(
12530 bind_role_versions_for(Some(sent.clone()), 141).await,
12531 Some(sent)
12532 );
12533 }
12534
12535 #[tokio::test]
12538 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12539 assert_eq!(bind_role_versions_for(None, 143).await, None);
12540 assert_eq!(
12541 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12542 None
12543 );
12544 }
12545
12546 #[tokio::test]
12547 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12548 let registry = Arc::new(Registry::default());
12549 let forwarding = Arc::new(ForwardingTable::default());
12550 let handler =
12551 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12552 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12553 hello_via_sink(
12554 &handler,
12555 &module_ctx,
12556 &mut module_rx,
12557 hello_frame("aft", PROTOCOL_VERSION, 7),
12558 )
12559 .await;
12560
12561 let nine: BTreeMap<String, String> = (0..9)
12562 .map(|index| (format!("role-{index}"), "v1".to_string()))
12563 .collect();
12564 for (label, malformed) in [
12565 (
12566 "invalid role name",
12567 role_versions(&[("Tool_Provider", "v1")]),
12568 ),
12569 ("invalid version", role_versions(&[("tool-provider", "v0")])),
12570 ("nine entries", nine),
12571 ] {
12572 let responses = tokio::time::timeout(
12575 Duration::from_secs(1),
12576 handler.handle_control_frame(
12577 &route_ctx(ConnectionId::new(148)).0,
12578 route_open_frame_with_role_versions(
12579 404,
12580 "aft",
12581 unique_project_root("role-versions-malformed"),
12582 Some(malformed),
12583 ),
12584 ),
12585 )
12586 .await
12587 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12588 .unwrap();
12589 assert_eq!(responses.len(), 1, "{label}");
12590 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12591 let error = parse_error(&responses[0]);
12592 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12593 assert_eq!(
12594 error["detail"]["field"], "role_versions",
12595 "{label}: {error}"
12596 );
12597 assert!(
12598 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12599 "{label}: a malformed declaration is terminal"
12600 );
12601 assert!(
12602 module_rx.try_recv().is_err(),
12603 "{label}: the module must never see a bind"
12604 );
12605 }
12606 }
12607
12608 #[tokio::test]
12612 async fn route_role_versions_capability_is_advertised() {
12613 let handler = ControlHandler::new(Arc::new(Registry::default()));
12614 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12615 let ack = hello_via_sink(
12616 &handler,
12617 &ctx,
12618 &mut rx,
12619 hello_frame("m", PROTOCOL_VERSION, 1),
12620 )
12621 .await;
12622 let ack = parse_ack(&ack);
12623 assert!(
12624 ack.subc_capabilities
12625 .iter()
12626 .any(|c| c == "route-role-versions/v1"),
12627 "{:?}",
12628 ack.subc_capabilities
12629 );
12630
12631 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12632 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12633 let reply = handler
12634 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12635 .await
12636 .unwrap()
12637 .pop()
12638 .unwrap();
12639 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12640 serde_json::from_slice(&reply.body).unwrap()
12641 else {
12642 panic!("not a server.describe reply");
12643 };
12644 assert!(
12645 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12646 "{capabilities:?}"
12647 );
12648 }
12649
12650 #[tokio::test]
12654 async fn catalog_events_capability_is_advertised() {
12655 let handler = ControlHandler::new(Arc::new(Registry::default()));
12656 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12657 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12658 let reply = handler
12659 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12660 .await
12661 .unwrap()
12662 .pop()
12663 .unwrap();
12664 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12665 serde_json::from_slice(&reply.body).unwrap()
12666 else {
12667 panic!("not a server.describe reply");
12668 };
12669 assert!(
12670 capabilities
12671 .iter()
12672 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12673 "{capabilities:?}"
12674 );
12675 }
12676
12677 #[tokio::test]
12678 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12679 let handler = ControlHandler::new(Arc::new(Registry::default()));
12680 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12681 let ack = hello_via_sink(
12682 &handler,
12683 &module,
12684 &mut module_rx,
12685 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12686 )
12687 .await;
12688 let ack = parse_ack(&ack);
12689 assert!(
12690 ack.subc_capabilities
12691 .iter()
12692 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12693 "{:?}",
12694 ack.subc_capabilities
12695 );
12696 assert!(ack
12697 .subc_capabilities
12698 .iter()
12699 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12700
12701 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12702 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12703 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12704 let reply = handler
12705 .handle_control_frame(&client, frame)
12706 .await
12707 .unwrap()
12708 .pop()
12709 .unwrap();
12710 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12711 serde_json::from_slice(&reply.body).unwrap()
12712 else {
12713 panic!("not a server.describe reply");
12714 };
12715 assert!(
12716 capabilities
12717 .iter()
12718 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12719 "{capabilities:?}"
12720 );
12721 assert!(capabilities
12722 .iter()
12723 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12724 }
12725
12726 #[tokio::test]
12727 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12728 let handler = ControlHandler::new(Arc::new(Registry::default()));
12729 let (module, mut rx) = route_ctx(ConnectionId::new(156));
12730 let ack = hello_via_sink(
12731 &handler,
12732 &module,
12733 &mut rx,
12734 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12735 )
12736 .await;
12737 assert!(parse_ack(&ack)
12738 .subc_capabilities
12739 .iter()
12740 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12741 assert!(parse_ack(&ack)
12742 .subc_capabilities
12743 .iter()
12744 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12745 let (client, _rx) = route_ctx(ConnectionId::new(157));
12746 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12747 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12748 let reply = handler
12749 .handle_control_frame(&client, frame)
12750 .await
12751 .unwrap()
12752 .pop()
12753 .unwrap();
12754 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12755 serde_json::from_slice(&reply.body).unwrap()
12756 else {
12757 panic!("server.describe reply")
12758 };
12759 assert!(capabilities
12760 .iter()
12761 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12762 assert!(capabilities
12763 .iter()
12764 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12765 }
12766
12767 #[tokio::test]
12768 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12769 let handler = ControlHandler::new(Arc::new(Registry::default()));
12770 let (module, mut rx) = route_ctx(ConnectionId::new(158));
12771 let ack = hello_via_sink(
12772 &handler,
12773 &module,
12774 &mut rx,
12775 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12776 )
12777 .await;
12778 assert!(parse_ack(&ack)
12779 .subc_capabilities
12780 .iter()
12781 .any(|cap| cap == "scope-workdir/v1"));
12782 let (client, _rx) = route_ctx(ConnectionId::new(159));
12783 let frame = Frame::build(
12784 FrameType::Request,
12785 control_flags(),
12786 0,
12787 0,
12788 5,
12789 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12790 )
12791 .unwrap();
12792 let reply = handler
12793 .handle_control_frame(&client, frame)
12794 .await
12795 .unwrap()
12796 .remove(0);
12797 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12798 serde_json::from_slice(&reply.body).unwrap()
12799 else {
12800 panic!("server.describe reply")
12801 };
12802 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12803 }
12804
12805 #[tokio::test]
12806 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
12807 let handler = ControlHandler::new(Arc::new(Registry::default()));
12808 let (module, mut rx) = route_ctx(ConnectionId::new(160));
12809 let ack = hello_via_sink(
12810 &handler,
12811 &module,
12812 &mut rx,
12813 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
12814 )
12815 .await;
12816 assert!(parse_ack(&ack)
12817 .subc_capabilities
12818 .iter()
12819 .any(|cap| cap == "worker-runs/v1"));
12820 let (client, _rx) = route_ctx(ConnectionId::new(161));
12821 let frame = Frame::build(
12822 FrameType::Request,
12823 control_flags(),
12824 0,
12825 0,
12826 5,
12827 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12828 )
12829 .unwrap();
12830 let reply = handler
12831 .handle_control_frame(&client, frame)
12832 .await
12833 .unwrap()
12834 .remove(0);
12835 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12836 serde_json::from_slice(&reply.body).unwrap()
12837 else {
12838 panic!("server.describe reply")
12839 };
12840 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
12841 }
12842
12843 #[tokio::test]
12844 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12845 let handler = ControlHandler::new(Arc::new(Registry::default()));
12846 let (module, mut rx) = route_ctx(ConnectionId::new(154));
12847 let ack = hello_via_sink(
12848 &handler,
12849 &module,
12850 &mut rx,
12851 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12852 )
12853 .await;
12854 assert!(parse_ack(&ack)
12855 .subc_capabilities
12856 .iter()
12857 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12858 let (client, _rx) = route_ctx(ConnectionId::new(155));
12859 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12860 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12861 let reply = handler
12862 .handle_control_frame(&client, frame)
12863 .await
12864 .unwrap()
12865 .pop()
12866 .unwrap();
12867 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12868 serde_json::from_slice(&reply.body).unwrap()
12869 else {
12870 panic!("not a server.describe reply")
12871 };
12872 assert!(capabilities
12873 .iter()
12874 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12875 }
12876
12877 #[tokio::test]
12878 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12879 let registry = Arc::new(Registry::default());
12880 let forwarding = Arc::new(ForwardingTable::default());
12881 let handler =
12882 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12883 .with_health_probe_timeout(Duration::from_secs(5));
12884 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12885 hello_via_sink(
12886 &handler,
12887 &module_ctx,
12888 &mut module_rx,
12889 non_routable_hello_frame_with_control_ops(
12890 "mcp",
12891 300,
12892 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12893 ),
12894 )
12895 .await;
12896 assert!(registry
12897 .get_module("mcp")
12898 .unwrap()
12899 .unwrap()
12900 .manifest
12901 .provides
12902 .is_empty());
12903
12904 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12905 let route_response = handler
12906 .handle_control_frame(
12907 &route_client_ctx,
12908 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12909 )
12910 .await
12911 .unwrap();
12912 assert_eq!(route_response[0].header.ty, FrameType::Error);
12913 assert_eq!(
12914 parse_error(&route_response[0])["code"],
12915 "target_unavailable"
12916 );
12917 assert!(parse_error(&route_response[0])["message"]
12918 .as_str()
12919 .unwrap()
12920 .contains("does not provide the requested target"));
12921 assert!(module_rx.try_recv().is_err());
12922
12923 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12924 let health_handler = handler.clone();
12925 let health_task = tokio::spawn(async move {
12926 health_handler
12927 .handle_control_frame(
12928 &health_client_ctx,
12929 supervisor_health_probe_frame(302, "mcp"),
12930 )
12931 .await
12932 .unwrap()
12933 });
12934 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12935 .await
12936 .unwrap()
12937 .unwrap();
12938 assert_eq!(
12939 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12940 ModuleControlRequest::HealthCheck {}
12941 );
12942 handler
12943 .handle_control_frame(
12944 &module_ctx,
12945 health_response(health_frame.header.corr, HealthStatus::Ok),
12946 )
12947 .await
12948 .unwrap();
12949 let health_response = health_task.await.unwrap();
12950 assert_eq!(health_response[0].header.ty, FrameType::Response);
12951 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12952 ClientControlResponse::SupervisorHealthProbe {
12953 module_id, status, ..
12954 } => {
12955 assert_eq!(module_id, "mcp");
12956 assert_eq!(status, HealthStatus::Ok);
12957 }
12958 other => panic!("unexpected health response: {other:?}"),
12959 }
12960
12961 forwarding
12966 .cleanup_connection(module_ctx.connection_id)
12967 .unwrap();
12968 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12969 let cleanup_response = tokio::time::timeout(
12970 Duration::from_millis(200),
12971 handler.handle_control_frame(
12972 &cleanup_probe_ctx,
12973 supervisor_health_probe_frame(303, "mcp"),
12974 ),
12975 )
12976 .await
12977 .expect("probe should fail immediately when the control lane is gone")
12978 .unwrap();
12979 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12980 assert_eq!(
12981 parse_error(&cleanup_response[0])["code"],
12982 "target_unavailable"
12983 );
12984 assert!(parse_error(&cleanup_response[0])["message"]
12985 .as_str()
12986 .unwrap()
12987 .contains("no module connection"));
12988
12989 handler
12990 .cleanup_connection(module_ctx.connection_id)
12991 .unwrap();
12992 }
12993
12994 #[tokio::test]
12995 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12996 let registry = Arc::new(Registry::default());
12997 let supervisor_handle = SupervisorHandle::new();
12998 let supervisor =
12999 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13000 .with_handle(supervisor_handle.clone())
13001 .with_connection_file_path(
13002 std::env::temp_dir()
13003 .join(format!("subc-route-open-warming-{}", std::process::id())),
13004 );
13005 let module = supervisor
13006 .supervise_configured(
13007 ModuleSpec {
13008 module_id: "warming".to_string(),
13009 program: fake_aft_stub_path(),
13010 args: Vec::new(),
13011 env: Vec::new(),
13012 reserved: false,
13013 reserved_prefixes: Vec::new(),
13014 protocol: ModuleProtocol::Subc,
13015 overlap: Default::default(),
13016 },
13017 true,
13018 )
13019 .unwrap();
13020 assert_eq!(module.state().unwrap(), ModuleState::Running);
13021
13022 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13023 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13024 let response = handler
13025 .handle_control_frame(
13026 &ctx,
13027 route_open_frame(304, "warming", unique_project_root("warming")),
13028 )
13029 .await
13030 .unwrap();
13031 module.stop().await.unwrap();
13032
13033 assert_eq!(response[0].header.ty, FrameType::Error);
13034 let error = parse_error(&response[0]);
13035 assert_eq!(error["code"], "module_warming");
13036 assert!(error["message"]
13037 .as_str()
13038 .unwrap()
13039 .contains("state=running, enabled=true, live=false"));
13040 }
13041
13042 #[test]
13043 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13044 let handler = ControlHandler::new(Arc::new(Registry::default()));
13045 let capture = EventCapture::default();
13046 let _subscriber =
13047 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13048 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13049 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13050 let pending = (0..limit).collect::<Vec<_>>();
13051 let response = handler
13052 .route_open_capacity_refusal(
13053 &ctx,
13054 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13055 "busy",
13056 pending.len(),
13057 limit,
13058 )
13059 .unwrap();
13060 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13061 let event = capture
13062 .events()
13063 .into_iter()
13064 .find(|event| {
13065 event.target == "control"
13066 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13067 })
13068 .expect("connection admission refusal event");
13069 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13070 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13071 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13074 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13075 }
13076
13077 #[test]
13078 fn route_open_target_cap_logs_admission_reason_and_capacity() {
13079 let handler = ControlHandler::new(Arc::new(Registry::default()));
13080 let capture = EventCapture::default();
13081 let _subscriber =
13082 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13083 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13084 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13085 let guards = (0..limit)
13086 .map(|_| {
13087 handler
13088 .route_bind_concurrency
13089 .try_admit("busy", limit)
13090 .unwrap()
13091 })
13092 .collect::<Vec<_>>();
13093 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13094 Err(in_flight) => in_flight,
13095 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13096 };
13097 let response = handler
13098 .route_open_target_capacity_refusal(
13099 &ctx,
13100 &RouteOpenCaller::unknown(),
13101 &route_open_frame(397, "busy", unique_project_root("target-cap")),
13102 "busy",
13103 in_flight,
13104 )
13105 .unwrap();
13106 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13107 let event = capture
13108 .events()
13109 .into_iter()
13110 .find(|event| {
13111 event.target == "control"
13112 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13113 })
13114 .expect("target admission refusal event");
13115 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13116 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13117 drop(guards);
13118 }
13119
13120 #[cfg(unix)]
13121 #[tokio::test(start_paused = true)]
13122 async fn supervisor_list_shows_a_failed_modules_next_retry() {
13123 let registry = Arc::new(Registry::default());
13124 let handle = SupervisorHandle::new();
13125 let supervisor =
13126 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13127 .with_handle(handle.clone());
13128 let module = supervisor
13129 .spawn(crate::ModuleSpec {
13130 module_id: "retry-list".into(),
13131 program: PathBuf::from("/bin/sh"),
13132 args: vec!["-c".into(), "exit 17".into()],
13133 env: vec![],
13134 reserved: false,
13135 reserved_prefixes: vec![],
13136 protocol: ModuleProtocol::None,
13137 overlap: Default::default(),
13138 })
13139 .unwrap();
13140 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13141 while module.state().unwrap() != crate::ModuleState::Failed {
13142 assert!(
13143 std::time::Instant::now() < deadline,
13144 "waiting for budget failure"
13145 );
13146 tokio::task::yield_now().await;
13147 }
13148 let handler = ControlHandler::new(registry).with_supervisor(handle);
13149 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13150 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13151 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13152 assert_eq!(body["modules"][0]["state"], "failed");
13153 let retry = body["modules"][0]["next_retry_at_ms"]
13154 .as_u64()
13155 .expect("scheduled retry on the wire");
13156 let now = std::time::SystemTime::now()
13157 .duration_since(std::time::UNIX_EPOCH)
13158 .unwrap()
13159 .as_millis() as u64;
13160 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13161 module.set_enabled(false).await.unwrap();
13162 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13163 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13164 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13165 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13166 }
13167
13168 #[tokio::test]
13169 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
13170 let registry = Arc::new(Registry::default());
13171 let handle = SupervisorHandle::new();
13172 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13173 .with_handle(handle.clone());
13174 supervisor
13175 .supervise_configured(
13176 crate::ModuleSpec {
13177 module_id: "drift".into(),
13178 program: PathBuf::from("disabled-module"),
13179 args: Vec::new(),
13180 env: Vec::new(),
13181 reserved: false,
13182 reserved_prefixes: Vec::new(),
13183 protocol: ModuleProtocol::Subc,
13184 overlap: Default::default(),
13185 },
13186 false,
13187 )
13188 .unwrap();
13189 let handler = ControlHandler::new(registry).with_supervisor(handle);
13190 for configured in [None, Some(false), Some(true)] {
13191 handler.capability_evaluator.configure(
13192 configured.map(|enabled| ("drift".to_string(), enabled)),
13193 BTreeMap::new(),
13194 );
13195 let frame =
13196 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13197 let replies = handler
13198 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
13199 .await
13200 .unwrap();
13201 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13202 assert!(body.get("configured_enabled").is_none());
13203 assert_eq!(body["op"], "supervisor.ack");
13204 let frame =
13205 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13206 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13207 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13208 assert_eq!(
13209 body["modules"][0].get("configured_enabled"),
13210 configured.map(Value::Bool).as_ref()
13211 );
13212 }
13213 }
13214
13215 #[tokio::test(flavor = "current_thread")]
13216 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
13217 let handler = ControlHandler::new(Arc::new(Registry::default()));
13218 let capture = EventCapture::default();
13219 let _subscriber =
13220 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13221 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
13222 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13223
13224 let _ = handler
13225 .handle_client_control_request(
13226 &ctx,
13227 frame,
13228 ClientControlRequest::SupervisorSetEnabled {
13229 module_id: "broca".to_string(),
13230 enabled: false,
13231 },
13232 &mut None,
13233 )
13234 .await
13235 .unwrap();
13236
13237 let events = capture
13238 .events()
13239 .into_iter()
13240 .filter(|event| {
13241 event.target == "control"
13242 && event.fields.get("message").map(String::as_str)
13243 == Some("supervisor request received")
13244 })
13245 .collect::<Vec<_>>();
13246 assert_eq!(events.len(), 1, "one supervisor request log line");
13247 let event = &events[0];
13248 assert_eq!(event.level, tracing::Level::INFO);
13249 assert_eq!(
13250 event.fields.get("op"),
13251 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
13252 );
13253 assert_eq!(
13254 event.fields.get("module_id"),
13255 Some(&"\"broca\"".to_string())
13256 );
13257 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
13258 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
13259 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
13260 }
13261
13262 #[tokio::test(flavor = "current_thread")]
13263 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
13264 let registry = Arc::new(Registry::default());
13265 let module_connection = ConnectionId::new(303);
13266 registry
13267 .register_with_control_ops(
13268 manifest("aft", PROTOCOL_VERSION),
13269 PROTOCOL_VERSION,
13270 module_connection,
13271 module_baseline_control_ops(),
13272 )
13273 .unwrap();
13274 let handler = ControlHandler::new(registry);
13275 let capture = EventCapture::default();
13276 let _subscriber =
13277 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13278 let (ctx, _rx) = route_ctx(module_connection);
13279 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13280
13281 let _ = handler
13282 .handle_client_control_request(
13283 &ctx,
13284 frame,
13285 ClientControlRequest::SupervisorSetEnabled {
13286 module_id: "broca".to_string(),
13287 enabled: false,
13288 },
13289 &mut None,
13290 )
13291 .await
13292 .unwrap();
13293
13294 let callers = capture
13295 .events()
13296 .into_iter()
13297 .filter(|event| {
13298 event.fields.get("message").map(String::as_str)
13299 == Some("supervisor request received")
13300 })
13301 .map(|event| event.fields.get("caller").cloned())
13302 .collect::<Vec<_>>();
13303 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13304 }
13305
13306 #[tokio::test(flavor = "current_thread")]
13307 async fn supervisor_list_does_not_log_request_received() {
13308 let handler = ControlHandler::new(Arc::new(Registry::default()));
13309 let capture = EventCapture::default();
13310 let _subscriber =
13311 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13312 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13313 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13314
13315 handler
13316 .handle_client_control_request(
13317 &ctx,
13318 frame,
13319 ClientControlRequest::SupervisorList {},
13320 &mut None,
13321 )
13322 .await
13323 .unwrap();
13324
13325 assert!(capture.events().into_iter().all(|event| {
13326 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13327 }));
13328 }
13329
13330 #[tokio::test(flavor = "current_thread")]
13331 async fn supervisor_rescan_preview_does_not_log_request_received() {
13332 let handler = ControlHandler::new(Arc::new(Registry::default()));
13333 let capture = EventCapture::default();
13334 let _subscriber =
13335 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13336 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13337 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13338
13339 handler
13340 .handle_client_control_request(
13341 &ctx,
13342 frame,
13343 ClientControlRequest::SupervisorRescan { preview: true },
13344 &mut None,
13345 )
13346 .await
13347 .unwrap();
13348
13349 assert!(capture.events().into_iter().all(|event| {
13350 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13351 }));
13352 }
13353
13354 struct ApproveOperator;
13355 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13356 fn prompt(
13357 &self,
13358 _: &str,
13359 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13360 ) -> crate::operator_confirm::ProviderResult {
13361 crate::operator_confirm::ProviderResult::Approved
13362 }
13363 }
13364
13365 #[tokio::test]
13366 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13367 assert_eq!(
13368 ControlHandler::default().privileged_operator_prompt,
13369 PrivilegedOperatorPrompt::Log
13370 );
13371 for (initial, changed, running) in [
13372 ("log", "enforce", PrivilegedOperatorPrompt::Log),
13373 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13374 ] {
13375 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13376 let path = dir.join("subc.jsonc");
13377 std::fs::write(
13378 &path,
13379 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13380 )
13381 .unwrap();
13382 let config = crate::daemon_config::load(&path).unwrap().unwrap();
13383 let registry = Arc::new(Registry::default());
13384 let handle = SupervisorHandle::new();
13385 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13386 .with_handle(handle.clone());
13387 let handler = ControlHandler::new(registry)
13388 .with_supervisor(handle)
13389 .with_privileged_operator_prompt(config.privileged_operator_prompt)
13390 .with_supervisor_rescan(supervisor, &path, config.port);
13391 handler.forwarding.operator_confirms().configure(
13392 Arc::new(ApproveOperator),
13393 crate::operator_confirm::OperatorLimits::default(),
13394 );
13395 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13396
13397 for (disk_mode, needs_restart) in [
13398 (Some(initial), false),
13399 (Some(changed), true),
13400 (Some(changed), true),
13401 (None, running == PrivilegedOperatorPrompt::Enforce),
13402 (Some(initial), false),
13403 ] {
13404 let mut doc = json!({"version":1});
13405 if let Some(mode) = disk_mode {
13406 doc["privileged_operator_prompt"] = json!(mode);
13407 }
13408 std::fs::write(&path, doc.to_string()).unwrap();
13409 for preview in [true, false] {
13410 let request = ClientControlRequest::SupervisorRescan { preview };
13411 let frame = Frame::build(
13412 FrameType::Request,
13413 control_flags(),
13414 0,
13415 0,
13416 1,
13417 serde_json::to_vec(&request).unwrap(),
13418 )
13419 .unwrap();
13420 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13421 assert_eq!(frames.len(), 1);
13422 let ClientControlResponse::SupervisorRescan { result } =
13423 serde_json::from_slice(&frames[0].body).unwrap()
13424 else {
13425 panic!("expected a rescan response");
13426 };
13427 let expected = if needs_restart {
13428 vec!["privileged_operator_prompt"]
13429 } else {
13430 Vec::new()
13431 };
13432 assert_eq!(result.restart_required, expected);
13433 assert_eq!(handler.privileged_operator_prompt, running);
13434 assert_eq!(
13435 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13436 running
13437 );
13438 }
13439 }
13440 }
13441 }
13442
13443 #[tokio::test]
13444 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13445 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13446 let path = dir.join("subc.jsonc");
13447 let registry = Arc::new(Registry::default());
13448 let handle = SupervisorHandle::new();
13449 let supervisor =
13450 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13451 let handler = ControlHandler::new(registry)
13452 .with_supervisor(handle)
13453 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13454 .with_supervisor_rescan(supervisor, &path, None);
13455 handler.forwarding.operator_confirms().configure(
13456 Arc::new(ApproveOperator),
13457 crate::operator_confirm::OperatorLimits::default(),
13458 );
13459 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13460 for preview in [true, false] {
13461 std::fs::write(
13462 &path,
13463 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13464 )
13465 .unwrap();
13466 let request = ClientControlRequest::SupervisorRescan { preview };
13467 let frame = Frame::build(
13468 FrameType::Request,
13469 control_flags(),
13470 0,
13471 0,
13472 1,
13473 serde_json::to_vec(&request).unwrap(),
13474 )
13475 .unwrap();
13476 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13477 assert_eq!(frames.len(), 1);
13478 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13479 assert_eq!(
13480 handler.privileged_operator_prompt,
13481 PrivilegedOperatorPrompt::Enforce
13482 );
13483 }
13484 }
13485
13486 #[tokio::test]
13491 async fn route_open_refusal_names_the_check_that_refused() {
13492 let handler = ControlHandler::new(Arc::new(Registry::default()));
13493 let capture = EventCapture::default();
13494 let _subscriber =
13495 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13496 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13497 let response = handler
13498 .handle_control_frame(
13499 &ctx,
13500 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13501 )
13502 .await
13503 .unwrap();
13504
13505 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13506 let event = capture
13507 .events()
13508 .into_iter()
13509 .find(|event| {
13510 event.target == "control"
13511 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13512 })
13513 .expect("route.open refusal event");
13514 assert_eq!(
13515 event.fields.get("reason"),
13516 Some(&"\"not_registered\"".to_string())
13517 );
13518 }
13519
13520 #[tokio::test]
13521 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13522 let registry = Arc::new(Registry::default());
13523 let supervisor_handle = SupervisorHandle::new();
13524 let supervisor =
13525 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13526 .with_handle(supervisor_handle.clone())
13527 .with_connection_file_path(std::env::temp_dir().join(format!(
13528 "subc-route-open-refusal-info-{}",
13529 std::process::id()
13530 )));
13531 let module = supervisor
13532 .supervise_configured(
13533 ModuleSpec {
13534 module_id: "warming".to_string(),
13535 program: fake_aft_stub_path(),
13536 args: Vec::new(),
13537 env: Vec::new(),
13538 reserved: false,
13539 reserved_prefixes: Vec::new(),
13540 protocol: ModuleProtocol::Subc,
13541 overlap: Default::default(),
13542 },
13543 true,
13544 )
13545 .unwrap();
13546 assert_eq!(module.state().unwrap(), ModuleState::Running);
13547
13548 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13549 assert!(handler
13550 .counters()
13551 .snapshot()
13552 .get("route_open_refused_by_code")
13553 .is_none());
13554 let capture = EventCapture::default();
13555 let _subscriber =
13556 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13557 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13558 let response = handler
13559 .handle_control_frame(
13560 &ctx,
13561 route_open_frame(394, "warming", unique_project_root("refusal-info")),
13562 )
13563 .await
13564 .unwrap();
13565 module.stop().await.unwrap();
13566
13567 assert_eq!(parse_error(&response[0])["code"], "module_warming");
13568 let event = capture
13569 .events()
13570 .into_iter()
13571 .find(|event| {
13572 event.target == "control"
13573 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13574 })
13575 .expect("route.open refusal event");
13576 assert_eq!(
13577 event.fields.get("module_id"),
13578 Some(&"\"warming\"".to_string())
13579 );
13580 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13581 assert_eq!(
13582 event.fields.get("reason"),
13583 Some(&"\"supervised_not_registered\"".to_string())
13584 );
13585 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13586 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13587 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13588 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13589 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13590 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13591 assert_eq!(
13592 handler.counters().snapshot()["route_open_refused_by_code"],
13593 json!({ "module_warming": 1 })
13594 );
13595 }
13596
13597 const OUTAGE_START: &str = "route.open refusing module: not serving";
13598 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13599
13600 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13601 capture
13602 .events()
13603 .into_iter()
13604 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13605 .collect()
13606 }
13607
13608 fn supervise_stub(
13609 registry: &Arc<Registry>,
13610 module_id: &str,
13611 enabled: bool,
13612 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13613 let supervisor_handle = SupervisorHandle::new();
13614 let supervisor =
13615 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13616 .with_handle(supervisor_handle.clone())
13617 .with_connection_file_path(std::env::temp_dir().join(format!(
13618 "subc-route-outage-{module_id}-{}",
13619 std::process::id()
13620 )));
13621 let module = supervisor
13622 .supervise_configured(
13623 ModuleSpec {
13624 module_id: module_id.to_string(),
13625 program: fake_aft_stub_path(),
13626 args: Vec::new(),
13627 env: Vec::new(),
13628 reserved: false,
13629 reserved_prefixes: Vec::new(),
13630 protocol: ModuleProtocol::Subc,
13631 overlap: Default::default(),
13632 },
13633 enabled,
13634 )
13635 .unwrap();
13636 (supervisor_handle, module)
13637 }
13638
13639 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13640 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13641 module_id: module_id.to_string(),
13642 drain_timeout_ms: Some(50),
13643 })
13644 .unwrap();
13645 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13646 }
13647
13648 #[test]
13652 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13653 let registry = Arc::new(Registry::default());
13654 let forwarding = Arc::new(ForwardingTable::default());
13655 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13656 let second = ControlHandler::with_forwarding(registry, forwarding);
13657 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13658 }
13659
13660 #[tokio::test(flavor = "current_thread")]
13664 async fn route_open_unknown_module_refusals_add_no_outage_state() {
13665 let handler = ControlHandler::new(Arc::new(Registry::default()));
13666 let capture = EventCapture::default();
13667 let _subscriber =
13668 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13669 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13670 for corr in 0..8 {
13671 let response = handler
13672 .handle_control_frame(
13673 &ctx,
13674 route_open_frame(
13675 380 + corr,
13676 &format!("nobody-{corr}"),
13677 unique_project_root("outage-unknown"),
13678 ),
13679 )
13680 .await
13681 .unwrap();
13682 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13683 }
13684
13685 assert_eq!(handler.route_outages.tracked_module_count(), 0);
13686 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13687 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13688 }
13689
13690 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13695 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13696 let registry = Arc::new(Registry::default());
13697 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
13698 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13699 let capture = EventCapture::default();
13700 let _subscriber =
13701 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13702 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13703 handler.route_outages.record_accepted("outage-restart");
13706
13707 let response = handler
13708 .handle_control_frame(
13709 &ctx,
13710 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13711 )
13712 .await
13713 .unwrap();
13714 assert_eq!(response[0].header.ty, FrameType::Error);
13715 let starts = outage_lines(&capture, OUTAGE_START);
13716 assert_eq!(starts.len(), 1, "{starts:?}");
13717 assert_eq!(starts[0].level, tracing::Level::WARN);
13718 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13719 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13720 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13721 handler.route_outages.record_accepted("outage-restart");
13722 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13723
13724 let restart = handler
13725 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13726 .await
13727 .unwrap();
13728 assert_eq!(
13729 restart[0].header.ty,
13730 FrameType::Response,
13731 "{:?}",
13732 parse_error(&restart[0])
13733 );
13734 handler
13735 .handle_control_frame(
13736 &ctx,
13737 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13738 )
13739 .await
13740 .unwrap();
13741 module.stop().await.unwrap();
13742
13743 let starts = outage_lines(&capture, OUTAGE_START);
13744 assert_eq!(starts.len(), 2, "{starts:?}");
13745 assert_eq!(starts[1].level, tracing::Level::INFO);
13746 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13747 }
13748
13749 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13753 async fn failed_operator_restart_leaves_no_operator_mark() {
13754 let registry = Arc::new(Registry::default());
13755 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
13756 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13757 let capture = EventCapture::default();
13758 let _subscriber =
13759 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13760 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13761 handler.route_outages.record_accepted("outage-disabled");
13762
13763 let restart = handler
13764 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13765 .await
13766 .unwrap();
13767 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13768 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13769
13770 handler
13771 .handle_control_frame(
13772 &ctx,
13773 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13774 )
13775 .await
13776 .unwrap();
13777 let starts = outage_lines(&capture, OUTAGE_START);
13778 assert_eq!(starts.len(), 1, "{starts:?}");
13779 assert_eq!(starts[0].level, tracing::Level::WARN);
13780 }
13781
13782 #[tokio::test(flavor = "current_thread")]
13783 async fn route_open_unknown_module_escapes_target_module_id() {
13784 let handler = ControlHandler::new(Arc::new(Registry::default()));
13785 let capture = EventCapture::default();
13786 let _subscriber =
13787 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13788 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13789 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13790 let response = handler
13791 .handle_control_frame(
13792 &ctx,
13793 route_open_frame(
13794 395,
13795 hostile_module_id,
13796 unique_project_root("hostile-target-module-id"),
13797 ),
13798 )
13799 .await
13800 .unwrap();
13801
13802 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13803 let event = capture
13804 .events()
13805 .into_iter()
13806 .find(|event| {
13807 event.target == "control"
13808 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13809 })
13810 .expect("route.open unknown-module refusal event");
13811 let logged = event.fields.get("module_id").expect("module_id field");
13812 assert!(!logged.bytes().any(|byte| byte < 0x20));
13813 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13814 }
13815
13816 #[tokio::test(flavor = "current_thread")]
13817 async fn route_open_module_rejection_uses_daemon_counter_key() {
13818 let registry = Arc::new(Registry::default());
13819 let forwarding = Arc::new(ForwardingTable::default());
13820 let handler =
13821 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13822 let module_connection = ConnectionId::new(95);
13823 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13824 hello_via_sink(
13825 &handler,
13826 &module_ctx,
13827 &mut module_rx,
13828 hello_frame("aft", PROTOCOL_VERSION, 395),
13829 )
13830 .await;
13831
13832 let client_connection = ConnectionId::new(96);
13833 let (client_ctx, _client_rx) = route_ctx(client_connection);
13834 let capture = EventCapture::default();
13835 let _subscriber =
13836 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13837 let (route_task, bind) = relay_route_open(
13838 &handler,
13839 client_connection,
13840 &client_ctx.egress,
13841 &mut module_rx,
13842 396,
13843 "aft",
13844 "hostile-module-code",
13845 )
13846 .await;
13847 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13848 let rejection = Frame::build(
13849 FrameType::Error,
13850 control_flags(),
13851 0,
13852 0,
13853 bind.header.corr,
13854 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13855 )
13856 .unwrap();
13857 handler
13858 .handle_control_frame(&module_ctx, rejection)
13859 .await
13860 .unwrap();
13861
13862 let response = route_task.await.unwrap();
13863 assert_eq!(parse_error(&response[0])["code"], hostile_code);
13864 let counters = handler.counters().snapshot();
13865 assert_eq!(
13866 counters["route_open_refused_by_code"],
13867 json!({ "module_rejected": 1 })
13868 );
13869 assert!(counters["route_open_refused_by_code"]
13870 .get(hostile_code)
13871 .is_none());
13872
13873 let event = capture
13874 .events()
13875 .into_iter()
13876 .find(|event| {
13877 event.target == "control"
13878 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13879 })
13880 .expect("route.open module-rejection refusal event");
13881 let logged = event.fields.get("module_code").expect("module_code field");
13882 assert!(!logged.bytes().any(|byte| byte < 0x20));
13883 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13884 }
13885
13886 #[tokio::test]
13887 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13888 let registry = Arc::new(Registry::default());
13889 let supervisor_handle = SupervisorHandle::new();
13890 let missing_program = std::env::temp_dir().join(format!(
13891 "subc-route-open-missing-program-{}",
13892 std::process::id()
13893 ));
13894 let supervisor =
13895 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13896 .with_handle(supervisor_handle.clone());
13897 let module = supervisor
13898 .supervise_configured(
13899 ModuleSpec {
13900 module_id: "failed".to_string(),
13901 program: missing_program,
13902 args: Vec::new(),
13903 env: Vec::new(),
13904 reserved: false,
13905 reserved_prefixes: Vec::new(),
13906 protocol: ModuleProtocol::Subc,
13907 overlap: Default::default(),
13908 },
13909 true,
13910 )
13911 .unwrap();
13912 assert_eq!(module.state().unwrap(), ModuleState::Failed);
13913
13914 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13915 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13916 let response = handler
13917 .handle_control_frame(
13918 &ctx,
13919 route_open_frame(305, "failed", unique_project_root("failed")),
13920 )
13921 .await
13922 .unwrap();
13923
13924 assert_eq!(response[0].header.ty, FrameType::Error);
13925 let error = parse_error(&response[0]);
13926 assert_eq!(error["code"], "target_unavailable");
13927 assert!(error["message"]
13928 .as_str()
13929 .unwrap()
13930 .contains("state=failed, enabled=true, live=false"));
13931 }
13932
13933 #[tokio::test]
13934 async fn route_open_role_mismatch_remains_target_unavailable() {
13935 let registry = Arc::new(Registry::default());
13936 let handler = ControlHandler::new(Arc::clone(®istry));
13937 handler
13938 .handle_control(
13939 ConnectionId::new(41),
13940 non_routable_hello_frame_with_control_ops("health-only", 306, None),
13941 )
13942 .unwrap();
13943
13944 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13945 let response = handler
13946 .handle_control_frame(
13947 &ctx,
13948 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13949 )
13950 .await
13951 .unwrap();
13952
13953 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13954 assert!(parse_error(&response[0])["message"]
13955 .as_str()
13956 .unwrap()
13957 .contains("does not provide the requested target"));
13958 }
13959
13960 #[tokio::test]
13961 async fn route_open_inactive_registration_remains_target_unavailable() {
13962 let registry = Arc::new(Registry::default());
13963 let handler = ControlHandler::new(Arc::clone(®istry));
13964 handler
13965 .handle_control(
13966 ConnectionId::new(43),
13967 hello_frame("inactive", PROTOCOL_VERSION, 308),
13968 )
13969 .unwrap();
13970 assert!(registry
13971 .set_module_state_for_test("inactive", ChannelState::Closed)
13972 .unwrap());
13973
13974 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13975 let response = handler
13976 .handle_control_frame(
13977 &ctx,
13978 route_open_frame(309, "inactive", unique_project_root("inactive")),
13979 )
13980 .await
13981 .unwrap();
13982
13983 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13984 assert!(parse_error(&response[0])["message"]
13985 .as_str()
13986 .unwrap()
13987 .contains("is not active"));
13988 }
13989
13990 #[tokio::test]
13991 async fn late_health_reply_is_recorded_through_the_module_response_path() {
13992 let registry = Arc::new(Registry::default());
13993 let forwarding = Arc::new(ForwardingTable::default());
13994 let supervisor_handle = SupervisorHandle::new();
13995 let supervisor =
13996 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
13997 .with_forwarding(Arc::clone(&forwarding))
13998 .with_handle(supervisor_handle.clone());
13999 let module = supervisor
14000 .supervise_configured(
14001 crate::ModuleSpec {
14002 module_id: "late-health-response".to_string(),
14003 program: PathBuf::from("disabled-module"),
14004 args: Vec::new(),
14005 env: Vec::new(),
14006 reserved: false,
14007 reserved_prefixes: Vec::new(),
14008 protocol: ModuleProtocol::Subc,
14009 overlap: Default::default(),
14010 },
14011 false,
14012 )
14013 .unwrap();
14014 let handler =
14015 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14016 .with_supervisor(supervisor_handle);
14017 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
14018 handler
14019 .handle_control_frame(
14020 &module_ctx,
14021 hello_frame_with_control_ops(
14022 "late-health-response",
14023 PROTOCOL_VERSION,
14024 7,
14025 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14026 ),
14027 )
14028 .await
14029 .unwrap();
14030 let probe_started_at = Instant::now() - Duration::from_millis(80);
14031 let pending = forwarding
14032 .begin_health_probe_rpc_for(
14033 "late-health-response",
14034 MODULE_CONTROL_OP_HEALTH_CHECK,
14035 probe_started_at,
14036 Instant::now() - Duration::from_millis(1),
14037 )
14038 .unwrap();
14039 assert!(forwarding
14040 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
14041 .unwrap());
14042
14043 let responses = handler
14044 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
14045 .await
14046 .unwrap();
14047
14048 assert!(responses.is_empty());
14049 let health = module.status().unwrap().health;
14050 assert_eq!(health.late_answer_count, 1);
14051 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
14052 }
14053
14054 #[tokio::test]
14055 async fn health_probe_timeout_and_module_death_are_typed() {
14056 let registry = Arc::new(Registry::default());
14057 let forwarding = Arc::new(ForwardingTable::default());
14058 let handler =
14059 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14060 .with_health_probe_timeout(Duration::from_millis(50));
14061 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14062 hello_via_sink(
14063 &handler,
14064 &module_ctx,
14065 &mut module_rx,
14066 hello_frame_with_control_ops(
14067 "aft",
14068 PROTOCOL_VERSION,
14069 7,
14070 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14071 ),
14072 )
14073 .await;
14074
14075 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14076 let responses = handler
14077 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
14078 .await
14079 .unwrap();
14080 assert_eq!(responses[0].header.ty, FrameType::Error);
14081 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
14082 let _ = module_rx.try_recv();
14083
14084 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
14085 let health_handler = handler.clone();
14086 let death_task = tokio::spawn(async move {
14087 health_handler
14088 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
14089 .await
14090 .unwrap()
14091 });
14092 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14093 .await
14094 .unwrap()
14095 .unwrap();
14096 handler
14097 .cleanup_connection(module_ctx.connection_id)
14098 .unwrap();
14099 let responses = death_task.await.unwrap();
14100 assert_eq!(responses[0].header.ty, FrameType::Error);
14101 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
14102 }
14103
14104 #[test]
14105 fn hello_requires_exact_protocol_version() {
14106 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
14107 let registry = Arc::new(Registry::default());
14108 let handler = ControlHandler::new(Arc::clone(®istry));
14109 let responses = handler
14110 .handle_control(
14111 ConnectionId::new(connection),
14112 hello_frame("aft", offered, 9),
14113 )
14114 .unwrap();
14115
14116 assert_eq!(responses.len(), 1);
14117 assert_eq!(responses[0].header.ty, FrameType::Error);
14118 let error = parse_error(&responses[0]);
14119 assert_eq!(error["code"], "version_unsupported");
14120 assert!(registry.get_module("aft").unwrap().is_none());
14121 assert_eq!(registry.active_registration_count().unwrap(), 0);
14122 }
14123 }
14124
14125 #[test]
14126 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
14127 let registry = Arc::new(Registry::default());
14128 let forwarding = Arc::new(ForwardingTable::default());
14129 let handler =
14130 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14131 let module_connection = ConnectionId::new(301);
14132 let registration = registry
14133 .register_with_control_ops(
14134 manifest("aft-push", PROTOCOL_VERSION),
14135 PROTOCOL_VERSION,
14136 module_connection,
14137 module_baseline_control_ops(),
14138 )
14139 .unwrap();
14140 let (module_tx, _module_rx) = mpsc::channel(8);
14141 let endpoint = forwarding
14142 .register_module_connection(
14143 module_connection,
14144 "aft-push".to_string(),
14145 PROTOCOL_VERSION,
14146 manifest_concurrency(®istration.manifest),
14147 FrameSink::new(module_tx),
14148 )
14149 .unwrap();
14150
14151 let unknown = Frame::build(
14153 FrameType::Push,
14154 control_flags(),
14155 0,
14156 0,
14157 5,
14158 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
14159 )
14160 .unwrap();
14161 let out = handler.handle_status_update(endpoint, unknown).unwrap();
14162 assert!(
14163 out.is_empty(),
14164 "unknown push op must be ignored, got {out:?}"
14165 );
14166
14167 let malformed = Frame::build(
14169 FrameType::Push,
14170 control_flags(),
14171 0,
14172 0,
14173 6,
14174 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
14175 )
14176 .unwrap();
14177 let out = handler.handle_status_update(endpoint, malformed).unwrap();
14178 assert_eq!(out.len(), 1);
14179 assert_eq!(out[0].header.ty, FrameType::Error);
14180 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
14181 }
14182
14183 #[test]
14184 fn hello_rejected_when_connection_already_owns_client_routes() {
14185 let registry = Arc::new(Registry::default());
14186 let forwarding = Arc::new(ForwardingTable::default());
14187 let handler =
14188 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14189 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
14191 let client_connection = ConnectionId::new(202);
14192
14193 let responses = handler
14196 .handle_control(
14197 client_connection,
14198 hello_frame("aft-second", PROTOCOL_VERSION, 9),
14199 )
14200 .unwrap();
14201 assert_eq!(responses[0].header.ty, FrameType::Error);
14202 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
14203 assert!(registry.get_module("aft-second").unwrap().is_none());
14204 }
14205
14206 #[tokio::test]
14207 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
14208 let registry = Arc::new(Registry::default());
14209 let forwarding = Arc::new(ForwardingTable::default());
14210 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
14211 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
14212 hello_via_sink(
14213 &handler,
14214 &module_ctx,
14215 &mut module_rx,
14216 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
14217 )
14218 .await;
14219 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
14220 let pending = forwarding
14221 .begin_route_bind_relay_for_test(
14222 client_ctx.connection_id,
14223 client_ctx.egress.clone(),
14224 2,
14225 "alpha",
14226 )
14227 .unwrap();
14228 forwarding
14229 .complete_pending_relay(
14230 module_ctx.connection_id,
14231 pending.corr,
14232 RouteBindRelayOutcome::Accepted,
14233 )
14234 .unwrap();
14235 client_rx.try_recv().unwrap();
14236 for module_id in ["beta", "alpha"] {
14237 let replies = handler
14238 .handle_control_frame(
14239 &module_ctx,
14240 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
14241 )
14242 .await
14243 .unwrap();
14244 assert_eq!(replies.len(), 1, "second HELLO must be refused");
14245 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
14246 }
14247 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
14248 assert!(registry.get_module("beta").unwrap().is_none());
14249 assert!(matches!(
14250 forwarding
14251 .lookup_data_route(
14252 client_ctx.connection_id,
14253 pending.client_channel,
14254 pending.client_epoch,
14255 )
14256 .unwrap(),
14257 DataRoute::Client(DataRouteState::Bound(_))
14258 ));
14259 assert!(handler
14260 .hello_launch_nonces
14261 .lock()
14262 .unwrap()
14263 .presented(module_ctx.connection_id, Some("alpha-nonce")));
14264 assert!(module_rx.try_recv().is_err());
14265 }
14266
14267 #[test]
14268 fn reserved_module_hello_requires_matching_launch_nonce() {
14269 let registry = Arc::new(Registry::default());
14270 let supervisor = SupervisorHandle::new();
14271 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
14274 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14275
14276 let no_nonce = handler
14278 .handle_control(
14279 ConnectionId::new(1),
14280 hello_frame("vault", PROTOCOL_VERSION, 1),
14281 )
14282 .unwrap();
14283 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
14284 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
14285 assert!(registry.get_module("vault").unwrap().is_none());
14286
14287 let wrong = handler
14289 .handle_control(
14290 ConnectionId::new(2),
14291 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
14292 )
14293 .unwrap();
14294 assert_eq!(wrong[0].header.ty, FrameType::Error);
14295 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14296 assert!(registry.get_module("vault").unwrap().is_none());
14297
14298 let ok = handler
14300 .handle_control(
14301 ConnectionId::new(3),
14302 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14303 )
14304 .unwrap();
14305 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14306 assert!(registry.get_module("vault").unwrap().is_some());
14307 }
14308
14309 #[test]
14310 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14311 let registry = Arc::new(Registry::default());
14312 let supervisor = SupervisorHandle::new();
14313 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14314 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14315 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14316
14317 let squat = handler
14318 .handle_control(
14319 ConnectionId::new(1),
14320 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14321 )
14322 .unwrap();
14323 assert_eq!(squat[0].header.ty, FrameType::Error);
14324 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14325 assert!(parse_error(&squat[0])["message"]
14326 .as_str()
14327 .unwrap()
14328 .contains("fed:"));
14329
14330 let accepted_peer = handler
14331 .handle_control(
14332 ConnectionId::new(2),
14333 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14334 )
14335 .unwrap();
14336 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14337
14338 let accepted_short = handler
14339 .handle_control(
14340 ConnectionId::new(3),
14341 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14342 )
14343 .unwrap();
14344 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14345
14346 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14347 let response = handler
14348 .handle_control(
14349 ConnectionId::new(conn),
14350 hello_frame(module_id, PROTOCOL_VERSION, conn),
14351 )
14352 .unwrap();
14353 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14354 }
14355 }
14356
14357 #[test]
14358 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14359 let registry = Arc::new(Registry::default());
14360 let supervisor = SupervisorHandle::new();
14361 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14362 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14363 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14364 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14365
14366 let owner_nonce = handler
14367 .handle_control(
14368 ConnectionId::new(1),
14369 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14370 )
14371 .unwrap();
14372 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14373 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14374 assert!(registry.get_module("fed:special").unwrap().is_none());
14375
14376 let exact_nonce = handler
14377 .handle_control(
14378 ConnectionId::new(2),
14379 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14380 )
14381 .unwrap();
14382 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14383 assert!(registry.get_module("fed:special").unwrap().is_some());
14384 }
14385
14386 #[test]
14387 fn non_reserved_module_ignores_launch_nonce() {
14388 let registry = Arc::new(Registry::default());
14389 let handler = ControlHandler::new(Arc::clone(®istry));
14392 let no_nonce = handler
14393 .handle_control(
14394 ConnectionId::new(1),
14395 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14396 )
14397 .unwrap();
14398 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14399 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14400
14401 let echoed_nonce = handler
14402 .handle_control(
14403 ConnectionId::new(2),
14404 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14405 )
14406 .unwrap();
14407 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14408 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14409 }
14410
14411 #[test]
14412 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14413 let handler = ControlHandler::default();
14414 let conn = ConnectionId::new(1);
14415 let malformed = Frame::build(
14416 FrameType::Hello,
14417 control_flags(),
14418 0,
14419 0,
14420 3,
14421 b"{not json".to_vec(),
14422 )
14423 .unwrap();
14424
14425 let error = handler.handle_control(conn, malformed).unwrap();
14426 assert_eq!(error[0].header.ty, FrameType::Error);
14427 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14428
14429 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14430 let pong = handler.handle_control(conn, ping).unwrap();
14431 assert_eq!(pong[0].header.ty, FrameType::Pong);
14432 assert_eq!(pong[0].header.corr, 4);
14433 }
14434
14435 #[test]
14436 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14437 let registry = Arc::new(Registry::default());
14438 let handler = ControlHandler::new(Arc::clone(®istry));
14439
14440 handler
14441 .handle_control(
14442 ConnectionId::new(1),
14443 hello_frame("aft", PROTOCOL_VERSION, 1),
14444 )
14445 .unwrap();
14446 let duplicate = handler
14447 .handle_control(
14448 ConnectionId::new(2),
14449 hello_frame("aft", PROTOCOL_VERSION, 2),
14450 )
14451 .unwrap();
14452
14453 assert_eq!(duplicate[0].header.ty, FrameType::Error);
14454 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14455 let registration = registry.get_module("aft").unwrap().unwrap();
14456 assert_eq!(registration.connection_id, ConnectionId::new(1));
14457 }
14458
14459 #[test]
14460 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14461 let registry = Arc::new(Registry::default());
14462 let forwarding = Arc::new(ForwardingTable::default());
14463 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14464 let handler =
14465 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14466 .with_process_liveness(process_liveness);
14467 let (ctx, route_channel, route_epoch) =
14468 bind_liveness_route(®istry, &forwarding, "aft-dead");
14469 let responses = handler
14470 .handle_route_poll(
14471 &ctx,
14472 route_poll_frame(41, PollKind::Liveness, route_channel),
14473 route_channel,
14474 route_epoch,
14475 PollKind::Liveness,
14476 )
14477 .unwrap();
14478
14479 assert_eq!(responses.len(), 1);
14480 assert_eq!(responses[0].header.ty, FrameType::Response);
14481 assert_route_poll_liveness(&responses[0], false);
14482 }
14483
14484 #[test]
14485 fn liveness_poll_without_process_source_uses_bound_route() {
14486 let registry = Arc::new(Registry::default());
14487 let forwarding = Arc::new(ForwardingTable::default());
14488 let handler =
14489 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14490 let (ctx, route_channel, route_epoch) =
14491 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
14492 let responses = handler
14493 .handle_route_poll(
14494 &ctx,
14495 route_poll_frame(42, PollKind::Liveness, route_channel),
14496 route_channel,
14497 route_epoch,
14498 PollKind::Liveness,
14499 )
14500 .unwrap();
14501
14502 assert_route_poll_liveness(&responses[0], true);
14503 }
14504
14505 #[test]
14506 fn liveness_poll_untracked_process_source_uses_bound_route() {
14507 let registry = Arc::new(Registry::default());
14508 let forwarding = Arc::new(ForwardingTable::default());
14509 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14510 let handler =
14511 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14512 .with_process_liveness(process_liveness);
14513 let (ctx, route_channel, route_epoch) =
14514 bind_liveness_route(®istry, &forwarding, "aft-untracked");
14515 let responses = handler
14516 .handle_route_poll(
14517 &ctx,
14518 route_poll_frame(43, PollKind::Liveness, route_channel),
14519 route_channel,
14520 route_epoch,
14521 PollKind::Liveness,
14522 )
14523 .unwrap();
14524
14525 assert_route_poll_liveness(&responses[0], true);
14526 }
14527
14528 #[tokio::test]
14529 async fn unknown_op_returns_unknown_control_op() {
14530 let handler = ControlHandler::default();
14531 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14532 let request = Frame::build(
14533 FrameType::Request,
14534 control_flags(),
14535 0,
14536 0,
14537 55,
14538 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14539 )
14540 .unwrap();
14541
14542 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14543
14544 assert_eq!(response.len(), 1);
14545 assert_eq!(response[0].header.ty, FrameType::Error);
14546 assert_eq!(response[0].header.corr, 55);
14547 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14548 }
14549
14550 #[tokio::test]
14551 async fn supervisor_provenance_rejects_unknown_exact_module() {
14552 let handler = ControlHandler::default();
14553 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14554 let request = Frame::build(
14555 FrameType::Request,
14556 control_flags(),
14557 0,
14558 0,
14559 57,
14560 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14561 )
14562 .unwrap();
14563
14564 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14565
14566 assert_eq!(response.len(), 1);
14567 assert_eq!(response[0].header.ty, FrameType::Error);
14568 assert_eq!(response[0].header.corr, 57);
14569 let error = parse_error(&response[0]);
14570 assert_eq!(error["code"], "unknown_module");
14571 assert_eq!(error["message"], "module_id 'missing' is not supervised");
14572 }
14573
14574 #[test]
14575 fn provenance_probe_override_keeps_handler_tests_deterministic() {
14576 let expected = subc_control::RunningImageAgreement::Unavailable {
14577 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14578 };
14579 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14580 assert_eq!(handler.provenance_probe_override, Some(expected));
14581 }
14582
14583 #[test]
14584 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14585 let verdict = reload_verdict(
14586 std::path::Path::new("/bin/new"),
14587 Some(std::path::Path::new("/bin/old")),
14588 subc_control::RunningImageAgreement::Unavailable {
14589 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14590 },
14591 );
14592 assert!(matches!(
14593 verdict.path,
14594 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14595 if configured == std::path::Path::new("/bin/new")
14596 && spawned_from == std::path::Path::new("/bin/old")
14597 ));
14598 }
14599
14600 #[test]
14601 fn launch_spec_without_running_digest_is_unavailable_not_match() {
14602 use subc_control::{
14603 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14604 };
14605 let configured = crate::supervise::ModuleSpec {
14606 module_id: "legacy".into(),
14607 program: "/module".into(),
14608 args: Vec::new(),
14609 env: Vec::new(),
14610 reserved: false,
14611 reserved_prefixes: Vec::new(),
14612 protocol: subc_control::ModuleProtocol::Subc,
14613 overlap: Default::default(),
14614 };
14615 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14616 assert_eq!(
14617 launch_spec_agreement(&configured, false, Some(&digest)),
14618 Agreement::Unavailable {
14619 reason: Reason::NotRunning
14620 }
14621 );
14622 assert_eq!(
14623 launch_spec_agreement(&configured, true, None),
14624 Agreement::Unavailable {
14625 reason: Reason::SpawnedSpecUnavailable
14626 }
14627 );
14628 assert_eq!(
14629 launch_spec_agreement(&configured, true, Some(&digest)),
14630 Agreement::Match
14631 );
14632 }
14633
14634 #[cfg(windows)]
14635 #[test]
14636 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14637 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14638 let configured = dir.join("module.exe");
14639 std::fs::write(&configured, b"executable path fixture").unwrap();
14640 let spawned = std::fs::canonicalize(&configured).unwrap();
14641 assert_ne!(
14642 configured, spawned,
14643 "fixture must exercise different path spellings"
14644 );
14645 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14646 let lower_case = configured.to_string_lossy().to_lowercase();
14647 for spelling in [
14648 &configured,
14649 Path::new(&forward_slashes),
14650 Path::new(&lower_case),
14651 ] {
14652 let verdict = reload_verdict(
14653 spelling,
14654 Some(&spawned),
14655 subc_control::RunningImageAgreement::Unavailable {
14656 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14657 },
14658 );
14659 assert_eq!(
14660 verdict.path,
14661 ReloadPathAgreement::Match,
14662 "configured {spelling:?}, spawned {spawned:?}"
14663 );
14664 }
14665 }
14666
14667 #[cfg(windows)]
14668 #[test]
14669 fn windows_reload_path_keeps_different_existing_program_pending() {
14670 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14671 let old = dir.join("old.exe");
14672 let new = dir.join("new.exe");
14673 std::fs::write(&old, b"same executable content").unwrap();
14674 std::fs::copy(&old, &new).unwrap();
14675 let spawned = std::fs::canonicalize(&old).unwrap();
14676 let image = subc_control::RunningImageAgreement::Unavailable {
14677 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14678 };
14679 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14680 assert!(
14681 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14682 if configured == new && spawned_from == spawned)
14683 );
14684 let missing = dir.join("missing.exe");
14685 assert!(matches!(
14686 reload_verdict(&missing, Some(&spawned), image).path,
14687 ReloadPathAgreement::Mismatch { .. }
14688 ));
14689 }
14690
14691 #[test]
14692 fn reload_verdict_detects_replaced_image_at_same_path() {
14693 let image = subc_control::RunningImageAgreement::Mismatch {
14694 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14695 digest: "old".into(),
14696 },
14697 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14698 digest: "new".into(),
14699 },
14700 };
14701 let verdict = reload_verdict(
14702 std::path::Path::new("/bin/same"),
14703 Some(std::path::Path::new("/bin/same")),
14704 image.clone(),
14705 );
14706 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14707 assert_eq!(verdict.image, image);
14708 }
14709
14710 #[test]
14711 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14712 let image = subc_control::RunningImageAgreement::Unavailable {
14713 reason: subc_control::RunningImageUnavailableReason::NotRunning,
14714 };
14715 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14716 assert_eq!(
14717 verdict.path,
14718 subc_control::ReloadPathAgreement::Unavailable {
14719 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14720 }
14721 );
14722 assert_eq!(verdict.image, image);
14723
14724 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14725 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14726 };
14727 let verdict = reload_verdict(
14728 std::path::Path::new("/bin/same"),
14729 Some(std::path::Path::new("/bin/same")),
14730 unconfirmed.clone(),
14731 );
14732 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14733 assert_eq!(verdict.image, unconfirmed);
14734 }
14735
14736 #[test]
14737 fn reload_verdict_preserves_each_image_unavailability_reason() {
14738 use subc_control::RunningImageUnavailableReason as Reason;
14739
14740 for reason in [
14741 Reason::NotRunning,
14742 Reason::UnsupportedPlatform,
14743 Reason::RunningExecutableUnreadable,
14744 Reason::SpawnedPathUnreadable,
14745 Reason::HashFailed,
14746 Reason::ProcessIdentityUnconfirmed,
14747 Reason::Unknown("future_probe_reason".to_string()),
14748 ] {
14749 let image = subc_control::RunningImageAgreement::Unavailable {
14750 reason: reason.clone(),
14751 };
14752 let verdict = reload_verdict(
14753 std::path::Path::new("/bin/same"),
14754 Some(std::path::Path::new("/bin/same")),
14755 image.clone(),
14756 );
14757 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14758 assert_eq!(verdict.image, image, "{reason:?}");
14759 }
14760 }
14761
14762 #[tokio::test]
14763 async fn malformed_control_bodies_return_invalid_control_body() {
14764 let handler = ControlHandler::default();
14765 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14766
14767 for (corr, body) in [
14768 (56, br#"{"route_channel":1}"#.as_slice()),
14769 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14770 (
14771 58,
14772 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14773 ),
14774 ] {
14775 let request = Frame::build(
14776 FrameType::Request,
14777 control_flags(),
14778 0,
14779 0,
14780 corr,
14781 body.to_vec(),
14782 )
14783 .unwrap();
14784 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14785
14786 assert_eq!(response.len(), 1);
14787 assert_eq!(response[0].header.ty, FrameType::Error);
14788 assert_eq!(response[0].header.corr, corr);
14789 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14790 }
14791 }
14792
14793 #[tokio::test]
14794 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14795 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14796 let registry = Arc::new(Registry::default());
14797 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14798 let router = Router::with_control_handler(Arc::clone(&control));
14799 let connection = router.begin_connection();
14800 let (ctx, mut rx) = route_ctx(connection.id());
14801
14802 router
14803 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14804 .await
14805 .unwrap();
14806 let response = rx.recv().await.unwrap();
14807 let ack = parse_ack(&response);
14808 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14809 let channel = 1;
14810
14811 let goodbye =
14812 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14813 router.route_for_connection(&ctx, goodbye).await.unwrap();
14814 assert!(rx.try_recv().is_err());
14815 assert!(registry.get_module("aft").unwrap().is_none());
14816
14817 router
14818 .route_for_connection(&ctx, channel_request(channel, 13))
14819 .await
14820 .unwrap();
14821 let error_frame = rx.recv().await.unwrap();
14822 assert_eq!(error_frame.header.ty, FrameType::Error);
14823 assert_eq!(error_frame.header.channel, channel);
14824 let captured = crate::router::test_log::captured_logs(&logs);
14825 assert_eq!(
14826 captured
14827 .lines()
14828 .filter(|line| {
14829 line.contains("module_id=aft")
14830 && line.contains("reason=explicit_goodbye")
14831 && line.contains("module registration ended")
14832 })
14833 .count(),
14834 1,
14835 "unexpected GOODBYE registry log: {captured}"
14836 );
14837 }
14838
14839 #[tokio::test]
14840 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14841 let registry = Arc::new(Registry::default());
14842 let handler = ControlHandler::new(registry).with_capability_config(
14843 [("prov".to_string(), true), ("cons".to_string(), true)],
14844 BTreeMap::new(),
14845 );
14846 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14847 register_capability_manifest(
14848 &handler,
14849 &provider_ctx,
14850 &mut provider_rx,
14851 capability_manifest("prov", &["thing/v1"], &[]),
14852 1,
14853 )
14854 .await;
14855 let mut consumer = capability_manifest("cons", &[], &[]);
14856 consumer.capabilities.as_mut().unwrap().requires.push(
14857 subc_protocol::manifest::CapabilityRequirement {
14858 capability: "thing/v1".to_string(),
14859 need: subc_protocol::manifest::CapabilityNeed::Required,
14860 },
14861 );
14862 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14863 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14864 assert_eq!(
14865 handler.capability_evaluator.verdict("cons", "thing/v1"),
14866 Some(CapabilityVerdict::Provided)
14867 );
14868 let (mut client_rx, _) = open_route_for_capability_test(
14869 &handler,
14870 &provider_ctx,
14871 &mut provider_rx,
14872 703,
14873 3,
14874 "prov",
14875 None,
14876 )
14877 .await;
14878 let goodbye =
14879 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14880 handler
14881 .handle_control_frame(&provider_ctx, goodbye)
14882 .await
14883 .unwrap();
14884 assert_eq!(
14885 handler.capability_evaluator.verdict("cons", "thing/v1"),
14886 Some(CapabilityVerdict::NeverProvided)
14887 );
14888 let closed = client_rx
14889 .try_recv()
14890 .expect("GOODBYE pushes route.closed before route GOODBYE");
14891 assert!(
14892 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14893 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14894 );
14895 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14896 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14897 }
14898
14899 struct RouteClosedFixture {
14900 handler: ControlHandler,
14901 #[cfg(unix)]
14902 supervisor: Supervisor,
14903 module_ctx: RouteCtx,
14904 #[cfg(unix)]
14905 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14906 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14907 route: Arc<crate::forwarding::RouteBinding>,
14908 }
14909
14910 async fn route_closed_fixture() -> RouteClosedFixture {
14911 let registry = Arc::new(Registry::default());
14912 let handler = ControlHandler::new(Arc::clone(®istry));
14913 #[cfg(unix)]
14914 let supervisor = Supervisor::new(registry, RestartPolicy::default())
14915 .with_forwarding(Arc::clone(&handler.forwarding));
14916 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14917 register_capability_manifest(
14918 &handler,
14919 &module_ctx,
14920 &mut module_rx,
14921 capability_manifest("prov", &[], &[]),
14922 1,
14923 )
14924 .await;
14925 let (client_rx, bind) = open_route_for_capability_test(
14926 &handler,
14927 &module_ctx,
14928 &mut module_rx,
14929 702,
14930 2,
14931 "prov",
14932 None,
14933 )
14934 .await;
14935 let ModuleControlRequest::RouteBind {
14936 route_channel: channel,
14937 epoch: route_epoch,
14938 ..
14939 } = bind
14940 else {
14941 panic!("expected route.bind");
14942 };
14943 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14944 handler
14945 .forwarding
14946 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14947 .unwrap()
14948 else {
14949 panic!("the acknowledged route must be bound");
14950 };
14951 RouteClosedFixture {
14952 handler,
14953 #[cfg(unix)]
14954 supervisor,
14955 module_ctx,
14956 #[cfg(unix)]
14957 module_rx,
14958 client_rx,
14959 route,
14960 }
14961 }
14962
14963 impl RouteClosedFixture {
14964 #[cfg(unix)]
14965 async fn cut(&mut self) {
14966 let notices = async {
14967 let mut draining = self.module_rx.recv().await.unwrap();
14968 assert!(matches!(
14969 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14970 &draining.body
14971 )
14972 .unwrap(),
14973 subc_protocol::session::ModuleControlCommand::Draining {
14974 reason: RouteCloseReason::Restart,
14975 ..
14976 }
14977 ));
14978 draining.flushed.take().unwrap().send(()).unwrap();
14979 let mut closing = self.client_rx.recv().await.unwrap();
14980 assert!(matches!(
14981 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14982 ClientControlPush::RouteClosing {
14983 reason: RouteCloseReason::Restart,
14984 ..
14985 }
14986 ));
14987 closing.flushed.take().unwrap().send(()).unwrap();
14988 };
14989 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14990 result.unwrap();
14991 }
14992
14993 fn disconnect(&mut self) -> ClientControlPush {
14994 self.handler
14995 .cleanup_connection(self.module_ctx.connection_id)
14996 .unwrap();
14997 let closed = self.client_rx.try_recv().unwrap();
14998 assert_eq!(closed.header.ty, FrameType::Push);
14999 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
15000 assert!(matches!(
15001 &push,
15002 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
15003 if channels == &[self.route.client_channel]
15004 ));
15005 assert_eq!(
15006 self.client_rx.try_recv().unwrap().header.ty,
15007 FrameType::Goodbye
15008 );
15009 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
15010 push
15011 }
15012 }
15013
15014 #[cfg(unix)]
15015 #[tokio::test(start_paused = true)]
15016 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
15017 let mut fixture = route_closed_fixture().await;
15018 fixture.cut().await;
15019 assert!(matches!(
15020 fixture.disconnect(),
15021 ClientControlPush::RouteClosed {
15022 reason: RouteCloseReason::Restart,
15023 drained: true,
15024 ..
15025 }
15026 ));
15027 }
15028
15029 #[cfg(unix)]
15030 #[tokio::test(start_paused = true)]
15031 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
15032 let mut fixture = route_closed_fixture().await;
15033 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
15034 let started = tokio::time::Instant::now();
15035 fixture.cut().await;
15036 assert!(started.elapsed() >= Duration::from_secs(2));
15037 assert_eq!(fixture.route.flow.in_flight(), 1);
15038 assert!(matches!(
15039 fixture.disconnect(),
15040 ClientControlPush::RouteClosed {
15041 reason: RouteCloseReason::Restart,
15042 drained: false,
15043 ..
15044 }
15045 ));
15046 }
15047
15048 #[tokio::test]
15049 async fn module_connection_crash_reports_crash_and_not_drained() {
15050 let mut fixture = route_closed_fixture().await;
15051 assert!(matches!(
15055 fixture.disconnect(),
15056 ClientControlPush::RouteClosed {
15057 reason: RouteCloseReason::Crash,
15058 drained: false,
15059 ..
15060 }
15061 ));
15062 }
15063
15064 #[tokio::test]
15065 async fn dropping_router_connection_releases_registration() {
15066 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15067 let registry = Arc::new(Registry::default());
15068 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15069 let router = Router::with_control_handler(Arc::clone(&control));
15070 let connection = router.begin_connection();
15071 let connection_id = connection.id();
15072 let (ctx, mut rx) = route_ctx(connection_id);
15073
15074 router
15075 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
15076 .await
15077 .unwrap();
15078 let response = rx.recv().await.unwrap();
15079 let ack = parse_ack(&response);
15080 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15081 assert!(registry.get_module("aft").unwrap().is_some());
15082
15083 drop(connection);
15084
15085 assert!(registry.get_module("aft").unwrap().is_none());
15086 assert_eq!(registry.active_registration_count().unwrap(), 0);
15087
15088 control
15089 .cleanup_connection(ConnectionId::new(u64::MAX))
15090 .unwrap();
15091 let captured = crate::router::test_log::captured_logs(&logs);
15092 println!("captured registry lifecycle logs:\n{captured}");
15093 let events: Vec<_> = captured
15094 .lines()
15095 .filter(|line| {
15096 line.contains("module registered module_id=aft ")
15097 || line.contains("module registration ended")
15098 })
15099 .collect();
15100 assert_eq!(
15101 events.len(),
15102 2,
15103 "unexpected registry lifecycle logs: {captured}"
15104 );
15105 assert!(events[0].contains("module registered module_id=aft "));
15106 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
15107 assert!(events[1].contains(&format!(
15108 "module_id=aft connection_id={}",
15109 connection_id.get()
15110 )));
15111 assert!(events[1].contains("reason=connection_closed"));
15112 assert!(events[1].contains("module registration ended"));
15113 assert_eq!(
15114 captured
15115 .lines()
15116 .filter(|line| line.contains("module registered module_id=aft "))
15117 .count(),
15118 1,
15119 "legacy registration admission line must appear once: {captured}"
15120 );
15121 }
15122
15123 #[tokio::test]
15124 async fn hello_registration_keeps_legacy_module_registered_line_once() {
15125 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15126 let registry = Arc::new(Registry::default());
15127 let control = ControlHandler::new(Arc::clone(®istry));
15128 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
15129 hello_via_sink(
15130 &control,
15131 &ctx,
15132 &mut rx,
15133 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
15134 )
15135 .await;
15136
15137 let captured = crate::router::test_log::captured_logs(&logs);
15138 let admissions: Vec<_> = captured
15139 .lines()
15140 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
15141 .collect();
15142 assert_eq!(
15143 admissions.len(),
15144 1,
15145 "legacy admission line must remain exactly once: {captured}"
15146 );
15147 assert!(admissions[0].contains("routable_provider=true"));
15148 assert!(admissions[0].contains("connection_id=777"));
15149 }
15150
15151 fn capability_manifest(
15152 module_id: &str,
15153 provides: &[&str],
15154 must_never_reach: &[&str],
15155 ) -> ModuleManifest {
15156 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
15157 manifest.capabilities = Some(CapabilityDeclarations {
15158 provides: provides
15159 .iter()
15160 .map(|capability| (*capability).to_string())
15161 .collect(),
15162 requires: Vec::new(),
15163 must_never_reach: must_never_reach
15164 .iter()
15165 .map(|capability| (*capability).to_string())
15166 .collect(),
15167 });
15168 manifest
15169 }
15170
15171 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
15172 Frame::build(
15173 FrameType::Hello,
15174 control_flags(),
15175 0,
15176 0,
15177 corr,
15178 serde_json::to_vec(&ModuleHelloBody {
15179 protocol_ver: manifest.protocol_ver,
15180 manifest,
15181 control_ops: None,
15182 launch_nonce: None,
15183 })
15184 .expect("capability test HELLO serializes"),
15185 )
15186 .expect("capability test HELLO frame builds")
15187 }
15188
15189 fn catalog_update_with_capabilities_frame(
15190 corr: u64,
15191 capabilities: CapabilityDeclarations,
15192 ) -> Frame {
15193 Frame::build(
15194 FrameType::Request,
15195 control_flags(),
15196 0,
15197 0,
15198 corr,
15199 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15200 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
15201 capabilities: Some(capabilities),
15202 ready: None,
15203 })
15204 .expect("capability catalog.update serializes"),
15205 )
15206 .expect("capability catalog.update frame builds")
15207 }
15208
15209 async fn register_capability_manifest(
15210 handler: &ControlHandler,
15211 ctx: &RouteCtx,
15212 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15213 manifest: ModuleManifest,
15214 corr: u64,
15215 ) {
15216 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
15217 }
15218
15219 async fn open_route_for_capability_test(
15220 handler: &ControlHandler,
15221 target_ctx: &RouteCtx,
15222 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15223 client_connection_id: u64,
15224 corr: u64,
15225 target_module_id: &str,
15226 consumer_identity: Option<ConsumerIdentity>,
15227 ) -> (
15228 mpsc::Receiver<crate::router::OutboundFrame>,
15229 ModuleControlRequest,
15230 ) {
15231 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
15232 let route_handler = handler.clone();
15233 let target_module_id = target_module_id.to_string();
15234 let route_task = tokio::spawn(async move {
15235 route_handler
15236 .handle_control_frame(
15237 &client_ctx,
15238 route_open_frame_with_admission_facts(
15239 corr,
15240 &target_module_id,
15241 unique_project_root("admission-facts"),
15242 consumer_identity,
15243 None,
15244 ),
15245 )
15246 .await
15247 .expect("capability test route.open succeeds")
15248 });
15249 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
15250 .await
15251 .expect("capability test route.open must reach route.bind")
15252 .expect("target control receiver stays open");
15253 let bind_request: ModuleControlRequest =
15254 serde_json::from_slice(&bind.body).expect("route.bind decodes");
15255 handler
15256 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
15257 .await
15258 .expect("capability test route.bind ACK succeeds");
15259 assert!(route_task.await.expect("route.open task joins").is_empty());
15260 let opened = client_rx
15261 .recv()
15262 .await
15263 .expect("successful route.open publishes a response");
15264 assert!(matches!(
15265 serde_json::from_slice::<ClientControlResponse>(&opened.body),
15266 Ok(ClientControlResponse::RouteOpen { .. })
15267 ));
15268 (client_rx, bind_request)
15269 }
15270
15271 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
15272 assert_eq!(frame.header.ty, FrameType::Push);
15273 assert_eq!(frame.header.channel, 0);
15274 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
15275 .expect("route.closed control push decodes");
15276 let ClientControlPush::RouteClosed { channels, .. } = &push else {
15277 panic!("expected route.closed");
15278 };
15279 assert_eq!(channels.len(), 1, "exactly one violating route closed");
15280 let closed_channel = channels[0];
15281 let channels = channels.clone();
15282 assert_eq!(
15283 push,
15284 ClientControlPush::RouteClosed {
15285 module_id: target_module_id.to_string(),
15286 channels,
15287 reason: RouteCloseReason::CapabilityDenied,
15288 drained: false,
15289 abandoned: 0,
15290 excluded_subscriptions: 0,
15291 terminal: Some(false),
15292 }
15293 );
15294 closed_channel
15295 }
15296
15297 #[tokio::test]
15298 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15299 let registry = Arc::new(Registry::default());
15300 let forwarding = Arc::new(ForwardingTable::default());
15301 let supervisor = SupervisorHandle::new();
15302 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15303 let handler =
15304 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15305 .with_supervisor(supervisor);
15306 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15307 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15308 register_capability_manifest(
15309 &handler,
15310 &target_ctx,
15311 &mut target_rx,
15312 capability_manifest("target", &["credentials-provider/v1"], &[]),
15313 1,
15314 )
15315 .await;
15316 register_capability_manifest(
15317 &handler,
15318 &opener_ctx,
15319 &mut opener_rx,
15320 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15321 2,
15322 )
15323 .await;
15324
15325 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15326 let replies = handler
15327 .handle_control_frame(
15328 &client_ctx,
15329 route_open_frame_with_admission_facts(
15330 3,
15331 "target",
15332 unique_project_root("admission-facts"),
15333 Some(ConsumerIdentity {
15334 module_id: "opener".to_string(),
15335 launch_nonce: "opener-nonce".to_string(),
15336 }),
15337 None,
15338 ),
15339 )
15340 .await
15341 .expect("denied route.open returns a typed frame");
15342 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15343 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15344 assert!(
15345 target_rx.try_recv().is_err(),
15346 "forbidden route.open must not relay route.bind"
15347 );
15348 }
15349
15350 #[tokio::test]
15351 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15352 let registry = Arc::new(Registry::default());
15353 let forwarding = Arc::new(ForwardingTable::default());
15354 let supervisor = SupervisorHandle::new();
15355 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15356 let handler =
15357 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15358 .with_supervisor(supervisor);
15359 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15360 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15361 register_capability_manifest(
15362 &handler,
15363 &target_ctx,
15364 &mut target_rx,
15365 capability_manifest("target", &["credentials-provider/v1"], &[]),
15366 1,
15367 )
15368 .await;
15369 register_capability_manifest(
15370 &handler,
15371 &old_opener_ctx,
15372 &mut old_opener_rx,
15373 capability_manifest("opener", &[], &[]),
15374 2,
15375 )
15376 .await;
15377 let (mut client_rx, _) = open_route_for_capability_test(
15378 &handler,
15379 &target_ctx,
15380 &mut target_rx,
15381 712,
15382 3,
15383 "target",
15384 Some(ConsumerIdentity {
15385 module_id: "opener".to_string(),
15386 launch_nonce: "opener-nonce".to_string(),
15387 }),
15388 )
15389 .await;
15390 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15391
15392 handler
15393 .cleanup_connection(old_opener_ctx.connection_id)
15394 .expect("old opener registration cleans up");
15395 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15396 register_capability_manifest(
15397 &handler,
15398 &new_opener_ctx,
15399 &mut new_opener_rx,
15400 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15401 4,
15402 )
15403 .await;
15404
15405 let closed_channel = assert_capability_denied_push(
15406 client_rx
15407 .try_recv()
15408 .expect("HELLO deny addition must emit route.closed")
15409 .frame,
15410 "target",
15411 );
15412 let goodbye = client_rx
15413 .try_recv()
15414 .expect("HELLO deny addition must send the client a route GOODBYE")
15415 .frame;
15416 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15417 assert_eq!(goodbye.header.channel, closed_channel);
15418 assert_ne!(goodbye.header.channel, 0);
15419 assert_ne!(goodbye.header.epoch, 0);
15420 assert!(client_rx.try_recv().is_err());
15421 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15422 assert!(matches!(
15423 target_rx.try_recv(),
15424 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15425 ));
15426 }
15427
15428 #[tokio::test]
15429 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15430 let registry = Arc::new(Registry::default());
15431 let forwarding = Arc::new(ForwardingTable::default());
15432 let supervisor = SupervisorHandle::new();
15433 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15434 let handler =
15435 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15436 .with_supervisor(supervisor);
15437 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15438 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15439 register_capability_manifest(
15440 &handler,
15441 &target_ctx,
15442 &mut target_rx,
15443 capability_manifest("target", &[], &[]),
15444 1,
15445 )
15446 .await;
15447 register_capability_manifest(
15448 &handler,
15449 &opener_ctx,
15450 &mut opener_rx,
15451 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15452 2,
15453 )
15454 .await;
15455 let (mut client_rx, _) = open_route_for_capability_test(
15456 &handler,
15457 &target_ctx,
15458 &mut target_rx,
15459 722,
15460 3,
15461 "target",
15462 Some(ConsumerIdentity {
15463 module_id: "opener".to_string(),
15464 launch_nonce: "opener-nonce".to_string(),
15465 }),
15466 )
15467 .await;
15468 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15469
15470 let replies = handler
15471 .handle_control_frame(
15472 &target_ctx,
15473 catalog_update_with_capabilities_frame(
15474 4,
15475 CapabilityDeclarations {
15476 provides: vec!["credentials-provider/v1".to_string()],
15477 requires: Vec::new(),
15478 must_never_reach: Vec::new(),
15479 },
15480 ),
15481 )
15482 .await
15483 .expect("claim catalog.update succeeds");
15484 assert!(matches!(
15485 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15486 Ok(ModuleControlResponseToModule::CatalogUpdate {})
15487 ));
15488 let closed_channel = assert_capability_denied_push(
15489 client_rx
15490 .try_recv()
15491 .expect("claim addition must emit route.closed")
15492 .frame,
15493 "target",
15494 );
15495 let goodbye = client_rx
15496 .try_recv()
15497 .expect("claim addition must send the client a route GOODBYE")
15498 .frame;
15499 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15500 assert_eq!(goodbye.header.channel, closed_channel);
15501 assert_ne!(goodbye.header.channel, 0);
15502 assert_ne!(goodbye.header.epoch, 0);
15503 assert!(client_rx.try_recv().is_err());
15504 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15505 assert!(matches!(
15506 target_rx.try_recv(),
15507 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15508 ));
15509 }
15510
15511 #[tokio::test]
15512 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15513 let registry = Arc::new(Registry::default());
15514 let forwarding = Arc::new(ForwardingTable::default());
15515 let supervisor = SupervisorHandle::new();
15516 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15517 let handler =
15518 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15519 .with_supervisor(supervisor);
15520 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15521 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15522 register_capability_manifest(
15523 &handler,
15524 &target_ctx,
15525 &mut target_rx,
15526 capability_manifest("target", &["credentials-provider/v1"], &[]),
15527 1,
15528 )
15529 .await;
15530 register_capability_manifest(
15531 &handler,
15532 &opener_ctx,
15533 &mut opener_rx,
15534 capability_manifest("opener", &[], &[]),
15535 2,
15536 )
15537 .await;
15538 let (mut client_rx, _) = open_route_for_capability_test(
15539 &handler,
15540 &target_ctx,
15541 &mut target_rx,
15542 732,
15543 3,
15544 "target",
15545 Some(ConsumerIdentity {
15546 module_id: "opener".to_string(),
15547 launch_nonce: "opener-nonce".to_string(),
15548 }),
15549 )
15550 .await;
15551 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15552
15553 handler
15554 .handle_control_frame(
15555 &target_ctx,
15556 catalog_update_with_capabilities_frame(
15557 4,
15558 CapabilityDeclarations {
15559 provides: Vec::new(),
15560 requires: Vec::new(),
15561 must_never_reach: Vec::new(),
15562 },
15563 ),
15564 )
15565 .await
15566 .expect("claim removal catalog.update succeeds");
15567 assert_eq!(
15568 forwarding.active_binding_count().unwrap(),
15569 1,
15570 "removing an attested target claim must leave the route census unchanged"
15571 );
15572 assert!(
15573 client_rx.try_recv().is_err(),
15574 "claim removal must not emit route.closed or a client route GOODBYE"
15575 );
15576 assert!(
15577 target_rx.try_recv().is_err(),
15578 "claim removal must not send the target a route GOODBYE"
15579 );
15580 }
15581
15582 #[tokio::test]
15585 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15586 let registry = Arc::new(Registry::default());
15587 let forwarding = Arc::new(ForwardingTable::default());
15588 let supervisor = SupervisorHandle::new();
15589 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15590 let handler =
15591 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15592 .with_supervisor(supervisor);
15593 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15594 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15595 register_capability_manifest(
15596 &handler,
15597 &target_ctx,
15598 &mut target_rx,
15599 capability_manifest("target", &["credentials-provider/v1"], &[]),
15600 1,
15601 )
15602 .await;
15603 register_capability_manifest(
15604 &handler,
15605 &opener_ctx,
15606 &mut opener_rx,
15607 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15608 2,
15609 )
15610 .await;
15611
15612 let (_client_rx, bind) = open_route_for_capability_test(
15613 &handler,
15614 &target_ctx,
15615 &mut target_rx,
15616 742,
15617 3,
15618 "target",
15619 None,
15620 )
15621 .await;
15622 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15623 panic!("direct scope-honesty route must bind");
15624 };
15625 assert_eq!(principal, Some(Principal::Direct));
15626 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15627 }
15628
15629 #[tokio::test]
15632 async fn must_never_reach_self_route_is_capability_forbidden() {
15633 let registry = Arc::new(Registry::default());
15634 let forwarding = Arc::new(ForwardingTable::default());
15635 let supervisor = SupervisorHandle::new();
15636 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15637 let handler =
15638 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15639 .with_supervisor(supervisor);
15640 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15641 register_capability_manifest(
15642 &handler,
15643 &self_ctx,
15644 &mut self_rx,
15645 capability_manifest(
15646 "self-provider",
15647 &["credentials-provider/v1"],
15648 &["credentials-provider/v1"],
15649 ),
15650 1,
15651 )
15652 .await;
15653
15654 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15655 let replies = handler
15656 .handle_control_frame(
15657 &client_ctx,
15658 route_open_frame_with_admission_facts(
15659 2,
15660 "self-provider",
15661 unique_project_root("admission-facts"),
15662 Some(ConsumerIdentity {
15663 module_id: "self-provider".to_string(),
15664 launch_nonce: "self-nonce".to_string(),
15665 }),
15666 None,
15667 ),
15668 )
15669 .await
15670 .expect("self-route refusal returns a typed frame");
15671 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15672 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15673 assert!(
15674 self_rx.try_recv().is_err(),
15675 "self denial must not relay route.bind"
15676 );
15677 }
15678
15679 #[test]
15680 fn unsupported_channel_zero_frame_returns_error() {
15681 let handler = ControlHandler::default();
15682 let request = Frame::build(
15683 FrameType::Request,
15684 control_flags(),
15685 0,
15686 0,
15687 21,
15688 b"opaque".to_vec(),
15689 )
15690 .unwrap();
15691
15692 let response = handler
15693 .handle_control(ConnectionId::new(1), request)
15694 .unwrap();
15695
15696 assert_eq!(response[0].header.ty, FrameType::Error);
15697 assert_eq!(
15698 parse_error(&response[0])["code"],
15699 "unsupported_control_frame"
15700 );
15701 }
15702
15703 mod swap {
15708 use super::*;
15709
15710 const INCUMBENT: ConnectionId = ConnectionId::new(30);
15711 const CANDIDATE: ConnectionId = ConnectionId::new(40);
15712
15713 struct Swap {
15714 registry: Arc<Registry>,
15715 forwarding: Arc<ForwardingTable>,
15716 handler: ControlHandler,
15717 incumbent_ctx: RouteCtx,
15718 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15719 candidate_ctx: RouteCtx,
15720 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15721 }
15722
15723 async fn swap_with_incumbent() -> Swap {
15724 let registry = Arc::new(Registry::default());
15725 let forwarding = Arc::new(ForwardingTable::default());
15726 let handler =
15727 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15728 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15729 hello_via_sink(
15730 &handler,
15731 &incumbent_ctx,
15732 &mut incumbent_rx,
15733 hello_frame("aft", PROTOCOL_VERSION, 7),
15734 )
15735 .await;
15736 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15737 Swap {
15738 registry,
15739 forwarding,
15740 handler,
15741 incumbent_ctx,
15742 incumbent_rx,
15743 candidate_ctx,
15744 candidate_rx,
15745 }
15746 }
15747
15748 fn register_candidate(swap: &Swap, ready: Option<bool>) {
15749 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15750 candidate_manifest.ready = ready;
15751 let registration = swap
15752 .registry
15753 .register_candidate_with_control_ops(
15754 candidate_manifest,
15755 PROTOCOL_VERSION,
15756 CANDIDATE,
15757 module_baseline_control_ops(),
15758 )
15759 .unwrap();
15760 swap.forwarding
15761 .register_candidate_module_connection(
15762 CANDIDATE,
15763 "aft".to_string(),
15764 PROTOCOL_VERSION,
15765 manifest_concurrency(®istration.manifest),
15766 swap.candidate_ctx.egress.clone(),
15767 )
15768 .unwrap();
15769 }
15770
15771 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15772 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15773 swap.registry.promote_candidate("aft").unwrap().unwrap();
15774 cutover.incumbent.unwrap()
15775 }
15776
15777 #[tokio::test]
15786 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15787 let mut swap = swap_with_incumbent().await;
15788 let hold = Arc::new(std::sync::Barrier::new(2));
15789 let handler = swap
15790 .handler
15791 .clone()
15792 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15793 handler
15794 .supervisor
15795 .open_swap("aft", "candidate-nonce".to_string());
15796 let hello = {
15797 let handler = handler.clone();
15798 let candidate_ctx = swap.candidate_ctx.clone();
15799 std::thread::spawn(move || {
15800 handler
15801 .handle_hello(
15802 candidate_ctx.connection_id,
15803 Some(candidate_ctx.egress.clone()),
15804 hello_frame_with_nonce(
15805 "aft",
15806 PROTOCOL_VERSION,
15807 8,
15808 Some("candidate-nonce"),
15809 ),
15810 )
15811 .unwrap()
15812 })
15813 };
15814
15815 hold.wait();
15816 assert!(swap
15819 .registry
15820 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15821 .unwrap()
15822 .is_some());
15823 assert!(swap
15824 .registry
15825 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15826 .unwrap()
15827 .is_none());
15828 assert!(swap
15829 .forwarding
15830 .module_endpoint_for_connection(CANDIDATE)
15831 .unwrap()
15832 .is_none());
15833
15834 hold.wait();
15835 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15836 assert_eq!(
15837 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15838 FrameType::HelloAck
15839 );
15840 let published = swap
15841 .registry
15842 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15843 .unwrap()
15844 .expect("candidate is published once its HELLO completes");
15845 assert_eq!(published.connection_id, CANDIDATE);
15846 assert!(swap
15847 .forwarding
15848 .module_endpoint_for_connection(CANDIDATE)
15849 .unwrap()
15850 .is_some());
15851 }
15852
15853 fn keyed_total(counters: &Value, key: &str) -> u64 {
15854 counters[key]
15855 .as_object()
15856 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15857 .unwrap_or(0)
15858 }
15859
15860 #[tokio::test]
15861 async fn replacement_logs_old_end_and_new_admission_once() {
15862 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15863 let mut swap = swap_with_incumbent().await;
15864 swap.handler
15865 .supervisor
15866 .open_swap("aft", "candidate-nonce".to_string());
15867 hello_via_sink(
15868 &swap.handler,
15869 &swap.candidate_ctx,
15870 &mut swap.candidate_rx,
15871 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15872 )
15873 .await;
15874 cutover(&swap);
15875 swap.handler.cleanup_connection(INCUMBENT).unwrap();
15876
15877 let captured = crate::router::test_log::captured_logs(&logs);
15878 println!("captured replacement registry logs:\n{captured}");
15879 let new_admissions: Vec<_> = captured
15880 .lines()
15881 .filter(|line| {
15882 line.contains("connection_id=40")
15883 && line.contains("swap candidate registered; not routable until cutover")
15884 })
15885 .collect();
15886 assert_eq!(
15887 new_admissions.len(),
15888 1,
15889 "unexpected admission logs: {captured}"
15890 );
15891 assert!(new_admissions[0].contains("module_id=aft"));
15892 assert!(new_admissions[0].contains("connection_id=40"));
15893 assert!(
15894 new_admissions[0].contains("swap candidate registered; not routable until cutover")
15895 );
15896 assert!(new_admissions[0].contains("ready=true"));
15897
15898 let old_admissions: Vec<_> = captured
15899 .lines()
15900 .filter(|line| {
15901 line.contains("module registered module_id=aft ")
15902 && line.contains("connection_id=30")
15903 })
15904 .collect();
15905 assert_eq!(
15906 old_admissions.len(),
15907 1,
15908 "unexpected incumbent admission: {captured}"
15909 );
15910
15911 let old_ends: Vec<_> = captured
15912 .lines()
15913 .filter(|line| {
15914 line.contains("connection_id=30") && line.contains("module registration ended")
15915 })
15916 .collect();
15917 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15918 assert!(old_ends[0].contains("module_id=aft"));
15919 assert!(old_ends[0].contains("connection_id=30"));
15920 assert!(old_ends[0].contains("reason=replaced"));
15921 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15922 assert_eq!(
15923 captured
15924 .lines()
15925 .filter(|line| line.contains("module registration promoted"))
15926 .count(),
15927 1,
15928 "unexpected promotion logs: {captured}"
15929 );
15930 }
15931
15932 #[tokio::test]
15938 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15939 let mut swap = swap_with_incumbent().await;
15940 let handler = swap.handler.clone();
15941
15942 let cotenant = ConnectionId::new(31);
15944 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15945 let (cotenant_task, cotenant_bind) = relay_route_open(
15946 &handler,
15947 cotenant,
15948 &cotenant_ctx.egress,
15949 &mut swap.incumbent_rx,
15950 100,
15951 "aft",
15952 "swap-cotenant",
15953 )
15954 .await;
15955 handler
15956 .handle_control_frame(
15957 &swap.incumbent_ctx,
15958 route_bind_ack(cotenant_bind.header.corr),
15959 )
15960 .await
15961 .unwrap();
15962 assert!(cotenant_task.await.unwrap().is_empty());
15963 let (cotenant_channel, cotenant_epoch) =
15964 published_route(&cotenant_rx.recv().await.unwrap());
15965
15966 let caller = ConnectionId::new(32);
15968 let (caller_ctx, mut caller_rx) = route_ctx(caller);
15969 let (caller_task, caller_bind) = relay_route_open(
15970 &handler,
15971 caller,
15972 &caller_ctx.egress,
15973 &mut swap.incumbent_rx,
15974 101,
15975 "aft",
15976 "swap-caller",
15977 )
15978 .await;
15979 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15980
15981 register_candidate(&swap, None);
15982 cutover(&swap);
15983
15984 let ack = handler
15986 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15987 .await;
15988 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15989 if module_loop_error.is_some() {
15990 handler.cleanup_connection(INCUMBENT).unwrap();
15993 }
15994
15995 assert!(
15997 cotenant_rx.try_recv().is_err(),
15998 "the co-tenant route on the incumbent was torn down by one late ack: \
15999 {module_loop_error:?}"
16000 );
16001 assert!(matches!(
16002 swap.forwarding
16003 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16004 .unwrap(),
16005 DataRoute::Client(DataRouteState::Bound(_))
16006 ));
16007 assert_eq!(module_loop_error, None);
16008 assert!(swap
16009 .registry
16010 .get_module_by_connection(INCUMBENT)
16011 .unwrap()
16012 .is_some());
16013
16014 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
16016 .await
16017 .expect("the incumbent is told to drop the abandoned binding")
16018 .unwrap()
16019 .frame;
16020 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16021 assert_eq!(goodbye.header.channel, abandoned_channel);
16022 assert_eq!(goodbye.header.epoch, abandoned_epoch);
16023 assert!(swap.incumbent_rx.try_recv().is_err());
16024
16025 let response = caller_task.await.unwrap();
16027 assert_eq!(response.len(), 1);
16028 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
16029 assert!(caller_rx.try_recv().is_err());
16030
16031 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
16035 let counters = handler.counters().snapshot();
16036 assert_eq!(
16037 keyed_total(&counters, "route_open_accepted_by_principal"),
16038 1
16039 );
16040 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
16041 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
16042 }
16043
16044 #[tokio::test]
16048 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
16049 let mut swap = swap_with_incumbent().await;
16050 register_candidate(&swap, None);
16051 let incumbent = cutover(&swap);
16052 swap.forwarding
16053 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
16054 .unwrap()
16055 .expect("the incumbent is still registered");
16056
16057 let client = ConnectionId::new(33);
16058 let (client_ctx, mut client_rx) = route_ctx(client);
16059 let route_handler = swap.handler.clone();
16060 let open_ctx = RouteCtx {
16061 connection_id: client,
16062 egress: client_ctx.egress.clone(),
16063 };
16064 let mut route_task = tokio::spawn(async move {
16065 route_handler
16066 .handle_control_frame(
16067 &open_ctx,
16068 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
16069 )
16070 .await
16071 .unwrap()
16072 });
16073 let bind = tokio::select! {
16074 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
16075 response = &mut route_task => {
16076 let response = response.unwrap();
16077 panic!(
16078 "post-cutover route.open was refused instead of relayed to the candidate: {}",
16079 parse_error(&response[0])["code"]
16080 );
16081 }
16082 };
16083 swap.handler
16084 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
16085 .await
16086 .unwrap();
16087 assert!(route_task.await.unwrap().is_empty());
16088 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
16089 match swap
16090 .forwarding
16091 .lookup_data_route(client, channel, epoch)
16092 .unwrap()
16093 {
16094 DataRoute::Client(DataRouteState::Bound(route)) => {
16095 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
16096 }
16097 other => panic!("expected a bound route on the candidate, got {other:?}"),
16098 }
16099 assert!(swap.incumbent_rx.try_recv().is_err());
16100 }
16101
16102 #[tokio::test]
16107 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
16108 let swap = swap_with_incumbent().await;
16109 register_candidate(&swap, Some(false));
16110 let update = Frame::build(
16111 FrameType::Request,
16112 control_flags(),
16113 0,
16114 0,
16115 55,
16116 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16117 provides: manifest("aft", PROTOCOL_VERSION).provides,
16118 capabilities: None,
16119 ready: Some(true),
16120 })
16121 .unwrap(),
16122 )
16123 .unwrap();
16124
16125 let replies = swap
16126 .handler
16127 .handle_control_frame(&swap.candidate_ctx, update)
16128 .await
16129 .unwrap();
16130
16131 assert_eq!(replies.len(), 1);
16132 assert_eq!(
16133 replies[0].header.ty,
16134 FrameType::Response,
16135 "candidate catalog.update was refused: {:?}",
16136 serde_json::from_slice::<Value>(&replies[0].body).ok()
16137 );
16138 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
16139 assert_eq!(
16140 swap.registry
16141 .get_module("aft")
16142 .unwrap()
16143 .unwrap()
16144 .connection_id,
16145 INCUMBENT
16146 );
16147 }
16148 }
16149
16150 mod swap_admission {
16154 use super::*;
16155
16156 const INCUMBENT_NONCE: &str = "incumbent-nonce";
16157 const CANDIDATE_NONCE: &str = "candidate-nonce";
16158
16159 fn handler_with_incumbent(
16160 module_id: &str,
16161 reserved: bool,
16162 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
16163 let registry = Arc::new(Registry::default());
16164 let supervisor = SupervisorHandle::new();
16165 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
16166 if reserved {
16167 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
16168 }
16169 let handler =
16170 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
16171 let incumbent = handler
16172 .handle_control(
16173 ConnectionId::new(1),
16174 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
16175 )
16176 .unwrap();
16177 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
16178 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
16179 (registry, supervisor, handler)
16180 }
16181
16182 #[test]
16188 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
16189 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
16190
16191 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
16192 let replies = handler
16193 .handle_control(
16194 ConnectionId::new(connection),
16195 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
16196 )
16197 .unwrap();
16198 assert_eq!(replies[0].header.ty, FrameType::Error);
16199 assert_eq!(
16200 parse_error(&replies[0])["code"],
16201 "swap_token_invalid",
16202 "nonce {nonce:?}"
16203 );
16204 }
16205 assert!(registry.get_candidate("aft").unwrap().is_none());
16206 assert_eq!(
16207 registry.get_module("aft").unwrap().unwrap().connection_id,
16208 ConnectionId::new(1)
16209 );
16210
16211 let admitted = handler
16214 .handle_control(
16215 ConnectionId::new(4),
16216 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
16217 )
16218 .unwrap();
16219 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
16220 assert_eq!(
16221 registry
16222 .get_candidate("aft")
16223 .unwrap()
16224 .unwrap()
16225 .connection_id,
16226 ConnectionId::new(4)
16227 );
16228 assert_eq!(
16229 registry.get_module("aft").unwrap().unwrap().connection_id,
16230 ConnectionId::new(1),
16231 "the candidate must not take the active slot"
16232 );
16233 let replayed = handler
16234 .handle_control(
16235 ConnectionId::new(5),
16236 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
16237 )
16238 .unwrap();
16239 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
16240
16241 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
16245 let squatter = handler
16246 .handle_control(
16247 ConnectionId::new(6),
16248 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
16249 )
16250 .unwrap();
16251 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
16252 assert!(
16253 registry.get_module("aft").unwrap().is_none(),
16254 "a squatter took the active slot of an id being swapped"
16255 );
16256 }
16257
16258 #[test]
16263 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
16264 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
16265
16266 let replies = handler
16267 .handle_control(
16268 ConnectionId::new(2),
16269 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16270 )
16271 .unwrap();
16272
16273 assert_eq!(
16274 replies[0].header.ty,
16275 FrameType::HelloAck,
16276 "reserved candidate refused: {:?}",
16277 serde_json::from_slice::<Value>(&replies[0].body).ok()
16278 );
16279 assert_eq!(
16280 registry
16281 .get_candidate("vault")
16282 .unwrap()
16283 .unwrap()
16284 .connection_id,
16285 ConnectionId::new(2)
16286 );
16287 }
16288
16289 #[test]
16292 fn without_an_open_swap_the_ordinary_gates_decide() {
16293 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16294 supervisor.close_swap("vault");
16295
16296 let candidate = handler
16297 .handle_control(
16298 ConnectionId::new(2),
16299 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16300 )
16301 .unwrap();
16302 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16303 let duplicate = handler
16304 .handle_control(
16305 ConnectionId::new(3),
16306 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16307 )
16308 .unwrap();
16309 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16310 assert!(registry.get_candidate("vault").unwrap().is_none());
16311 }
16312 }
16313
16314 mod scopes {
16318 mod idle_tests {
16319 use super::*;
16320 include!("scope_idle_tests.rs");
16321 }
16322 mod route_open_timings {
16323 use super::*;
16324 include!("route_open_timing_tests.rs");
16325 }
16326 mod route_open_refusal_log {
16327 use super::*;
16328 include!("route_open_refusal_log_tests.rs");
16329 }
16330 mod activity {
16331 include!("activity_scope_tests.rs");
16332 }
16333 mod delta {
16334 include!("scope_handler_delta_tests.rs");
16335 }
16336 mod call_modes {
16337 include!("scope_call_mode_tests.rs");
16338 }
16339 #[tokio::test]
16340 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16341 let rig = rig().await;
16342 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16343 "s".into(),
16344 1,
16345 "tool".into(),
16346 ))
16347 .unwrap();
16348 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16349 let (direct, _rx) = wide_ctx(999);
16350 for (ctx, expected) in [
16351 (&rig.owner, FrameType::Response),
16352 (&direct, FrameType::Error),
16353 ] {
16354 let frame =
16355 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16356 .unwrap();
16357 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16358 assert_eq!(replies[0].header.ty, expected);
16359 if expected == FrameType::Response {
16360 let reply: subc_control::ScopeServedByReply =
16361 serde_json::from_slice(&replies[0].body).unwrap();
16362 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16363 } else {
16364 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16365 }
16366 }
16367 let reply = call(
16368 &rig.handler,
16369 &rig.owner,
16370 &ModuleControlRequestFromModule::ScopeDescribe {
16371 owner: Principal::Reserved {
16372 module_id: OWNER.into(),
16373 },
16374 scope_ref: "s".into(),
16375 },
16376 )
16377 .await;
16378 assert!(matches!(
16379 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16380 ModuleControlResponseToModule::ScopeDescribe { .. }
16381 ));
16382 }
16383
16384 use subc_protocol::scope::{
16385 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16386 ScopeStatus,
16387 };
16388
16389 use super::*;
16390
16391 const OWNER: &str = "prefrontal-core";
16392
16393 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16394 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16395 }
16396
16397 async fn call(
16398 handler: &ControlHandler,
16399 ctx: &RouteCtx,
16400 request: &ModuleControlRequestFromModule,
16401 ) -> Frame {
16402 let body = serde_json::to_vec(request).unwrap();
16403 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16404 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16405 assert_eq!(replies.len(), 1, "{replies:?}");
16406 replies.pop().unwrap()
16407 }
16408
16409 async fn sync(
16410 handler: &ControlHandler,
16411 ctx: &RouteCtx,
16412 generation: u64,
16413 scopes: Vec<ScopeRecord>,
16414 ) -> Result<ModuleControlResponseToModule, String> {
16415 let reply = call(
16416 handler,
16417 ctx,
16418 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16419 )
16420 .await;
16421 match reply.header.ty {
16422 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16423 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16424 }
16425 }
16426
16427 async fn describe(
16428 handler: &ControlHandler,
16429 ctx: &RouteCtx,
16430 owner: &str,
16431 scope_ref: &str,
16432 ) -> ModuleControlResponseToModule {
16433 let reply = call(
16434 handler,
16435 ctx,
16436 &ModuleControlRequestFromModule::ScopeDescribe {
16437 owner: Principal::Reserved {
16438 module_id: owner.to_string(),
16439 },
16440 scope_ref: scope_ref.to_string(),
16441 },
16442 )
16443 .await;
16444 assert_eq!(
16445 reply.header.ty,
16446 FrameType::Response,
16447 "{:?}",
16448 parse_error(&reply)
16449 );
16450 serde_json::from_slice(&reply.body).unwrap()
16451 }
16452
16453 #[tokio::test]
16454 async fn scope_describe_never_echoes_bind_proof_or_hash() {
16455 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16456 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16457 let mut rig = rig().await;
16458 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16459 .await;
16460 let _bound = rig
16461 .bound(
16462 Some(AFT),
16463 PLEXUS,
16464 Some(
16465 rig.selector("s", Some(1))
16466 .with_bind_proof(Some(PROOF.into())),
16467 ),
16468 )
16469 .await;
16470 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16471 let output = serde_json::to_string(&reply).unwrap();
16472 assert!(output.contains("\"status\":\"live\""));
16473 assert!(!output.contains(PROOF));
16474 assert!(!output.contains(HASH));
16475 }
16476
16477 async fn module(
16479 handler: &ControlHandler,
16480 connection: u64,
16481 module_id: &str,
16482 nonce: Option<&str>,
16483 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16484 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16485 hello_via_sink(
16486 handler,
16487 &ctx,
16488 &mut rx,
16489 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16490 )
16491 .await;
16492 (ctx, rx)
16493 }
16494
16495 #[tokio::test]
16498 async fn a_client_connection_cannot_sync_or_describe() {
16499 let handler = ControlHandler::new(Arc::new(Registry::default()));
16500 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16501 for request in [
16502 ModuleControlRequestFromModule::ScopeSync {
16503 generation: 1,
16504 scopes: vec![head("s", 1)],
16505 },
16506 ModuleControlRequestFromModule::ScopeDescribe {
16507 owner: Principal::Direct,
16508 scope_ref: "s".to_string(),
16509 },
16510 ] {
16511 let reply = call(&handler, &ctx, &request).await;
16512 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16513 }
16514 assert!(
16515 !handler
16516 .scopes
16517 .read()
16518 .unwrap()
16519 .describe(
16520 &Principal::Reserved {
16521 module_id: OWNER.to_string()
16522 },
16523 "s"
16524 )
16525 .owner_synced
16526 );
16527 }
16528
16529 #[tokio::test]
16532 async fn a_module_without_a_supervised_launch_cannot_sync() {
16533 let handler = ControlHandler::new(Arc::new(Registry::default()));
16534 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16535 assert_eq!(
16536 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16537 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16538 );
16539 }
16540
16541 #[tokio::test]
16542 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16543 let supervisor = SupervisorHandle::new();
16544 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16545 let handler = ControlHandler::new(Arc::new(Registry::default()))
16546 .with_supervisor(supervisor.clone());
16547 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16548 sync(&handler, &incumbent, 1, vec![head("s", 1)])
16549 .await
16550 .expect("the current launch syncs");
16551
16552 supervisor.open_swap(OWNER, "n2".to_string());
16555 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16556 assert_eq!(
16557 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16558 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16559 );
16560 sync(&handler, &incumbent, 2, vec![head("s", 1)])
16561 .await
16562 .expect("the serving owner syncs during the swap");
16563
16564 supervisor.close_swap(OWNER);
16567 assert_eq!(
16568 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16569 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16570 );
16571 sync(&handler, &incumbent, 3, vec![head("s", 1)])
16572 .await
16573 .expect("the serving owner syncs after the rollback");
16574 handler.cleanup_connection(candidate.connection_id).unwrap();
16575
16576 supervisor.open_swap(OWNER, "n3".to_string());
16581 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16582 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16583 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16584 .await
16585 .expect("the promoted launch takes authority");
16586 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16587 panic!("unexpected reply {reply:?}");
16588 };
16589 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16590 assert_eq!(
16591 sync(&handler, &incumbent, 4, Vec::new()).await,
16592 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16593 );
16594 }
16595
16596 #[tokio::test]
16599 async fn closing_the_authority_connection_frees_sync_authority() {
16600 let supervisor = SupervisorHandle::new();
16601 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16602 let handler = ControlHandler::new(Arc::new(Registry::default()))
16603 .with_supervisor(supervisor.clone());
16604 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16605 sync(&handler, &first, 10, vec![head("s", 1)])
16606 .await
16607 .unwrap();
16608 handler.cleanup_connection(first.connection_id).unwrap();
16609
16610 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16611 sync(&handler, &second, 1, vec![head("s", 1)])
16612 .await
16613 .expect("the next connection takes the released authority");
16614 }
16615
16616 #[tokio::test]
16617 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16618 let supervisor = SupervisorHandle::new();
16619 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16620 let handler =
16621 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16622 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16623 sync(&handler, &first, 10, vec![head("s", 1)])
16624 .await
16625 .unwrap();
16626 handler
16627 .handle_control_frame(
16628 &first,
16629 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16630 )
16631 .await
16632 .unwrap();
16633 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16634 sync(&handler, &second, 1, vec![head("s", 1)])
16635 .await
16636 .expect("GOODBYE releases authority even if the old socket remains open");
16637 }
16638
16639 #[tokio::test]
16640 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16641 let registry = Arc::new(Registry::default());
16642 let supervisor_handle = SupervisorHandle::new();
16643 let supervisor =
16644 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16645 .with_handle(supervisor_handle.clone())
16646 .with_daemon_incarnation("incarnation-7".to_string());
16647 supervisor
16650 .supervise_configured(
16651 ModuleSpec {
16652 module_id: OWNER.to_string(),
16653 program: PathBuf::from("/nonexistent/prefrontal-core"),
16654 args: Vec::new(),
16655 env: Vec::new(),
16656 reserved: false,
16657 reserved_prefixes: Vec::new(),
16658 protocol: ModuleProtocol::Subc,
16659 overlap: Default::default(),
16660 },
16661 false,
16662 )
16663 .unwrap();
16664 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16665 let handler =
16666 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16667 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16668
16669 let ModuleControlResponseToModule::ScopeDescribe {
16671 status,
16672 daemon_incarnation,
16673 owner_synced,
16674 owner_configured,
16675 scope,
16676 ..
16677 } = describe(&handler, &reader, OWNER, "s").await
16678 else {
16679 panic!("not a describe reply");
16680 };
16681 assert_eq!(status, ScopeStatus::NotLive);
16682 assert_eq!(daemon_incarnation, "incarnation-7");
16683 assert!(!owner_synced);
16684 assert!(owner_configured);
16685 assert!(scope.is_none());
16686
16687 let ModuleControlResponseToModule::ScopeDescribe {
16690 status,
16691 owner_configured,
16692 ..
16693 } = describe(&handler, &reader, "ghost", "s").await
16694 else {
16695 panic!("not a describe reply");
16696 };
16697 assert_eq!(status, ScopeStatus::NotLive);
16698 assert!(!owner_configured);
16699
16700 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16702 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16703 let ModuleControlResponseToModule::ScopeDescribe {
16704 status,
16705 scope_epoch,
16706 owner_synced,
16707 scope,
16708 ..
16709 } = describe(&handler, &reader, OWNER, "s").await
16710 else {
16711 panic!("not a describe reply");
16712 };
16713 assert_eq!(status, ScopeStatus::Live);
16714 assert_eq!(scope_epoch, Some(4));
16715 assert!(owner_synced);
16716 let stamp = scope.expect("a live scope carries its stamp");
16717 assert!(
16718 stamp.owner_authorized,
16719 "prefrontal-core is the default authority"
16720 );
16721 assert_eq!(stamp.kind, ScopeKind::Head);
16722 }
16723
16724 #[tokio::test]
16725 async fn scope_authority_owners_decides_owner_authorized() {
16726 let supervisor = SupervisorHandle::new();
16727 supervisor.set_spawn_nonce("broca", "b1".to_string());
16728 let handler = ControlHandler::new(Arc::new(Registry::default()))
16729 .with_supervisor(supervisor)
16730 .with_scope_authority_owners(vec!["broca".to_string()]);
16731 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16732 let mut gated = head("s", 1);
16733 gated.attributes.agent_id = Some("agent".to_string());
16734 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16735 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16736 describe(&handler, &broca, "broca", "s").await
16737 else {
16738 panic!("not a describe reply");
16739 };
16740 assert!(scope.unwrap().owner_authorized);
16741 }
16742
16743 #[tokio::test]
16747 async fn scope_ops_and_the_scopes_capability_are_advertised() {
16748 let handler = ControlHandler::new(Arc::new(Registry::default()));
16749 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16750 let ack = hello_via_sink(
16751 &handler,
16752 &ctx,
16753 &mut rx,
16754 hello_frame("m", PROTOCOL_VERSION, 1),
16755 )
16756 .await;
16757 let ack = parse_ack(&ack);
16758 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16759 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16760 }
16761 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16762
16763 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16764 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16765 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16766 let reply = handler
16767 .handle_control_frame(&client, frame)
16768 .await
16769 .unwrap()
16770 .pop()
16771 .unwrap();
16772 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16773 serde_json::from_slice(&reply.body).unwrap()
16774 else {
16775 panic!("not a server.describe reply");
16776 };
16777 assert!(
16778 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16779 "{capabilities:?}"
16780 );
16781 }
16782
16783 const PLEXUS: &str = "plexus";
16786 const OTHER: &str = "other";
16787 const AFT: &str = "aft";
16788 const BROCA: &str = "broca";
16789 const MAGIC: &str = "magic-context";
16790
16791 fn nonce(module_id: &str) -> String {
16792 format!("nonce-{module_id}")
16793 }
16794
16795 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16796 let (tx, rx) = mpsc::channel(64);
16797 (
16798 RouteCtx {
16799 connection_id: ConnectionId::new(connection),
16800 egress: FrameSink::new(tx),
16801 },
16802 rx,
16803 )
16804 }
16805
16806 struct Rig {
16810 handler: ControlHandler,
16811 forwarding: Arc<ForwardingTable>,
16812 owner: RouteCtx,
16813 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16814 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16815 generation: u64,
16816 next_connection: u64,
16817 _supervisor: Supervisor,
16818 }
16819
16820 async fn rig() -> Rig {
16821 rig_with_flow_support(true).await
16822 }
16823
16824 async fn rig_with_flow_support(flow_support: bool) -> Rig {
16825 rig_with_scope_capabilities(flow_support, false).await
16826 }
16827
16828 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16829 let mut provides = Vec::new();
16830 if flow_support {
16831 provides.push("flow-scopes/v1");
16832 }
16833 if run_support {
16834 provides.push("agent-run-scopes/v1");
16835 }
16836 rig_with_target_capabilities(&provides).await
16837 }
16838
16839 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16840 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16841 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16842 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16845 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16846 Frame::build(
16847 FrameType::Hello,
16848 control_flags(),
16849 0,
16850 0,
16851 connection,
16852 serde_json::to_vec(&body).unwrap(),
16853 )
16854 .unwrap()
16855 }
16856
16857 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16858 let registry = Arc::new(Registry::default());
16859 let forwarding = Arc::new(ForwardingTable::default());
16860 let supervisor_handle = SupervisorHandle::new();
16861 let supervisor =
16862 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16863 .with_handle(supervisor_handle.clone());
16864 supervisor
16865 .supervise_configured(
16866 ModuleSpec {
16867 module_id: OWNER.to_string(),
16868 program: PathBuf::from("/nonexistent/prefrontal-core"),
16869 args: Vec::new(),
16870 env: Vec::new(),
16871 reserved: false,
16872 reserved_prefixes: Vec::new(),
16873 protocol: ModuleProtocol::Subc,
16874 overlap: Default::default(),
16875 },
16876 false,
16877 )
16878 .unwrap();
16879 for module_id in [OWNER, AFT, BROCA, MAGIC] {
16880 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16881 }
16882 let handler =
16883 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16884 .with_supervisor(supervisor_handle);
16885 let (owner, mut owner_rx) = wide_ctx(1);
16886 hello_via_sink(
16887 &handler,
16888 &owner,
16889 &mut owner_rx,
16890 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16891 )
16892 .await;
16893 let mut modules = BTreeMap::new();
16894 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16895 let (ctx, mut rx) = wide_ctx(connection);
16896 let hello = hello_with_capabilities(module_id, connection, provides);
16897 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16898 modules.insert(module_id.to_string(), (ctx, rx));
16899 }
16900 Rig {
16901 handler,
16902 forwarding,
16903 owner,
16904 _owner_rx: owner_rx,
16905 modules,
16906 generation: 0,
16907 next_connection: 100,
16908 _supervisor: supervisor,
16909 }
16910 }
16911
16912 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16913 ScopeCarrier::new(Principal::Reserved {
16914 module_id: module_id.to_string(),
16915 })
16916 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16917 }
16918
16919 fn session(scope_epoch: u64) -> ScopeRecord {
16922 let mut record = head("s", scope_epoch);
16923 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16924 record.attributes.agent_id = Some("agent-1".to_string());
16925 record.attributes.delegates = true;
16926 record
16927 }
16928
16929 impl Rig {
16930 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16931 self.generation += 1;
16932 sync(&self.handler, &self.owner, self.generation, scopes)
16933 .await
16934 .expect("the owner's sync is accepted");
16935 }
16936
16937 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16938 ScopeSelector::new(
16939 Principal::Reserved {
16940 module_id: OWNER.to_string(),
16941 },
16942 scope_ref.to_string(),
16943 scope_epoch,
16944 )
16945 }
16946
16947 fn open_frame(
16948 &mut self,
16949 opener: Option<&str>,
16950 target: &str,
16951 scope: Option<ScopeSelector>,
16952 ) -> (
16953 RouteCtx,
16954 mpsc::Receiver<crate::router::OutboundFrame>,
16955 Frame,
16956 ) {
16957 self.next_connection += 1;
16958 let (ctx, rx) = wide_ctx(self.next_connection);
16959 let root = unique_project_root("scoped-open");
16960 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16961 target: RouteTarget::ToolProvider {
16962 module_id: target.to_string(),
16963 },
16964 identity: BindIdentity::new(
16965 root.path().to_path_buf(),
16966 "unit".to_string(),
16967 "session".to_string(),
16968 ),
16969 consumer_identity: opener.map(|module_id| ConsumerIdentity {
16970 module_id: module_id.to_string(),
16971 launch_nonce: nonce(module_id),
16972 }),
16973 consumer_capabilities: None,
16974 role_versions: None,
16975 admission_facts: None,
16976 scope,
16977 })
16978 .unwrap();
16979 let frame = Frame::build(
16980 FrameType::Request,
16981 control_flags(),
16982 0,
16983 0,
16984 self.next_connection,
16985 body,
16986 )
16987 .unwrap();
16988 (ctx, rx, frame)
16989 }
16990
16991 async fn refused(
16993 &mut self,
16994 opener: Option<&str>,
16995 target: &str,
16996 scope: Option<ScopeSelector>,
16997 ) -> String {
16998 self.refusal_body(opener, target, scope).await["code"]
16999 .as_str()
17000 .unwrap()
17001 .to_string()
17002 }
17003
17004 async fn refusal_body(
17005 &mut self,
17006 opener: Option<&str>,
17007 target: &str,
17008 scope: Option<ScopeSelector>,
17009 ) -> Value {
17010 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
17011 let replies = tokio::time::timeout(
17012 Duration::from_secs(2),
17013 self.handler.handle_control_frame(&ctx, frame),
17014 )
17015 .await
17016 .expect("the open must be refused before waiting for a bind ack")
17017 .unwrap();
17018 assert_eq!(replies.len(), 1, "{replies:?}");
17019 assert_eq!(replies[0].header.ty, FrameType::Error);
17020 let (_, module_rx) = self.modules.get_mut(target).unwrap();
17021 assert!(
17022 module_rx.try_recv().is_err(),
17023 "a refused open relays nothing"
17024 );
17025 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
17026 parse_error(&replies[0])
17027 }
17028
17029 async fn relayed(
17031 &mut self,
17032 opener: Option<&str>,
17033 target: &str,
17034 scope: Option<ScopeSelector>,
17035 ) -> Relayed {
17036 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
17037 let handler = self.handler.clone();
17038 let task_ctx = ctx.clone();
17039 let task = tokio::spawn(async move {
17040 handler
17041 .handle_control_frame(&task_ctx, frame)
17042 .await
17043 .unwrap()
17044 });
17045 let (_, module_rx) = self.modules.get_mut(target).unwrap();
17046 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
17047 .await
17048 .expect("the target receives the relayed route.bind")
17049 .unwrap()
17050 .frame;
17051 Relayed {
17052 target: target.to_string(),
17053 client: ctx,
17054 client_rx: rx,
17055 task,
17056 bind,
17057 }
17058 }
17059
17060 async fn ack(&self, relayed: &Relayed) {
17061 let (module, _) = &self.modules[&relayed.target];
17062 self.handler
17063 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
17064 .await
17065 .unwrap();
17066 }
17067
17068 async fn bound(
17070 &mut self,
17071 opener: Option<&str>,
17072 target: &str,
17073 scope: Option<ScopeSelector>,
17074 ) -> Bound {
17075 let relayed = self.relayed(opener, target, scope).await;
17076 self.ack(&relayed).await;
17077 let Relayed {
17078 target,
17079 client,
17080 mut client_rx,
17081 task,
17082 bind,
17083 } = relayed;
17084 assert!(
17085 task.await.unwrap().is_empty(),
17086 "the open is answered by commit"
17087 );
17088 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
17089 Bound {
17090 target,
17091 client,
17092 client_rx,
17093 channel,
17094 epoch,
17095 bind,
17096 }
17097 }
17098
17099 fn live(&self, route: &Bound) -> bool {
17100 matches!(
17101 self.forwarding
17102 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17103 .unwrap(),
17104 DataRoute::Client(DataRouteState::Bound(_))
17105 )
17106 }
17107 }
17108
17109 struct Relayed {
17110 target: String,
17111 client: RouteCtx,
17112 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17113 task: tokio::task::JoinHandle<Vec<Frame>>,
17114 bind: Frame,
17115 }
17116
17117 struct Bound {
17118 target: String,
17119 client: RouteCtx,
17120 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17121 channel: u16,
17122 epoch: u32,
17123 bind: Frame,
17124 }
17125
17126 impl Bound {
17127 fn closed_reason(&mut self) -> RouteCloseReason {
17130 let mut reason = None;
17131 let mut goodbye = false;
17132 while let Ok(outbound) = self.client_rx.try_recv() {
17133 let frame = outbound.frame;
17134 match frame.header.ty {
17135 FrameType::Goodbye => {
17136 assert_eq!(
17137 (frame.header.channel, frame.header.epoch),
17138 (self.channel, self.epoch)
17139 );
17140 goodbye = true;
17141 }
17142 FrameType::Push => {
17143 let ClientControlPush::RouteClosed {
17144 reason: r,
17145 module_id,
17146 ..
17147 } = serde_json::from_slice(&frame.body).unwrap()
17148 else {
17149 panic!("unexpected push");
17150 };
17151 assert_eq!(module_id, self.target);
17152 reason = Some(r);
17153 }
17154 other => panic!("unexpected frame {other:?}"),
17155 }
17156 }
17157 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
17158 reason.expect("the client is told why the route closed")
17159 }
17160
17161 fn untouched(&mut self) -> bool {
17162 self.client_rx.try_recv().is_err()
17163 }
17164
17165 fn stamp(&self) -> Option<ScopeStamp> {
17166 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
17167 ModuleControlRequest::RouteBind { scope, .. } => scope,
17168 other => panic!("expected a route.bind, got {other:?}"),
17169 }
17170 }
17171 }
17172
17173 #[tokio::test]
17174 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
17175 ) {
17176 let mut rig = rig().await;
17177 let mut record = session(1);
17178 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
17179 record.child_owners = vec![Principal::Reserved {
17180 module_id: MAGIC.to_string(),
17181 }];
17182 rig.sync(vec![record]).await;
17183 let scope = || Some(rig_selector("s", Some(1)));
17184
17185 rig.bound(Some(OWNER), PLEXUS, scope()).await;
17188 rig.bound(Some(AFT), OTHER, scope()).await;
17189 rig.bound(Some(BROCA), PLEXUS, scope()).await;
17190
17191 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
17195 assert_eq!(
17196 rig.refused(opener, target, scope()).await,
17197 error_codes::SCOPE_NOT_CARRIER,
17198 "{opener:?} -> {target}"
17199 );
17200 }
17201 }
17202
17203 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17204 ScopeSelector::new(
17205 Principal::Reserved {
17206 module_id: OWNER.to_string(),
17207 },
17208 scope_ref.to_string(),
17209 scope_epoch,
17210 )
17211 }
17212
17213 #[tokio::test]
17214 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
17215 let mut rig = rig().await;
17216 rig.sync(vec![session(1)]).await;
17217 for opener in [OWNER, AFT] {
17218 assert_eq!(
17219 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
17220 .await,
17221 error_codes::SCOPE_EPOCH_REQUIRED,
17222 "{opener}"
17223 );
17224 }
17225 }
17226
17227 #[tokio::test]
17228 async fn admission_separates_not_synced_not_live_and_ended() {
17229 let mut rig = rig().await;
17230 let code = rig
17232 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17233 .await;
17234 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
17235 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
17236
17237 let ghost = ScopeSelector::new(
17239 Principal::Reserved {
17240 module_id: "ghost".to_string(),
17241 },
17242 "s".to_string(),
17243 Some(1),
17244 );
17245 assert_eq!(
17246 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
17247 error_codes::SCOPE_NOT_LIVE
17248 );
17249
17250 rig.sync(vec![session(2)]).await;
17251 assert_eq!(
17252 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
17253 .await,
17254 error_codes::SCOPE_NOT_LIVE
17255 );
17256 for epoch in [1, 3] {
17257 assert_eq!(
17258 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
17259 .await,
17260 error_codes::SCOPE_ENDED,
17261 "epoch {epoch}"
17262 );
17263 }
17264 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
17266 .await;
17267 }
17268
17269 #[tokio::test]
17270 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
17271 let mut rig = rig().await;
17272 rig.sync(vec![session(1)]).await;
17273 let route = rig
17274 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17275 .await;
17276 let stamp = route.stamp().expect("a scoped bind carries the stamp");
17277 assert_eq!(stamp.scope_ref, "s");
17278 assert_eq!(stamp.scope_epoch, 1);
17279 assert_eq!(stamp.kind, ScopeKind::Head);
17280 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
17281 assert!(stamp.attributes.delegates);
17282 assert!(stamp.owner_authorized);
17283 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17284 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
17285
17286 let (broca, mut broca_rx) = wide_ctx(50);
17289 hello_via_sink(
17290 &rig.handler,
17291 &broca,
17292 &mut broca_rx,
17293 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17294 )
17295 .await;
17296 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17297 .await
17298 .unwrap();
17299 let own = ScopeSelector::new(
17300 Principal::Reserved {
17301 module_id: BROCA.to_string(),
17302 },
17303 "b".to_string(),
17304 Some(1),
17305 );
17306 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17307 assert!(!route.stamp().unwrap().owner_authorized);
17308 }
17309
17310 #[tokio::test]
17311 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17312 let mut rig = rig().await;
17313 let mut record = head("s", 1);
17314 record.carriers = vec![carrier(AFT, None)];
17315 let flow_id = "Flow:run-7/step_2!~";
17316 record.attributes.flow_id = Some(flow_id.to_string());
17317 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17318 .await
17319 .unwrap();
17320 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17321 panic!("not a sync reply");
17322 };
17323 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17324 let route = rig
17325 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17326 .await;
17327 assert!(rig.live(&route), "the stamped bind committed");
17328 let stamp = route.stamp().expect("a flow scope carries a stamp");
17329 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17330 assert_eq!(stamp.attributes.agent_id, None);
17331 assert!(!stamp.attributes.delegates);
17332 assert!(stamp.owner_authorized);
17333 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17334 assert_eq!(unscoped.stamp(), None);
17335 }
17336
17337 #[tokio::test]
17338 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17339 let mut rig = rig_with_flow_support(false).await;
17340 let mut record = session(1);
17341 record.attributes.flow_id = Some("flow:7".to_string());
17342 rig.sync(vec![record]).await;
17343 for opener in [OWNER, AFT] {
17344 let body = rig
17345 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17346 .await;
17347 assert_eq!(body["code"], "target_flow_unsupported");
17348 let message = body["message"].as_str().unwrap();
17349 for required in [PLEXUS, "flow-scopes/v1"] {
17350 assert!(message.contains(required), "{message}");
17351 }
17352 }
17353 }
17354
17355 #[tokio::test]
17356 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17357 let mut rig = rig_with_flow_support(true).await;
17358 let mut record = session(1);
17359 record.attributes.flow_id = Some("flow:7".to_string());
17360 rig.sync(vec![record]).await;
17361 for opener in [OWNER, AFT] {
17362 let route = rig
17363 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17364 .await;
17365 assert!(rig.live(&route));
17366 assert_eq!(
17367 route.stamp().unwrap().attributes.flow_id.as_deref(),
17368 Some("flow:7")
17369 );
17370 }
17371 }
17372
17373 #[tokio::test]
17374 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17375 use std::future::Future;
17376
17377 let mut rig = rig_with_flow_support(true).await;
17378 let mut record = session(1);
17379 record.attributes.flow_id = Some("flow:7".to_string());
17380 rig.sync(vec![record]).await;
17381 let (client, mut client_rx, frame) =
17382 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17383 for _ in 0..64 {
17386 client.egress.try_send(route_bind_ack(1)).unwrap();
17387 }
17388 let handler = rig.handler.clone();
17389 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17390 std::future::poll_fn(|cx| {
17391 assert!(open.as_mut().poll(cx).is_pending());
17392 std::task::Poll::Ready(())
17393 })
17394 .await;
17395 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17396
17397 let old_connection = rig.modules[PLEXUS].0.connection_id;
17398 rig.handler.cleanup_connection(old_connection).unwrap();
17399 let (replacement, mut replacement_rx) = wide_ctx(200);
17400 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17401 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17402 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17403 let hello = Frame::build(
17404 FrameType::Hello,
17405 control_flags(),
17406 0,
17407 0,
17408 200,
17409 serde_json::to_vec(&body).unwrap(),
17410 )
17411 .unwrap();
17412 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17413
17414 client_rx.try_recv().unwrap();
17415 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17416 .await
17417 .expect("the replacement is refused without waiting for a bind ack")
17418 .unwrap();
17419 assert_eq!(replies.len(), 1);
17420 let body = parse_error(&replies[0]);
17421 assert_eq!(body["code"], "target_flow_unsupported");
17422 for required in [PLEXUS, "flow-scopes/v1"] {
17423 assert!(body["message"].as_str().unwrap().contains(required));
17424 }
17425 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17426 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17427 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17428 assert!(rig
17429 .handler
17430 .registry
17431 .get_module_by_connection(replacement.connection_id)
17432 .unwrap()
17433 .is_some());
17434 }
17435
17436 #[tokio::test]
17437 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17438 use std::future::Future;
17439 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17440 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17441 const NEXT_HASH: &str =
17442 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17443 let mut rig = rig().await;
17444 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17445 record.carriers.push(ScopeCarrier::new(Principal::Direct));
17446 rig.sync(vec![record.clone()]).await;
17447 let (client, mut rx, frame) = rig.open_frame(
17448 None,
17449 PLEXUS,
17450 Some(
17451 rig.selector("s", Some(1))
17452 .with_bind_proof(Some(PROOF.into())),
17453 ),
17454 );
17455 for _ in 0..64 {
17458 client.egress.try_send(route_bind_ack(1)).unwrap();
17459 }
17460 let handler = rig.handler.clone();
17461 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17462 std::future::poll_fn(|cx| {
17463 assert!(open.as_mut().poll(cx).is_pending());
17464 std::task::Poll::Ready(())
17465 })
17466 .await;
17467 record.bind_proof_sha256 = Some(NEXT_HASH.into());
17468 rig.sync(vec![record]).await;
17469 rx.try_recv().unwrap();
17470 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17471 .await
17472 .unwrap()
17473 .unwrap();
17474 assert_eq!(
17475 parse_error(&replies[0])["code"],
17476 error_codes::SCOPE_BIND_PROOF_MISMATCH
17477 );
17478 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17479 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17480 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17481 }
17482
17483 #[tokio::test]
17484 async fn scope_bind_proof_reserved_openers_ignore_proof() {
17485 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17486 let mut rig = rig().await;
17487 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17488 .await;
17489 for opener in [OWNER, AFT] {
17490 for proof in [None, Some("malformed+proof=")] {
17491 let route = rig
17492 .bound(
17493 Some(opener),
17494 PLEXUS,
17495 Some(
17496 rig.selector("s", Some(1))
17497 .with_bind_proof(proof.map(str::to_string)),
17498 ),
17499 )
17500 .await;
17501 assert!(rig.live(&route));
17502 }
17503 }
17504 }
17505
17506 #[tokio::test]
17507 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17508 use std::future::Future;
17509 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17510 const NEXT_HASH: &str =
17511 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17512 let mut rig = rig().await;
17513 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17514 .await;
17515 let (client, mut rx, frame) =
17516 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17517 for _ in 0..64 {
17518 client.egress.try_send(route_bind_ack(1)).unwrap();
17519 }
17520 let handler = rig.handler.clone();
17521 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17522 std::future::poll_fn(|cx| {
17523 assert!(open.as_mut().poll(cx).is_pending());
17524 std::task::Poll::Ready(())
17525 })
17526 .await;
17527 rig.sync(vec![
17528 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17529 ])
17530 .await;
17531 rx.try_recv().unwrap();
17532 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17533 let bind = std::future::poll_fn(|cx| {
17534 assert!(open.as_mut().poll(cx).is_pending());
17535 module_rx.poll_recv(cx)
17536 })
17537 .await
17538 .unwrap()
17539 .frame;
17540 handler
17541 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17542 .await
17543 .unwrap();
17544 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17545 .await
17546 .unwrap()
17547 .unwrap();
17548 assert_eq!(replies.len(), 1);
17549 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17552 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17553 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17554 let route = rig
17555 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17556 .await;
17557 assert!(rig.live(&route));
17558 }
17559
17560 #[tokio::test]
17561 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17562 {
17563 let mut rig = rig_with_flow_support(false).await;
17564 rig.sync(vec![session(1)]).await;
17565 let route = rig
17566 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17567 .await;
17568 assert!(rig.live(&route));
17569 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17570 let mut record = session(1);
17571 record.attributes.flow_id = Some("flow:7".to_string());
17572 rig.sync(vec![record]).await;
17573 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17574 assert!(rig.live(&unscoped));
17575 assert_eq!(unscoped.stamp(), None);
17576 }
17577
17578 #[tokio::test]
17579 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17580 let mut rig = rig().await;
17581 let mut record = session(1);
17582 record.attributes.flow_id = Some("flow:7".to_string());
17583 rig.sync(vec![record.clone()]).await;
17584 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17585 let mut owner_route = rig
17586 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17587 .await;
17588 let mut carrier_route = rig
17589 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17590 .await;
17591 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17592 rig.sync(vec![record.clone()]).await;
17593 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17594 assert!(rig.live(&owner_route) && owner_route.untouched());
17595 assert!(rig.live(&carrier_route) && carrier_route.untouched());
17596
17597 record.attributes.flow_id = Some("flow:8".to_string());
17598 rig.sync(vec![record]).await;
17599 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17600 let before = before.unwrap();
17601 assert_eq!(after.scope_epoch, before.scope_epoch);
17602 assert!(after.version > before.version);
17603 for route in [&mut owner_route, &mut carrier_route] {
17604 assert!(!rig.live(route));
17605 assert_eq!(
17606 route.closed_reason(),
17607 RouteCloseReason::ScopeDelegationChanged
17608 );
17609 }
17610 assert!(rig.live(&unscoped) && unscoped.untouched());
17611 for target in [PLEXUS, OTHER] {
17614 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17615 let goodbye = module_rx
17616 .try_recv()
17617 .expect("the provider sees the drain")
17618 .frame;
17619 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17620 assert!(module_rx.try_recv().is_err());
17621 }
17622 let rebound = rig
17623 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17624 .await;
17625 assert_eq!(
17626 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17627 Some("flow:8")
17628 );
17629 }
17630
17631 #[tokio::test]
17636 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17637 let mut rig = rig().await;
17638 rig.sync(vec![session(1)]).await;
17639 let mut cotenant = rig
17640 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17641 .await;
17642
17643 let mut changed = session(1);
17644 changed.child_owners.push(Principal::Reserved {
17645 module_id: MAGIC.to_string(),
17646 });
17647 let mut ended = None;
17648 for (code, next) in [
17649 (error_codes::SCOPE_CHANGED, vec![changed]),
17650 (error_codes::SCOPE_ENDED, Vec::new()),
17651 ] {
17652 let relayed = rig
17653 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17654 .await;
17655 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17656 ended = Some(next.is_empty());
17657 rig.sync(next).await;
17658 rig.ack(&relayed).await;
17659 let replies = relayed.task.await.unwrap();
17660 assert_eq!(replies.len(), 1, "{replies:?}");
17661 assert_eq!(parse_error(&replies[0])["code"], code);
17662 assert_eq!(
17663 subc_protocol::error_codes::is_retryable_route_open(code),
17664 code == error_codes::SCOPE_CHANGED
17665 );
17666 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17667 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17669 let mut goodbyes = Vec::new();
17672 while let Ok(outbound) = plexus_rx.try_recv() {
17673 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17674 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17675 }
17676 assert!(
17677 goodbyes.contains(&(bind_channel, bind_epoch)),
17678 "{goodbyes:?}"
17679 );
17680 assert!(rig
17681 .handler
17682 .registry
17683 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17684 .unwrap()
17685 .is_some());
17686 }
17687 assert_eq!(ended, Some(true));
17688 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17691 }
17692
17693 #[tokio::test]
17696 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17697 struct Case {
17698 name: &'static str,
17699 change: fn(&mut ScopeRecord),
17700 closed: [Option<RouteCloseReason>; 4],
17703 }
17704 use RouteCloseReason::*;
17705 let cases = [
17706 Case {
17707 name: "a carrier entry removed",
17708 change: |r| {
17709 r.carriers.retain(|c| {
17710 c.principal
17711 != Principal::Reserved {
17712 module_id: AFT.to_string(),
17713 }
17714 })
17715 },
17716 closed: [None, Some(ScopeCarrierRemoved), None, None],
17717 },
17718 Case {
17719 name: "a target removed from a carrier",
17720 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17721 closed: [None, None, None, Some(ScopeCarrierRemoved)],
17722 },
17723 Case {
17724 name: "a bare carrier narrowed to targets",
17725 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17726 closed: [None, Some(ScopeCarrierRemoved), None, None],
17727 },
17728 Case {
17729 name: "delegates turned off",
17730 change: |r| r.attributes.delegates = false,
17731 closed: [Some(ScopeDelegationChanged); 4],
17732 },
17733 Case {
17734 name: "agent_id changed",
17735 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17736 closed: [Some(ScopeDelegationChanged); 4],
17737 },
17738 Case {
17739 name: "a carrier added, child owners changed, the record re-sent",
17740 change: |r| {
17741 r.carriers.push(carrier(MAGIC, None));
17742 r.child_owners.push(Principal::Reserved {
17743 module_id: MAGIC.to_string(),
17744 });
17745 },
17746 closed: [None; 4],
17747 },
17748 Case {
17749 name: "a target added",
17750 change: |r| {
17751 r.carriers[1]
17752 .targets
17753 .as_mut()
17754 .unwrap()
17755 .push("third".to_string())
17756 },
17757 closed: [None; 4],
17758 },
17759 Case {
17760 name: "delegates turned on",
17761 change: |r| r.attributes.delegates = true,
17762 closed: [None; 4],
17763 },
17764 ];
17765 for case in cases {
17766 let mut rig = rig().await;
17767 rig.sync(vec![session(1)]).await;
17768 let scope = || Some(rig_selector("s", Some(1)));
17769 let mut routes = [
17770 rig.bound(Some(OWNER), PLEXUS, scope()).await,
17771 rig.bound(Some(AFT), PLEXUS, scope()).await,
17772 rig.bound(Some(BROCA), PLEXUS, scope()).await,
17773 rig.bound(Some(BROCA), OTHER, scope()).await,
17774 ];
17775 let mut record = session(1);
17776 (case.change)(&mut record);
17777 rig.sync(vec![record]).await;
17778 for (index, expected) in case.closed.iter().enumerate() {
17779 let route = &mut routes[index];
17780 match expected {
17781 Some(reason) => {
17782 assert!(!rig.live(route), "{}: route {index} still live", case.name);
17783 assert_eq!(
17784 route.closed_reason(),
17785 *reason,
17786 "{}: route {index}",
17787 case.name
17788 );
17789 }
17790 None => {
17791 assert!(rig.live(route), "{}: route {index} closed", case.name);
17792 assert!(
17793 route.untouched(),
17794 "{}: route {index} was told something",
17795 case.name
17796 );
17797 }
17798 }
17799 }
17800 }
17801 }
17802
17803 #[tokio::test]
17804 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17805 for next in [Vec::new(), vec![session(2)]] {
17807 let mut rig = rig().await;
17808 rig.sync(vec![session(1)]).await;
17809 let mut route = rig
17810 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17811 .await;
17812 rig.sync(next).await;
17813 assert!(!rig.live(&route));
17814 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17815 }
17816
17817 let mut rig = rig().await;
17820 let mut child = session(1);
17821 child.scope_ref = "child".to_string();
17822 child.kind = ScopeKind::Worker;
17823 child.parent = Some(ScopeParent::new(
17824 Principal::Reserved {
17825 module_id: OWNER.to_string(),
17826 },
17827 "s".to_string(),
17828 1,
17829 ));
17830 rig.sync(vec![session(1), child.clone()]).await;
17831 let mut child_route = rig
17832 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17833 .await;
17834 assert_eq!(
17835 child_route.stamp().unwrap().parent_state,
17836 Some(ParentState::Linked)
17837 );
17838 rig.sync(vec![child]).await;
17839 assert!(!rig.live(&child_route));
17840 assert_eq!(
17841 child_route.closed_reason(),
17842 RouteCloseReason::ScopeParentEnded
17843 );
17844 }
17845
17846 #[tokio::test]
17847 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17848 ) {
17849 let mut rig = rig().await;
17850 rig.sync(vec![session(1)]).await;
17851 let mut route = rig
17852 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17853 .await;
17854 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17855 rig.sync(vec![session(1)]).await;
17856 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17857 assert!(rig.live(&route) && route.untouched());
17858
17859 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17863 .forwarding
17864 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17865 .unwrap()
17866 else {
17867 panic!("the route is bound");
17868 };
17869 binding.flow.acquire_tagged(9, false).await.unwrap();
17870 let mut widened = session(1);
17871 widened.carriers.push(carrier(MAGIC, None));
17872 rig.sync(vec![widened]).await;
17873 assert!(rig.live(&route) && route.untouched());
17874 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17875 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17876 assert_eq!(binding.flow.in_flight(), 1);
17879 binding
17880 .flow
17881 .acquire_tagged(10, false)
17882 .await
17883 .expect("the flow is still open");
17884 }
17885
17886 #[tokio::test]
17889 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17890 let mut rig = rig().await;
17891 rig.sync(vec![session(1)]).await;
17892 let mut on_incumbent = rig
17893 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17894 .await;
17895
17896 let (candidate, _candidate_rx) = wide_ctx(9);
17899 let registration = rig
17900 .handler
17901 .registry
17902 .register_candidate_with_control_ops(
17903 manifest(PLEXUS, PROTOCOL_VERSION),
17904 PROTOCOL_VERSION,
17905 candidate.connection_id,
17906 module_baseline_control_ops(),
17907 )
17908 .unwrap();
17909 rig.forwarding
17910 .register_candidate_module_connection(
17911 candidate.connection_id,
17912 PLEXUS.to_string(),
17913 PROTOCOL_VERSION,
17914 manifest_concurrency(®istration.manifest),
17915 candidate.egress.clone(),
17916 )
17917 .unwrap();
17918 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17919 rig.handler
17920 .registry
17921 .promote_candidate(PLEXUS)
17922 .unwrap()
17923 .unwrap();
17924 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17925
17926 rig.sync(Vec::new()).await;
17927 assert!(!rig.live(&on_incumbent));
17928 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17929 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17930 let goodbye = incumbent_rx
17931 .try_recv()
17932 .expect("the superseded endpoint is told")
17933 .frame;
17934 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17935 }
17936 }
17937}
17938
17939#[cfg(test)]
17940mod concurrency_default_exposure_tests {
17941 use super::*;
17942
17943 fn hello_body(role_json: &str) -> Vec<u8> {
17944 format!(
17945 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17946 )
17947 .into_bytes()
17948 }
17949
17950 fn manifest_from(body: &[u8]) -> ModuleManifest {
17951 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17952 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17953 .expect("manifest parses")
17954 }
17955
17956 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17957
17958 #[test]
17959 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17960 let body = hello_body(&format!(
17961 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17962 ));
17963 let manifest = manifest_from(&body);
17964 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17967 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17968 }
17969
17970 #[test]
17971 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17972 let body = hello_body(&format!(
17973 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17974 ));
17975 let manifest = manifest_from(&body);
17976 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17977 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17978 }
17979
17980 #[test]
17981 fn non_management_roles_are_never_reported() {
17982 let body = hello_body(
17983 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17984 );
17985 let manifest = manifest_from(&body);
17986 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17987 }
17988}