1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_flow_operations, validate_hello_rich_ui_descriptor,
26 validate_hello_role_declarations, validate_hello_self_signal_declarations,
27 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
28 ProviderRole,
29 },
30 scope::{
31 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
32 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
33 SCOPE_SYNC_OP,
34 },
35 session::{
36 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
37 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
38 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
39 ROLE_VERSIONS_FIELD,
40 },
41 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
42 Priority, RouteTarget, PROTOCOL_VERSION,
43};
44use tokio::time::{timeout_at, Instant};
45use tracing::{debug, info, warn};
46
47use crate::{
48 caller_identity::CallerIdentity,
49 capability_requirements::{
50 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
51 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
52 RequirementStatus, RuntimeModule,
53 },
54 child_roster::DaemonShutdownFlag,
55 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
56 forwarding::{
57 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
58 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
59 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
60 },
61 observability::{
62 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
63 },
64 privileged_prompt::{prompt_text, PrivilegedOperation},
65 provenance::{
66 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
67 },
68 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
69 router::{RouteCtx, RouterError},
70 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
71 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
72 stderr_tail::{CaptureState, TailEntry},
73 supervise::{
74 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
75 SupervisorHandle, SwapHelloAdmission,
76 },
77 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
78};
79
80pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
86
87const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
88const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
89const CAP_PING_PONG: &str = "ping_pong_v1";
90const CAP_SESSION_ATTACH: &str = "session_attach_v1";
91const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
92
93const SUBC_CONTROL_OPS: &[&str] = &[
94 ops::ACTIVITY_RECORD,
95 ops::ACTIVITY_QUERY,
96 ops::ACTIVITY_SUBSCRIBE,
97 ops::SERVER_DESCRIBE,
98 ops::CATALOG_LIST,
99 ops::ROUTE_OPEN,
100 ops::ROUTE_POLL,
101 ops::ROUTE_CLOSING,
102 ops::ROUTE_CLOSED,
103 ops::SUPERVISOR_LIST,
104 ops::SUPERVISOR_RESTART,
105 ops::SUPERVISOR_SWAP,
106 ops::SUPERVISOR_RELOAD,
107 ops::SUPERVISOR_RESCAN,
108 ops::SUPERVISOR_RELEASE_RESERVED,
109 ops::SUPERVISOR_SET_ENABLED,
110 ops::SUPERVISOR_HEALTH_PROBE,
111 ops::SUPERVISOR_HEALTH,
112 ops::SUPERVISOR_STDERR_TAIL,
113 ops::SUPERVISOR_TERMINALS,
114 ops::SUPERVISOR_ROUTES,
115 ops::SUPERVISOR_PROVENANCE,
116 ops::SUPERVISOR_SPAWN_SNAPSHOT,
117 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
118];
119
120const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
127
128mod rescan_checked;
129use rescan_checked::{RescanMode, RescanPlan};
130
131const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
132 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
133 "supervisor.live_roots",
134 SCOPE_SYNC_OP,
135 SCOPE_APPLY_OP,
136 SCOPE_DESCRIBE_OP,
137 "operator.confirm",
138];
139
140const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
144
145const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
146
147pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
155
156pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
171
172pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
190
191const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
192const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
193
194fn reload_verdict(
195 configured: &Path,
196 spawned_from: Option<&Path>,
197 image: subc_control::RunningImageAgreement,
198) -> PendingReloadVerdict {
199 let path = match spawned_from {
200 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
201 ReloadPathAgreement::Match
202 }
203 Some(spawned_from) => ReloadPathAgreement::Mismatch {
204 configured: configured.to_path_buf(),
205 spawned_from: spawned_from.to_path_buf(),
206 },
207 None => ReloadPathAgreement::Unavailable {
208 reason: if matches!(
209 image,
210 subc_control::RunningImageAgreement::Unavailable {
211 reason: subc_control::RunningImageUnavailableReason::NotRunning
212 }
213 ) {
214 ReloadPathUnavailableReason::NotRunning
215 } else {
216 ReloadPathUnavailableReason::SpawnedPathUnavailable
217 },
218 },
219 };
220 PendingReloadVerdict {
221 path,
222 image,
223 spec: None,
224 }
225}
226
227fn launch_spec_agreement(
228 configured: &crate::supervise::ModuleSpec,
229 running: bool,
230 spawned: Option<&crate::supervise::LaunchSpecDigest>,
231) -> subc_control::LaunchSpecAgreement {
232 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
233 if !running {
234 return Agreement::Unavailable {
235 reason: Reason::NotRunning,
236 };
237 }
238 match spawned {
239 None => Agreement::Unavailable {
240 reason: Reason::SpawnedSpecUnavailable,
241 },
242 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
243 Agreement::Match
244 }
245 Some(_) => Agreement::Mismatch,
246 }
247}
248
249fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
250 if configured == spawned_from {
251 return true;
252 }
253 #[cfg(windows)]
254 {
255 match (
261 std::fs::canonicalize(configured),
262 std::fs::canonicalize(spawned_from),
263 ) {
264 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
265 _ => false,
266 }
267 }
268 #[cfg(not(windows))]
269 {
270 false
271 }
272}
273
274#[derive(Clone)]
275struct DaemonProvenanceFacts {
276 build: DaemonBuildProvenance,
277 pid: Option<u32>,
278 started_at_ms: Option<u64>,
279 start_clock: Option<crate::clock::StartClock>,
280 executable_path: Option<PathBuf>,
281 executable_identity: Option<SpawnedFileIdentity>,
282 process_start_time: Option<u64>,
283 probe: ExecutableIdentityProbe,
284}
285
286impl Default for DaemonProvenanceFacts {
287 fn default() -> Self {
288 Self {
289 build: DaemonBuildProvenance {
290 build_git_sha: None,
291 build_lock_digest: None,
292 },
293 pid: None,
294 started_at_ms: None,
295 start_clock: None,
296 executable_path: None,
297 executable_identity: None,
298 process_start_time: None,
299 probe: ExecutableIdentityProbe::default(),
300 }
301 }
302}
303
304#[derive(Debug, Clone)]
305struct SupervisorRescanContext {
306 supervisor: Supervisor,
307 config_path: PathBuf,
308 configured_port: Option<u16>,
309 storage_config: Option<crate::daemon_config::StorageConfig>,
310 admission_facts_carrier_module_id: Option<String>,
311 admission_facts_targets: Option<Vec<String>>,
312 scope_authority_owners: Vec<String>,
313 privileged_operator_prompt: PrivilegedOperatorPrompt,
314 generation: Arc<std::sync::atomic::AtomicU64>,
320}
321
322const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
343 "reloading",
344 "supervisor_not_live",
345 "registration_not_active",
346 "no_forwarding_connection",
347 "relay_send_failed",
348];
349
350#[derive(Clone)]
352pub struct ControlHandler {
353 registry: Arc<Registry>,
354 activity: Option<crate::activity::ActivityLedger>,
355 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
356 forwarding: Arc<ForwardingTable>,
357 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
358 supervisor: SupervisorHandle,
359 subc_capabilities: Arc<[String]>,
360 route_bind_relay_timeout: Duration,
364 route_bind_relay_timeouts: BTreeMap<String, Duration>,
368 route_bind_breakers: RouteBindBreakers,
371 route_bind_concurrency: RouteBindConcurrency,
374 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
379 route_bind_breaker_threshold: u32,
381 route_bind_breaker_cooldown: Duration,
383 health_probe_timeout: Duration,
384 storage_config: Option<crate::daemon_config::StorageConfig>,
387 machine_id: Option<crate::machine_id::MachineId>,
391 admission_facts_carrier_module_id: Option<String>,
392 admission_facts_targets: Option<Vec<String>>,
393 scopes: Arc<RwLock<ScopeTable>>,
397 scope_deadline_added: Arc<tokio::sync::Notify>,
401 scope_authority_owners: Vec<String>,
404 privileged_operator_prompt: PrivilegedOperatorPrompt,
406 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
409 #[cfg(test)]
410 caller_lookup: Option<TestCallerLookup>,
411 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
414 rescan: Option<SupervisorRescanContext>,
415 connected_clients: ConnectedClients,
416 counters: DaemonCounters,
417 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
418 daemon_shutdown: DaemonShutdownFlag,
423 daemon_provenance: DaemonProvenanceFacts,
424 #[cfg(test)]
425 control_dispatch_delay: Option<Duration>,
426 #[cfg(test)]
427 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
428 #[cfg(test)]
432 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
433 #[cfg(test)]
439 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
440}
441
442impl fmt::Debug for ControlHandler {
443 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
444 f.debug_struct("ControlHandler")
445 .field("registry", &self.registry)
446 .field("forwarding", &self.forwarding)
447 .field("process_liveness", &self.process_liveness.is_some())
448 .field("supervisor", &self.supervisor)
449 .field("subc_capabilities", &self.subc_capabilities)
450 .finish()
451 }
452}
453
454struct RouteOpenRequest {
455 target: RouteTarget,
456 identity: BindIdentity,
457 consumer_identity: Option<ConsumerIdentity>,
458 consumer_capabilities: Option<Vec<String>>,
459 role_versions: Option<BTreeMap<String, String>>,
460 admission_facts: Option<serde_json::Value>,
461 scope: Option<ScopeSelector>,
462}
463
464#[derive(Clone, Copy)]
465enum RouteOpenPhase {
466 Admission,
467 ProjectRoot,
468 EgressReservation,
469 BindSend,
470 BindAckWait,
471}
472
473struct RouteOpenTimings {
477 module_id: String,
478 principal: String,
479 admission: Duration,
480 project_root: Duration,
481 egress_reservation: Duration,
482 bind_send: Duration,
483 bind_ack_wait: Duration,
484 phase: Option<RouteOpenPhase>,
485 phase_started_at: Instant,
486}
487
488impl RouteOpenTimings {
489 fn new() -> Self {
490 Self {
491 module_id: String::new(),
492 principal: "unresolved".to_string(),
494 admission: Duration::ZERO,
495 project_root: Duration::ZERO,
496 egress_reservation: Duration::ZERO,
497 bind_send: Duration::ZERO,
498 bind_ack_wait: Duration::ZERO,
499 phase: Some(RouteOpenPhase::Admission),
500 phase_started_at: Instant::now(),
501 }
502 }
503
504 fn finish(&mut self) {
505 let elapsed = self.phase_started_at.elapsed();
506 match self.phase.take() {
507 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
508 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
509 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
510 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
511 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
512 None => {}
513 }
514 }
515
516 fn enter(&mut self, phase: RouteOpenPhase) {
517 self.finish();
518 self.phase = Some(phase);
519 self.phase_started_at = Instant::now();
520 }
521}
522
523struct RouteBindReservationGuard {
524 forwarding: Arc<ForwardingTable>,
525 endpoint: ModuleEndpointId,
526 relay_corr: u64,
527 armed: bool,
528}
529
530struct ModuleControlRpcGuard {
531 forwarding: Arc<ForwardingTable>,
532 endpoint: ModuleEndpointId,
533 corr: u64,
534 armed: bool,
535}
536
537impl ModuleControlRpcGuard {
538 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
539 Self {
540 forwarding,
541 endpoint,
542 corr,
543 armed: true,
544 }
545 }
546
547 fn disarm(&mut self) {
548 self.armed = false;
549 }
550}
551
552impl Drop for ModuleControlRpcGuard {
553 fn drop(&mut self) {
554 if self.armed {
555 let _ = self
556 .forwarding
557 .cancel_module_control_rpc(self.endpoint, self.corr);
558 }
559 }
560}
561
562impl RouteBindReservationGuard {
563 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
564 Self {
565 forwarding,
566 endpoint,
567 relay_corr,
568 armed: true,
569 }
570 }
571
572 fn release_and_disarm(&mut self) {
573 if !self.armed {
574 return;
575 }
576 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
577 self.endpoint,
578 self.relay_corr,
579 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
580 ) {
581 send_goodbye_target_best_effort(
582 &self.forwarding.counters(),
583 &target,
584 "canceled route.bind",
585 );
586 }
587 self.armed = false;
588 }
589
590 fn disarm(&mut self) {
591 self.armed = false;
592 }
593}
594
595impl Drop for RouteBindReservationGuard {
596 fn drop(&mut self) {
597 self.release_and_disarm();
598 }
599}
600
601#[derive(Debug, Clone, Default)]
631pub(crate) struct RouteBindBreakers {
632 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
633}
634
635#[derive(Debug, Clone, Default)]
636pub(crate) struct RouteBindConcurrency {
637 modules: Arc<Mutex<HashMap<String, usize>>>,
638}
639
640struct RouteBindConcurrencyGuard {
641 concurrency: RouteBindConcurrency,
642 module_id: String,
643}
644
645impl RouteBindConcurrency {
646 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
649 let mut modules = self
650 .modules
651 .lock()
652 .expect("route.bind concurrency mutex poisoned");
653 let in_flight = modules.entry(module_id.to_string()).or_default();
654 if *in_flight >= limit {
655 return Err(*in_flight);
656 }
657 *in_flight += 1;
658 Ok(RouteBindConcurrencyGuard {
659 concurrency: self.clone(),
660 module_id: module_id.to_string(),
661 })
662 }
663}
664
665impl Drop for RouteBindConcurrencyGuard {
666 fn drop(&mut self) {
667 let mut modules = self
668 .concurrency
669 .modules
670 .lock()
671 .expect("route.bind concurrency mutex poisoned");
672 let remove = {
673 let in_flight = modules
674 .get_mut(&self.module_id)
675 .expect("admitted route.bind has a concurrency entry");
676 *in_flight -= 1;
677 *in_flight == 0
678 };
679 if remove {
680 modules.remove(&self.module_id);
681 }
682 }
683}
684
685#[derive(Debug, Default)]
686struct ModuleBreakerState {
687 consecutive_timeouts: u32,
689 cooldown_until: Option<Instant>,
692 probe_in_flight: Option<Arc<()>>,
697}
698
699enum RouteBindAdmission<'a> {
701 Admitted {
702 guard: RouteBindBreakerGuard<'a>,
703 probe: bool,
706 },
707 Refused {
708 consecutive_timeouts: u32,
709 retry_in: Duration,
713 probe_in_flight: bool,
714 },
715}
716
717struct RouteBindBreakerGuard<'a> {
724 breakers: RouteBindBreakers,
725 module_id: &'a str,
726 probe_token: Option<Arc<()>>,
727 settled: bool,
728}
729
730impl RouteBindBreakerGuard<'_> {
731 fn record_accepted(&mut self) -> bool {
735 self.settled = true;
736 self.breakers.record_accepted(self.module_id)
737 }
738
739 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
742 self.settled = true;
743 self.breakers.record_timeout(
744 self.module_id,
745 self.probe_token.as_ref(),
746 threshold,
747 cooldown,
748 )
749 }
750
751 fn record_inconclusive(&mut self) {
760 self.settled = true;
761 self.breakers
762 .record_inconclusive(self.module_id, self.probe_token.as_ref());
763 }
764}
765
766impl Drop for RouteBindBreakerGuard<'_> {
767 fn drop(&mut self) {
768 if !self.settled {
769 self.breakers
770 .record_inconclusive(self.module_id, self.probe_token.as_ref());
771 }
772 }
773}
774
775struct BreakerOpened {
779 consecutive_timeouts: u32,
780 reopened_after_probe: bool,
783}
784
785impl RouteBindBreakers {
786 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
787 self.modules
788 .lock()
789 .expect("route.bind breaker mutex poisoned")
790 }
791
792 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
795 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
796 probe: probe_token.is_some(),
797 guard: RouteBindBreakerGuard {
798 breakers: self.clone(),
799 module_id,
800 probe_token,
801 settled: false,
802 },
803 };
804
805 let mut modules = self.lock();
806 let Some(state) = modules.get_mut(module_id) else {
807 return admitted(None);
808 };
809 let Some(cooldown_until) = state.cooldown_until else {
810 return admitted(None);
811 };
812 if state.probe_in_flight.is_some() {
813 return RouteBindAdmission::Refused {
814 consecutive_timeouts: state.consecutive_timeouts,
815 retry_in: Duration::ZERO,
816 probe_in_flight: true,
817 };
818 }
819 let now = Instant::now();
820 if now < cooldown_until {
821 return RouteBindAdmission::Refused {
822 consecutive_timeouts: state.consecutive_timeouts,
823 retry_in: cooldown_until - now,
824 probe_in_flight: false,
825 };
826 }
827 let token = Arc::new(());
828 state.probe_in_flight = Some(Arc::clone(&token));
829 admitted(Some(token))
830 }
831
832 fn record_accepted(&self, module_id: &str) -> bool {
833 self.lock()
834 .remove(module_id)
835 .is_some_and(|state| state.cooldown_until.is_some())
836 }
837
838 fn record_timeout(
839 &self,
840 module_id: &str,
841 probe_token: Option<&Arc<()>>,
842 threshold: u32,
843 cooldown: Duration,
844 ) -> Option<BreakerOpened> {
845 let mut modules = self.lock();
846 let state = modules.entry(module_id.to_string()).or_default();
847 let was_open = state.cooldown_until.is_some();
848 let was_probe = Self::owns_probe(state, probe_token);
849 if was_probe {
850 state.probe_in_flight = None;
851 }
852 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
853 if state.consecutive_timeouts < threshold {
854 return None;
855 }
856 state.cooldown_until = Some(Instant::now() + cooldown);
857 Some(BreakerOpened {
858 consecutive_timeouts: state.consecutive_timeouts,
859 reopened_after_probe: was_open && was_probe,
860 })
861 }
862
863 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
864 state
868 .probe_in_flight
869 .as_ref()
870 .zip(token)
871 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
872 }
873
874 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
875 if let Some(state) = self.lock().get_mut(module_id) {
876 if Self::owns_probe(state, probe_token) {
877 state.probe_in_flight = None;
878 }
879 }
880 }
881
882 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
897 self.lock()
898 .remove(module_id)
899 .map(|state| state.consecutive_timeouts)
900 .filter(|discarded| *discarded > 0)
901 }
902
903 fn open_snapshot(&self) -> Option<serde_json::Value> {
910 let now = Instant::now();
911 let modules = self.lock();
912 let open = modules
913 .iter()
914 .filter_map(|(module_id, state)| {
915 let cooldown_until = state.cooldown_until?;
916 Some((
917 module_id.clone(),
918 serde_json::json!({
919 "consecutive_timeouts": state.consecutive_timeouts,
920 "cooldown_remaining_ms":
921 cooldown_until.saturating_duration_since(now).as_millis() as u64,
922 "probe_in_flight": state.probe_in_flight.is_some(),
923 }),
924 ))
925 })
926 .collect::<serde_json::Map<String, serde_json::Value>>();
927 (!open.is_empty()).then_some(serde_json::Value::Object(open))
928 }
929}
930
931impl ControlHandler {
932 pub fn new(registry: Arc<Registry>) -> Self {
933 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
934 }
935
936 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
937 let counters = forwarding.counters();
938 let route_bind_breakers = forwarding.route_bind_breakers();
942 let route_bind_concurrency = forwarding.route_bind_concurrency();
943 let route_outages = forwarding.route_outages();
944 Self {
945 registry,
946 activity: None,
947 activity_streams: Arc::new(Mutex::new(HashMap::new())),
948 forwarding,
949 process_liveness: None,
950 supervisor: SupervisorHandle::new(),
951 subc_capabilities: Arc::from([
952 CAP_MANIFEST_REGISTRATION.to_string(),
953 CAP_CHANNEL_LIFECYCLE.to_string(),
954 CAP_PING_PONG.to_string(),
955 CAP_SESSION_ATTACH.to_string(),
956 CAP_ADMISSION_FACTS_RELAY.to_string(),
957 CAP_SCOPES_V1.to_string(),
958 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
959 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
960 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
961 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
962 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
963 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
964 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
965 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
966 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
967 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
968 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
969 ]),
970 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
971 route_bind_relay_timeouts: BTreeMap::new(),
972 route_bind_breakers,
973 route_bind_concurrency,
974 route_outages,
975 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
976 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
977 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
978 storage_config: None,
979 machine_id: None,
980 admission_facts_carrier_module_id: None,
981 admission_facts_targets: None,
982 scopes: Arc::new(RwLock::new(ScopeTable::new(
983 crate::daemon_config::default_scope_authority_owners(),
984 ))),
985 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
986 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
987 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
988 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
989 #[cfg(test)]
990 caller_lookup: None,
991 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
992 rescan: None,
993 connected_clients: ConnectedClients::new(),
994 counters,
995 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
996 daemon_shutdown: DaemonShutdownFlag::default(),
997 daemon_provenance: DaemonProvenanceFacts::default(),
998 #[cfg(test)]
999 control_dispatch_delay: None,
1000 #[cfg(test)]
1001 provenance_probe_override: None,
1002 #[cfg(test)]
1003 route_bind_relay_sent_hold: None,
1004 #[cfg(test)]
1005 hello_registry_to_forwarding_hold: None,
1006 }
1007 }
1008
1009 pub fn with_storage_config(
1012 mut self,
1013 storage_config: Option<crate::daemon_config::StorageConfig>,
1014 ) -> Self {
1015 self.storage_config = storage_config;
1016 self
1017 }
1018
1019 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1022 self.activity = Some(crate::activity::ActivityLedger::open(
1023 run_dir.join("activity.jsonl"),
1024 incarnation,
1025 ));
1026 self.supervisor.set_activity(self.activity.clone());
1027 self
1028 }
1029
1030 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1033 self.machine_id = machine_id;
1034 self
1035 }
1036
1037 pub fn with_admission_facts_config(
1041 mut self,
1042 carrier_module_id: Option<String>,
1043 targets: Option<Vec<String>>,
1044 ) -> Self {
1045 self.admission_facts_carrier_module_id = carrier_module_id;
1046 self.admission_facts_targets = targets;
1047 self
1048 }
1049
1050 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1054 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1055 self.scope_authority_owners = owners;
1056 self
1057 }
1058
1059 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1061 self.privileged_operator_prompt = mode;
1062 self
1063 }
1064
1065 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1068 self.route_bind_relay_timeout = timeout;
1069 self
1070 }
1071
1072 pub fn with_route_bind_relay_timeouts(
1078 mut self,
1079 timeouts: impl IntoIterator<Item = (String, Duration)>,
1080 ) -> Self {
1081 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1082 self
1083 }
1084
1085 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1091 self.route_bind_relay_timeouts
1092 .get(module_id)
1093 .copied()
1094 .unwrap_or(self.route_bind_relay_timeout)
1095 }
1096
1097 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1105 self.route_bind_breaker_threshold = threshold.max(1);
1106 self.route_bind_breaker_cooldown = cooldown;
1107 self
1108 }
1109
1110 #[cfg(test)]
1111 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1112 self.health_probe_timeout = timeout;
1113 self
1114 }
1115
1116 #[cfg(test)]
1117 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1118 self.control_dispatch_delay = Some(delay);
1119 self
1120 }
1121
1122 #[cfg(test)]
1126 pub(crate) fn with_route_bind_relay_sent_hold(
1127 mut self,
1128 hold: Arc<tokio::sync::Barrier>,
1129 ) -> Self {
1130 self.route_bind_relay_sent_hold = Some(hold);
1131 self
1132 }
1133
1134 #[cfg(test)]
1139 pub(crate) fn with_hello_registry_to_forwarding_hold(
1140 mut self,
1141 hold: Arc<std::sync::Barrier>,
1142 ) -> Self {
1143 self.hello_registry_to_forwarding_hold = Some(hold);
1144 self
1145 }
1146
1147 #[cfg(test)]
1148 fn hold_between_registry_and_forwarding(&self) {
1149 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1150 hold.wait();
1151 hold.wait();
1152 }
1153 }
1154
1155 pub fn with_process_liveness(
1156 mut self,
1157 process_liveness: Arc<dyn ModuleProcessLiveness>,
1158 ) -> Self {
1159 self.process_liveness = Some(process_liveness);
1160 self
1161 }
1162
1163 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1164 self.supervisor = supervisor;
1165 self.supervisor.set_activity(self.activity.clone());
1166 self
1167 }
1168
1169 pub fn with_daemon_provenance(
1170 mut self,
1171 pid: u32,
1172 started_at_ms: u64,
1173 executable_path: Option<PathBuf>,
1174 build_git_sha: Option<String>,
1175 build_lock_digest: Option<String>,
1176 ) -> Self {
1177 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1178 let process_start_time = process_start_time(pid);
1179 self.daemon_provenance = DaemonProvenanceFacts {
1180 build: DaemonBuildProvenance {
1181 build_git_sha,
1182 build_lock_digest,
1183 },
1184 pid: Some(pid),
1185 started_at_ms: Some(started_at_ms),
1186 start_clock: None,
1187 executable_path,
1188 executable_identity,
1189 process_start_time,
1190 probe: ExecutableIdentityProbe::default(),
1191 };
1192 self
1193 }
1194
1195 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1196 self.daemon_provenance.start_clock = Some(clock);
1197 self
1198 }
1199
1200 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1202 self.scopes
1203 .write()
1204 .unwrap_or_else(|poisoned| poisoned.into_inner())
1205 .set_wall_clock(Arc::new(clock));
1206 self
1207 }
1208
1209 #[cfg(test)]
1210 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1211 self.provenance_probe_override = Some(result);
1212 self
1213 }
1214
1215 pub fn with_capability_config(
1219 self,
1220 modules: impl IntoIterator<Item = (String, bool)>,
1221 reserved_capabilities: BTreeMap<String, String>,
1222 ) -> Self {
1223 self.capability_evaluator
1224 .configure(modules, reserved_capabilities);
1225 self
1226 }
1227
1228 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1232 self.daemon_shutdown = flag;
1233 self
1234 }
1235
1236 pub fn with_supervisor_rescan(
1237 mut self,
1238 supervisor: Supervisor,
1239 config_path: impl Into<PathBuf>,
1240 configured_port: Option<u16>,
1241 ) -> Self {
1242 self.rescan = Some(SupervisorRescanContext {
1243 supervisor,
1244 config_path: config_path.into(),
1245 configured_port,
1246 storage_config: self.storage_config.clone(),
1247 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1248 admission_facts_targets: self.admission_facts_targets.clone(),
1249 scope_authority_owners: self.scope_authority_owners.clone(),
1250 privileged_operator_prompt: self.privileged_operator_prompt,
1251 generation: Arc::default(),
1252 });
1253 self
1254 }
1255
1256 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1257 self.connected_clients = connected_clients;
1258 self
1259 }
1260
1261 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1262 Arc::clone(&self.forwarding)
1263 }
1264
1265 pub(crate) fn counters(&self) -> DaemonCounters {
1266 self.counters.clone()
1267 }
1268
1269 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1272 tokio::spawn(async move {
1273 loop {
1274 self.capability_evaluator
1275 .wait_for_change_or_deadline()
1276 .await;
1277 self.refresh_capability_requirements();
1278 }
1279 });
1280 }
1281
1282 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1288 tokio::spawn(async move {
1289 loop {
1290 let has_deadlines = self
1291 .scopes
1292 .read()
1293 .unwrap_or_else(|poisoned| poisoned.into_inner())
1294 .has_deadlines();
1295 if !has_deadlines {
1296 self.scope_deadline_added.notified().await;
1297 continue;
1298 }
1299 tokio::time::sleep(Duration::from_secs(1)).await;
1300 if let Err(error) = self.sweep_expired_scopes() {
1301 warn!(%error, "scope expiry sweep failed");
1302 }
1303 }
1304 });
1305 }
1306
1307 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1308 let mut table = self
1309 .scopes
1310 .write()
1311 .unwrap_or_else(|poisoned| poisoned.into_inner());
1312 let swept = table.sweep_expired();
1313 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1314 drop(table);
1315 self.close_scope_drained_routes(drained);
1316 Ok(())
1317 }
1318
1319 fn publish_scope_changes(
1320 &self,
1321 changes: &[crate::scopes::ScopeTagChange],
1322 expired: &[crate::scopes::ScopeExpired],
1323 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1324 if changes.is_empty() {
1325 return Ok(Vec::new());
1326 }
1327 let drained = self
1328 .forwarding
1329 .publish_scope_changes(changes)
1330 .map_err(RouterError::Forwarding)?;
1331 let mut counts = HashMap::new();
1332 for route in &drained {
1333 *counts
1334 .entry((
1335 route.scope.owner.as_str(),
1336 route.scope.scope_ref.as_str(),
1337 route.scope.tag.scope_epoch,
1338 ))
1339 .or_insert(0usize) += 1;
1340 }
1341 for scope in expired {
1342 let routes_closed = counts
1343 .get(&(
1344 scope.owner.as_str(),
1345 scope.scope_ref.as_str(),
1346 scope.scope_epoch,
1347 ))
1348 .copied()
1349 .unwrap_or(0);
1350 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1351 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1352 routes_closed, "scope expired");
1353 }
1354 Ok(drained)
1355 }
1356
1357 fn runtime_capability_snapshot(
1358 &self,
1359 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1360 let runtime = self
1361 .supervisor
1362 .list()
1363 .into_iter()
1364 .map(|module| {
1365 let status = module.status().map_err(|err| {
1366 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1367 })?;
1368 Ok(RuntimeModule {
1369 module_id: status.module_id,
1370 state: status.state,
1371 enabled: status.enabled,
1372 })
1373 })
1374 .collect::<Result<Vec<_>, RouterError>>()?;
1375 let registrations = self.registry.list_registered_modules().map_err(|err| {
1381 RouterError::backend(
1382 0,
1383 0,
1384 format!("failed to list capability registrations: {err}"),
1385 )
1386 })?;
1387 let registrations = registrations
1388 .into_iter()
1389 .map(|registration| RegisteredModule {
1390 module_id: registration.manifest.module_id,
1391 module_version: registration.manifest.module_version,
1392 capabilities: registration.manifest.capabilities,
1393 })
1394 .collect();
1395 Ok((runtime, registrations))
1396 }
1397
1398 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1405 let cached_registration = RegisteredModule {
1406 module_id: registration.manifest.module_id.clone(),
1407 module_version: registration.manifest.module_version.clone(),
1408 capabilities: registration.manifest.capabilities.clone(),
1409 };
1410 let change = self.capability_evaluator.record_hello(&cached_registration);
1411 log_manifest_change(&cached_registration.module_id, &change);
1412 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1413 self.enforce_capability_denies();
1414 }
1415 self.refresh_capability_requirements();
1416 }
1417
1418 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1422 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1423 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1424 self.supervisor.set_swap_promotion_observer(observer);
1425 }
1426
1427 pub fn refresh_capability_requirements(&self) {
1428 match self.runtime_capability_snapshot() {
1429 Ok((runtime, registrations)) => {
1430 log_requirement_events(
1431 self.capability_evaluator
1432 .evaluate_now(&runtime, ®istrations),
1433 &self.daemon_shutdown,
1434 );
1435 }
1436 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1437 }
1438 }
1439
1440 fn enforce_capability_denies(&self) {
1444 let registrations = match self.registry.list_registered_modules() {
1445 Ok(snapshot) => snapshot,
1446 Err(err) => {
1447 warn!(error = %err, "failed to read registrations for capability deny census");
1448 return;
1449 }
1450 };
1451 let manifests = registrations
1452 .into_iter()
1453 .map(|registration| {
1454 (
1455 registration.manifest.module_id.clone(),
1456 registration.manifest,
1457 )
1458 })
1459 .collect::<BTreeMap<_, _>>();
1460 let census = match self.forwarding.route_census(None) {
1461 Ok(census) => census,
1462 Err(err) => {
1463 warn!(error = %err, "failed to read route census for capability deny enforcement");
1464 return;
1465 }
1466 };
1467
1468 for (target_module_id, routes) in census {
1469 let Some(target_manifest) = manifests.get(&target_module_id) else {
1470 continue;
1471 };
1472 let mut closed_routes = Vec::new();
1473 let mut goodbyes = Vec::new();
1474 for route in routes {
1475 let Principal::Reserved {
1476 module_id: opening_module_id,
1477 } = &route.principal
1478 else {
1479 continue;
1480 };
1481 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1482 continue;
1483 };
1484 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1485 continue;
1486 };
1487
1488 match self.forwarding.release_client_route(
1489 route.goodbye_target.connection_id,
1490 route.goodbye_target.channel,
1491 route.goodbye_target.epoch,
1492 ) {
1493 Ok(RouteRelease::Removed(module_goodbye)) => {
1494 warn!(
1495 opening_module_id,
1496 target_module_id,
1497 capability,
1498 "force-closing route because an attested capability deny edge now matches"
1499 );
1500 goodbyes.push(module_goodbye);
1501 goodbyes.push(route.goodbye_target.clone());
1502 closed_routes.push(route);
1503 }
1504 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1505 Err(err) => warn!(
1506 opening_module_id,
1507 target_module_id,
1508 capability,
1509 error = %err,
1510 "failed to force-close capability-denied route"
1511 ),
1512 }
1513 }
1514
1515 if closed_routes.is_empty() {
1516 continue;
1517 }
1518 send_route_control_pushes(
1519 &self.forwarding,
1520 closed_routes,
1521 ClientControlPush::RouteClosed {
1522 module_id: target_module_id,
1523 channels: Vec::new(),
1524 reason: RouteCloseReason::CapabilityDenied,
1525 drained: false,
1526 abandoned: 0,
1527 excluded_subscriptions: 0,
1528 terminal: Some(false),
1529 },
1530 );
1531 self.emit_route_goodbyes(goodbyes);
1534 }
1535 }
1536
1537 fn not_ready_reason(
1543 &self,
1544 registration: &crate::registry::ModuleRegistration,
1545 ) -> Option<NotReadyReason> {
1546 if !registration.ready {
1547 return Some(NotReadyReason {
1548 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1549 capability: None,
1550 });
1551 }
1552 self.first_unprovided_required_capability(registration)
1553 .map(|capability| NotReadyReason {
1554 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1555 capability: Some(capability),
1556 })
1557 }
1558
1559 fn first_unprovided_required_capability(
1577 &self,
1578 registration: &crate::registry::ModuleRegistration,
1579 ) -> Option<String> {
1580 let required = registration
1581 .manifest
1582 .capabilities
1583 .iter()
1584 .flat_map(|declarations| declarations.requires.iter())
1585 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1586 .map(|requirement| requirement.capability.as_str())
1587 .collect::<BTreeSet<_>>();
1588 if required.is_empty() {
1589 return None;
1590 }
1591 let module_id = registration.manifest.module_id.as_str();
1592 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1593 if required
1594 .iter()
1595 .any(|capability| verdict(capability).is_none())
1596 {
1597 self.refresh_capability_requirements();
1598 }
1599 required
1600 .into_iter()
1601 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1602 .map(str::to_string)
1603 }
1604
1605 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1606 self.capability_evaluator
1607 .statuses()
1608 .into_iter()
1609 .map(capability_requirement_status)
1610 .collect()
1611 }
1612
1613 fn deregister_connection(
1620 &self,
1621 connection_id: ConnectionId,
1622 reason: RegistrationEndReason,
1623 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1624 self.registry
1625 .deregister_connection_with_reason(connection_id, reason)
1626 }
1627
1628 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1629 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1630 return None;
1631 }
1632 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1633 parse_client_control_request(&frame.body)
1634 else {
1635 return None;
1636 };
1637 Some(target_module_id(&target).to_string())
1638 }
1639
1640 pub(crate) fn route_open_capacity_refusal(
1641 &self,
1642 ctx: &RouteCtx,
1643 frame: &Frame,
1644 target_module_id: &str,
1645 in_flight: usize,
1646 limit: usize,
1647 ) -> Result<Frame, RouterError> {
1648 let caller = self.route_open_caller_from_frame(frame);
1651 self.route_open_admission_refusal_frame(
1652 ctx,
1653 &caller,
1654 frame,
1655 target_module_id,
1656 "open_admission_full",
1657 (in_flight, limit),
1658 format!(
1659 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1660 ),
1661 )
1662 }
1663
1664 fn route_open_target_capacity_refusal(
1665 &self,
1666 ctx: &RouteCtx,
1667 caller: &RouteOpenCaller,
1668 frame: &Frame,
1669 target_module_id: &str,
1670 in_flight: usize,
1671 ) -> Result<Frame, RouterError> {
1672 self.route_open_admission_refusal_frame(
1673 ctx,
1674 caller,
1675 frame,
1676 target_module_id,
1677 "target_binds_full",
1678 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1679 format!(
1680 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1681 ),
1682 )
1683 }
1684
1685 #[allow(clippy::too_many_arguments)]
1693 fn route_open_admission_refusal_frame(
1694 &self,
1695 ctx: &RouteCtx,
1696 caller: &RouteOpenCaller,
1697 frame: &Frame,
1698 target_module_id: &str,
1699 reason: &'static str,
1700 (in_flight, limit): (usize, usize),
1701 message: impl Into<String>,
1702 ) -> Result<Frame, RouterError> {
1703 let code = error_codes::TARGET_UNAVAILABLE;
1704 self.counters.increment_route_open_refused(code);
1705 info!(
1706 target: "control",
1707 code,
1708 reason,
1709 module_id = ?target_module_id,
1710 connection_id = ctx.connection_id.get(),
1711 principal = %caller.principal,
1712 scope = %caller.scope,
1713 harness = %caller.harness,
1714 in_flight,
1715 limit,
1716 "route.open refused"
1717 );
1718 control_error_frame(frame, code, message.into())
1719 }
1720
1721 #[cfg(test)]
1727 pub fn handle_control(
1728 &self,
1729 connection_id: ConnectionId,
1730 frame: Frame,
1731 ) -> Result<Vec<Frame>, RouterError> {
1732 match frame.header.ty {
1733 FrameType::Ping => Ok(vec![pong(&frame)?]),
1734 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1735 FrameType::Goodbye => self.handle_goodbye(connection_id),
1736 ty => Ok(vec![control_error_frame(
1737 &frame,
1738 "unsupported_control_frame",
1739 format!("unsupported channel-0 frame {ty:?}"),
1740 )?]),
1741 }
1742 }
1743
1744 pub async fn handle_control_frame(
1745 &self,
1746 ctx: &RouteCtx,
1747 frame: Frame,
1748 ) -> Result<Vec<Frame>, RouterError> {
1749 self.handle_control_frame_timed(ctx, frame, None).await
1750 }
1751
1752 pub(crate) fn record_caller_endpoints(
1753 &self,
1754 connection: ConnectionId,
1755 endpoints: Option<(SocketAddr, SocketAddr)>,
1756 ) {
1757 let mut callers = self
1758 .caller_endpoints
1759 .lock()
1760 .unwrap_or_else(|p| p.into_inner());
1761 callers.remove(&connection);
1762 if let Some(endpoints) = endpoints {
1763 callers.insert(connection, endpoints);
1764 }
1765 }
1766
1767 #[cfg(test)]
1768 pub(crate) fn with_caller_lookup_for_test(
1769 mut self,
1770 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1771 ) -> Self {
1772 self.caller_lookup = Some(Arc::new(lookup));
1773 self
1774 }
1775
1776 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1777 let endpoints = self
1778 .caller_endpoints
1779 .lock()
1780 .unwrap_or_else(|p| p.into_inner())
1781 .get(&connection)
1782 .copied();
1783 #[cfg(test)]
1784 let lookup = self.caller_lookup.clone();
1785 tokio::task::spawn_blocking(move || {
1786 #[cfg(test)]
1787 if let Some(lookup) = lookup {
1788 return lookup(endpoints);
1789 }
1790 endpoints
1791 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1792 .unwrap_or_default()
1793 })
1794 .await
1795 .unwrap_or_default()
1796 }
1797
1798 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1799 frame.header.channel == 0
1800 && frame.header.ty == FrameType::Request
1801 && self
1802 .forwarding
1803 .module_endpoint_for_connection(ctx.connection_id)
1804 .is_ok_and(|endpoint| endpoint.is_none())
1805 && parse_client_control_request(&frame.body)
1806 .ok()
1807 .is_some_and(|request| privileged_operation(&request).is_some())
1808 }
1809
1810 pub(crate) async fn handle_control_frame_timed(
1811 &self,
1812 ctx: &RouteCtx,
1813 frame: Frame,
1814 dispatch_started_at: Option<StdInstant>,
1815 ) -> Result<Vec<Frame>, RouterError> {
1816 match frame.header.ty {
1817 FrameType::Ping => Ok(vec![pong(&frame)?]),
1818 FrameType::Hello => {
1819 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1820 }
1821 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1822 FrameType::Cancel => {
1823 if self
1826 .forwarding
1827 .operator_confirms()
1828 .cancel(ctx.connection_id, frame.header.corr)
1829 || self
1830 .supervisor
1831 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1832 || self.cancel_activity_subscription(ctx, &frame)?
1833 {
1834 Ok(Vec::new())
1835 } else {
1836 Ok(vec![control_error_frame(
1837 &frame,
1838 "unknown_subscription",
1839 "no subscription has this correlation id",
1840 )?])
1841 }
1842 }
1843 FrameType::Request => {
1844 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1847 if op
1848 .as_ref()
1849 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1850 {
1851 return self.handle_activity_record(ctx, frame).await;
1852 }
1853 if op.as_ref().is_some_and(|probe| {
1854 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1855 }) {
1856 return self.handle_activity_read(ctx, frame).await;
1857 }
1858 if op
1859 .as_ref()
1860 .is_some_and(|probe| probe.op == "operator.confirm")
1861 {
1862 return self.handle_operator_confirm(ctx, frame);
1863 }
1864 if op
1865 .as_ref()
1866 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1867 {
1868 return self.handle_scope_served_by(ctx.connection_id, frame);
1869 }
1870 if self
1871 .forwarding
1872 .module_endpoint_for_connection(ctx.connection_id)
1873 .map_err(RouterError::Forwarding)?
1874 .is_some()
1875 {
1876 if !is_known_module_request_op(&frame.body) {
1877 return Ok(vec![control_error_frame(
1878 &frame,
1879 "unsupported_control_frame",
1880 "module-originated channel-0 REQUEST is not supported",
1881 )?]);
1882 }
1883 let request = match parse_module_control_request_from_module(&frame.body) {
1884 Ok(request) => request,
1885 Err((err, ControlRequestBodyError::UnknownOp)) => {
1886 return Ok(vec![control_error_frame(
1887 &frame,
1888 "unsupported_control_frame",
1889 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1890 )?])
1891 }
1892 Err((err, ControlRequestBodyError::InvalidBody)) => {
1893 return Ok(vec![control_error_frame(
1894 &frame,
1895 "invalid_control_body",
1896 format!("malformed module control body: {err}"),
1897 )?])
1898 }
1899 };
1900 let op = module_control_request_op(&request);
1901 let corr = frame.header.corr;
1902 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1903 let result =
1904 self.handle_module_control_request(ctx.connection_id, frame, request);
1905 log_slow_control_dispatch(
1906 dispatch_started_at,
1907 op,
1908 ctx.connection_id,
1909 corr,
1910 None,
1911 );
1912 return result;
1913 }
1914
1915 if is_known_module_request_op(&frame.body) {
1916 return Ok(vec![control_error_frame(
1917 &frame,
1918 "not_registered",
1919 "catalog.update requires an active module registration owned by this connection",
1920 )?]);
1921 }
1922
1923 let request = match parse_client_control_request(&frame.body) {
1924 Ok(request) => request,
1925 Err((err, ControlRequestBodyError::UnknownOp)) => {
1926 return Ok(vec![control_error_frame(
1927 &frame,
1928 "unknown_control_op",
1929 format!("unknown client control op: {err}"),
1930 )?])
1931 }
1932 Err((err, ControlRequestBodyError::InvalidBody)) => {
1933 return Ok(vec![control_error_frame(
1934 &frame,
1935 "invalid_control_body",
1936 format!("malformed client control body: {err}"),
1937 )?])
1938 }
1939 };
1940 let op = client_control_request_op(&request);
1941 let corr = frame.header.corr;
1942 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1943 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1944 #[cfg(test)]
1945 if let Some(delay) = self.control_dispatch_delay {
1946 tokio::time::sleep(delay).await;
1947 }
1948 let result = self
1949 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1950 .await;
1951 if let Some(timings) = route_open_timings.as_mut() {
1952 timings.finish();
1953 }
1954 log_slow_control_dispatch(
1955 dispatch_started_at,
1956 op,
1957 ctx.connection_id,
1958 corr,
1959 route_open_timings.as_ref(),
1960 );
1961 result
1962 }
1963 FrameType::Push => {
1964 let Some(endpoint) = self
1965 .forwarding
1966 .module_endpoint_for_connection(ctx.connection_id)
1967 .map_err(RouterError::Forwarding)?
1968 else {
1969 return Ok(vec![control_error_frame(
1970 &frame,
1971 "unsupported_control_frame",
1972 "client-originated channel-0 PUSH is not supported",
1973 )?]);
1974 };
1975 self.handle_status_update(endpoint, frame)
1976 }
1977 FrameType::Response | FrameType::Error
1978 if self
1979 .forwarding
1980 .module_endpoint_for_connection(ctx.connection_id)
1981 .map_err(RouterError::Forwarding)?
1982 .is_some() =>
1983 {
1984 self.handle_module_relay_response(ctx.connection_id, frame)
1985 }
1986 ty => Ok(vec![control_error_frame(
1987 &frame,
1988 "unsupported_control_frame",
1989 format!("unsupported channel-0 frame {ty:?}"),
1990 )?]),
1991 }
1992 }
1993
1994 pub fn cleanup_connection(
1995 &self,
1996 connection_id: ConnectionId,
1997 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1998 self.caller_endpoints
1999 .lock()
2000 .unwrap_or_else(|p| p.into_inner())
2001 .remove(&connection_id);
2002 self.forwarding
2003 .operator_confirms()
2004 .connection_closed(connection_id);
2005 self.cleanup_connection_with_end_reason(
2006 connection_id,
2007 RegistrationEndReason::ConnectionClosed,
2008 )
2009 }
2010
2011 fn cleanup_connection_with_end_reason(
2012 &self,
2013 connection_id: ConnectionId,
2014 requested_reason: RegistrationEndReason,
2015 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2016 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2017 self.registry
2018 .get_module_by_connection(connection_id)?
2019 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2020 .and_then(|module| module.registration_end_reason().ok().flatten())
2021 .unwrap_or(requested_reason)
2022 } else {
2023 requested_reason
2024 };
2025 let crash_closed = self
2026 .registry
2027 .get_module_by_connection(connection_id)?
2028 .and_then(|registration| {
2029 self.forwarding
2030 .module_endpoint_for_connection(connection_id)
2031 .ok()
2032 .flatten()
2033 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2034 .map(|routes| (registration.manifest.module_id, routes))
2035 });
2036 let crash_closed = crash_closed.map(|(module_id, routes)| {
2037 let terminal = match self.supervisor.get(&module_id) {
2038 None => false,
2039 Some(module) => match module.will_recover_after_connection_loss() {
2040 Ok(will_recover) => !will_recover,
2041 Err(err) => {
2042 warn!(
2043 %module_id,
2044 error = %err,
2045 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2046 );
2047 false
2048 }
2049 },
2050 };
2051 let (reason, drained) = match self
2055 .forwarding
2056 .daemon_drain_result_for_connection(connection_id)
2057 {
2058 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2059 Ok(None) => (RouteCloseReason::Crash, false),
2060 Err(err) => {
2061 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2062 (RouteCloseReason::Crash, false)
2063 }
2064 };
2065 (module_id, routes, reason, drained, terminal)
2066 });
2067 let registrations = self.deregister_connection(connection_id, end_reason);
2068 let cleanup = if crash_closed.is_some() {
2069 self.forwarding.cleanup_connection_counted(connection_id)
2070 } else {
2071 self.forwarding
2077 .cleanup_connection(connection_id)
2078 .map(|released| crate::forwarding::ConnectionCleanup {
2079 released,
2080 abandoned_relays: 0,
2081 })
2082 };
2083 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2088 let abandoned = cleanup
2089 .as_ref()
2090 .map_or(0, |cleanup| cleanup.abandoned_relays);
2091 send_route_control_pushes(
2092 &self.forwarding,
2093 routes,
2094 ClientControlPush::RouteClosed {
2095 module_id,
2096 channels: Vec::new(),
2097 reason,
2098 drained,
2099 abandoned,
2100 excluded_subscriptions: 0,
2101 terminal: Some(terminal),
2102 },
2103 );
2104 }
2105 if let Ok(cleanup) = cleanup {
2106 self.emit_route_goodbyes(cleanup.released);
2107 }
2108 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2112 crate::supervise::notify_registration_release();
2113 self.capability_evaluator.wake_deadline_loop();
2114 self.refresh_capability_requirements();
2115 }
2116 self.supervisor.remove_spawn_subscribers(connection_id);
2117 self.activity_streams
2118 .lock()
2119 .unwrap_or_else(|p| p.into_inner())
2120 .retain(|(id, _), task| {
2121 if *id == connection_id {
2122 task.abort();
2123 false
2124 } else {
2125 true
2126 }
2127 });
2128 self.hello_launch_nonces
2131 .lock()
2132 .unwrap_or_else(|poisoned| poisoned.into_inner())
2133 .forget(connection_id);
2134 self.scopes
2135 .write()
2136 .unwrap_or_else(|poisoned| poisoned.into_inner())
2137 .release_connection(connection_id);
2138 registrations
2139 }
2140
2141 pub(crate) fn handle_route_goodbye(
2142 &self,
2143 connection_id: ConnectionId,
2144 route_channel: u16,
2145 route_epoch: u32,
2146 ) -> Result<bool, RouterError> {
2147 debug!(
2148 connection_id = connection_id.get(),
2149 route_channel, route_epoch, "handling route GOODBYE"
2150 );
2151 let RouteRelease::Removed(released_route) = self
2152 .forwarding
2153 .release_client_route(connection_id, route_channel, route_epoch)
2154 .map_err(RouterError::Forwarding)?
2155 else {
2156 return Ok(false);
2157 };
2158 self.emit_route_goodbyes(vec![released_route]);
2159 Ok(true)
2160 }
2161
2162 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2163 for released in released_routes {
2164 let frame = match Frame::build_with_version(
2165 released.negotiated_ver,
2166 FrameType::Goodbye,
2167 control_flags(),
2168 released.channel,
2169 released.epoch,
2170 0,
2171 Vec::new(),
2172 ) {
2173 Ok(frame) => frame,
2174 Err(err) => {
2175 warn!(
2176 route_channel = released.channel,
2177 error = %err,
2178 "failed to build route GOODBYE frame"
2179 );
2180 continue;
2181 }
2182 };
2183 if !released.close_on_delivery_failure() {
2184 crate::forwarding::send_module_route_goodbye(
2185 &self.counters,
2186 &released.sink,
2187 frame,
2188 released.module_id.as_deref(),
2189 "client route released",
2190 );
2191 continue;
2192 }
2193 if let Err(err) = released.sink.try_send(frame) {
2194 warn!(
2195 target_connection_id = released.connection_id.get(),
2196 route_channel = released.channel,
2197 error = %err,
2198 "route GOODBYE was not delivered to client; closing target connection"
2199 );
2200 if self
2201 .forwarding
2202 .escalate_client_delivery_failure(
2203 released.connection_id,
2204 released.channel,
2205 released.epoch,
2206 CloseReason::new(
2207 "route_goodbye_delivery_failed",
2208 format!(
2209 "failed to enqueue route GOODBYE for channel {}: {err}",
2210 released.channel
2211 ),
2212 ),
2213 crate::forwarding::UndeliveredFrame {
2214 module_id: released.module_id.as_deref(),
2215 sink: &released.sink,
2216 },
2217 )
2218 .unwrap_or(false)
2219 {
2220 self.counters.increment_goodbye_relay_client_failed();
2221 }
2222 }
2223 }
2224 }
2225
2226 fn send_abandoned_route_bind_goodbye(
2238 &self,
2239 module_sink: &crate::FrameSink,
2240 negotiated_ver: u8,
2241 module_channel: u16,
2242 module_epoch: u32,
2243 ) {
2244 let frame = match Frame::build_with_version(
2245 negotiated_ver,
2246 FrameType::Goodbye,
2247 control_flags(),
2248 module_channel,
2249 module_epoch,
2250 0,
2251 Vec::new(),
2252 ) {
2253 Ok(frame) => frame,
2254 Err(err) => {
2255 warn!(
2256 route_channel = module_channel,
2257 error = %err,
2258 "failed to build GOODBYE for abandoned route.bind"
2259 );
2260 return;
2261 }
2262 };
2263 crate::forwarding::send_module_route_goodbye(
2264 &self.counters,
2265 module_sink,
2266 frame,
2267 None,
2268 "abandoned route.bind",
2269 );
2270 }
2271
2272 fn handle_hello(
2273 &self,
2274 connection_id: ConnectionId,
2275 sink: Option<crate::FrameSink>,
2276 frame: Frame,
2277 ) -> Result<Vec<Frame>, RouterError> {
2278 debug!(
2279 connection_id = connection_id.get(),
2280 corr = frame.header.corr,
2281 "handling HELLO"
2282 );
2283 if self
2287 .registry
2288 .get_module_by_connection(connection_id)
2289 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2290 .is_some()
2291 {
2292 return Ok(vec![control_error_frame(
2293 &frame,
2294 "invalid_hello",
2295 "connection is already registered as a module",
2296 )?]);
2297 }
2298 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2299 Ok(value) => value,
2300 Err(err) => {
2301 return Ok(vec![control_error_frame(
2302 &frame,
2303 "invalid_hello",
2304 format!("malformed HELLO body: {err}"),
2305 )?])
2306 }
2307 };
2308 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2309 return Ok(vec![control_error_frame(
2310 &frame,
2311 "invalid_capability_grammar",
2312 err.to_string(),
2313 )?]);
2314 }
2315 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2316 return Ok(vec![control_error_frame(
2317 &frame,
2318 "invalid_manifest",
2319 err.to_string(),
2320 )?]);
2321 }
2322 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2323 return Ok(vec![control_error_frame(
2324 &frame,
2325 "invalid_event_declaration",
2326 err.to_string(),
2327 )?]);
2328 }
2329 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2330 return Ok(vec![control_error_frame(
2331 &frame,
2332 "invalid_manifest",
2333 err.to_string(),
2334 )?]);
2335 }
2336 if let Some(provenance) = hello_value
2337 .get("manifest")
2338 .and_then(|manifest| manifest.get("provenance"))
2339 {
2340 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2341 return Ok(vec![control_error_frame(
2342 &frame,
2343 "invalid_manifest",
2344 format!("malformed manifest provenance: {err}"),
2345 )?]);
2346 }
2347 }
2348 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2349 let module_id = hello_value["manifest"]["module_id"]
2350 .as_str()
2351 .unwrap_or("<unknown>");
2352 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2353 }
2354 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2355 let module_id = hello_value["manifest"]["module_id"]
2356 .as_str()
2357 .unwrap_or("<unknown>");
2358 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2359 }
2360 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2361 Ok(hello) => hello,
2362 Err(err) => {
2363 return Ok(vec![control_error_frame(
2364 &frame,
2365 "invalid_hello",
2366 format!("malformed HELLO body: {err}"),
2367 )?])
2368 }
2369 };
2370
2371 if hello.protocol_ver != hello.manifest.protocol_ver {
2372 return Ok(vec![control_error_frame(
2373 &frame,
2374 "invalid_manifest",
2375 format!(
2376 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2377 hello.protocol_ver, hello.manifest.protocol_ver
2378 ),
2379 )?]);
2380 }
2381
2382 if hello.manifest.module_id.trim().is_empty() {
2383 return Ok(vec![control_error_frame(
2384 &frame,
2385 "invalid_manifest",
2386 "manifest module_id must not be empty",
2387 )?]);
2388 }
2389
2390 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2391 Ok(negotiated_ver) => negotiated_ver,
2392 Err(message) => {
2393 return Ok(vec![control_error_frame(
2394 &frame,
2395 "version_unsupported",
2396 message,
2397 )?])
2398 }
2399 };
2400
2401 let swap_admission = self
2409 .supervisor
2410 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2411 if swap_admission == SwapHelloAdmission::Refused {
2412 warn!(
2413 module_id = %hello.manifest.module_id,
2414 connection_id = connection_id.get(),
2415 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2416 );
2417 return Ok(vec![control_error_frame(
2418 &frame,
2419 "swap_token_invalid",
2420 format!(
2421 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2422 hello.manifest.module_id
2423 ),
2424 )?]);
2425 }
2426 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2427
2428 let nonce_verified = if swap_candidate {
2433 true
2434 } else {
2435 match self.supervisor.reserved_hello_verification(
2436 &hello.manifest.module_id,
2437 hello.launch_nonce.as_deref(),
2438 ) {
2439 Ok(verified) => verified,
2440 Err(rejection) => {
2441 let message = match rejection {
2442 ReservedHelloRejection::Exact { module_id } => format!(
2443 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2444 ),
2445 ReservedHelloRejection::Prefix {
2446 prefix,
2447 owner_module_id,
2448 } => format!(
2449 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2450 hello.manifest.module_id
2451 ),
2452 };
2453 return Ok(vec![control_error_frame(
2454 &frame,
2455 "reserved_module",
2456 message,
2457 )?]);
2458 }
2459 }
2460 };
2461
2462 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2463 &hello.manifest.module_id,
2464 hello.manifest.capabilities.as_ref(),
2465 );
2466 if let Some(refusal) = reserved_capability_refusals.first() {
2467 let capability = refusal.capability.clone();
2468 let bound_module = refusal.claimants[0].clone();
2469 log_duplicate_claim_events(reserved_capability_refusals);
2470 return Ok(vec![control_error_frame(
2471 &frame,
2472 "reserved_capability",
2473 format!(
2474 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2475 capability, bound_module, hello.manifest.module_id
2476 ),
2477 )?]);
2478 }
2479
2480 if self
2483 .forwarding
2484 .connection_has_client_routes(connection_id)
2485 .map_err(RouterError::Forwarding)?
2486 {
2487 return Ok(vec![control_error_frame(
2488 &frame,
2489 "invalid_hello",
2490 "connection has open client routes and cannot also register as a module",
2491 )?]);
2492 }
2493
2494 self.hello_launch_nonces
2499 .lock()
2500 .unwrap_or_else(|poisoned| poisoned.into_inner())
2501 .record(connection_id, hello.launch_nonce.as_deref());
2502 let control_ops = effective_module_control_ops(hello.control_ops);
2503 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2506 if swap_candidate {
2507 return self.register_swap_candidate(
2508 connection_id,
2509 sink,
2510 &frame,
2511 hello.manifest,
2512 negotiated_ver,
2513 control_ops,
2514 hello_ack,
2515 );
2516 }
2517 let registration = match self
2524 .registry
2525 .register_unpublished_verified_with_control_ops(
2526 hello.manifest,
2527 negotiated_ver,
2528 connection_id,
2529 control_ops,
2530 nonce_verified,
2531 ) {
2532 Ok(registration) => registration,
2533 Err(RegistryError::DuplicateModuleId { module_id }) => {
2534 return Ok(vec![control_error_frame(
2535 &frame,
2536 "duplicate_module_id",
2537 format!(
2538 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2539 ),
2540 )?])
2541 }
2542 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2543 return Ok(vec![control_error_frame(
2544 &frame,
2545 "invalid_module_id",
2546 err.to_string(),
2547 )?])
2548 }
2549 Err(err) => {
2550 return Ok(vec![control_error_frame(
2551 &frame,
2552 "registry_error",
2553 err.to_string(),
2554 )?])
2555 }
2556 };
2557 #[cfg(test)]
2558 self.hold_between_registry_and_forwarding();
2559
2560 let reply = if let Some(sink) = sink {
2561 let concurrency = manifest_concurrency(®istration.manifest);
2577 if let Err(err) = self.forwarding.register_module_connection_acked(
2578 connection_id,
2579 registration.manifest.module_id.clone(),
2580 negotiated_ver,
2581 concurrency,
2582 sink,
2583 hello_ack,
2584 ) {
2585 if matches!(
2590 self.deregister_connection(
2591 connection_id,
2592 RegistrationEndReason::RegistrationFailed,
2593 ),
2594 Ok(r) if !r.is_empty()
2595 ) {
2596 crate::supervise::notify_registration_release();
2597 }
2598 return Ok(vec![control_error_frame(
2599 &frame,
2600 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2601 "invalid_hello"
2602 } else {
2603 forwarding_error_code(&err)
2604 },
2605 err.to_string(),
2606 )?]);
2607 }
2608 Vec::new()
2609 } else {
2610 vec![hello_ack]
2613 };
2614
2615 if !self
2620 .registry
2621 .publish_registration(connection_id)
2622 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2623 {
2624 debug!(
2625 module_id = %registration.manifest.module_id,
2626 connection_id = connection_id.get(),
2627 "registration was removed before it could be published"
2628 );
2629 }
2630
2631 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2640 info!(
2641 module_id = %registration.manifest.module_id,
2642 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2643 );
2644 }
2645
2646 self.apply_registration_capabilities(®istration);
2647
2648 self.supervisor
2649 .record_activity_registration(®istration.manifest.module_id);
2650
2651 info!(
2652 module_id = %registration.manifest.module_id,
2653 module_version = %registration.manifest.module_version,
2654 negotiated_ver,
2655 routable_provider = manifest_provides_routable_role(®istration.manifest),
2656 connection_id = connection_id.get(),
2657 "module registered"
2658 );
2659
2660 Ok(reply)
2661 }
2662
2663 #[allow(clippy::too_many_arguments)]
2680 fn register_swap_candidate(
2681 &self,
2682 connection_id: ConnectionId,
2683 sink: Option<crate::FrameSink>,
2684 frame: &Frame,
2685 manifest: ModuleManifest,
2686 negotiated_ver: u8,
2687 control_ops: Vec<String>,
2688 hello_ack: Frame,
2689 ) -> Result<Vec<Frame>, RouterError> {
2690 let module_id = manifest.module_id.clone();
2691 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2692 manifest,
2693 negotiated_ver,
2694 connection_id,
2695 control_ops,
2696 ) {
2697 Ok(registration) => registration,
2698 Err(RegistryError::DuplicateModuleId { module_id }) => {
2699 return Ok(vec![control_error_frame(
2700 frame,
2701 "duplicate_module_id",
2702 format!(
2703 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2704 ),
2705 )?])
2706 }
2707 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2708 return Ok(vec![control_error_frame(
2709 frame,
2710 "invalid_module_id",
2711 err.to_string(),
2712 )?])
2713 }
2714 Err(err) => {
2715 return Ok(vec![control_error_frame(
2716 frame,
2717 "registry_error",
2718 err.to_string(),
2719 )?])
2720 }
2721 };
2722 #[cfg(test)]
2723 self.hold_between_registry_and_forwarding();
2724 let reply = if let Some(sink) = sink {
2725 let concurrency = manifest_concurrency(®istration.manifest);
2729 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2730 connection_id,
2731 module_id.clone(),
2732 negotiated_ver,
2733 concurrency,
2734 sink,
2735 hello_ack,
2736 ) {
2737 if matches!(
2738 self.deregister_connection(
2739 connection_id,
2740 RegistrationEndReason::RegistrationFailed,
2741 ),
2742 Ok(r) if !r.is_empty()
2743 ) {
2744 crate::supervise::notify_registration_release();
2745 }
2746 return Ok(vec![control_error_frame(
2747 frame,
2748 forwarding_error_code(&err),
2749 err.to_string(),
2750 )?]);
2751 }
2752 Vec::new()
2753 } else {
2754 vec![hello_ack]
2755 };
2756 if !self
2757 .registry
2758 .publish_registration(connection_id)
2759 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2760 {
2761 debug!(
2762 module_id = %module_id,
2763 connection_id = connection_id.get(),
2764 "swap candidate registration was removed before it could be published"
2765 );
2766 }
2767 self.supervisor.mark_swap_candidate_admitted(&module_id);
2768 info!(
2769 module_id = %module_id,
2770 module_version = %registration.manifest.module_version,
2771 negotiated_ver,
2772 ready = registration.ready,
2773 connection_id = connection_id.get(),
2774 "swap candidate registered; not routable until cutover"
2775 );
2776 Ok(reply)
2777 }
2778
2779 fn build_hello_ack(
2780 &self,
2781 frame: &Frame,
2782 negotiated_ver: u8,
2783 module_id: &str,
2784 ) -> Result<Frame, RouterError> {
2785 let ack = ModuleHelloAckBody {
2786 negotiated_ver,
2787 subc_ops: module_subc_ops(),
2788 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2789 storage: self
2790 .storage_config
2791 .as_ref()
2792 .map(|cfg| cfg.descriptor_for(module_id)),
2793 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2794 };
2795 let body = serde_json::to_vec(&ack).map_err(|err| {
2796 RouterError::backend(
2797 0,
2798 frame.header.corr,
2799 format!("failed to encode HELLO_ACK: {err}"),
2800 )
2801 })?;
2802
2803 Frame::build_with_version(
2804 negotiated_ver,
2805 FrameType::HelloAck,
2806 control_flags(),
2807 0,
2808 0,
2809 frame.header.corr,
2810 body,
2811 )
2812 .map_err(RouterError::FrameBuild)
2813 }
2814
2815 async fn handle_client_control_request(
2816 &self,
2817 ctx: &RouteCtx,
2818 frame: Frame,
2819 request: ClientControlRequest,
2820 route_open_timings: &mut Option<RouteOpenTimings>,
2821 ) -> Result<Vec<Frame>, RouterError> {
2822 if let Some(operation) = privileged_operation(&request) {
2823 let caller = self.resolve_caller(ctx.connection_id).await;
2824 let op = client_control_request_op(&request);
2825 let text = prompt_text(operation, &caller);
2826 match self.privileged_operator_prompt {
2827 PrivilegedOperatorPrompt::Log => {
2828 info!(target: "subc_daemon::privileged_prompt", operation = op,
2829 target = ?operation, caller = %caller.description(), summary = %text,
2830 "privileged request would prompt the operator");
2831 }
2832 PrivilegedOperatorPrompt::Enforce => {
2833 let key =
2834 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2835 let outcome = self
2836 .forwarding
2837 .operator_confirms()
2838 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2839 .await;
2840 if outcome.code != "confirmed" {
2841 return Ok(vec![outcome.frame(&frame)]);
2842 }
2843 }
2844 }
2845 }
2846 match request {
2847 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2848 ClientControlRequest::CatalogList { module_id } => {
2849 self.handle_catalog_list(frame, module_id)
2850 }
2851 ClientControlRequest::RouteOpen {
2852 target,
2853 identity,
2854 consumer_identity,
2855 consumer_capabilities,
2856 role_versions,
2857 admission_facts,
2858 scope,
2859 } => {
2860 self.handle_route_open(
2861 ctx,
2862 frame,
2863 RouteOpenRequest {
2864 target,
2865 identity,
2866 consumer_identity,
2867 consumer_capabilities,
2868 role_versions,
2869 admission_facts,
2870 scope,
2871 },
2872 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2873 )
2874 .await
2875 }
2876 ClientControlRequest::RoutePoll {
2877 route_channel,
2878 route_epoch,
2879 kind,
2880 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2881 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2882 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2883 self.handle_supervisor_spawn_snapshot(frame)
2884 }
2885 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2886 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2887 }
2888 ClientControlRequest::SupervisorRestart {
2889 module_id,
2890 drain_timeout_ms,
2891 } => {
2892 self.log_supervisor_request_received(
2893 ctx,
2894 frame.header.corr,
2895 ops::SUPERVISOR_RESTART,
2896 Some(&module_id),
2897 None,
2898 )?;
2899 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2900 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2901 .await
2902 }
2903 ClientControlRequest::SupervisorSwap {
2904 module_id,
2905 ready_timeout_ms,
2906 } => {
2907 self.log_supervisor_request_received(
2908 ctx,
2909 frame.header.corr,
2910 ops::SUPERVISOR_SWAP,
2911 Some(&module_id),
2912 None,
2913 )?;
2914 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2915 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2916 .await
2917 }
2918 ClientControlRequest::SupervisorReload { module_id } => {
2919 self.log_supervisor_request_received(
2920 ctx,
2921 frame.header.corr,
2922 ops::SUPERVISOR_RELOAD,
2923 Some(&module_id),
2924 None,
2925 )?;
2926 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2927 self.handle_supervisor_reload(frame, module_id, requester)
2928 .await
2929 }
2930 ClientControlRequest::SupervisorRescan { preview } => {
2931 if !preview {
2932 self.log_supervisor_request_received(
2933 ctx,
2934 frame.header.corr,
2935 ops::SUPERVISOR_RESCAN,
2936 None,
2937 None,
2938 )?;
2939 }
2940 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2941 self.handle_supervisor_rescan(
2942 frame,
2943 if preview {
2944 RescanMode::Preview
2945 } else {
2946 RescanMode::Execute
2947 },
2948 requester,
2949 )
2950 .await
2951 }
2952 ClientControlRequest::SupervisorRescanChecked {
2953 preview,
2954 candidate,
2955 expect,
2956 } => {
2957 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2958 .await
2959 }
2960 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2961 self.log_supervisor_request_received(
2962 ctx,
2963 frame.header.corr,
2964 ops::SUPERVISOR_RELEASE_RESERVED,
2965 Some(&module_id),
2966 None,
2967 )?;
2968 self.handle_supervisor_release_reserved(frame, module_id)
2969 .await
2970 }
2971 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2972 self.log_supervisor_request_received(
2973 ctx,
2974 frame.header.corr,
2975 ops::SUPERVISOR_SET_ENABLED,
2976 Some(&module_id),
2977 Some(enabled),
2978 )?;
2979 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2980 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2981 .await
2982 }
2983 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2984 self.handle_supervisor_health_probe(frame, module_id).await
2985 }
2986 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2987 ClientControlRequest::SupervisorRoutes { module_id } => {
2988 self.handle_supervisor_routes(frame, module_id)
2989 }
2990 ClientControlRequest::SupervisorProvenance { module_id } => {
2991 self.handle_supervisor_provenance(frame, module_id).await
2992 }
2993 ClientControlRequest::SupervisorStderrTail {
2994 module_id,
2995 max_lines,
2996 max_bytes,
2997 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2998 ClientControlRequest::SupervisorTerminals { module_id } => {
2999 self.handle_supervisor_terminals(frame, module_id).await
3000 }
3001 }
3002 }
3003
3004 fn log_supervisor_request_received(
3005 &self,
3006 ctx: &RouteCtx,
3007 corr: u64,
3008 op: &'static str,
3009 module_id: Option<&str>,
3010 enabled: Option<bool>,
3011 ) -> Result<(), RouterError> {
3012 let caller = self.supervisor_requester(ctx, corr)?;
3013 let caller = principal_label(&caller);
3014 info!(
3017 target: "control",
3018 op,
3019 module_id,
3020 enabled,
3021 connection_id = ctx.connection_id.get(),
3022 caller = %caller,
3023 "supervisor request received"
3024 );
3025 Ok(())
3026 }
3027
3028 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3029 Ok(self
3030 .registry
3031 .get_module_by_connection(ctx.connection_id)
3032 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3033 .map(|registration| Principal::Reserved {
3034 module_id: registration.manifest.module_id,
3035 })
3036 .unwrap_or(Principal::Direct))
3037 }
3038
3039 fn handle_module_control_request(
3040 &self,
3041 connection_id: ConnectionId,
3042 frame: Frame,
3043 request: ModuleControlRequestFromModule,
3044 ) -> Result<Vec<Frame>, RouterError> {
3045 match request {
3046 ModuleControlRequestFromModule::CatalogUpdate {
3047 provides,
3048 capabilities,
3049 ready,
3050 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3051 ModuleControlRequestFromModule::LiveRoots {} => {
3052 let registered = self
3053 .registry
3054 .get_module_by_connection(connection_id)
3055 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3056 let Some(registration) = registered else {
3057 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3058 };
3059 let response = self
3060 .forwarding
3061 .live_roots(®istration.manifest.module_id)
3062 .map_err(RouterError::Forwarding)?;
3063 Ok(vec![control_response_body_frame(
3064 &frame,
3065 &response,
3066 "ModuleControlResponseToModule::LiveRoots",
3067 )?])
3068 }
3069 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3070 self.handle_scope_sync(connection_id, frame, generation, scopes)
3071 }
3072 ModuleControlRequestFromModule::ScopeApply {
3073 generation,
3074 upsert,
3075 end,
3076 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3077 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3078 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3079 }
3080 }
3081 }
3082
3083 fn handle_scope_served_by(
3084 &self,
3085 connection_id: ConnectionId,
3086 frame: Frame,
3087 ) -> Result<Vec<Frame>, RouterError> {
3088 let registered = self
3089 .registry
3090 .get_module_by_connection(connection_id)
3091 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3092 let Some(registered) = registered else {
3093 return Ok(vec![control_error_frame(
3094 &frame,
3095 "not_registered",
3096 "scope.served_by requires an active module registration owned by this connection",
3097 )?]);
3098 };
3099 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3100 {
3101 Ok(request) => request,
3102 Err(err) => {
3103 return Ok(vec![control_error_frame(
3104 &frame,
3105 "invalid_control_body",
3106 format!("malformed scope.served_by body: {err}"),
3107 )?])
3108 }
3109 };
3110 let answer = self
3111 .scopes
3112 .read()
3113 .unwrap_or_else(|poisoned| poisoned.into_inner())
3114 .served_by(
3115 ®istered.manifest.module_id,
3116 &request.scope_ref,
3117 request.scope_epoch,
3118 &request.module_id,
3119 );
3120 let reply = subc_control::ScopeServedByReply::new(
3121 answer,
3122 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3123 );
3124 Ok(vec![control_response_body_frame(
3125 &frame,
3126 &reply,
3127 "ScopeServedByReply",
3128 )?])
3129 }
3130
3131 fn handle_operator_confirm(
3132 &self,
3133 ctx: &RouteCtx,
3134 frame: Frame,
3135 ) -> Result<Vec<Frame>, RouterError> {
3136 use crate::operator_confirm::{audit, Outcome};
3137 let registration = self
3138 .registry
3139 .get_module_by_connection(ctx.connection_id)
3140 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3141 let Some(registration) = registration else {
3142 let outcome = Outcome::refusal("not_registered");
3143 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3144 return Ok(vec![outcome.frame(&frame)]);
3145 };
3146 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3147 Ok(request) => request,
3148 Err(_) => {
3149 let outcome = Outcome::refusal("invalid_control_body");
3150 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3151 return Ok(vec![outcome.frame(&frame)]);
3152 }
3153 };
3154 let module_id = registration.manifest.module_id;
3155 let nonce = self
3159 .hello_launch_nonces
3160 .lock()
3161 .unwrap_or_else(|p| p.into_inner())
3162 .nonce(ctx.connection_id)
3163 .map(str::to_owned);
3164 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3165 self.supervisor
3166 .spawned_consumer_authorized(&module_id, nonce)
3167 });
3168 let confirms = self.forwarding.operator_confirms();
3169 self.forwarding
3170 .with_operator_route(
3171 ctx.connection_id,
3172 request.route_channel,
3173 request.route_epoch,
3174 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3175 )
3176 .map_err(RouterError::Forwarding)
3177 }
3178
3179 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3182 let mut scopes = ScopeRecords::from(scopes);
3183 for (index, record) in scopes.records.iter_mut().enumerate() {
3184 if let Some(refusal) = Self::canonicalize_workdir(record) {
3185 scopes.refusals.insert(index, refusal);
3186 }
3187 }
3188 scopes
3189 }
3190
3191 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3192 let workdir = record.workdir.as_mut()?;
3193 let path = std::path::Path::new(workdir);
3194 if !path.is_absolute() {
3195 return Some((
3196 error_codes::INVALID_WORKDIR,
3197 "workdir must be absolute".to_string(),
3198 ));
3199 }
3200 match ProjectRootId::from_path_allowing_missing(path) {
3201 Ok(root) => match root.as_path().to_str() {
3202 Some(canonical) => {
3203 *workdir = canonical.to_string();
3204 None
3205 }
3206 None => Some((
3207 error_codes::INVALID_WORKDIR,
3208 "canonical workdir is not UTF-8".to_string(),
3209 )),
3210 },
3211 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3212 }
3213 }
3214
3215 fn handle_scope_sync(
3219 &self,
3220 connection_id: ConnectionId,
3221 frame: Frame,
3222 generation: u64,
3223 scopes: Vec<ScopeRecord>,
3224 ) -> Result<Vec<Frame>, RouterError> {
3225 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3226 }
3227
3228 fn handle_scope_change(
3229 &self,
3230 connection_id: ConnectionId,
3231 frame: Frame,
3232 generation: u64,
3233 scopes: Vec<ScopeRecord>,
3234 end: Option<Vec<ScopeEnd>>,
3235 ) -> Result<Vec<Frame>, RouterError> {
3236 let is_apply = end.is_some();
3237 let Some(registration) = self
3238 .registry
3239 .get_module_by_connection(connection_id)
3240 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3241 else {
3242 return Ok(vec![control_error_frame(
3243 &frame,
3244 "not_registered",
3245 "scope.sync and scope.apply require an active module registration owned by this connection",
3246 )?]);
3247 };
3248 let owner = registration.manifest.module_id;
3249 let scopes = Self::prepare_scope_records(scopes);
3250 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3251 let is_current_launch = |connection: ConnectionId| {
3252 self.hello_launch_nonces
3253 .lock()
3254 .unwrap_or_else(|poisoned| poisoned.into_inner())
3255 .presented(connection, current_nonce.as_deref())
3256 };
3257 let mut table = self
3262 .scopes
3263 .write()
3264 .unwrap_or_else(|poisoned| poisoned.into_inner());
3265 let outcome = match end {
3266 Some(end) => table.apply(
3267 &owner,
3268 connection_id,
3269 is_current_launch,
3270 generation,
3271 scopes,
3272 end,
3273 ),
3274 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3275 };
3276 let drained = match &outcome {
3277 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3278 Err(_) => Vec::new(),
3279 };
3280 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3281 drop(table);
3282 if has_deadlines {
3283 self.scope_deadline_added.notify_one();
3284 }
3285 match outcome {
3286 Ok(applied) => {
3287 let counts = ScopeOutcomeCounts::of(&applied.results);
3288 info!(
3289 owner = %owner,
3290 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3291 generation,
3292 records = applied.results.len(),
3293 created = counts.created,
3294 replaced = counts.replaced,
3295 updated = counts.updated,
3296 unchanged = counts.unchanged,
3297 refused = counts.refused,
3298 ended = applied.ended.len(),
3299 tag_changes = applied.tag_changes.len(),
3300 routes_closed = drained.len(),
3301 "scope change accepted"
3302 );
3303 for refused in applied
3309 .results
3310 .iter()
3311 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3312 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3313 {
3314 warn!(
3315 owner = %owner,
3316 generation,
3317 scope_ref = %refused.scope_ref,
3318 scope_epoch = refused.scope_epoch,
3319 code = refused.code.as_deref().unwrap_or(""),
3320 "scope record refused"
3321 );
3322 }
3323 self.close_scope_drained_routes(drained);
3324 let response = if is_apply {
3325 ModuleControlResponseToModule::ScopeApply {
3326 generation,
3327 results: applied.results,
3328 end_results: applied.end_results,
3329 ended: applied.ended,
3330 }
3331 } else {
3332 ModuleControlResponseToModule::ScopeSync {
3333 generation,
3334 results: applied.results,
3335 ended: applied.ended,
3336 }
3337 };
3338 Ok(vec![control_response_body_frame(
3339 &frame,
3340 &response,
3341 "scope change response",
3342 )?])
3343 }
3344 Err(refusal) => {
3345 info!(
3346 owner = %owner,
3347 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3348 generation,
3349 code = refusal.code,
3350 "scope change refused"
3351 );
3352 Ok(vec![control_error_frame(
3353 &frame,
3354 refusal.code,
3355 refusal.message,
3356 )?])
3357 }
3358 }
3359 }
3360
3361 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3367 if drained.is_empty() {
3368 return;
3369 }
3370 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3371 BTreeMap::new();
3372 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3373 for route in drained {
3374 warn!(
3375 module_id = %route.module_id,
3376 reason = ?route.reason,
3377 client_connection_id = route.client.connection_id.get(),
3378 route_channel = route.client.channel,
3379 "closing route because its scope changed"
3380 );
3381 pushes
3382 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3383 .or_insert_with(|| (route.reason, Vec::new()))
3384 .1
3385 .push(EndpointRoute {
3386 goodbye_target: route.client.clone(),
3387 principal: Principal::Unverified,
3388 bound_at: Instant::now(),
3389 draining: false,
3390 drain_reason: None,
3391 });
3392 goodbyes.push(route.module);
3393 goodbyes.push(route.client);
3394 }
3395 for ((module_id, _), (reason, routes)) in pushes {
3396 send_route_control_pushes(
3397 &self.forwarding,
3398 routes,
3399 ClientControlPush::RouteClosed {
3400 module_id,
3401 channels: Vec::new(),
3402 reason,
3403 drained: false,
3404 abandoned: 0,
3405 excluded_subscriptions: 0,
3406 terminal: Some(false),
3407 },
3408 );
3409 }
3410 self.emit_route_goodbyes(goodbyes);
3411 }
3412
3413 fn handle_scope_describe(
3416 &self,
3417 connection_id: ConnectionId,
3418 frame: Frame,
3419 owner: Principal,
3420 scope_ref: String,
3421 ) -> Result<Vec<Frame>, RouterError> {
3422 let registered = self
3423 .registry
3424 .get_module_by_connection(connection_id)
3425 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3426 let Some(registered) = registered else {
3427 return Ok(vec![control_error_frame(
3428 &frame,
3429 "not_registered",
3430 "scope.describe requires an active module registration owned by this connection",
3431 )?]);
3432 };
3433 let mut description = self
3434 .scopes
3435 .read()
3436 .unwrap_or_else(|poisoned| poisoned.into_inner())
3437 .describe(&owner, &scope_ref);
3438 if !registered
3441 .manifest
3442 .capabilities
3443 .as_ref()
3444 .is_some_and(|caps| {
3445 caps.provides
3446 .iter()
3447 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3448 })
3449 {
3450 if let Some(stamp) = description.stamp.as_mut() {
3451 stamp.attributes.on_behalf_of_agent = None;
3452 }
3453 }
3454 if !registered
3455 .manifest
3456 .capabilities
3457 .as_ref()
3458 .is_some_and(|caps| {
3459 caps.provides
3460 .iter()
3461 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3462 })
3463 {
3464 if let Some(stamp) = description.stamp.as_mut() {
3465 stamp.attributes.flow_version = None;
3466 stamp.attributes.flow_code_hash = None;
3467 stamp.attributes.flow_install_id = None;
3468 }
3469 }
3470 let owner_configured = match &owner {
3471 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3476 _ => false,
3477 };
3478 let response = ModuleControlResponseToModule::ScopeDescribe {
3479 status: description.status,
3480 scope_epoch: description.scope_epoch,
3481 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3482 owner_synced: description.owner_synced,
3483 owner_configured,
3484 scope: description.stamp,
3485 };
3486 Ok(vec![control_response_body_frame(
3487 &frame,
3488 &response,
3489 "ModuleControlResponseToModule::ScopeDescribe",
3490 )?])
3491 }
3492
3493 fn handle_catalog_update(
3494 &self,
3495 connection_id: ConnectionId,
3496 frame: Frame,
3497 provides: Vec<ProviderRole>,
3498 capabilities: Option<CapabilityDeclarations>,
3499 ready: Option<bool>,
3500 ) -> Result<Vec<Frame>, RouterError> {
3501 self.refresh_capability_requirements();
3502 let Some(registration) = self
3503 .registry
3504 .get_module_by_connection(connection_id)
3505 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3506 else {
3507 return Ok(vec![control_error_frame(
3508 &frame,
3509 "not_registered",
3510 "catalog.update requires an active module registration owned by this connection",
3511 )?]);
3512 };
3513
3514 if let Some(message) =
3515 catalog_update_frozen_field_message(®istration.manifest, &provides)
3516 {
3517 return Ok(vec![control_error_frame(
3518 &frame,
3519 "catalog_update_frozen_field",
3520 message,
3521 )?]);
3522 }
3523
3524 let mut candidate = registration.manifest.clone();
3525 candidate.provides = provides.clone();
3526 candidate.capabilities = capabilities
3527 .clone()
3528 .or_else(|| registration.manifest.capabilities.clone());
3529 if let Some(rich_ui) = &candidate.rich_ui {
3530 if let Err(err) = rich_ui.validate() {
3531 return Ok(vec![control_error_frame(
3532 &frame,
3533 "invalid_manifest",
3534 err.to_string(),
3535 )?]);
3536 }
3537 }
3538 if let Err(err) = candidate.validate_capability_grammar() {
3539 return Ok(vec![control_error_frame(
3540 &frame,
3541 "invalid_capability_grammar",
3542 err.to_string(),
3543 )?]);
3544 }
3545
3546 let mut conflicts = self
3549 .capability_evaluator
3550 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3551 if let Some(conflict) = conflicts.first() {
3552 let message = format!(
3553 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3554 conflict.capability, conflict.claimants[0], candidate.module_id
3555 );
3556 for conflict in &mut conflicts {
3557 conflict.source = DuplicateClaimSource::CatalogUpdate;
3558 }
3559 log_duplicate_claim_events(conflicts);
3560 return Ok(vec![control_error_frame(
3561 &frame,
3562 "reserved_capability",
3563 message,
3564 )?]);
3565 }
3566
3567 let updated = self
3568 .registry
3569 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3570 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3571 if updated.is_none() {
3572 return Ok(vec![control_error_frame(
3573 &frame,
3574 "not_registered",
3575 "catalog.update requires an active module registration owned by this connection",
3576 )?]);
3577 }
3578 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3579 log_duplicate_claim_events(
3580 self.capability_evaluator
3581 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3582 );
3583 }
3584 if capability_census_trigger(
3585 registration.manifest.capabilities.as_ref(),
3586 updated
3587 .as_ref()
3588 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3589 ) {
3590 self.enforce_capability_denies();
3591 }
3592 self.refresh_capability_requirements();
3593
3594 let response = ModuleControlResponseToModule::CatalogUpdate {};
3595 control_response_body_frame(
3596 &frame,
3597 &response,
3598 "ModuleControlResponseToModule::CatalogUpdate",
3599 )
3600 .map(|frame| vec![frame])
3601 }
3602
3603 pub(crate) fn with_activity_ledger_config(
3604 mut self,
3605 run_dir: &Path,
3606 incarnation: String,
3607 config: crate::activity::ActivityConfig,
3608 ) -> Self {
3609 self.activity = Some(crate::activity::ActivityLedger::open_config(
3610 run_dir.join("activity.jsonl"),
3611 incarnation,
3612 config,
3613 ));
3614 self.supervisor.set_activity(self.activity.clone());
3615 self
3616 }
3617
3618 pub(crate) async fn configure_activity(
3619 &self,
3620 config: crate::activity::ActivityConfig,
3621 ) -> Result<(), &'static str> {
3622 match &self.activity {
3623 Some(ledger) => ledger.configure(config).await,
3624 None => Ok(()),
3625 }
3626 }
3627
3628 fn cancel_activity_subscription(
3629 &self,
3630 ctx: &RouteCtx,
3631 frame: &Frame,
3632 ) -> Result<bool, RouterError> {
3633 let task = self
3634 .activity_streams
3635 .lock()
3636 .unwrap_or_else(|p| p.into_inner())
3637 .remove(&(ctx.connection_id, frame.header.corr));
3638 let Some(task) = task else {
3639 return Ok(false);
3640 };
3641 task.abort();
3642 let end = Frame::build_with_version(
3643 response_version(frame),
3644 FrameType::StreamEnd,
3645 control_flags(),
3646 0,
3647 0,
3648 frame.header.corr,
3649 Vec::new(),
3650 )
3651 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3652 let sink = ctx.egress.clone();
3653 tokio::spawn(async move {
3654 let _ = sink.send(end).await;
3655 });
3656 Ok(true)
3657 }
3658
3659 async fn handle_activity_read(
3660 &self,
3661 ctx: &RouteCtx,
3662 frame: Frame,
3663 ) -> Result<Vec<Frame>, RouterError> {
3664 use subc_control::{
3665 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3666 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3667 };
3668 #[derive(Deserialize)]
3669 struct Identity {
3670 #[serde(default)]
3671 consumer_identity: Option<ConsumerIdentity>,
3672 }
3673 let principal = match self
3677 .forwarding
3678 .module_id_for_connection(ctx.connection_id)
3679 .map_err(RouterError::Forwarding)?
3680 {
3681 Some(module_id) => Ok(Principal::Reserved { module_id }),
3682 None => {
3683 let identity: Identity = match serde_json::from_slice(&frame.body) {
3684 Ok(identity) => identity,
3685 Err(_) => {
3686 return Ok(vec![control_error_frame(
3687 &frame,
3688 ACTIVITY_MALFORMED,
3689 "malformed activity read",
3690 )?])
3691 }
3692 };
3693 self.route_open_principal(&frame, identity.consumer_identity)?
3694 }
3695 };
3696 let admitted = matches!(principal, Ok(Principal::Direct))
3697 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3698 if !admitted {
3699 return Ok(vec![control_error_frame(
3700 &frame,
3701 ACTIVITY_READ_FORBIDDEN,
3702 "activity reads require an operator or trusted mirror principal",
3703 )?]);
3704 }
3705 let Some(ledger) = &self.activity else {
3706 return Ok(vec![control_error_frame(
3707 &frame,
3708 ACTIVITY_UNAVAILABLE,
3709 "activity ledger unavailable",
3710 )?]);
3711 };
3712 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3713 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3714 {
3715 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3716 Ok(request) => ledger.query(request).await,
3717 Err(_) => Err(ACTIVITY_MALFORMED),
3718 };
3719 return match result {
3720 Ok(reply) => Ok(vec![control_response_body_frame(
3721 &frame,
3722 &reply,
3723 ops::ACTIVITY_QUERY,
3724 )?]),
3725 Err(code) => Ok(vec![control_error_frame(
3726 &frame,
3727 code,
3728 "activity query refused",
3729 )?]),
3730 };
3731 }
3732 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3733 Ok(request) => request,
3734 Err(_) => {
3735 return Ok(vec![control_error_frame(
3736 &frame,
3737 ACTIVITY_MALFORMED,
3738 "malformed activity subscription",
3739 )?])
3740 }
3741 };
3742 let mut subscription = match ledger.subscribe(request.since).await {
3743 Ok(subscription) => subscription,
3744 Err(code) => {
3745 return Ok(vec![control_error_frame(
3746 &frame,
3747 code,
3748 "activity subscription refused",
3749 )?])
3750 }
3751 };
3752 let key = (ctx.connection_id, frame.header.corr);
3753 let sink = ctx.egress.clone();
3754 let mut streams = self
3755 .activity_streams
3756 .lock()
3757 .unwrap_or_else(|p| p.into_inner());
3758 streams.retain(|_, task| !task.is_finished());
3759 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3760 return Ok(vec![control_error_frame(
3761 &frame,
3762 "duplicate_subscription",
3763 "activity subscription correlation id already in use",
3764 )?]);
3765 }
3766 let task = tokio::spawn(async move {
3767 loop {
3768 let entry = if let Some(entry) = subscription.replay.pop_front() {
3769 Some(entry)
3770 } else {
3771 subscription.receiver.recv().await
3772 };
3773 let Some(entry) = entry else {
3774 break;
3775 };
3776 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3777 return;
3778 };
3779 let Ok(next) = Frame::build_with_version(
3780 response_version(&frame),
3781 FrameType::StreamData,
3782 control_flags(),
3783 0,
3784 0,
3785 frame.header.corr,
3786 body,
3787 ) else {
3788 return;
3789 };
3790 if sink.send(next).await.is_err() {
3791 return;
3792 }
3793 }
3794 if let Some(code) = subscription.terminal_code() {
3797 if let Ok(end) =
3798 control_error_frame(&frame, code, "activity subscriber fell behind")
3799 {
3800 let _ = sink.send(end).await;
3801 }
3802 }
3803 });
3804 streams.insert(key, task.abort_handle());
3805 Ok(Vec::new())
3806 }
3807
3808 async fn handle_activity_record(
3809 &self,
3810 ctx: &RouteCtx,
3811 frame: Frame,
3812 ) -> Result<Vec<Frame>, RouterError> {
3813 use subc_control::{
3814 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3815 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3816 };
3817 let result = async {
3818 let module = self
3819 .forwarding
3820 .module_id_for_connection(ctx.connection_id)
3821 .ok()
3822 .flatten()
3823 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3824 let request: ActivityRecordRequest =
3825 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3826 crate::activity::validate(&module, &request)?;
3827 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3828 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3829 let at_ms = std::time::SystemTime::now()
3830 .duration_since(std::time::UNIX_EPOCH)
3831 .unwrap_or_default()
3832 .as_millis() as u64;
3833 ledger.record(module, request, actor, at_ms).await
3834 }
3835 .await;
3836 match result {
3837 Ok(entry) => Ok(vec![control_response_body_frame(
3838 &frame,
3839 &ActivityRecordResponse { seq: entry.seq },
3840 "activity.record",
3841 )?]),
3842 Err(code) => Ok(vec![control_error_frame(
3843 &frame,
3844 code,
3845 "activity recording refused",
3846 )?]),
3847 }
3848 }
3849
3850 fn activity_actor(
3851 &self,
3852 connection: ConnectionId,
3853 module: &str,
3854 on: Option<&subc_control::ActivityOn>,
3855 ) -> Result<subc_control::ActivityActor, &'static str> {
3856 use crate::forwarding::{DataRoute, DataRouteState};
3857 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3858 let principal = |id: &str| Principal::Reserved {
3859 module_id: id.to_owned(),
3860 };
3861 let mut actor = ActivityActor {
3862 principal: principal(module),
3863 scope_owner: None,
3864 agent_id: None,
3865 run_id: None,
3866 flow_id: None,
3867 project_root: None,
3868 workdir: None,
3869 project_id: None,
3870 };
3871 let scope = match on {
3872 None => return Ok(actor),
3873 Some(ActivityOn::Scope(scope)) => {
3874 let selector = subc_protocol::scope::ScopeSelector::new(
3875 principal(module),
3876 scope.scope_ref.clone(),
3877 Some(scope.epoch),
3878 );
3879 self.scopes
3880 .read()
3881 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3882 .attribute(&principal(module), module, &selector, true)
3883 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3884 }
3885 Some(ActivityOn::Route(route)) => {
3886 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3887 .forwarding
3888 .lookup_data_route(connection, route.channel, route.epoch)
3889 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3890 else {
3891 return Err(ACTIVITY_ACTOR_UNKNOWN);
3892 };
3893 if binding.module_id != module {
3894 return Err(ACTIVITY_ACTOR_UNKNOWN);
3895 }
3896 actor.principal = binding.principal.clone();
3897 actor.project_root = binding
3898 .project_root
3899 .as_ref()
3900 .map(|root| root.as_path().to_path_buf());
3901 let Some(bound) = &binding.scope else {
3902 return Ok(actor);
3903 };
3904 let owner = principal(&bound.owner);
3905 let selector = subc_protocol::scope::ScopeSelector::new(
3906 owner.clone(),
3907 bound.scope_ref.clone(),
3908 Some(bound.tag.scope_epoch),
3909 );
3910 let admitted = self
3911 .scopes
3912 .read()
3913 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3914 .attribute(&owner, module, &selector, true)
3915 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3916 if admitted.tag != bound.tag {
3917 return Err(ACTIVITY_ACTOR_UNKNOWN);
3918 }
3919 admitted
3920 }
3921 };
3922 actor.scope_owner = Some(scope.stamp.owner);
3923 actor.agent_id = scope.stamp.attributes.agent_id;
3924 actor.run_id = scope.stamp.attributes.run_id;
3925 actor.flow_id = scope.stamp.attributes.flow_id;
3926 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3927 actor.project_id = scope.stamp.project_id;
3928 Ok(actor)
3929 }
3930
3931 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3932 self.refresh_capability_requirements();
3933 let mut counters = self.counters.snapshot();
3939 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3940 obj.insert(
3941 "activity_unparseable_lines".into(),
3942 ledger.read_errors().into(),
3943 );
3944 obj.insert(
3945 "activity_lifecycle_append_failures".into(),
3946 ledger.lifecycle_failures().into(),
3947 );
3948 obj.insert(
3949 "activity_started_digest_skips".into(),
3950 ledger.lifecycle_digest_skips().into(),
3951 );
3952 }
3953 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3954 self.forwarding.client_route_concentration(),
3955 counters.as_object_mut(),
3956 ) {
3957 obj.insert(
3958 "client_connections_with_routes".into(),
3959 connections_with_routes.into(),
3960 );
3961 obj.insert("max_routes_on_one_connection".into(), max.into());
3962 }
3963 if let (Some(open_breakers), Some(obj)) = (
3969 self.route_bind_breakers.open_snapshot(),
3970 counters.as_object_mut(),
3971 ) {
3972 obj.insert("route_bind_breakers_open".into(), open_breakers);
3973 }
3974 let response = ClientControlResponse::ServerDescribe {
3975 protocol_ver: PROTOCOL_VERSION,
3976 subc_ops: subc_ops(),
3977 capabilities: self.describe_capabilities(),
3978 connected_clients: self.connected_clients.count(),
3979 counters: Some(counters),
3980 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3981 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3982 capability_requirements: self.capability_requirement_statuses(),
3983 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3984 config_path: self
3985 .rescan
3986 .as_ref()
3987 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3988 };
3989 Ok(vec![control_response_body_frame(
3990 &frame,
3991 &response,
3992 "ClientControlResponse::ServerDescribe",
3993 )?])
3994 }
3995
3996 fn handle_catalog_list(
3997 &self,
3998 frame: Frame,
3999 module_id: Option<String>,
4000 ) -> Result<Vec<Frame>, RouterError> {
4001 let (generation, modules) = self
4002 .registry
4003 .list_modules_with_nonce_verification()
4004 .map_err(|err| {
4005 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4006 })?;
4007 let entries = modules
4008 .into_iter()
4009 .filter(|(registration, _)| {
4010 module_id
4011 .as_deref()
4012 .map(|wanted| registration.manifest.module_id == wanted)
4013 .unwrap_or(true)
4014 })
4015 .map(|(registration, nonce_verified)| {
4016 let not_ready = self.not_ready_reason(®istration);
4017 let roles = registration.manifest.provides;
4018 CatalogEntry::new(
4019 registration.manifest.module_id,
4020 roles,
4021 registration.control_ops,
4022 )
4023 .with_ready(not_ready.is_none())
4024 .with_not_ready(not_ready)
4025 .with_module_version(Some(registration.manifest.module_version))
4026 .with_capabilities(registration.manifest.capabilities)
4027 .with_self_signals(registration.manifest.self_signals)
4028 .with_events(registration.manifest.events)
4029 .with_operation_delivery(registration.manifest.operation_delivery)
4030 .with_operation_payload(registration.manifest.operation_payload)
4031 .with_flow_operations(registration.manifest.flow_operations)
4032 .with_rich_ui(registration.manifest.rich_ui)
4033 .with_role_declarations(registration.manifest.role_declarations)
4034 .with_nonce_verified(nonce_verified)
4035 })
4036 .collect();
4037 let response = ClientControlResponse::CatalogList {
4038 generation,
4039 modules: entries,
4040 subc_ops: subc_ops(),
4041 };
4042 Ok(vec![control_response_body_frame(
4043 &frame,
4044 &response,
4045 "ClientControlResponse::CatalogList",
4046 )?])
4047 }
4048
4049 fn attest_route_open_consumer(
4054 &self,
4055 consumer_identity: Option<&ConsumerIdentity>,
4056 ) -> Option<Principal> {
4057 let Some(consumer_identity) = consumer_identity else {
4058 return Some(Principal::Direct);
4059 };
4060 self.supervisor
4061 .spawned_consumer_authorized(
4062 &consumer_identity.module_id,
4063 &consumer_identity.launch_nonce,
4064 )
4065 .then(|| Principal::Reserved {
4066 module_id: consumer_identity.module_id.clone(),
4067 })
4068 }
4069
4070 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4074 let Ok(ClientControlRequest::RouteOpen {
4075 identity,
4076 consumer_identity,
4077 scope,
4078 ..
4079 }) = parse_client_control_request(&frame.body)
4080 else {
4081 return RouteOpenCaller::unknown();
4082 };
4083 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4084 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4085 }
4086
4087 fn route_open_principal(
4088 &self,
4089 frame: &Frame,
4090 consumer_identity: Option<ConsumerIdentity>,
4091 ) -> Result<Result<Principal, Frame>, RouterError> {
4092 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4093 self.route_open_principal_attested(frame, consumer_identity, attested)
4094 }
4095
4096 fn route_open_principal_attested(
4100 &self,
4101 frame: &Frame,
4102 consumer_identity: Option<ConsumerIdentity>,
4103 attested: Option<Principal>,
4104 ) -> Result<Result<Principal, Frame>, RouterError> {
4105 if let Some(principal) = attested {
4106 return Ok(Ok(principal));
4107 }
4108 let Some(consumer_identity) = consumer_identity else {
4109 return Ok(Ok(Principal::Direct));
4110 };
4111
4112 Ok(Err(control_error_frame(
4113 frame,
4114 "bad_consumer_identity",
4115 format!(
4116 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4117 consumer_identity.module_id
4118 ),
4119 )?))
4120 }
4121
4122 #[allow(clippy::too_many_arguments)]
4129 fn route_open_refusal_frame(
4130 &self,
4131 ctx: &RouteCtx,
4132 caller: &RouteOpenCaller,
4133 frame: &Frame,
4134 module_id: &str,
4135 reason: &'static str,
4136 code: &'static str,
4137 message: impl Into<String>,
4138 ) -> Result<Frame, RouterError> {
4139 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4140 control_error_frame(frame, code, message.into())
4141 }
4142
4143 #[allow(clippy::too_many_arguments)]
4161 fn route_open_breaker_refusal_frame(
4162 &self,
4163 ctx: &RouteCtx,
4164 caller: &RouteOpenCaller,
4165 frame: &Frame,
4166 module_id: &str,
4167 consecutive_timeouts: u32,
4168 retry_in: Duration,
4169 probe_in_flight: bool,
4170 ) -> Result<Frame, RouterError> {
4171 self.counters
4172 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4173 debug!(
4174 target: "control",
4175 code = "module_timeout",
4176 module_id = ?module_id,
4177 connection_id = ctx.connection_id.get(),
4178 principal = %caller.principal,
4179 scope = %caller.scope,
4180 harness = %caller.harness,
4181 consecutive_timeouts,
4182 retry_in_ms = retry_in.as_millis() as u64,
4183 probe_in_flight,
4184 "route.open refused by open bind-relay breaker"
4185 );
4186 let detail = if probe_in_flight {
4193 "one test route is already being tried; retry once it settles".to_string()
4194 } else {
4195 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4196 };
4197 control_error_frame(
4198 frame,
4199 "module_timeout",
4200 format!(
4201 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4202 timed out in a row); {detail}; its established routes are unaffected"
4203 ),
4204 )
4205 }
4206
4207 fn observe_route_open_refusal(
4220 &self,
4221 ctx: &RouteCtx,
4222 caller: &RouteOpenCaller,
4223 module_id: &str,
4224 reason: &'static str,
4225 code: &'static str,
4226 ) {
4227 self.counters.increment_route_open_refused(code);
4228 info!(
4229 target: "control",
4230 code,
4231 reason,
4232 module_id = ?module_id,
4233 connection_id = ctx.connection_id.get(),
4234 principal = %caller.principal,
4235 scope = %caller.scope,
4236 harness = %caller.harness,
4237 "route.open refused"
4238 );
4239 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4240 self.route_outages.record_not_serving(module_id, reason);
4241 }
4242 }
4243
4244 fn observe_route_open_accept(
4289 &self,
4290 ctx: &RouteCtx,
4291 module_id: &str,
4292 principal: &str,
4293 scope: &str,
4294 ) {
4295 self.route_outages.record_accepted(module_id);
4296 self.counters.increment_route_open_accepted(principal);
4297 info!(
4298 target: "control",
4299 principal,
4300 module_id,
4301 connection_id = ctx.connection_id.get(),
4302 scope = %scope,
4303 "route.open accepted"
4304 );
4305 }
4306
4307 fn supervised_absent_route_open_refusal_frame(
4308 &self,
4309 ctx: &RouteCtx,
4310 caller: &RouteOpenCaller,
4311 frame: &Frame,
4312 module_id: &str,
4313 code: &'static str,
4314 status: &crate::supervise::ModuleStatus,
4315 ) -> Result<Frame, RouterError> {
4316 self.counters.increment_route_open_refused(code);
4317 info!(
4318 target: "control",
4319 code,
4320 reason = "supervised_not_registered",
4321 module_id = ?module_id,
4322 connection_id = ctx.connection_id.get(),
4323 principal = %caller.principal,
4324 scope = %caller.scope,
4325 harness = %caller.harness,
4326 state = %status.state,
4327 enabled = status.enabled,
4328 live = status.live,
4329 "route.open refused"
4330 );
4331 self.route_outages
4335 .record_not_serving(module_id, "supervised_not_registered");
4336 let message = if let Some(pid) = status.waiting_for_previous_pid {
4337 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4338 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4339 })
4340 } else {
4341 format!(
4342 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4343 status.state, status.enabled, status.live
4344 )
4345 };
4346 control_error_frame(frame, code, message)
4347 }
4348
4349 async fn handle_route_open(
4350 &self,
4351 ctx: &RouteCtx,
4352 frame: Frame,
4353 request: RouteOpenRequest,
4354 timings: &mut RouteOpenTimings,
4355 ) -> Result<Vec<Frame>, RouterError> {
4356 let RouteOpenRequest {
4357 target,
4358 mut identity,
4359 consumer_identity,
4360 consumer_capabilities,
4361 role_versions,
4362 admission_facts,
4363 scope,
4364 } = request;
4365 let target_module_id = target_module_id(&target).to_string();
4366 timings.module_id.clone_from(&target_module_id);
4367 if self
4368 .registry
4369 .get_module_by_connection(ctx.connection_id)
4370 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4371 .is_some()
4372 {
4373 return Ok(vec![control_error_frame(
4374 &frame,
4375 "invalid_request",
4376 "module connections cannot open client routes",
4377 )?]);
4378 }
4379 debug!(
4380 connection_id = ctx.connection_id.get(),
4381 corr = frame.header.corr,
4382 module_id = %target_module_id,
4383 "handling route.open"
4384 );
4385
4386 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4391 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4392
4393 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4399 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4400 self.observe_route_open_refusal(
4401 ctx,
4402 &caller,
4403 &target_module_id,
4404 "invalid_role_versions",
4405 error_codes::INVALID_REQUEST,
4406 );
4407 return Ok(vec![control_error_body_frame(
4408 &frame,
4409 ErrorBody {
4410 code: error_codes::INVALID_REQUEST.to_string(),
4411 message: error.to_string(),
4412 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4413 },
4414 )?]);
4415 }
4416
4417 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4418 if status.waiting_for_previous_pid.is_some() {
4419 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4420 ctx,
4421 &caller,
4422 &frame,
4423 &target_module_id,
4424 "module_warming",
4425 &status,
4426 )?]);
4427 }
4428 }
4429
4430 let Some(registration) = self
4449 .registry
4450 .get_module(&target_module_id)
4451 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4452 else {
4453 if let Some((status, warming)) =
4454 self.supervisor_status(&target_module_id, frame.header.corr)?
4455 {
4456 if status.protocol == ModuleProtocol::None {
4463 return Ok(vec![self.route_open_refusal_frame(
4464 ctx,
4465 &caller,
4466 &frame,
4467 &target_module_id,
4468 "protocol_none",
4469 error_codes::MODULE_NO_PROTOCOL,
4470 format!(
4471 "module_id '{target_module_id}' is declared protocol: none; \
4472 it speaks no subc wire and serves no routes"
4473 ),
4474 )?]);
4475 }
4476 let code = if warming {
4477 "module_warming"
4478 } else {
4479 "target_unavailable"
4480 };
4481 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4482 ctx,
4483 &caller,
4484 &frame,
4485 &target_module_id,
4486 code,
4487 &status,
4488 )?]);
4489 }
4490 if let Some(removed_ago_ms) =
4491 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4492 {
4493 return Ok(vec![self.route_open_refusal_frame(
4494 ctx,
4495 &caller,
4496 &frame,
4497 &target_module_id,
4498 "removed",
4499 error_codes::MODULE_REMOVED,
4500 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4501 )?]);
4502 }
4503 return Ok(vec![self.route_open_refusal_frame(
4504 ctx,
4505 &caller,
4506 &frame,
4507 &target_module_id,
4508 "not_registered",
4509 error_codes::UNKNOWN_MODULE,
4510 format!("module_id '{target_module_id}' is not registered"),
4511 )?]);
4512 };
4513
4514 if !registration.ready {
4518 self.counters
4519 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4520 info!(
4521 target: "control",
4522 code = error_codes::MODULE_WARMING,
4523 module_id = ?target_module_id,
4524 connection_id = ctx.connection_id.get(),
4525 principal = %caller.principal,
4526 scope = %caller.scope,
4527 harness = %caller.harness,
4528 reason = "declared_not_ready",
4529 "route.open refused"
4530 );
4531 self.route_outages
4534 .record_not_serving(&target_module_id, "declared_not_ready");
4535 return Ok(vec![control_error_body_frame(
4536 &frame,
4537 ErrorBody {
4538 code: error_codes::MODULE_WARMING.to_string(),
4539 message: format!(
4540 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4541 ),
4542 detail: Some(serde_json::json!({
4543 "reason": "declared_not_ready"
4544 })),
4545 },
4546 )?]);
4547 }
4548
4549 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4568 self.counters
4569 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4570 info!(
4571 target: "control",
4572 code = error_codes::MODULE_WARMING,
4573 module_id = ?target_module_id,
4574 connection_id = ctx.connection_id.get(),
4575 principal = %caller.principal,
4576 scope = %caller.scope,
4577 harness = %caller.harness,
4578 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4579 capability = %capability,
4580 "route.open refused"
4581 );
4582 return Ok(vec![control_error_body_frame(
4583 &frame,
4584 ErrorBody {
4585 code: error_codes::MODULE_WARMING.to_string(),
4586 message: format!(
4587 "module_id '{target_module_id}' requires capability '{capability}', \
4588 which no registered module provides; retry"
4589 ),
4590 detail: Some(serde_json::json!({
4591 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4592 "capability": capability,
4593 })),
4594 },
4595 )?]);
4596 }
4597
4598 if !target_has_required_role(&target, ®istration.manifest.provides) {
4599 return Ok(vec![self.route_open_refusal_frame(
4600 ctx,
4601 &caller,
4602 &frame,
4603 &target_module_id,
4604 "role_not_provided",
4605 "target_unavailable",
4606 format!("module_id '{target_module_id}' does not provide the requested target"),
4607 )?]);
4608 }
4609
4610 if registration.state != ChannelState::Active {
4611 return Ok(vec![self.route_open_refusal_frame(
4612 ctx,
4613 &caller,
4614 &frame,
4615 &target_module_id,
4616 "registration_not_active",
4617 "target_unavailable",
4618 format!("module_id '{target_module_id}' is not active"),
4619 )?]);
4620 }
4621
4622 if self
4623 .forwarding
4624 .module_is_draining(&target_module_id)
4625 .map_err(RouterError::Forwarding)?
4626 {
4627 return Ok(vec![self.route_open_refusal_frame(
4628 ctx,
4629 &caller,
4630 &frame,
4631 &target_module_id,
4632 "reloading",
4633 "module_reloading",
4634 format!("module_id '{target_module_id}' is reloading"),
4635 )?]);
4636 }
4637
4638 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4639 process_liveness.process_live(&target_module_id) == Some(false)
4640 }) {
4641 if process_liveness.process_replacing(&target_module_id) {
4648 return Ok(vec![self.route_open_refusal_frame(
4649 ctx,
4650 &caller,
4651 &frame,
4652 &target_module_id,
4653 "reloading",
4654 "module_reloading",
4655 format!("module_id '{target_module_id}' is reloading"),
4656 )?]);
4657 }
4658 return Ok(vec![self.route_open_refusal_frame(
4659 ctx,
4660 &caller,
4661 &frame,
4662 &target_module_id,
4663 "supervisor_not_live",
4664 "target_unavailable",
4665 format!("module_id '{target_module_id}' is not live"),
4666 )?]);
4667 }
4668
4669 if !self
4670 .forwarding
4671 .has_live_module_connection(&target_module_id)
4672 .map_err(RouterError::Forwarding)?
4673 {
4674 return Ok(vec![self.route_open_refusal_frame(
4675 ctx,
4676 &caller,
4677 &frame,
4678 &target_module_id,
4679 "no_forwarding_connection",
4680 "target_unavailable",
4681 format!("module_id '{target_module_id}' has no live forwarding connection"),
4682 )?]);
4683 }
4684
4685 if let Some(error) =
4686 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4687 {
4688 self.observe_route_open_refusal(
4689 ctx,
4690 &caller,
4691 &target_module_id,
4692 "op_not_allowed",
4693 "op_not_allowed",
4694 );
4695 return Ok(vec![error]);
4696 }
4697
4698 let principal =
4699 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4700 Ok(principal) => principal,
4701 Err(error) => {
4702 self.observe_route_open_refusal(
4703 ctx,
4704 &caller,
4705 &target_module_id,
4706 "bad_consumer_identity",
4707 "bad_consumer_identity",
4708 );
4709 return Ok(vec![error]);
4710 }
4711 };
4712 timings.principal = principal_label(&principal);
4713
4714 if let Principal::Reserved {
4718 module_id: opening_module_id,
4719 } = &principal
4720 {
4721 if let Some(opening_registration) = self
4722 .registry
4723 .get_module(opening_module_id)
4724 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4725 {
4726 if let Some(capability) =
4727 denied_capability(&opening_registration.manifest, ®istration.manifest)
4728 {
4729 warn!(
4730 opening_module_id,
4731 target_module_id,
4732 capability,
4733 "refusing route.open because an attested capability deny edge matches"
4734 );
4735 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4736 &frame,
4737 &target_module_id,
4738 "capability_deny_edge",
4739 "capability_forbidden",
4740 format!(
4741 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4742 ),
4743 )?]);
4744 }
4745 }
4746 }
4747
4748 if admission_facts.is_some() {
4749 let carrier_matches = matches!(
4750 &principal,
4751 Principal::Reserved { module_id }
4752 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4753 );
4754 if !carrier_matches {
4755 return Ok(vec![self.route_open_refusal_frame(
4756 ctx,
4757 &caller,
4758 &frame,
4759 &target_module_id,
4760 "admission_facts_carrier_not_permitted",
4761 "admission_facts_not_permitted",
4762 "admission facts may only be carried by the configured reserved module",
4763 )?]);
4764 }
4765
4766 let target_allowed = self
4767 .admission_facts_targets
4768 .as_ref()
4769 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4770 if !target_allowed {
4771 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4772 &frame,
4773 &target_module_id,
4774 "admission_facts_target_not_listed",
4775 "admission_facts_target_not_allowed",
4776 format!(
4777 "admission facts are not permitted for target module_id '{target_module_id}'"
4778 ),
4779 )?]);
4780 }
4781
4782 }
4786
4787 let (bound_scope, mut scope_stamp) = match &scope {
4793 None => (None, None),
4794 Some(selector) => {
4795 let owner_configured = match &selector.owner {
4796 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4802 _ => false,
4803 };
4804 let mut scopes = self
4805 .scopes
4806 .write()
4807 .unwrap_or_else(|poisoned| poisoned.into_inner());
4808 let admitted = scopes
4809 .admit(&principal, &target_module_id, selector, owner_configured)
4810 .and_then(|admission| {
4811 crate::scopes::check_target_flow_support(
4812 &admission.stamp,
4813 &target_module_id,
4814 registration.manifest.capabilities.as_ref(),
4815 )?;
4816 crate::scopes::check_target_agent_run_support(
4817 &admission.stamp,
4818 &target_module_id,
4819 registration.manifest.capabilities.as_ref(),
4820 )?;
4821 let bound = scopes.reserve_bind(
4822 &admission,
4823 &principal,
4824 ctx.connection_id,
4825 selector.hold_presence,
4826 );
4827 Ok((admission, bound))
4828 });
4829 match admitted {
4830 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4831 Err(refusal) => {
4832 return Ok(vec![self.route_open_refusal_frame(
4833 ctx,
4834 &caller,
4835 &frame,
4836 &target_module_id,
4837 refusal.code,
4838 refusal.code,
4839 refusal.message,
4840 )?]);
4841 }
4842 }
4843 }
4844 };
4845 if let Some(stamp) = &scope_stamp {
4848 caller.scope = route_open_scope_label(stamp);
4849 }
4850
4851 timings.enter(RouteOpenPhase::ProjectRoot);
4873 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4874 Ok(project_root) => project_root,
4875 Err(err) => {
4876 return Ok(vec![control_error_frame(
4877 &frame,
4878 "invalid_project_root",
4879 err.to_string(),
4880 )?])
4881 }
4882 };
4883 timings.enter(RouteOpenPhase::Admission);
4884 identity.project_root = project_root.as_path().to_path_buf();
4885
4886 let _concurrency_guard = match self
4897 .route_bind_concurrency
4898 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4899 {
4900 Ok(guard) => guard,
4901 Err(in_flight) => {
4902 return Ok(vec![self.route_open_target_capacity_refusal(
4903 ctx,
4904 &caller,
4905 &frame,
4906 &target_module_id,
4907 in_flight,
4908 )?]);
4909 }
4910 };
4911
4912 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4915 RouteBindAdmission::Admitted { guard, probe } => {
4916 if probe {
4917 info!(
4918 module_id = %target_module_id,
4919 connection_id = ctx.connection_id.get(),
4920 "route.bind breaker half-open: admitting one probe"
4921 );
4922 }
4923 guard
4924 }
4925 RouteBindAdmission::Refused {
4926 consecutive_timeouts,
4927 retry_in,
4928 probe_in_flight,
4929 } => {
4930 return Ok(vec![self.route_open_breaker_refusal_frame(
4931 ctx,
4932 &caller,
4933 &frame,
4934 &target_module_id,
4935 consecutive_timeouts,
4936 retry_in,
4937 probe_in_flight,
4938 )?]);
4939 }
4940 };
4941
4942 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4947 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4948 timings.enter(RouteOpenPhase::EgressReservation);
4949 let pending = match self
4950 .forwarding
4951 .begin_route_bind_relay_for(
4952 ctx.connection_id,
4953 ctx.egress.clone(),
4954 response_version(&frame),
4955 frame.header.corr,
4956 &target_module_id,
4957 principal.clone(),
4958 bound_scope,
4959 Some(project_root),
4960 relay_deadline,
4961 )
4962 .await
4963 {
4964 Ok(pending) => pending,
4965 Err(err) => {
4966 return Ok(vec![self.route_open_refusal_frame(
4967 ctx,
4968 &caller,
4969 &frame,
4970 &target_module_id,
4971 "relay_reservation_failed",
4972 forwarding_error_code(&err),
4973 err.to_string(),
4974 )?])
4975 }
4976 };
4977 timings.enter(RouteOpenPhase::Admission);
4978 let crate::forwarding::PendingRouteBindRelay {
4979 endpoint,
4980 module_sink,
4981 negotiated_ver,
4982 client_channel,
4983 client_epoch,
4984 module_channel,
4985 module_epoch,
4986 corr: relay_corr,
4987 receiver,
4988 } = pending;
4989 let mut reservation =
4990 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4991
4992 let relay_registration = self
4997 .registry
4998 .get_module_by_connection(endpoint.connection_id)
4999 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5000 let relay_capabilities = relay_registration
5001 .as_ref()
5002 .and_then(|registration| registration.manifest.capabilities.as_ref());
5003 if let Some(selector) = &scope {
5004 if let Err(refusal) = self
5007 .scopes
5008 .read()
5009 .unwrap_or_else(|poisoned| poisoned.into_inner())
5010 .check_bind_proof(&principal, selector)
5011 {
5012 reservation.release_and_disarm();
5013 return Ok(vec![self.route_open_refusal_frame(
5014 ctx,
5015 &caller,
5016 &frame,
5017 &target_module_id,
5018 refusal.code,
5019 refusal.code,
5020 refusal.message,
5021 )?]);
5022 }
5023 }
5024 if let Some(stamp) = scope_stamp
5025 .as_ref()
5026 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5027 {
5028 if let Err(refusal) = crate::scopes::check_target_flow_support(
5029 stamp,
5030 &target_module_id,
5031 relay_capabilities,
5032 )
5033 .and_then(|()| {
5034 crate::scopes::check_target_agent_run_support(
5035 stamp,
5036 &target_module_id,
5037 relay_capabilities,
5038 )
5039 }) {
5040 reservation.release_and_disarm();
5041 return Ok(vec![self.route_open_refusal_frame(
5042 ctx,
5043 &caller,
5044 &frame,
5045 &target_module_id,
5046 refusal.code,
5047 refusal.code,
5048 refusal.message,
5049 )?]);
5050 }
5051 }
5052
5053 if !relay_capabilities.is_some_and(|caps| {
5056 caps.provides
5057 .iter()
5058 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5059 }) {
5060 if let Some(stamp) = scope_stamp.as_mut() {
5061 stamp.workdir = None;
5062 stamp.project_id = None;
5063 }
5064 }
5065 let worker_run = scope_stamp
5070 .as_ref()
5071 .is_some_and(crate::scopes::is_worker_run)
5072 && relay_capabilities.is_some_and(|caps| {
5073 caps.provides
5074 .iter()
5075 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5076 });
5077 if !worker_run
5078 && !relay_capabilities.is_some_and(|caps| {
5079 caps.provides
5080 .iter()
5081 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5082 })
5083 {
5084 if let Some(stamp) = scope_stamp.as_mut() {
5085 stamp.attributes.on_behalf_of_agent = None;
5086 }
5087 }
5088 if !relay_capabilities.is_some_and(|caps| {
5089 caps.provides
5090 .iter()
5091 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5092 }) {
5093 if let Some(stamp) = scope_stamp.as_mut() {
5094 stamp.attributes.flow_version = None;
5095 stamp.attributes.flow_code_hash = None;
5096 stamp.attributes.flow_install_id = None;
5097 }
5098 }
5099 let call_mode = if worker_run
5100 || relay_capabilities
5101 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5102 {
5103 Some(derive_call_mode(scope_stamp.as_ref()))
5104 } else {
5105 if let Some(stamp) = scope_stamp.as_mut() {
5106 stamp.expires_at_ms = None;
5107 }
5108 None
5109 };
5110
5111 debug!(
5112 connection_id = ctx.connection_id.get(),
5113 client_channel,
5114 client_epoch,
5115 module_channel,
5116 module_epoch,
5117 "reserved route handle pair"
5118 );
5119 let route_principal_label = principal_label(&principal);
5122 identity.build_class = None;
5126 let relay = ModuleControlRequest::RouteBind {
5127 route_channel: module_channel,
5128 epoch: module_epoch,
5129 target,
5130 identity,
5131 principal: Some(principal),
5132 consumer_capabilities,
5133 role_versions,
5134 admission_facts,
5135 scope: scope_stamp,
5136 call_mode,
5137 };
5138 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5139 RouterError::backend(
5140 0,
5141 frame.header.corr,
5142 format!("failed to encode route.bind request: {err}"),
5143 )
5144 })?;
5145 let relay_frame = Frame::build_with_version(
5146 negotiated_ver,
5147 FrameType::Request,
5148 control_flags(),
5149 0,
5150 0,
5151 relay_corr,
5152 relay_body,
5153 )
5154 .map_err(RouterError::FrameBuild)?;
5155
5156 timings.enter(RouteOpenPhase::BindSend);
5157 let sent = module_sink.send(relay_frame).await;
5158 timings.enter(RouteOpenPhase::Admission);
5159 if let Err(err) = sent {
5160 reservation.release_and_disarm();
5161 return Ok(vec![self.route_open_refusal_frame(
5162 ctx,
5163 &caller,
5164 &frame,
5165 &target_module_id,
5166 "relay_send_failed",
5167 "target_unavailable",
5168 err.to_string(),
5169 )?]);
5170 }
5171
5172 #[cfg(test)]
5173 if let Some(hold) = &self.route_bind_relay_sent_hold {
5174 hold.wait().await;
5178 hold.wait().await;
5179 }
5180
5181 if self
5186 .forwarding
5187 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5188 .map_err(RouterError::Forwarding)?
5189 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5190 {
5191 self.send_abandoned_route_bind_goodbye(
5192 &module_sink,
5193 negotiated_ver,
5194 module_channel,
5195 module_epoch,
5196 );
5197 }
5198
5199 timings.enter(RouteOpenPhase::BindAckWait);
5200 let outcome = timeout_at(relay_deadline, receiver).await;
5201 timings.enter(RouteOpenPhase::Admission);
5202 match outcome {
5203 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5204 reservation.disarm();
5205 if breaker.record_accepted() {
5206 info!(
5207 module_id = %target_module_id,
5208 "route.bind breaker closed: the probe was accepted"
5209 );
5210 }
5211 self.observe_route_open_accept(
5212 ctx,
5213 &target_module_id,
5214 &route_principal_label,
5215 &caller.scope,
5216 );
5217 Ok(Vec::new())
5218 }
5219 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5220 reservation.release_and_disarm();
5221 breaker.record_inconclusive();
5225 let scope_code = match body.code.as_str() {
5229 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5230 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5231 _ => None,
5232 };
5233 if let Some(code) = scope_code {
5234 self.observe_route_open_refusal(
5235 ctx,
5236 &caller,
5237 &target_module_id,
5238 "scope_changed_before_commit",
5239 code,
5240 );
5241 return Ok(vec![control_error_body_frame(&frame, body)?]);
5242 }
5243 self.counters
5244 .increment_route_open_refused("module_rejected");
5245 info!(
5246 target: "control",
5247 code = "module_rejected",
5248 module_code = ?body.code,
5249 module_id = ?target_module_id,
5250 connection_id = ctx.connection_id.get(),
5251 principal = %caller.principal,
5252 scope = %caller.scope,
5253 harness = %caller.harness,
5254 "route.open refused"
5255 );
5256 Ok(vec![control_error_body_frame(&frame, body)?])
5257 }
5258 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5259 reservation.release_and_disarm();
5260 breaker.record_inconclusive();
5261 tracing::warn!(
5268 module_id = %target_module_id,
5269 "route.bind relay abandoned: {message}"
5270 );
5271 Ok(vec![self.route_open_refusal_frame(
5272 ctx,
5273 &caller,
5274 &frame,
5275 &target_module_id,
5276 "relay_abandoned",
5277 "target_unavailable",
5278 message,
5279 )?])
5280 }
5281 Ok(Err(_)) => {
5282 reservation.release_and_disarm();
5283 breaker.record_inconclusive();
5284 Ok(vec![self.route_open_refusal_frame(
5285 ctx,
5286 &caller,
5287 &frame,
5288 &target_module_id,
5289 "relay_waiter_canceled",
5290 "target_unavailable",
5291 "route.bind relay waiter was canceled before the module responded",
5292 )?])
5293 }
5294 Err(_) => {
5295 reservation.release_and_disarm();
5296 if let Some(opened) = breaker.record_timeout(
5300 self.route_bind_breaker_threshold,
5301 self.route_bind_breaker_cooldown,
5302 ) {
5303 warn!(
5304 module_id = %target_module_id,
5305 consecutive_timeouts = opened.consecutive_timeouts,
5306 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5307 reopened_after_probe = opened.reopened_after_probe,
5308 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5309 );
5310 }
5311 tracing::warn!(
5319 module_id = %target_module_id,
5320 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5321 "route.bind relay timed out: module did not ack within budget"
5322 );
5323 Ok(vec![self.route_open_refusal_frame(
5324 ctx,
5325 &caller,
5326 &frame,
5327 &target_module_id,
5328 "relay_timed_out",
5329 "module_timeout",
5330 format!(
5331 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5332 route_bind_relay_timeout
5333 ),
5334 )?])
5335 }
5336 }
5337 }
5338
5339 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5340 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5341 snapshot: self.supervisor.spawn_snapshot(),
5342 };
5343 Ok(vec![control_response_body_frame(
5344 &frame,
5345 &response,
5346 "ClientControlResponse::SupervisorSpawnSnapshot",
5347 )?])
5348 }
5349
5350 fn handle_supervisor_spawn_subscribe(
5351 &self,
5352 ctx: &RouteCtx,
5353 frame: Frame,
5354 since: Option<SpawnCursor>,
5355 ) -> Result<Vec<Frame>, RouterError> {
5356 match self.supervisor.subscribe_spawns(
5357 ctx.connection_id,
5358 frame.header.corr,
5359 response_version(&frame),
5360 since,
5361 ctx.egress.clone(),
5362 ) {
5363 Ok(()) => Ok(Vec::new()),
5364 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5365 Ok(vec![control_error_body_frame(
5366 &frame,
5367 ErrorBody {
5368 code: "spawn_cursor_incarnation_mismatch".to_string(),
5369 message: "spawn cursor belongs to a different daemon incarnation"
5370 .to_string(),
5371 detail: Some(serde_json::json!({
5372 "current_daemon_incarnation": current
5373 })),
5374 },
5375 )?])
5376 }
5377 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5378 &frame,
5379 ErrorBody {
5380 code: "spawn_cursor_too_old".to_string(),
5381 message: "spawn cursor predates the retained event ring".to_string(),
5382 detail: Some(serde_json::json!({
5383 "oldest_retained_cursor": oldest
5384 })),
5385 },
5386 )?]),
5387 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5388 0,
5389 frame.header.corr,
5390 format!("failed to open supervisor spawn subscription: {error}"),
5391 )),
5392 }
5393 }
5394
5395 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5396 let generation = self
5397 .registry
5398 .generation()
5399 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5400 let mut modules = Vec::new();
5401 for module in self.supervisor.list() {
5402 let status = module.status_for_control("list").map_err(|err| {
5403 RouterError::backend(
5404 0,
5405 frame.header.corr,
5406 format!("failed to read supervisor status: {err}"),
5407 )
5408 })?;
5409 let (configured, _) = module.configuration().map_err(|err| {
5410 RouterError::backend(
5411 0,
5412 frame.header.corr,
5413 format!("failed to read module configuration: {err}"),
5414 )
5415 })?;
5416 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5417 RouterError::backend(
5418 0,
5419 frame.header.corr,
5420 format!("failed to read retry time: {err}"),
5421 )
5422 })?;
5423 let image = module.running_image_agreement().await;
5425 let resources = Some(module.child_resource_usage());
5428 let mut verdict =
5429 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5430 verdict.spec = Some(launch_spec_agreement(
5431 &configured,
5432 status.process_alive,
5433 status.launch_spec_digest.as_ref(),
5434 ));
5435 let pending_reload = Some(verdict);
5436 let configured_enabled = self
5437 .capability_evaluator
5438 .configured_enabled(&status.module_id);
5439 modules.push(
5442 SupervisorEntry::new(
5443 status.module_id,
5444 status.state.to_string(),
5445 status.enabled,
5446 status.live,
5447 status.health.status,
5448 )
5449 .with_configured_enabled(configured_enabled)
5450 .with_next_retry_at_ms(next_retry_at_ms)
5451 .with_launch_nonce_env(Some(!cfg!(unix)))
5452 .with_launch_nonce_source(status.launch_nonce_source)
5453 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5454 .with_protocol(status.protocol)
5455 .with_pending_reload(pending_reload)
5456 .with_last_probe_ms(status.health.last_probe_ms)
5457 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5458 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5459 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5460 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5461 .with_restart_count(Some(status.restart_count))
5462 .with_max_restarts(Some(status.max_restarts))
5463 .with_lifetime_restarts(Some(status.lifetime_restarts))
5464 .with_spawn_generation(Some(status.spawn_generation))
5465 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5466 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5467 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5468 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5469 .with_resources(resources),
5470 );
5471 }
5472 let response = ClientControlResponse::SupervisorList {
5473 generation,
5474 modules,
5475 };
5476 Ok(vec![control_response_body_frame(
5477 &frame,
5478 &response,
5479 "ClientControlResponse::SupervisorList",
5480 )?])
5481 }
5482
5483 fn handle_supervisor_stderr_tail(
5484 &self,
5485 frame: Frame,
5486 module_id: String,
5487 max_lines: Option<u32>,
5488 max_bytes: Option<u32>,
5489 ) -> Result<Vec<Frame>, RouterError> {
5490 let Some(module) = self.supervisor.get(&module_id) else {
5491 return Ok(vec![control_error_frame(
5492 &frame,
5493 "unknown_module",
5494 format!("module_id '{module_id}' is not supervised"),
5495 )?]);
5496 };
5497
5498 let snapshot = module.stderr_tail(
5499 max_lines.map(|value| value as usize),
5500 max_bytes.map(|value| value as usize),
5501 );
5502
5503 let response = ClientControlResponse::SupervisorStderrTail {
5504 module_id,
5505 tail: StderrTail {
5506 capture: match snapshot.capture {
5507 CaptureState::Captured => StderrCaptureState::Captured,
5508 CaptureState::Incomplete { reason } => {
5509 StderrCaptureState::Incomplete { reason }
5510 }
5511 CaptureState::NotCaptured { reason } => {
5512 StderrCaptureState::NotCaptured { reason }
5513 }
5514 },
5515 entries: snapshot
5516 .entries
5517 .into_iter()
5518 .map(|entry| match entry {
5519 TailEntry::Line {
5520 text,
5521 truncated,
5522 at_ms,
5523 } => StderrTailEntry::Line {
5524 text,
5525 truncated,
5526 at_ms,
5527 },
5528 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5529 })
5530 .collect(),
5531 dropped_lines: snapshot.dropped_lines,
5532 },
5533 };
5534 Ok(vec![control_response_body_frame(
5535 &frame,
5536 &response,
5537 "ClientControlResponse::SupervisorStderrTail",
5538 )?])
5539 }
5540
5541 async fn handle_supervisor_terminals(
5542 &self,
5543 frame: Frame,
5544 module_id: String,
5545 ) -> Result<Vec<Frame>, RouterError> {
5546 let Some(module) = self.supervisor.get(&module_id) else {
5547 return Ok(vec![control_error_frame(
5548 &frame,
5549 "unknown_module",
5550 format!("module_id '{module_id}' is not supervised"),
5551 )?]);
5552 };
5553
5554 let terminals = module
5557 .read_durable_terminal_history()
5558 .await
5559 .map_err(|error| {
5560 RouterError::backend(
5561 0,
5562 frame.header.corr,
5563 format!("failed to read terminal history: {error}"),
5564 )
5565 })?;
5566 let response = ClientControlResponse::SupervisorTerminals {
5567 module_id,
5568 terminals,
5569 };
5570 Ok(vec![control_response_body_frame(
5571 &frame,
5572 &response,
5573 "ClientControlResponse::SupervisorTerminals",
5574 )?])
5575 }
5576
5577 fn handle_supervisor_routes(
5578 &self,
5579 frame: Frame,
5580 module_id: Option<String>,
5581 ) -> Result<Vec<Frame>, RouterError> {
5582 let modules = self
5583 .forwarding
5584 .route_census(module_id.as_deref())
5585 .map_err(RouterError::Forwarding)?
5586 .into_iter()
5587 .map(|(module_id, routes)| SupervisorRouteModule {
5588 module_id,
5589 routes: routes
5590 .into_iter()
5591 .map(|route| SupervisorRoute {
5592 consumer: match route.principal {
5593 Principal::Reserved { module_id } => {
5594 SupervisorRouteConsumer::Reserved { module_id }
5595 }
5596 Principal::Direct | Principal::Unverified => {
5597 SupervisorRouteConsumer::Direct {
5598 connection_id: route.goodbye_target.connection_id.get(),
5599 }
5600 }
5601 },
5602 age_ms: Instant::now()
5603 .saturating_duration_since(route.bound_at)
5604 .as_millis()
5605 .try_into()
5606 .unwrap_or(u64::MAX),
5607 draining: route.draining,
5608 drain_reason: route.drain_reason,
5609 })
5610 .collect(),
5611 })
5612 .collect();
5613 let response = ClientControlResponse::SupervisorRoutes { modules };
5614 Ok(vec![control_response_body_frame(
5615 &frame,
5616 &response,
5617 "ClientControlResponse::SupervisorRoutes",
5618 )?])
5619 }
5620
5621 async fn handle_supervisor_provenance(
5622 &self,
5623 frame: Frame,
5624 module_id: Option<String>,
5625 ) -> Result<Vec<Frame>, RouterError> {
5626 let mut selected = if let Some(module_id) = module_id {
5627 let Some(module) = self.supervisor.get(&module_id) else {
5628 return Ok(vec![control_error_frame(
5629 &frame,
5630 "unknown_module",
5631 format!("module_id '{module_id}' is not supervised"),
5632 )?]);
5633 };
5634 vec![module]
5635 } else {
5636 self.supervisor.list()
5637 };
5638
5639 let mut modules = Vec::with_capacity(selected.len());
5640 for module in selected.drain(..) {
5641 let (status, observed_image) = module
5642 .status_and_running_image_agreement()
5643 .await
5644 .map_err(|err| {
5645 RouterError::backend(
5646 0,
5647 frame.header.corr,
5648 format!("failed to read supervisor status: {err}"),
5649 )
5650 })?;
5651 let module_declared = self
5652 .registry
5653 .get_module(&status.module_id)
5654 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5655 .and_then(|registration| registration.manifest.provenance)
5656 .map(|build| ModuleDeclaredProvenance::Reported { build })
5657 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5658 #[cfg(test)]
5659 let running_image = match &self.provenance_probe_override {
5660 Some(result) => result.clone(),
5661 None => observed_image,
5662 };
5663 #[cfg(not(test))]
5664 let running_image = observed_image;
5665 modules.push(SupervisorModuleProvenance {
5666 module_id: status.module_id,
5667 module_declared,
5668 daemon_observed: SupervisorObservedProcess {
5669 launch_nonce_source: status.launch_nonce_source,
5670 pid: status.pid,
5671 spawned_at_ms: status.spawned_at_ms,
5672 spawned_from: status.spawned_from,
5673 running_image,
5674 },
5675 });
5676 }
5677 let daemon = SupervisorDaemonProvenance {
5678 daemon_build: self.daemon_provenance.build.clone(),
5679 daemon_observed: DaemonObservedProcess {
5680 pid: self.daemon_provenance.pid,
5681 started_at_ms: self
5682 .daemon_provenance
5683 .start_clock
5684 .map(|clock| clock.started_at_ms())
5685 .or(self.daemon_provenance.started_at_ms),
5686 running_image: self
5687 .daemon_provenance
5688 .probe
5689 .observe(
5690 self.daemon_provenance.pid,
5691 self.daemon_provenance.executable_path.as_deref(),
5692 self.daemon_provenance.executable_identity.clone(),
5693 self.daemon_provenance.process_start_time,
5694 )
5695 .await,
5696 },
5697 };
5698 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5699 Ok(vec![control_response_body_frame(
5700 &frame,
5701 &response,
5702 "ClientControlResponse::SupervisorProvenance",
5703 )?])
5704 }
5705
5706 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5707 self.refresh_capability_requirements();
5708 let generation = self
5709 .registry
5710 .generation()
5711 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5712 let modules = self
5713 .supervisor
5714 .list()
5715 .into_iter()
5716 .map(|module| {
5717 let status = module.status_for_control("health").map_err(|err| {
5718 RouterError::backend(
5719 0,
5720 frame.header.corr,
5721 format!("failed to read supervisor health: {err}"),
5722 )
5723 })?;
5724 let module_id = status.module_id;
5725 let capability_detail = self
5726 .capability_evaluator
5727 .required_problem_detail(&module_id);
5728 Ok(SupervisorHealthEntry {
5729 module_id,
5730 status: status.health.status,
5731 detail: append_capability_problem_detail(
5732 status.health.detail,
5733 capability_detail,
5734 ),
5735 metrics: status.health.metrics,
5736 consecutive_failures: status.health.consecutive_failures,
5737 late_answer_count: status.health.late_answer_count,
5738 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5739 last_action: status.health.last_action,
5740 last_action_ms: status.health.last_action_ms,
5741 last_probe_ms: status.health.last_probe_ms,
5742 })
5743 })
5744 .collect::<Result<Vec<_>, RouterError>>()?;
5745 let response = ClientControlResponse::SupervisorHealth {
5746 generation,
5747 modules,
5748 };
5749 Ok(vec![control_response_body_frame(
5750 &frame,
5751 &response,
5752 "ClientControlResponse::SupervisorHealth",
5753 )?])
5754 }
5755
5756 async fn handle_supervisor_restart(
5757 &self,
5758 frame: Frame,
5759 module_id: String,
5760 drain_timeout_ms: Option<u64>,
5761 requester: Principal,
5762 ) -> Result<Vec<Frame>, RouterError> {
5763 let operation_lock = self.supervisor.operation_lock();
5764 let _operation_guard = operation_lock.lock().await;
5765 let Some(module) = self.supervisor.get(&module_id) else {
5766 return Ok(vec![control_error_frame(
5767 &frame,
5768 "unknown_module",
5769 format!("module_id '{module_id}' is not supervised"),
5770 )?]);
5771 };
5772
5773 self.route_outages.mark_operator_action(&module_id);
5774 if let Err(err) = module
5775 .restart_requested(drain_timeout_ms, Some(requester))
5776 .await
5777 {
5778 self.route_outages
5779 .operator_action_ended_unrefused(&module_id);
5780 let (code, message) = match err {
5781 crate::supervise::SuperviseError::Disabled { .. } => {
5782 ("module_disabled", err.to_string())
5783 }
5784 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5785 ("swap_in_progress", err.to_string())
5786 }
5787 _ => (
5788 "target_unavailable",
5789 format!("failed to restart module_id '{module_id}': {err}"),
5790 ),
5791 };
5792 return Ok(vec![control_error_frame(&frame, code, message)?]);
5793 }
5794
5795 let response = ClientControlResponse::SupervisorAck {
5796 module_id,
5797 applied: true,
5798 };
5799 Ok(vec![control_response_body_frame(
5800 &frame,
5801 &response,
5802 "ClientControlResponse::SupervisorAck",
5803 )?])
5804 }
5805
5806 async fn handle_supervisor_swap(
5810 &self,
5811 frame: Frame,
5812 module_id: String,
5813 ready_timeout_ms: Option<u64>,
5814 requester: Principal,
5815 ) -> Result<Vec<Frame>, RouterError> {
5816 let module = {
5823 let operation_lock = self.supervisor.operation_lock();
5824 let _operation_guard = operation_lock.lock().await;
5825 self.supervisor.get(&module_id)
5826 };
5827 let Some(module) = module else {
5828 return Ok(vec![control_error_frame(
5829 &frame,
5830 "unknown_module",
5831 format!("module_id '{module_id}' is not supervised"),
5832 )?]);
5833 };
5834
5835 self.route_outages.mark_operator_action(&module_id);
5836 if let Err(err) = module
5837 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5838 .await
5839 {
5840 self.route_outages
5841 .operator_action_ended_unrefused(&module_id);
5842 use crate::supervise::SuperviseError;
5843 let message = err.to_string();
5844 let error = match err {
5845 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5846 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5847 code: "swap_refused".to_string(),
5848 message,
5849 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5850 },
5851 SuperviseError::SwapFailed {
5852 arm,
5853 candidate_exit,
5854 ..
5855 } => ErrorBody {
5856 code: "swap_failed".to_string(),
5857 message,
5858 detail: Some(serde_json::json!({
5859 "arm": arm.as_str(),
5860 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5861 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5862 })),
5863 },
5864 _ => ErrorBody::new(
5865 "target_unavailable",
5866 format!("failed to swap module_id '{module_id}': {message}"),
5867 ),
5868 };
5869 return Ok(vec![control_error_body_frame(&frame, error)?]);
5870 }
5871 self.route_outages
5875 .operator_action_ended_unrefused(&module_id);
5876
5877 let response = ClientControlResponse::SupervisorAck {
5878 module_id,
5879 applied: true,
5880 };
5881 Ok(vec![control_response_body_frame(
5882 &frame,
5883 &response,
5884 "ClientControlResponse::SupervisorAck",
5885 )?])
5886 }
5887
5888 async fn handle_supervisor_reload(
5889 &self,
5890 frame: Frame,
5891 module_id: String,
5892 requester: Principal,
5893 ) -> Result<Vec<Frame>, RouterError> {
5894 let operation_lock = self.supervisor.operation_lock();
5895 let _operation_guard = operation_lock.lock().await;
5896 let Some(module) = self.supervisor.get(&module_id) else {
5897 return Ok(vec![control_error_frame(
5898 &frame,
5899 "unknown_module",
5900 format!("module_id '{module_id}' is not supervised"),
5901 )?]);
5902 };
5903
5904 self.route_outages.mark_operator_action(&module_id);
5905 if let Err(err) = module.reload_requested(Some(requester)).await {
5906 self.route_outages
5907 .operator_action_ended_unrefused(&module_id);
5908 let (code, message) = match err {
5909 crate::supervise::SuperviseError::Disabled { .. } => {
5910 ("module_disabled", err.to_string())
5911 }
5912 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5913 ("swap_in_progress", err.to_string())
5914 }
5915 _ => (
5916 "reload_failed",
5917 format!("failed to reload module_id '{module_id}': {err}"),
5918 ),
5919 };
5920 return Ok(vec![control_error_frame(&frame, code, message)?]);
5921 }
5922
5923 let response = ClientControlResponse::SupervisorAck {
5924 module_id,
5925 applied: true,
5926 };
5927 Ok(vec![control_response_body_frame(
5928 &frame,
5929 &response,
5930 "ClientControlResponse::SupervisorAck",
5931 )?])
5932 }
5933
5934 async fn handle_supervisor_rescan(
5938 &self,
5939 frame: Frame,
5940 mode: RescanMode,
5941 requester: Principal,
5942 ) -> Result<Vec<Frame>, RouterError> {
5943 let preview = mode.previews();
5944 let Some(context) = self.rescan.clone() else {
5945 return Ok(vec![control_error_frame(
5946 &frame,
5947 "rescan_unavailable",
5948 "the daemon was not started with a reloadable config path".to_string(),
5949 )?]);
5950 };
5951
5952 let operation_lock = self.supervisor.operation_lock();
5953 let _operation_guard = operation_lock.lock().await;
5954 let loaded = match self.load_rescan_config(&mode, &context) {
5958 Ok(config) => config,
5959 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5960 };
5961 let Some((config, activity_config)) = loaded else {
5969 return Ok(vec![control_error_frame(
5970 &frame,
5971 "invalid_daemon_config",
5972 format!(
5973 "daemon config not found at {}; refusing to rescan (an absent config would \
5974 retire every supervised module)",
5975 context.config_path.display()
5976 ),
5977 )?]);
5978 };
5979 let (
5980 configured_port,
5981 storage_config,
5982 admission_facts_carrier_module_id,
5983 admission_facts_targets,
5984 scope_authority_owners,
5985 privileged_operator_prompt,
5986 modules,
5987 reserved_capabilities,
5988 ) = (
5989 config.port,
5990 config.storage,
5991 config.admission_facts_carrier_module_id,
5992 config.admission_facts_targets,
5993 config.scope_authority_owners,
5994 config.privileged_operator_prompt,
5995 config.modules,
5996 config.reserved_capabilities,
5997 );
5998
5999 let mut restart_required = Vec::new();
6008 for section in RestartRequiredSection::ALL {
6009 let changed = match section {
6010 RestartRequiredSection::Port => configured_port != context.configured_port,
6011 RestartRequiredSection::Storage => storage_config != context.storage_config,
6012 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6013 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6014 }
6015 RestartRequiredSection::AdmissionFactsTargets => {
6016 admission_facts_targets != context.admission_facts_targets
6017 }
6018 RestartRequiredSection::ScopeAuthorityOwners => {
6019 scope_authority_owners != context.scope_authority_owners
6020 }
6021 RestartRequiredSection::PrivilegedOperatorPrompt => {
6022 privileged_operator_prompt != context.privileged_operator_prompt
6023 }
6024 };
6025 if changed {
6026 restart_required.push(section.label().to_string());
6027 }
6028 }
6029 if !restart_required.is_empty() {
6030 warn!(
6031 config_path = %context.config_path.display(),
6032 sections = %restart_required.join(", "),
6033 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6034 );
6035 }
6036
6037 for configured in &modules {
6038 if let Err(err) = validate_spec(&configured.module_spec()) {
6039 return Ok(vec![control_error_frame(
6040 &frame,
6041 "invalid_daemon_config",
6042 format!("supervisor rescan rejected daemon config: {err}"),
6043 )?]);
6044 }
6045 }
6046
6047 let configured_capabilities = modules
6048 .iter()
6049 .map(|module| (module.module_id.clone(), module.enabled))
6050 .collect::<Vec<_>>();
6051 let preview_capability_warnings = if preview {
6052 let (_, registrations) = self.runtime_capability_snapshot()?;
6053 let current_modules = self
6054 .supervisor
6055 .list()
6056 .into_iter()
6057 .map(|module| module.module_id().to_string())
6058 .collect::<BTreeSet<_>>();
6059 let resulting_modules = configured_capabilities.clone();
6060 let removed = current_modules
6061 .into_iter()
6062 .filter(|module_id| {
6063 !resulting_modules
6064 .iter()
6065 .any(|(configured_id, _)| configured_id == module_id)
6066 })
6067 .collect::<Vec<_>>();
6068 let disabled = match &mode {
6071 RescanMode::Candidate(_) => resulting_modules
6072 .iter()
6073 .filter(|(_, enabled)| !enabled)
6074 .map(|(module_id, _)| module_id.clone())
6075 .collect::<Vec<_>>(),
6076 _ => Vec::new(),
6077 };
6078 self.capability_evaluator.preview_warnings(
6079 resulting_modules,
6080 &removed,
6081 &disabled,
6082 ®istrations,
6083 )
6084 } else {
6085 Vec::new()
6086 };
6087 let reconciled = match &mode {
6088 RescanMode::Candidate(candidate) => self
6089 .preview_candidate(&context, modules, candidate)
6090 .map(|result| (result, Vec::new())),
6091 _ => {
6092 self.reconcile_supervised_modules(
6093 &context.supervisor,
6094 modules,
6095 preview,
6096 requester.clone(),
6097 )
6098 .await
6099 }
6100 };
6101 let (result, changed) = match reconciled {
6102 Ok(result) => result,
6103 Err(message) => {
6104 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6105 }
6106 };
6107 if !preview {
6108 if let Some(ledger) = &self.activity {
6109 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6110 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6111 }
6112 if let Err(code) = self.configure_activity(activity_config).await {
6113 warn!(
6114 code,
6115 "activity retention could not be applied during rescan"
6116 );
6117 }
6118 self.capability_evaluator
6119 .configure(configured_capabilities, reserved_capabilities);
6120 self.capability_evaluator.wake_deadline_loop();
6121 self.refresh_capability_requirements();
6122 }
6123 let mut result = result;
6124 result.restart_required = restart_required;
6125 result.capability_warnings = preview_capability_warnings;
6126 let response = ClientControlResponse::SupervisorRescan { result };
6127 Ok(vec![control_response_body_frame(
6128 &frame,
6129 &response,
6130 "ClientControlResponse::SupervisorRescan",
6131 )?])
6132 }
6133
6134 async fn handle_supervisor_release_reserved(
6135 &self,
6136 frame: Frame,
6137 module_id: String,
6138 ) -> Result<Vec<Frame>, RouterError> {
6139 let Some(context) = self.rescan.clone() else {
6140 return Ok(vec![control_error_frame(
6141 &frame,
6142 "release_unavailable",
6143 "reserved-id release requires a daemon started with a reloadable config path",
6144 )?]);
6145 };
6146 let operation_lock = self.supervisor.operation_lock();
6147 let _operation_guard = operation_lock.lock().await;
6148 let loaded = match crate::daemon_config::load(&context.config_path) {
6149 Ok(Some(config)) => config,
6150 Ok(None) => {
6151 return Ok(vec![control_error_frame(
6152 &frame,
6153 "invalid_daemon_config",
6154 format!(
6155 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6156 context.config_path.display()
6157 ),
6158 )?])
6159 }
6160 Err(err) => {
6161 return Ok(vec![control_error_frame(
6162 &frame,
6163 "invalid_daemon_config",
6164 format!("unable to verify reserved-id release against daemon config: {err}"),
6165 )?])
6166 }
6167 };
6168 if loaded
6169 .modules
6170 .iter()
6171 .any(|configured| configured.module_id == module_id)
6172 {
6173 return Ok(vec![control_error_frame(
6174 &frame,
6175 "reserved_module_configured",
6176 format!(
6177 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6178 ),
6179 )?]);
6180 }
6181 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6182 return Ok(vec![control_error_frame(
6183 &frame,
6184 "reserved_gate_not_retained",
6185 format!(
6186 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6187 ),
6188 )?]);
6189 }
6190
6191 let response = ClientControlResponse::SupervisorAck {
6192 module_id,
6193 applied: true,
6194 };
6195 Ok(vec![control_response_body_frame(
6196 &frame,
6197 &response,
6198 "ClientControlResponse::SupervisorAck",
6199 )?])
6200 }
6201
6202 fn plan_rescan(
6206 &self,
6207 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6208 ) -> Result<RescanPlan, String> {
6209 let mut current = BTreeMap::new();
6210 for module in self.supervisor.list() {
6211 let (spec, health) = module.configuration().map_err(|err| {
6212 format!(
6213 "failed to read configuration for module_id '{}': {err}",
6214 module.module_id()
6215 )
6216 })?;
6217 let enabled = module
6218 .status()
6219 .map_err(|err| {
6220 format!(
6221 "failed to read status for module_id '{}': {err}",
6222 module.module_id()
6223 )
6224 })?
6225 .enabled;
6226 current.insert(
6227 module.module_id().to_string(),
6228 (module, spec, health, enabled),
6229 );
6230 }
6231 let configured = configured_modules
6232 .into_iter()
6233 .map(|module| (module.module_id.clone(), module))
6234 .collect::<BTreeMap<_, _>>();
6235
6236 let added = configured
6237 .keys()
6238 .filter(|module_id| !current.contains_key(*module_id))
6239 .cloned()
6240 .collect::<Vec<_>>();
6241 let removed = current
6242 .keys()
6243 .filter(|module_id| !configured.contains_key(*module_id))
6244 .cloned()
6245 .collect::<Vec<_>>();
6246 let mut changed_pending_reload = Vec::new();
6247 let mut configuration_changes = BTreeSet::new();
6248 let mut enabled_changes = BTreeSet::new();
6249 let mut unchanged = 0_u32;
6250
6251 for (module_id, configured_module) in &configured {
6252 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6253 else {
6254 continue;
6255 };
6256 let launch_changed = *current_spec != configured_module.module_spec();
6260 let configuration_changed =
6261 launch_changed || *current_health != configured_module.health;
6262 let enabled_changed = *current_enabled != configured_module.enabled;
6263 if configuration_changed {
6264 configuration_changes.insert(module_id.clone());
6265 }
6266 if launch_changed {
6267 changed_pending_reload.push(module_id.clone());
6268 }
6269 if enabled_changed {
6270 enabled_changes.insert(module_id.clone());
6271 }
6272 if !configuration_changed && !enabled_changed {
6273 unchanged = unchanged.saturating_add(1);
6274 }
6275 }
6276 Ok(RescanPlan {
6277 current,
6278 configured,
6279 added,
6280 removed,
6281 changed_pending_reload,
6282 configuration_changes,
6283 enabled_changes,
6284 unchanged,
6285 })
6286 }
6287
6288 async fn reconcile_supervised_modules(
6297 &self,
6298 supervisor: &Supervisor,
6299 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6300 preview: bool,
6301 requester: Principal,
6302 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6303 let plan = self.plan_rescan(configured_modules)?;
6304
6305 if preview {
6311 return Ok((plan.preview_result(), Vec::new()));
6312 }
6313 let changed = plan
6314 .configuration_changes
6315 .union(&plan.enabled_changes)
6316 .cloned()
6317 .collect::<Vec<_>>();
6318 let RescanPlan {
6319 current,
6320 configured,
6321 added,
6322 removed,
6323 changed_pending_reload,
6324 configuration_changes,
6325 enabled_changes,
6326 unchanged,
6327 } = plan;
6328 if !(added.is_empty()
6333 && removed.is_empty()
6334 && configuration_changes.is_empty()
6335 && enabled_changes.is_empty())
6336 {
6337 if let Some(context) = &self.rescan {
6338 context
6339 .generation
6340 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6341 }
6342 }
6343
6344 for module_id in &removed {
6345 let module = ¤t
6346 .get(module_id)
6347 .expect("removed module came from current supervisor state")
6348 .0;
6349 module
6350 .retire_requested(Some(requester.clone()))
6351 .await
6352 .map_err(|err| {
6353 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6354 })?;
6355 self.supervisor.record_rescan_removal(module_id);
6389 self.supervisor.retire(module_id);
6390 self.route_outages.forget(module_id);
6391 }
6392
6393 for module_id in configured.keys() {
6394 let Some((module, _, _, _)) = current.get(module_id) else {
6395 continue;
6396 };
6397 let configured_module = configured
6398 .get(module_id)
6399 .expect("configured module id came from configured map");
6400 if configuration_changes.contains(module_id) {
6401 module
6402 .update_configuration(
6403 configured_module.module_spec(),
6404 configured_module.health.clone(),
6405 configured_module.drain_timeout_ms,
6406 )
6407 .await
6408 .map_err(|err| {
6409 format!(
6410 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6411 )
6412 })?;
6413 }
6414 if enabled_changes.contains(module_id) {
6415 self.route_outages.mark_operator_action(module_id);
6418 module
6419 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6420 .await
6421 .map_err(|err| {
6422 self.route_outages.operator_action_ended_unrefused(module_id);
6423 format!(
6424 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6425 configured_module.enabled
6426 )
6427 })?;
6428 }
6429 }
6430
6431 for module_id in &added {
6432 let configured_module = configured
6433 .get(module_id)
6434 .expect("added module id came from configured map");
6435 supervisor
6436 .supervise_configured_requested(
6437 configured_module.module_spec(),
6438 configured_module.enabled,
6439 configured_module.health.clone(),
6440 configured_module.drain_timeout_ms,
6441 configured_module.restart,
6442 configured_module.register_timeout,
6443 Some(requester.clone()),
6444 )
6445 .map_err(|err| {
6446 format!("failed to add module_id '{module_id}' during rescan: {err}")
6447 })?;
6448 }
6449
6450 Ok((
6451 SupervisorRescanResult {
6452 added,
6453 removed,
6454 changed_pending_reload,
6455 enabled_changes: enabled_changes.iter().cloned().collect(),
6456 unchanged,
6457 preview: false,
6458 restart_required: Vec::new(),
6461 capability_warnings: Vec::new(),
6462 module_actions: Vec::new(),
6463 earlier_pending: Vec::new(),
6464 expect: None,
6465 },
6466 changed,
6467 ))
6468 }
6469
6470 async fn handle_supervisor_set_enabled(
6471 &self,
6472 frame: Frame,
6473 module_id: String,
6474 enabled: bool,
6475 requester: Principal,
6476 ) -> Result<Vec<Frame>, RouterError> {
6477 let operation_lock = self.supervisor.operation_lock();
6478 let _operation_guard = operation_lock.lock().await;
6479 let Some(module) = self.supervisor.get(&module_id) else {
6480 return Ok(vec![control_error_frame(
6481 &frame,
6482 "unknown_module",
6483 format!("module_id '{module_id}' is not supervised"),
6484 )?]);
6485 };
6486
6487 self.route_outages.mark_operator_action(&module_id);
6490 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6491 Ok(applied) => applied,
6492 Err(err) => {
6493 self.route_outages
6494 .operator_action_ended_unrefused(&module_id);
6495 return Ok(vec![control_error_frame(
6496 &frame,
6497 "target_unavailable",
6498 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6499 )?]);
6500 }
6501 };
6502 if !applied {
6503 self.route_outages
6506 .operator_action_ended_unrefused(&module_id);
6507 }
6508
6509 self.capability_evaluator.wake_deadline_loop();
6510 self.refresh_capability_requirements();
6511 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6512 Ok(vec![control_response_body_frame(
6513 &frame,
6514 &response,
6515 "ClientControlResponse::SupervisorAck",
6516 )?])
6517 }
6518
6519 async fn handle_supervisor_health_probe(
6520 &self,
6521 frame: Frame,
6522 module_id: String,
6523 ) -> Result<Vec<Frame>, RouterError> {
6524 self.refresh_capability_requirements();
6525 let Some(registration) = self
6526 .registry
6527 .get_module(&module_id)
6528 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6529 else {
6530 return Ok(vec![control_error_frame(
6531 &frame,
6532 "unknown_module",
6533 format!("module_id '{module_id}' is not registered"),
6534 )?]);
6535 };
6536
6537 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6550 {
6551 return Ok(vec![control_error_frame(
6552 &frame,
6553 "health_not_advertised",
6554 format!("module_id '{module_id}' did not advertise health.check"),
6555 )?]);
6556 }
6557
6558 let deadline = Instant::now() + self.health_probe_timeout;
6559 let pending = match self.forwarding.begin_module_control_rpc_for(
6560 &module_id,
6561 MODULE_CONTROL_OP_HEALTH_CHECK,
6562 deadline,
6563 ) {
6564 Ok(pending) => pending,
6565 Err(err) => {
6566 return Ok(vec![control_error_frame(
6567 &frame,
6568 forwarding_error_code(&err),
6569 err.to_string(),
6570 )?])
6571 }
6572 };
6573
6574 let PendingModuleControlRpc {
6575 endpoint,
6576 module_sink,
6577 negotiated_ver,
6578 corr: probe_corr,
6579 receiver,
6580 } = pending;
6581 let mut guard =
6582 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6583 let probe_body =
6584 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6585 RouterError::backend(
6586 0,
6587 frame.header.corr,
6588 format!("failed to encode health.check request: {err}"),
6589 )
6590 })?;
6591 let probe_frame = Frame::build_with_version(
6592 negotiated_ver,
6593 FrameType::Request,
6594 control_flags(),
6595 0,
6596 0,
6597 probe_corr,
6598 probe_body,
6599 )
6600 .map_err(RouterError::FrameBuild)?;
6601
6602 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6605 Ok(Ok(())) => {}
6606 Ok(Err(err)) => {
6607 return Ok(vec![control_error_frame(
6608 &frame,
6609 "target_unavailable",
6610 err.to_string(),
6611 )?]);
6612 }
6613 Err(_) => {
6614 return Ok(vec![control_error_frame(
6615 &frame,
6616 "module_timeout",
6617 format!(
6618 "module_id '{module_id}' did not answer health.check within {:?}",
6619 self.health_probe_timeout
6620 ),
6621 )?]);
6622 }
6623 }
6624
6625 match timeout_at(deadline, receiver).await {
6626 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6627 guard.disarm();
6628 let Some(report) = response.health_report() else {
6629 return Ok(vec![control_error_frame(
6630 &frame,
6631 "invalid_control_body",
6632 "health.check RPC returned a non-health response",
6633 )?]);
6634 };
6635 let HealthReport {
6640 status,
6641 detail,
6642 metrics,
6643 } = report;
6644 let capability_detail = self
6645 .capability_evaluator
6646 .required_problem_detail(&module_id);
6647 let response = ClientControlResponse::SupervisorHealthProbe {
6648 module_id,
6649 status,
6650 detail: append_capability_problem_detail(detail, capability_detail),
6651 metrics,
6652 };
6653 Ok(vec![control_response_body_frame(
6654 &frame,
6655 &response,
6656 "ClientControlResponse::SupervisorHealthProbe",
6657 )?])
6658 }
6659 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6660 guard.disarm();
6661 Ok(vec![control_error_body_frame(&frame, body)?])
6662 }
6663 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6664 guard.disarm();
6665 Ok(vec![control_error_frame(
6666 &frame,
6667 "target_unavailable",
6668 message,
6669 )?])
6670 }
6671 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6672 guard.disarm();
6673 Ok(vec![control_error_frame(
6674 &frame,
6675 "invalid_control_body",
6676 message,
6677 )?])
6678 }
6679 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6680 guard.disarm();
6681 Ok(vec![control_error_frame(
6682 &frame,
6683 "invalid_control_body",
6684 format!("expected module-control op '{expected}', got '{actual}'"),
6685 )?])
6686 }
6687 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6688 guard.disarm();
6689 Ok(vec![control_error_frame(
6690 &frame,
6691 "module_timeout",
6692 format!(
6693 "module_id '{module_id}' answered health.check after {:?}",
6694 self.health_probe_timeout
6695 ),
6696 )?])
6697 }
6698 Ok(Err(_)) => Ok(vec![control_error_frame(
6699 &frame,
6700 "target_unavailable",
6701 "health.check waiter was canceled before the module responded",
6702 )?]),
6703 Err(_) => Ok(vec![control_error_frame(
6704 &frame,
6705 "module_timeout",
6706 format!(
6707 "module_id '{module_id}' did not answer health.check within {:?}",
6708 self.health_probe_timeout
6709 ),
6710 )?]),
6711 }
6712 }
6713
6714 fn supervisor_status(
6715 &self,
6716 module_id: &str,
6717 corr: u64,
6718 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6719 self.supervisor
6720 .get(module_id)
6721 .map(|module| {
6722 let warming = module.is_warming_for_control("status").map_err(|err| {
6723 RouterError::backend(
6724 0,
6725 corr,
6726 format!(
6727 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6728 ),
6729 )
6730 })?;
6731 module.status_for_control("status").map_err(|err| {
6732 RouterError::backend(
6733 0,
6734 corr,
6735 format!(
6736 "failed to read supervisor status for module_id '{module_id}': {err}"
6737 ),
6738 )
6739 }).map(|status| (status, warming))
6740 })
6741 .transpose()
6742 }
6743
6744 fn guard_module_control_op(
6745 &self,
6746 frame: &Frame,
6747 module_id: &str,
6748 op: &str,
6749 ) -> Result<Option<Frame>, RouterError> {
6750 if self.module_grants_op(module_id, op, frame.header.corr)? {
6751 return Ok(None);
6752 }
6753
6754 Ok(Some(control_error_frame(
6755 frame,
6756 "op_not_allowed",
6757 format!("module_id '{module_id}' did not grant control op '{op}'"),
6758 )?))
6759 }
6760
6761 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6762 let Some(registration) = self
6763 .registry
6764 .get_module(module_id)
6765 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6766 else {
6767 return Ok(false);
6768 };
6769 Ok(module_registration_grants_op(®istration.control_ops, op))
6770 }
6771
6772 fn handle_status_update(
6773 &self,
6774 endpoint: ModuleEndpointId,
6775 frame: Frame,
6776 ) -> Result<Vec<Frame>, RouterError> {
6777 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6778 Ok(update) => update,
6779 Err(err) => {
6780 if is_known_module_push_op(&frame.body) {
6785 return Ok(vec![control_error_frame(
6786 &frame,
6787 "invalid_control_body",
6788 format!("malformed module control push body: {err}"),
6789 )?]);
6790 }
6791 return Ok(Vec::new());
6792 }
6793 };
6794
6795 match update {
6796 ModuleControlPush::RouteStatus {
6797 route_channel,
6798 route_epoch,
6799 status,
6800 } => {
6801 self.forwarding
6802 .cache_status(endpoint, route_channel, route_epoch, status)
6803 .map_err(RouterError::Forwarding)?;
6804 }
6805 }
6806 Ok(Vec::new())
6807 }
6808
6809 fn handle_route_poll(
6810 &self,
6811 ctx: &RouteCtx,
6812 frame: Frame,
6813 route_channel: u16,
6814 route_epoch: u32,
6815 kind: PollKind,
6816 ) -> Result<Vec<Frame>, RouterError> {
6817 let snapshot = self
6818 .forwarding
6819 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6820 .map_err(RouterError::Forwarding)?;
6821 let response = match (kind, snapshot) {
6822 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6823 ClientControlResponse::RoutePoll {
6824 route_channel,
6825 route_epoch,
6826 status,
6827 live: None,
6828 }
6829 }
6830 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6831 route_channel,
6832 route_epoch,
6833 status: None,
6834 live: None,
6835 },
6836 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6837 let live = self
6863 .process_liveness
6864 .as_ref()
6865 .and_then(|source| source.process_live(&module_id))
6866 .unwrap_or(true);
6867 ClientControlResponse::RoutePoll {
6868 route_channel,
6869 route_epoch,
6870 status: None,
6871 live: Some(live),
6872 }
6873 }
6874 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6875 route_channel,
6876 route_epoch,
6877 status: None,
6878 live: Some(false),
6879 },
6880 };
6881
6882 Ok(vec![control_response_body_frame(
6883 &frame,
6884 &response,
6885 "ClientControlResponse::RoutePoll",
6886 )?])
6887 }
6888
6889 pub(crate) fn observe_module_control_completion(
6890 &self,
6891 completion: ModuleControlRpcCompletion,
6892 ) -> bool {
6893 match completion {
6894 ModuleControlRpcCompletion::Unknown => false,
6895 ModuleControlRpcCompletion::Settled => true,
6896 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6897 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6898 info!(
6899 module_id = %module_id,
6900 latency_ms,
6901 "late health.check answer proves the module is alive"
6902 );
6903 match self
6904 .supervisor
6905 .record_late_health_answer(&module_id, latency_ms)
6906 {
6907 Ok(true) => {}
6908 Ok(false) => debug!(
6909 module_id = %module_id,
6910 latency_ms,
6911 "late health.check answer has no active supervisor snapshot"
6912 ),
6913 Err(err) => warn!(
6914 module_id = %module_id,
6915 latency_ms,
6916 error = %err,
6917 "failed to record late health.check answer"
6918 ),
6919 }
6920 true
6921 }
6922 }
6923 }
6924
6925 fn refuse_to_end_module_connection_for_a_client(
6946 &self,
6947 module_connection_id: ConnectionId,
6948 corr: u64,
6949 err: ForwardingError,
6950 ) -> Result<(), RouterError> {
6951 if let ForwardingError::ConnectionClosing { connection_id } = err {
6952 if connection_id != module_connection_id {
6953 warn!(
6954 module_connection_id = module_connection_id.get(),
6955 client_connection_id = connection_id.get(),
6956 corr,
6957 "dropping a route.bind response for a closing client; the module connection keeps serving"
6958 );
6959 return Ok(());
6960 }
6961 }
6962 Err(RouterError::Forwarding(err))
6963 }
6964
6965 fn handle_module_relay_response(
6966 &self,
6967 connection_id: ConnectionId,
6968 frame: Frame,
6969 ) -> Result<Vec<Frame>, RouterError> {
6970 let mut secondary_error = None;
6971 let outcome = match frame.header.ty {
6972 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6973 Ok(probe) if probe.op == "route.bind" => {
6974 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6975 Ok(ModuleControlResponse::RouteBindAck {}) => {
6976 RouteBindRelayOutcome::Accepted
6977 }
6978 Ok(other) => {
6979 let message =
6980 format!("route.bind response carried unexpected body: {other:?}");
6981 secondary_error = Some(control_error_frame(
6982 &frame,
6983 "invalid_control_body",
6984 message.clone(),
6985 )?);
6986 RouteBindRelayOutcome::ModuleGone(message)
6987 }
6988 Err(err) => {
6989 let message = format!("malformed route.bind response body: {err}");
6990 secondary_error = Some(control_error_frame(
6991 &frame,
6992 "invalid_control_body",
6993 message.clone(),
6994 )?);
6995 RouteBindRelayOutcome::ModuleGone(message)
6996 }
6997 }
6998 }
6999 Ok(probe) => {
7000 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7001 {
7002 Ok(response) => ModuleControlRpcOutcome::Response(response),
7003 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7004 "malformed {} response body: {err}",
7005 probe.op
7006 )),
7007 };
7008 let completion = self
7009 .forwarding
7010 .complete_module_control_rpc(
7011 connection_id,
7012 frame.header.corr,
7013 Some(&probe.op),
7014 outcome,
7015 )
7016 .map_err(RouterError::Forwarding)?;
7017 if !self.observe_module_control_completion(completion) {
7018 debug!(
7019 connection_id = connection_id.get(),
7020 corr = frame.header.corr,
7021 op = %probe.op,
7022 "dropping late or unknown module-control RPC response"
7023 );
7024 }
7025 return Ok(Vec::new());
7026 }
7027 Err(err) => {
7028 if let Some(expected_op) = self
7029 .forwarding
7030 .pending_module_control_op(connection_id, frame.header.corr)
7031 .map_err(RouterError::Forwarding)?
7032 {
7033 let completion = self
7034 .forwarding
7035 .complete_module_control_rpc(
7036 connection_id,
7037 frame.header.corr,
7038 None,
7039 ModuleControlRpcOutcome::MalformedResponse(format!(
7040 "malformed {expected_op} response body: {err}"
7041 )),
7042 )
7043 .map_err(RouterError::Forwarding)?;
7044 if !self.observe_module_control_completion(completion) {
7045 debug!(
7046 connection_id = connection_id.get(),
7047 corr = frame.header.corr,
7048 "dropping late malformed module-control RPC response"
7049 );
7050 }
7051 return Ok(Vec::new());
7052 }
7053 let message = format!("malformed route.bind response body: {err}");
7054 secondary_error = Some(control_error_frame(
7055 &frame,
7056 "invalid_control_body",
7057 message.clone(),
7058 )?);
7059 RouteBindRelayOutcome::ModuleGone(message)
7060 }
7061 },
7062 FrameType::Error => {
7063 if self
7064 .forwarding
7065 .pending_module_control_op(connection_id, frame.header.corr)
7066 .map_err(RouterError::Forwarding)?
7067 .is_some()
7068 {
7069 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7070 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7071 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7072 "malformed module-control ERROR body: {err}"
7073 )),
7074 };
7075 let completion = self
7076 .forwarding
7077 .complete_module_control_rpc(
7078 connection_id,
7079 frame.header.corr,
7080 None,
7081 outcome,
7082 )
7083 .map_err(RouterError::Forwarding)?;
7084 if !self.observe_module_control_completion(completion) {
7085 debug!(
7086 connection_id = connection_id.get(),
7087 corr = frame.header.corr,
7088 "dropping late or unknown module-control RPC error"
7089 );
7090 }
7091 return Ok(Vec::new());
7092 }
7093 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7094 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7095 Err(err) => {
7096 let message = format!("malformed route.bind ERROR body: {err}");
7097 secondary_error = Some(control_error_frame(
7098 &frame,
7099 "invalid_control_body",
7100 message.clone(),
7101 )?);
7102 RouteBindRelayOutcome::ModuleGone(message)
7103 }
7104 }
7105 }
7106 ty => {
7107 return Ok(vec![control_error_frame(
7108 &frame,
7109 "unsupported_control_frame",
7110 format!("unsupported module channel-0 frame {ty:?}"),
7111 )?])
7112 }
7113 };
7114
7115 let settled =
7116 self.forwarding
7117 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7118 let completion = match settled {
7119 Ok(completion) => completion,
7120 Err(err) => {
7121 self.refuse_to_end_module_connection_for_a_client(
7122 connection_id,
7123 frame.header.corr,
7124 err,
7125 )?;
7126 return Ok(secondary_error.into_iter().collect());
7127 }
7128 };
7129 if let Some(target) = completion.abandoned.as_ref() {
7130 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7131 }
7132 if !completion.settled {
7133 debug!(
7134 connection_id = connection_id.get(),
7135 corr = frame.header.corr,
7136 frame_type = ?frame.header.ty,
7137 "dropping late or unknown route.bind relay response"
7138 );
7139 }
7140 Ok(secondary_error.into_iter().collect())
7141 }
7142
7143 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7144 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7145 self.cleanup_connection_with_end_reason(
7149 connection_id,
7150 RegistrationEndReason::ExplicitGoodbye,
7151 )
7152 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7153 Ok(Vec::new())
7154 }
7155}
7156
7157impl Default for ControlHandler {
7158 fn default() -> Self {
7159 Self::new(Arc::new(Registry::default()))
7160 }
7161}
7162
7163impl crate::supervise::SwapPromotionObserver for ControlHandler {
7164 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7165 self.apply_registration_capabilities(registration);
7166 }
7167}
7168
7169fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7170 CapabilityRequirementStatus {
7171 consumer: status.consumer,
7172 capability: status.capability,
7173 need: match status.need {
7174 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7175 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7176 },
7177 verdict: status.verdict.as_str().to_string(),
7178 episode_seq: status.episode_seq,
7179 config_satisfiable: status.config_satisfiable,
7180 runtime_available: status.runtime_available,
7181 detail: status.detail,
7182 }
7183}
7184
7185fn append_capability_problem_detail(
7186 detail: Option<String>,
7187 capability_detail: Option<String>,
7188) -> Option<String> {
7189 match (detail, capability_detail) {
7190 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7191 (Some(detail), None) => Some(detail),
7192 (None, Some(capability_detail)) => Some(capability_detail),
7193 (None, None) => None,
7194 }
7195}
7196
7197fn subc_ops() -> Vec<String> {
7198 SUBC_CONTROL_OPS
7199 .iter()
7200 .map(|op| (*op).to_string())
7201 .collect()
7202}
7203
7204fn module_subc_ops() -> Vec<String> {
7205 SUBC_CONTROL_OPS
7206 .iter()
7207 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7208 .map(|op| (*op).to_string())
7209 .collect()
7210}
7211
7212#[cfg(test)]
7213fn module_baseline_control_ops() -> Vec<String> {
7214 MODULE_BASELINE_CONTROL_OPS
7215 .iter()
7216 .map(|op| (*op).to_string())
7217 .collect()
7218}
7219
7220fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7221 let mut seen = HashSet::new();
7222 let mut effective = Vec::new();
7223 for op in MODULE_BASELINE_CONTROL_OPS {
7224 if seen.insert((*op).to_string()) {
7225 effective.push((*op).to_string());
7226 }
7227 }
7228 for op in declared.unwrap_or_default() {
7229 if seen.insert(op.clone()) {
7230 effective.push(op);
7231 }
7232 }
7233 effective
7234}
7235
7236fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7237 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7238}
7239
7240fn target_module_id(target: &RouteTarget) -> &str {
7241 match target {
7242 RouteTarget::ToolProvider { module_id }
7243 | RouteTarget::ManagementSurface { module_id }
7244 | RouteTarget::InternalService { module_id, .. } => module_id,
7245 }
7246}
7247
7248fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7249 roles.iter().any(|role| match (target, role) {
7250 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7251 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7252 (
7253 RouteTarget::InternalService { service_id, .. },
7254 ProviderRole::InternalService {
7255 service_id: provided,
7256 ..
7257 },
7258 ) => service_id == provided,
7259 _ => false,
7260 })
7261}
7262
7263fn is_routable_role(role: &ProviderRole) -> bool {
7264 matches!(
7265 role,
7266 ProviderRole::ToolProvider { .. }
7267 | ProviderRole::ManagementSurface { .. }
7268 | ProviderRole::InternalService { .. }
7269 )
7270}
7271
7272#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7273enum ControlRequestBodyError {
7274 UnknownOp,
7275 InvalidBody,
7276}
7277
7278#[derive(Debug, Deserialize)]
7279struct ControlOpProbe {
7280 op: String,
7281}
7282
7283const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7286
7287fn is_known_module_push_op(body: &[u8]) -> bool {
7288 serde_json::from_slice::<ControlOpProbe>(body)
7289 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7290 .unwrap_or(false)
7291}
7292
7293fn is_known_module_request_op(body: &[u8]) -> bool {
7294 serde_json::from_slice::<ControlOpProbe>(body)
7295 .map(|probe| is_module_to_subc_op(&probe.op))
7296 .unwrap_or(false)
7297}
7298
7299fn is_module_to_subc_op(op: &str) -> bool {
7300 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7301}
7302
7303fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7304 debug!(
7305 op = %op,
7306 connection_id = connection_id.get(),
7307 corr,
7308 "control dispatch"
7309 );
7310}
7311
7312fn log_slow_control_dispatch(
7313 dispatch_started_at: Option<StdInstant>,
7314 op: &'static str,
7315 connection_id: ConnectionId,
7316 corr: u64,
7317 route_open_timings: Option<&RouteOpenTimings>,
7318) {
7319 let Some(dispatch_started_at) = dispatch_started_at else {
7320 return;
7321 };
7322 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7324 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7325 if let Some(timings) = route_open_timings {
7326 warn!(
7327 op = %op,
7328 connection_id = connection_id.get(),
7329 corr,
7330 elapsed_ms = elapsed.as_millis() as u64,
7331 module_id = ?timings.module_id,
7332 principal = %timings.principal,
7333 admission_ms = timings.admission.as_millis() as u64,
7334 project_root_ms = timings.project_root.as_millis() as u64,
7335 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7336 bind_send_ms = timings.bind_send.as_millis() as u64,
7337 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7338 "slow control dispatch"
7339 );
7340 return;
7341 }
7342 warn!(
7343 op = %op,
7344 connection_id = connection_id.get(),
7345 corr,
7346 elapsed_ms = elapsed.as_millis() as u64,
7347 "slow control dispatch"
7348 );
7349 }
7350}
7351
7352#[cfg(test)]
7354type TestCallerLookup =
7355 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7356
7357fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7359 match request {
7360 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7361 Some(PrivilegedOperation::Restart { module_id })
7362 }
7363 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7364 Some(PrivilegedOperation::Swap { module_id })
7365 }
7366 ClientControlRequest::SupervisorReload { module_id } => {
7367 Some(PrivilegedOperation::Reload { module_id })
7368 }
7369 ClientControlRequest::SupervisorRescan { preview } => {
7370 if *preview {
7371 None
7372 } else {
7373 Some(PrivilegedOperation::Rescan)
7374 }
7375 }
7376 ClientControlRequest::SupervisorRescanChecked {
7381 preview,
7382 candidate,
7383 expect,
7384 } => (!*preview && candidate.is_none() && expect.is_some())
7385 .then_some(PrivilegedOperation::Rescan),
7386 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7387 Some(PrivilegedOperation::SetEnabled {
7388 module_id,
7389 enabled: *enabled,
7390 })
7391 }
7392 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7393 Some(PrivilegedOperation::ReleaseReserved { module_id })
7394 }
7395 ClientControlRequest::ServerDescribe {}
7396 | ClientControlRequest::CatalogList { .. }
7397 | ClientControlRequest::RouteOpen { .. }
7398 | ClientControlRequest::RoutePoll { .. }
7399 | ClientControlRequest::SupervisorList {}
7400 | ClientControlRequest::SupervisorHealth {}
7401 | ClientControlRequest::SupervisorHealthProbe { .. }
7402 | ClientControlRequest::SupervisorRoutes { .. }
7403 | ClientControlRequest::SupervisorProvenance { .. }
7404 | ClientControlRequest::SupervisorStderrTail { .. }
7405 | ClientControlRequest::SupervisorTerminals { .. }
7406 | ClientControlRequest::SupervisorSpawnSnapshot {}
7407 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7408 }
7409}
7410
7411fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7412 match request {
7413 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7414 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7415 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7416 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7417 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7418 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7419 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7420 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7421 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7422 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7423 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7424 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7425 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7426 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7427 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7428 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7429 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7430 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7431 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7432 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7433 }
7434}
7435
7436struct RouteOpenCaller {
7444 principal: String,
7448 scope: String,
7451 harness: String,
7454}
7455
7456impl RouteOpenCaller {
7457 fn new(
7458 principal: Option<&Principal>,
7459 scope: Option<&ScopeSelector>,
7460 identity: &BindIdentity,
7461 ) -> Self {
7462 Self {
7463 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7464 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7465 harness: bounded_log_text(&identity.harness),
7466 }
7467 }
7468
7469 fn unknown() -> Self {
7472 Self {
7473 principal: "unknown".to_string(),
7474 scope: "unknown".to_string(),
7475 harness: "unknown".to_string(),
7476 }
7477 }
7478}
7479
7480const MAX_LOGGED_TEXT_CHARS: usize = 64;
7483
7484fn bounded_log_text(value: &str) -> String {
7488 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7489 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7490 bounded.push('…');
7491 }
7492 format!("{bounded:?}")
7493}
7494
7495fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7501 use subc_protocol::scope::ScopeKind;
7502 let kind = match stamp.kind {
7503 ScopeKind::Head => "head",
7504 ScopeKind::Worker => "worker",
7505 ScopeKind::Ephemeral => "ephemeral",
7506 };
7507 format!(
7508 "{kind}/{}/{}@{}",
7509 principal_label(&stamp.owner),
7510 bounded_log_text(&stamp.scope_ref),
7511 stamp.scope_epoch
7512 )
7513}
7514
7515fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7522 let epoch = selector
7523 .scope_epoch
7524 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7525 format!(
7526 "requested/{}/{}@{epoch}",
7527 bounded_log_text(&principal_label(&selector.owner)),
7528 bounded_log_text(&selector.scope_ref),
7529 )
7530}
7531
7532fn principal_label(principal: &Principal) -> String {
7533 match principal {
7534 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7535 Principal::Direct => "direct".to_string(),
7536 other => format!("{other:?}"),
7537 }
7538}
7539
7540fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7541 match request {
7542 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7543 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7544 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7545 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7546 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7547 }
7548}
7549
7550fn parse_client_control_request(
7551 body: &[u8],
7552) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7553 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7554 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7555 Ok(probe)
7556 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7557 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7558 {
7559 ControlRequestBodyError::InvalidBody
7560 }
7561 Ok(_) => ControlRequestBodyError::UnknownOp,
7562 Err(_) => ControlRequestBodyError::InvalidBody,
7563 };
7564 (err, classification)
7565 })
7566}
7567
7568fn parse_module_control_request_from_module(
7569 body: &[u8],
7570) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7571 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7572 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7573 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7574 Ok(_) => ControlRequestBodyError::UnknownOp,
7575 Err(_) => ControlRequestBodyError::InvalidBody,
7576 };
7577 (err, classification)
7578 })
7579}
7580
7581#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7582enum ProviderRoleKind {
7583 ToolProvider,
7584 PipelineStage,
7585 ManagementSurface,
7586 InternalService,
7587}
7588
7589fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7590 match role {
7591 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7592 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7593 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7594 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7595 }
7596}
7597
7598fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7599 roles.iter().map(provider_role_kind).collect()
7600}
7601
7602const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7605
7606#[derive(Debug, Default, PartialEq, Eq)]
7608struct ScopeOutcomeCounts {
7609 created: usize,
7610 replaced: usize,
7611 updated: usize,
7612 unchanged: usize,
7613 refused: usize,
7614}
7615
7616impl ScopeOutcomeCounts {
7617 fn of(results: &[ScopeRecordResult]) -> Self {
7618 let mut counts = Self::default();
7619 for result in results {
7620 let slot = match result.outcome {
7621 ScopeRecordOutcome::Created => &mut counts.created,
7622 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7623 ScopeRecordOutcome::Updated => &mut counts.updated,
7624 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7625 ScopeRecordOutcome::Refused => &mut counts.refused,
7626 };
7627 *slot += 1;
7628 }
7629 counts
7630 }
7631}
7632
7633#[cfg(test)]
7634mod scope_outcome_count_tests {
7635 use super::*;
7636
7637 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7638 ScopeRecordResult {
7639 scope_ref: "r".to_string(),
7640 scope_epoch: 1,
7641 outcome,
7642 code: None,
7643 message: None,
7644 version: None,
7645 parent_state: None,
7646 }
7647 }
7648
7649 #[test]
7652 fn every_outcome_is_counted_in_its_own_field() {
7653 let results = [
7654 result(ScopeRecordOutcome::Created),
7655 result(ScopeRecordOutcome::Created),
7656 result(ScopeRecordOutcome::Replaced),
7657 result(ScopeRecordOutcome::Updated),
7658 result(ScopeRecordOutcome::Unchanged),
7659 result(ScopeRecordOutcome::Refused),
7660 result(ScopeRecordOutcome::Refused),
7661 result(ScopeRecordOutcome::Refused),
7662 ];
7663 assert_eq!(
7664 ScopeOutcomeCounts::of(&results),
7665 ScopeOutcomeCounts {
7666 created: 2,
7667 replaced: 1,
7668 updated: 1,
7669 unchanged: 1,
7670 refused: 3,
7671 }
7672 );
7673 }
7674}
7675
7676fn capability_census_trigger(
7680 old: Option<&CapabilityDeclarations>,
7681 new: Option<&CapabilityDeclarations>,
7682) -> bool {
7683 let old_provides = old
7684 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7685 .unwrap_or_default();
7686 let old_denies = old
7687 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7688 .unwrap_or_default();
7689 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7690 provides: Vec::new(),
7691 requires: Vec::new(),
7692 must_never_reach: Vec::new(),
7693 });
7694
7695 new.provides
7696 .iter()
7697 .any(|capability| !old_provides.contains(capability))
7698 || new
7699 .must_never_reach
7700 .iter()
7701 .any(|capability| !old_denies.contains(capability))
7702}
7703
7704fn denied_capability<'a>(
7707 opening_manifest: &'a ModuleManifest,
7708 target_manifest: &ModuleManifest,
7709) -> Option<&'a str> {
7710 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7711 let target_capabilities = target_manifest.capabilities.as_ref()?;
7712 opening_capabilities
7713 .must_never_reach
7714 .iter()
7715 .find(|denied| {
7716 target_capabilities
7717 .provides
7718 .iter()
7719 .any(|provided| provided == *denied)
7720 })
7721 .map(String::as_str)
7722}
7723
7724fn catalog_update_frozen_field_message(
7725 registered: &ModuleManifest,
7726 provides: &[ProviderRole],
7727) -> Option<String> {
7728 let old_has_provides = !registered.provides.is_empty();
7729 let new_has_provides = !provides.is_empty();
7730 if old_has_provides != new_has_provides {
7731 return Some(format!(
7732 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7733 registered.module_id
7734 ));
7735 }
7736
7737 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7738 return Some(format!(
7739 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7740 registered.module_id
7741 ));
7742 }
7743
7744 let registered_concurrency = manifest_concurrency(registered);
7745 let mut candidate = registered.clone();
7746 candidate.provides = provides.to_vec();
7747 let candidate_concurrency = manifest_concurrency(&candidate);
7748 if candidate_concurrency != registered_concurrency {
7749 return Some(format!(
7750 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7751 registered.module_id, registered_concurrency, candidate_concurrency
7752 ));
7753 }
7754
7755 None
7758}
7759
7760fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7761 manifest.provides.iter().any(is_routable_role)
7762}
7763
7764fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7770 manifest
7771 .provides
7772 .iter()
7773 .find_map(|provider| match provider {
7774 ProviderRole::ToolProvider { concurrency, .. }
7775 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7776 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7777 })
7778 .unwrap_or(Concurrency::ModuleManaged)
7779}
7780
7781fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7788 let has_management_surface = manifest
7789 .provides
7790 .iter()
7791 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7792 if !has_management_surface {
7793 return false;
7794 }
7795 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7796 return false;
7797 };
7798 let Some(provides) = raw
7799 .get("manifest")
7800 .and_then(|manifest| manifest.get("provides"))
7801 .and_then(serde_json::Value::as_array)
7802 else {
7803 return false;
7804 };
7805 provides.iter().any(|role| {
7810 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7811 && role.get("concurrency").is_none()
7812 })
7813}
7814
7815fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7816 if peer_version != PROTOCOL_VERSION {
7817 return Err(format!(
7818 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7819 ));
7820 }
7821 Ok(PROTOCOL_VERSION)
7822}
7823
7824fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7825 Frame::build_with_version(
7826 response_version(frame),
7827 FrameType::Pong,
7828 frame.header.flags,
7829 0,
7830 0,
7831 frame.header.corr,
7832 Vec::new(),
7833 )
7834 .map_err(RouterError::FrameBuild)
7835}
7836
7837pub(crate) fn control_error_frame(
7838 frame: &Frame,
7839 code: &'static str,
7840 message: impl Into<String>,
7841) -> Result<Frame, RouterError> {
7842 control_error_body_frame(
7843 frame,
7844 ErrorBody {
7845 code: code.to_string(),
7846 message: message.into(),
7847 detail: None,
7848 },
7849 )
7850}
7851
7852fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7853 let body = serde_json::to_vec(&error).map_err(|err| {
7854 RouterError::backend(
7855 0,
7856 frame.header.corr,
7857 format!("failed to encode control ERROR: {err}"),
7858 )
7859 })?;
7860
7861 Frame::build_with_version(
7862 response_version(frame),
7863 FrameType::Error,
7864 control_flags(),
7865 0,
7866 0,
7867 frame.header.corr,
7868 body,
7869 )
7870 .map_err(RouterError::FrameBuild)
7871}
7872
7873fn control_response_body_frame<T: Serialize>(
7874 frame: &Frame,
7875 reply: &T,
7876 label: &'static str,
7877) -> Result<Frame, RouterError> {
7878 let body = serde_json::to_vec(reply).map_err(|err| {
7879 RouterError::backend(
7880 0,
7881 frame.header.corr,
7882 format!("failed to encode {label}: {err}"),
7883 )
7884 })?;
7885
7886 Frame::build_with_version(
7887 response_version(frame),
7888 FrameType::Response,
7889 control_flags(),
7890 0,
7891 0,
7892 frame.header.corr,
7893 body,
7894 )
7895 .map_err(RouterError::FrameBuild)
7896}
7897
7898fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7928 match err {
7929 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7930 ForwardingError::NoModuleConnection => "target_unavailable",
7931 ForwardingError::ModuleReloading { .. } => "module_reloading",
7932 ForwardingError::ClientRouteChannelExhausted { .. }
7933 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7934 ForwardingError::StaleModuleEndpoint
7935 | ForwardingError::UnknownReservation { .. }
7936 | ForwardingError::ConnectionClosing { .. }
7937 | ForwardingError::ClientEgressClosed { .. }
7938 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7939 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7942 ForwardingError::RelayCorrelationExhausted
7943 | ForwardingError::RouteOpenBuild(_)
7944 | ForwardingError::Poisoned => "forwarding_error",
7945 }
7946}
7947
7948fn response_version(frame: &Frame) -> u8 {
7949 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7950 frame.header.ver
7951 } else {
7952 PROTOCOL_VERSION
7953 }
7954}
7955
7956fn control_flags() -> Flags {
7957 Flags::new(false, Priority::Passive, false)
7958}
7959
7960fn send_goodbye_target_best_effort(
7965 counters: &DaemonCounters,
7966 target: &GoodbyeTarget,
7967 context: &'static str,
7968) {
7969 let Ok(frame) = Frame::build_with_version(
7970 target.negotiated_ver,
7971 FrameType::Goodbye,
7972 control_flags(),
7973 target.channel,
7974 target.epoch,
7975 0,
7976 Vec::new(),
7977 ) else {
7978 return;
7979 };
7980 crate::forwarding::send_module_route_goodbye(
7981 counters,
7982 &target.sink,
7983 frame,
7984 target.module_id.as_deref(),
7985 context,
7986 );
7987}
7988
7989pub(crate) fn send_route_control_pushes(
7990 forwarding: &ForwardingTable,
7991 routes: Vec<EndpointRoute>,
7992 push: ClientControlPush,
7993) {
7994 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7995 for route in routes {
7996 let target = route.goodbye_target;
7997 if let Some((existing, channels)) = targets
7998 .iter_mut()
7999 .find(|(existing, _)| existing.connection_id == target.connection_id)
8000 {
8001 debug_assert_eq!(
8002 existing.negotiated_ver, target.negotiated_ver,
8003 "one connection cannot negotiate multiple frame versions"
8004 );
8005 if !channels.contains(&target.channel) {
8006 channels.push(target.channel);
8007 }
8008 continue;
8009 }
8010 let channel = target.channel;
8011 targets.push((target, vec![channel]));
8012 }
8013 for (target, mut channels) in targets {
8014 channels.sort_unstable();
8015 let mut push = push.clone();
8016 match &mut push {
8017 ClientControlPush::RouteClosing {
8018 channels: covered, ..
8019 }
8020 | ClientControlPush::RouteClosed {
8021 channels: covered, ..
8022 } => *covered = channels,
8023 }
8024 let body = match serde_json::to_vec(&push) {
8025 Ok(body) => body,
8026 Err(err) => {
8027 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8028 continue;
8029 }
8030 };
8031 let frame = match Frame::build_with_version(
8032 target.negotiated_ver,
8033 FrameType::Push,
8034 control_flags(),
8035 0,
8036 0,
8037 0,
8038 body.clone(),
8039 ) {
8040 Ok(frame) => frame,
8041 Err(err) => {
8042 warn!(
8043 route_channel = target.channel,
8044 error = %err,
8045 "failed to build route lifecycle control PUSH frame"
8046 );
8047 continue;
8048 }
8049 };
8050 if let Err(err) = target.sink.try_send(frame) {
8051 if target.close_on_delivery_failure() {
8052 warn!(
8053 target_connection_id = target.connection_id.get(),
8054 route_channel = target.channel,
8055 error = %err,
8056 "route lifecycle control PUSH was not delivered to client; closing target connection"
8057 );
8058 let _ = forwarding.escalate_client_delivery_failure(
8059 target.connection_id,
8060 target.channel,
8061 target.epoch,
8062 CloseReason::new(
8063 "route_lifecycle_push_delivery_failed",
8064 format!(
8065 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8066 target.channel
8067 ),
8068 ),
8069 crate::forwarding::UndeliveredFrame {
8070 module_id: target.module_id.as_deref(),
8071 sink: &target.sink,
8072 },
8073 );
8074 }
8075 }
8076 }
8077}
8078
8079#[cfg(test)]
8080mod tests {
8081 mod activity_read {
8082 include!("activity_read_tests.rs");
8083 }
8084
8085 mod lifecycle_activity {
8086 include!("control_lifecycle_tests.rs");
8087 }
8088 #[cfg(unix)]
8089 #[tokio::test]
8090 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8091 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8092 let path = dir.join("subc.jsonc");
8093 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8094 "program":"/bin/sleep","args":["60"],"protocol":"none",
8095 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8096 }}}).to_string()).unwrap();
8097 let mut configured = crate::daemon_config::load(&path)
8098 .unwrap()
8099 .unwrap()
8100 .modules
8101 .pop()
8102 .unwrap();
8103 let registry = std::sync::Arc::new(crate::Registry::default());
8104 let handle = crate::SupervisorHandle::new();
8105 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8106 .with_handle(handle.clone());
8107 let module = supervisor
8108 .supervise_configured_with_health(
8109 configured.module_spec(),
8110 true,
8111 configured.health.clone(),
8112 None,
8113 configured.restart,
8114 )
8115 .unwrap();
8116 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8117 let before = module.status().unwrap().pid;
8118 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8119 configured.health.cadence = std::time::Duration::from_secs(3600);
8120 let health_only = handler
8121 .reconcile_supervised_modules(
8122 &supervisor,
8123 vec![configured.clone()],
8124 false,
8125 Principal::Direct,
8126 )
8127 .await
8128 .unwrap();
8129 assert!(
8130 health_only.0.changed_pending_reload.is_empty(),
8131 "health policy is already applied live"
8132 );
8133 assert_eq!(module.status().unwrap().pid, before);
8134 assert_eq!(
8135 module.configuration().unwrap().1.http,
8136 configured.health.http
8137 );
8138 configured.args = vec!["61".into()];
8139 let launch = handler
8140 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8141 .await
8142 .unwrap();
8143 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8144 assert_eq!(
8145 module.status().unwrap().pid,
8146 before,
8147 "a launch edit is stored until reload"
8148 );
8149 module.drain().await.unwrap();
8150 }
8151 use cortexkit_test_support::ScratchDir;
8152 use std::{
8153 collections::BTreeMap,
8154 fmt,
8155 path::PathBuf,
8156 sync::{Arc, Mutex},
8157 time::Duration,
8158 };
8159
8160 use serde_json::{json, Value};
8161 use subc_protocol::{
8162 manifest::{
8163 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8164 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8165 },
8166 session::HealthStatus,
8167 FrameType,
8168 };
8169
8170 use super::*;
8171 use crate::{
8172 forwarding::{DataRoute, DataRouteState},
8173 registry::ChannelState,
8174 router::FrameSink,
8175 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8176 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8177 RouteCtx, Router,
8178 };
8179 use tokio::{
8180 sync::mpsc,
8181 time::{sleep, Instant},
8182 };
8183 use tracing::{
8184 field::{Field, Visit},
8185 Event, Subscriber,
8186 };
8187 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8188
8189 fn fake_aft_stub_path() -> PathBuf {
8208 let mut path = std::env::current_exe().expect("current_exe available in tests");
8209 path.pop(); path.pop(); path.push(if cfg!(windows) {
8212 "fake-aft-stub.exe"
8213 } else {
8214 "fake-aft-stub"
8215 });
8216 assert!(
8217 path.exists(),
8218 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8219 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8220 path.display()
8221 );
8222 path
8223 }
8224
8225 fn client_retries(code: &str) -> bool {
8230 subc_protocol::error_codes::is_retryable_route_open(code)
8231 }
8232
8233 #[test]
8244 fn retryability_of_forwarding_codes_matches_the_failure() {
8245 let transient = [
8248 ForwardingError::NoModuleConnection,
8249 ForwardingError::ModuleReloading {
8250 module_id: "m".into(),
8251 },
8252 ForwardingError::StaleModuleEndpoint,
8253 ForwardingError::UnknownReservation {
8254 client_channel: 1,
8255 module_channel: 1,
8256 },
8257 ForwardingError::ConnectionClosing {
8258 connection_id: ConnectionId::new(1),
8259 },
8260 ForwardingError::ClientEgressClosed {
8261 connection_id: ConnectionId::new(1),
8262 },
8263 ForwardingError::ModuleEgressUnavailable {
8264 connection_id: ConnectionId::new(1),
8265 },
8266 ];
8267 for err in transient {
8268 let code = forwarding_error_code(&err);
8269 assert!(
8270 client_retries(code),
8271 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8272 );
8273 }
8274
8275 let permanent = [
8280 ForwardingError::ConnectionRoleConflict {
8281 connection_id: ConnectionId::new(1),
8282 },
8283 ForwardingError::ClientRouteChannelExhausted {
8284 connection_id: ConnectionId::new(1),
8285 },
8286 ForwardingError::ModuleRouteChannelExhausted {
8287 endpoint: ModuleEndpointId {
8288 connection_id: ConnectionId::new(1),
8289 generation: 1,
8290 },
8291 },
8292 ForwardingError::RelayCorrelationExhausted,
8293 ForwardingError::RouteOpenBuild("x".into()),
8294 ForwardingError::Poisoned,
8295 ];
8296 for err in permanent {
8297 let code = forwarding_error_code(&err);
8298 assert!(
8299 !client_retries(code),
8300 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8301 );
8302 }
8303 }
8304
8305 #[tokio::test]
8333 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8334 let handler = ControlHandler::default();
8335 let frame =
8336 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8337
8338 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8342 assert_eq!(
8343 stamped,
8344 Principal::Direct,
8345 "a caller that proved nothing must not be stamped as a supervised module"
8346 );
8347
8348 let forged = handler
8353 .route_open_principal(
8354 &frame,
8355 Some(ConsumerIdentity {
8356 module_id: "aft".to_string(),
8357 launch_nonce: "not-a-real-nonce".to_string(),
8358 }),
8359 )
8360 .unwrap();
8361 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8362 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8363 }
8364
8365 #[test]
8382 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8383 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8384 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8385 let ClientControlRequest::RouteOpen {
8386 consumer_identity, ..
8387 } = parsed
8388 else {
8389 panic!("route.open body must parse as RouteOpen");
8390 };
8391 assert_eq!(
8392 consumer_identity,
8393 Some(ConsumerIdentity {
8394 module_id: "aft".to_string(),
8395 launch_nonce: "n".to_string(),
8396 }),
8397 "the wire field name must reach the value route_open_principal reads"
8398 );
8399 }
8400
8401 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8402 ModuleManifest::builder(module_id, "0.1.0")
8403 .protocol_ver(protocol_ver)
8404 .provides(vec![ProviderRole::ToolProvider {
8405 tools: vec![Tool {
8406 name: "read".to_string(),
8407 description: None,
8408 execution_mode: ExecutionMode::Pure,
8409 schema: json!({"type": "object"}),
8410 }],
8411 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8412 concurrency: Concurrency::ModuleManaged,
8413 emits_push: true,
8414 sub_supervises: true,
8415 }])
8416 .build()
8417 }
8418
8419 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8420 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8421 }
8422
8423 fn hello_frame_with_control_ops(
8424 module_id: &str,
8425 protocol_ver: u8,
8426 corr: u64,
8427 control_ops: Option<Vec<String>>,
8428 ) -> Frame {
8429 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8430 }
8431
8432 fn hello_frame_with_nonce(
8433 module_id: &str,
8434 protocol_ver: u8,
8435 corr: u64,
8436 launch_nonce: Option<&str>,
8437 ) -> Frame {
8438 hello_frame_full(
8439 module_id,
8440 protocol_ver,
8441 corr,
8442 None,
8443 launch_nonce.map(ToOwned::to_owned),
8444 )
8445 }
8446
8447 fn hello_frame_full(
8448 module_id: &str,
8449 protocol_ver: u8,
8450 corr: u64,
8451 control_ops: Option<Vec<String>>,
8452 launch_nonce: Option<String>,
8453 ) -> Frame {
8454 let body = serde_json::to_vec(&ModuleHelloBody {
8455 manifest: manifest(module_id, protocol_ver),
8456 protocol_ver,
8457 control_ops,
8458 launch_nonce,
8459 })
8460 .unwrap();
8461 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8462 }
8463
8464 fn hello_frame_with_rich_ui(
8465 module_id: &str,
8466 protocol_ver: u8,
8467 corr: u64,
8468 launch_nonce: Option<&str>,
8469 rich_ui: Value,
8470 ) -> Frame {
8471 let mut body = serde_json::to_value(ModuleHelloBody {
8472 manifest: manifest(module_id, protocol_ver),
8473 protocol_ver,
8474 control_ops: None,
8475 launch_nonce: launch_nonce.map(str::to_string),
8476 })
8477 .unwrap();
8478 body["manifest"]["rich_ui"] = rich_ui;
8479 Frame::build(
8480 FrameType::Hello,
8481 control_flags(),
8482 0,
8483 0,
8484 corr,
8485 serde_json::to_vec(&body).unwrap(),
8486 )
8487 .unwrap()
8488 }
8489
8490 fn non_routable_hello_frame_with_control_ops(
8491 module_id: &str,
8492 corr: u64,
8493 control_ops: Option<Vec<String>>,
8494 ) -> Frame {
8495 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8496 manifest.provides.clear();
8497 let body = serde_json::to_vec(&ModuleHelloBody {
8498 manifest,
8499 protocol_ver: PROTOCOL_VERSION,
8500 control_ops,
8501 launch_nonce: None,
8502 })
8503 .unwrap();
8504 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8505 }
8506
8507 fn capability_grammar_hello_frame(
8508 capabilities: Value,
8509 runtime_computed: Option<Value>,
8510 corr: u64,
8511 ) -> Frame {
8512 let mut body = serde_json::to_value(ModuleHelloBody {
8513 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8514 protocol_ver: PROTOCOL_VERSION,
8515 control_ops: None,
8516 launch_nonce: None,
8517 })
8518 .expect("HELLO body serializes");
8519 body["manifest"]["capabilities"] = capabilities;
8520 if let Some(runtime_computed) = runtime_computed {
8521 body["runtime_computed"] = runtime_computed;
8522 }
8523 Frame::build(
8524 FrameType::Hello,
8525 control_flags(),
8526 0,
8527 0,
8528 corr,
8529 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8530 )
8531 .expect("HELLO frame builds")
8532 }
8533
8534 fn channel_request(channel: u16, corr: u64) -> Frame {
8535 Frame::build(
8536 FrameType::Request,
8537 Flags::new(true, Priority::Interactive, false),
8538 channel,
8539 0,
8540 corr,
8541 b"opaque".to_vec(),
8542 )
8543 .unwrap()
8544 }
8545
8546 fn route_ctx(
8547 connection_id: ConnectionId,
8548 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8549 let (tx, rx) = mpsc::channel(8);
8550 (
8551 RouteCtx {
8552 connection_id,
8553 egress: FrameSink::new(tx),
8554 },
8555 rx,
8556 )
8557 }
8558
8559 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8560 serde_json::from_slice(&frame.body).unwrap()
8561 }
8562
8563 async fn hello_via_sink(
8568 handler: &ControlHandler,
8569 ctx: &RouteCtx,
8570 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8571 hello: Frame,
8572 ) -> Frame {
8573 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8574 assert!(
8575 replies.is_empty(),
8576 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8577 );
8578 let ack = rx
8579 .try_recv()
8580 .expect("HELLO_ACK is queued on the module sink")
8581 .frame;
8582 assert_eq!(ack.header.ty, FrameType::HelloAck);
8583 ack
8584 }
8585
8586 fn parse_error(frame: &Frame) -> Value {
8587 serde_json::from_slice(&frame.body).unwrap()
8588 }
8589
8590 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8591 serde_json::from_slice(&frame.body).unwrap()
8592 }
8593
8594 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8595 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8596 route_channel,
8597 route_epoch: 0,
8598 kind,
8599 })
8600 .unwrap();
8601 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8602 }
8603
8604 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8605 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8606 module_id: module_id.to_string(),
8607 })
8608 .unwrap();
8609 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8610 }
8611
8612 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8613 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8614 }
8615
8616 fn route_open_frame_with_consumer_capabilities(
8617 corr: u64,
8618 module_id: &str,
8619 project_root: ScratchDir,
8620 consumer_capabilities: Option<Vec<String>>,
8621 ) -> Frame {
8622 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8623 target: RouteTarget::ToolProvider {
8624 module_id: module_id.to_string(),
8625 },
8626 identity: BindIdentity::new(
8627 project_root.path().to_path_buf(),
8628 "unit".to_string(),
8629 "session".to_string(),
8630 ),
8631 consumer_identity: None,
8632 consumer_capabilities,
8633 role_versions: None,
8634 admission_facts: None,
8635 scope: None,
8636 })
8637 .unwrap();
8638 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8639 }
8640
8641 fn route_open_frame_with_role_versions(
8642 corr: u64,
8643 module_id: &str,
8644 project_root: ScratchDir,
8645 role_versions: Option<BTreeMap<String, String>>,
8646 ) -> Frame {
8647 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8648 target: RouteTarget::ToolProvider {
8649 module_id: module_id.to_string(),
8650 },
8651 identity: BindIdentity::new(
8652 project_root.path().to_path_buf(),
8653 "unit".to_string(),
8654 format!("session-{corr}"),
8655 ),
8656 consumer_identity: None,
8657 consumer_capabilities: None,
8658 role_versions,
8659 admission_facts: None,
8660 scope: None,
8661 })
8662 .unwrap();
8663 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8664 }
8665
8666 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8667 entries
8668 .iter()
8669 .map(|(role, version)| (role.to_string(), version.to_string()))
8670 .collect()
8671 }
8672
8673 fn route_open_frame_with_admission_facts(
8674 corr: u64,
8675 module_id: &str,
8676 project_root: ScratchDir,
8677 consumer_identity: Option<subc_control::ConsumerIdentity>,
8678 facts: Option<Value>,
8679 ) -> Frame {
8680 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8681 target: RouteTarget::ToolProvider {
8682 module_id: module_id.to_string(),
8683 },
8684 identity: BindIdentity::new(
8685 project_root.path().to_path_buf(),
8686 "unit".to_string(),
8687 format!("session-{corr}"),
8688 ),
8689 consumer_identity,
8690 consumer_capabilities: None,
8691 role_versions: None,
8692 admission_facts: facts,
8693 scope: None,
8694 })
8695 .unwrap();
8696 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8697 }
8698
8699 #[derive(Clone, Default)]
8700 struct EventCapture {
8701 events: Arc<Mutex<Vec<CapturedEvent>>>,
8702 }
8703
8704 #[derive(Clone, Debug)]
8705 struct CapturedEvent {
8706 target: String,
8707 level: tracing::Level,
8708 fields: BTreeMap<String, String>,
8709 }
8710
8711 impl EventCapture {
8712 fn events(&self) -> Vec<CapturedEvent> {
8713 self.events.lock().unwrap().clone()
8714 }
8715 }
8716
8717 impl<S> Layer<S> for EventCapture
8718 where
8719 S: Subscriber,
8720 {
8721 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8722 let mut visitor = EventFieldVisitor::default();
8723 event.record(&mut visitor);
8724 self.events.lock().unwrap().push(CapturedEvent {
8725 target: event.metadata().target().to_string(),
8726 level: *event.metadata().level(),
8727 fields: visitor.fields,
8728 });
8729 }
8730 }
8731
8732 #[derive(Default)]
8733 struct EventFieldVisitor {
8734 fields: BTreeMap<String, String>,
8735 }
8736
8737 impl Visit for EventFieldVisitor {
8738 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8739 self.fields
8740 .insert(field.name().to_string(), format!("{value:?}"));
8741 }
8742 }
8743
8744 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8745 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8746 status,
8747 detail: Some("warming".to_string()),
8748 metrics: Some(json!({"queue_depth": 3})),
8749 })
8750 .unwrap();
8751 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8752 }
8753
8754 fn route_bind_ack(corr: u64) -> Frame {
8755 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8756 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8757 }
8758
8759 fn unique_project_root(label: &str) -> ScratchDir {
8760 ScratchDir::new(label)
8761 }
8762
8763 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8764 match parse_route_poll(frame) {
8765 ClientControlResponse::RoutePoll {
8766 status: None,
8767 live: Some(live),
8768 ..
8769 } => assert_eq!(live, expected_live),
8770 other => panic!("unexpected route.poll response: {other:?}"),
8771 }
8772 }
8773
8774 fn bind_liveness_route(
8775 registry: &Registry,
8776 forwarding: &ForwardingTable,
8777 module_id: &str,
8778 ) -> (RouteCtx, u16, u32) {
8779 let module_connection = ConnectionId::new(101);
8780 let client_connection = ConnectionId::new(202);
8781 let registration = registry
8782 .register_with_control_ops(
8783 manifest(module_id, PROTOCOL_VERSION),
8784 PROTOCOL_VERSION,
8785 module_connection,
8786 module_baseline_control_ops(),
8787 )
8788 .unwrap();
8789 let (module_tx, _module_rx) = mpsc::channel(8);
8790 let endpoint = forwarding
8791 .register_module_connection(
8792 module_connection,
8793 module_id.to_string(),
8794 PROTOCOL_VERSION,
8795 manifest_concurrency(®istration.manifest),
8796 FrameSink::new(module_tx),
8797 )
8798 .unwrap();
8799 let (client_ctx, _client_rx) = route_ctx(client_connection);
8800 let pending = forwarding
8801 .begin_route_bind_relay_for_test(
8802 client_connection,
8803 client_ctx.egress.clone(),
8804 1,
8805 module_id,
8806 )
8807 .unwrap();
8808 assert_eq!(pending.endpoint, endpoint);
8809 let route_channel = pending.client_channel;
8810 let route_epoch = pending.client_epoch;
8811 forwarding
8812 .complete_pending_relay(
8813 module_connection,
8814 pending.corr,
8815 RouteBindRelayOutcome::Accepted,
8816 )
8817 .unwrap();
8818 (client_ctx, route_channel, route_epoch)
8819 }
8820
8821 struct FakeProcessLiveness {
8822 live: Option<bool>,
8823 }
8824
8825 impl ModuleProcessLiveness for FakeProcessLiveness {
8826 fn process_live(&self, _module_id: &str) -> Option<bool> {
8827 self.live
8828 }
8829 }
8830
8831 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8832 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8833 {
8834 let registry = Arc::new(Registry::default());
8835 let supervisor_handle = SupervisorHandle::new();
8836 let supervisor = Supervisor::new_for_test(
8837 Arc::clone(®istry),
8838 RestartPolicy::new(1, Duration::from_millis(10)),
8839 )
8840 .with_handle(supervisor_handle.clone());
8841 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8842 let module = supervisor
8843 .spawn(ModuleSpec {
8844 module_id: "stderr-tail-wire".to_string(),
8845 program: fake_aft_stub_path(),
8846 args: Vec::new(),
8847 env: vec![
8848 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8849 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8850 ],
8851 reserved: false,
8852 reserved_prefixes: Vec::new(),
8853 protocol: ModuleProtocol::Subc,
8854 overlap: Default::default(),
8855 })
8856 .unwrap();
8857
8858 let deadline = Instant::now() + Duration::from_secs(5);
8859 loop {
8860 let tail = module.stderr_tail(None, None);
8861 if tail
8862 .entries
8863 .iter()
8864 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8865 && tail.entries.iter().any(|entry| {
8866 matches!(
8867 entry,
8868 TailEntry::Line {
8869 truncated: true,
8870 ..
8871 }
8872 )
8873 })
8874 {
8875 break;
8876 }
8877 assert!(
8878 Instant::now() < deadline,
8879 "module did not produce a truncated line and restart boundary: {tail:?}"
8880 );
8881 sleep(Duration::from_millis(10)).await;
8882 }
8883
8884 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8885 let request = ClientControlRequest::SupervisorStderrTail {
8886 module_id: "stderr-tail-wire".to_string(),
8887 max_lines: None,
8888 max_bytes: None,
8889 };
8890 let frame = Frame::build(
8891 FrameType::Request,
8892 control_flags(),
8893 0,
8894 0,
8895 1,
8896 serde_json::to_vec(&request).unwrap(),
8897 )
8898 .unwrap();
8899 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8900 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8901 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8902 serde_json::from_slice(&responses[0].body).unwrap()
8903 else {
8904 panic!("expected supervisor.stderr_tail response");
8905 };
8906
8907 assert!(
8908 tail.entries
8909 .iter()
8910 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8911 "the control response lost the restart boundary"
8912 );
8913 let Some(StderrTailEntry::Line {
8914 text,
8915 truncated,
8916 at_ms,
8917 }) = tail.entries.iter().find(|entry| {
8918 matches!(
8919 entry,
8920 StderrTailEntry::Line {
8921 truncated: true,
8922 ..
8923 }
8924 )
8925 })
8926 else {
8927 panic!("the control response lost the truncated line");
8928 };
8929 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8930 assert!(*truncated);
8931 assert!(
8932 at_ms.is_some(),
8933 "the control response lost the line's capture time"
8934 );
8935 }
8936
8937 #[tokio::test(flavor = "current_thread")]
8942 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8943 let dir = ScratchDir::new("terminals-off-worker");
8944 let journal_path = dir.join("terminals.jsonl");
8945 let registry = Arc::new(Registry::default());
8946 let supervisor_handle = SupervisorHandle::new();
8947 let supervisor =
8948 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8949 .with_handle(supervisor_handle.clone())
8950 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8951 let module = supervisor
8952 .spawn(ModuleSpec {
8953 module_id: "terminal-off-worker".to_string(),
8954 program: fake_aft_stub_path(),
8955 args: Vec::new(),
8956 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8957 reserved: false,
8958 reserved_prefixes: Vec::new(),
8959 protocol: ModuleProtocol::Subc,
8960 overlap: Default::default(),
8961 })
8962 .unwrap();
8963 let deadline = Instant::now() + Duration::from_secs(5);
8964 while module.terminal_history().entries.len() != 2 {
8965 assert!(Instant::now() < deadline, "module did not record two exits");
8966 sleep(Duration::from_millis(10)).await;
8967 }
8968
8969 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8970 let handler =
8971 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8972 let frame = Frame::build(
8973 FrameType::Request,
8974 control_flags(),
8975 0,
8976 0,
8977 1,
8978 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8979 module_id: "terminal-off-worker".to_string(),
8980 })
8981 .unwrap(),
8982 )
8983 .unwrap();
8984 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8985 let spawned_at = std::time::Instant::now();
8986 let read = tokio::spawn({
8987 let handler = Arc::clone(&handler);
8988 async move { handler.handle_control_frame(&ctx, frame).await }
8989 });
8990 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8993 .await
8994 .unwrap()
8995 .expect("the history read reached its pause");
8996 let elapsed = spawned_at.elapsed();
8997 assert!(
8998 elapsed < Duration::from_secs(2) && !read.is_finished(),
8999 "this task could not run while the history read was paused \
9000 (resumed after {elapsed:?}, read finished: {})",
9001 read.is_finished()
9002 );
9003
9004 drop(release);
9005 let responses = read.await.unwrap().unwrap();
9006 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9007 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9008 panic!("expected supervisor.terminals response");
9009 };
9010 assert_eq!(terminals.entries.len(), 2);
9011 assert_eq!(terminals.journal_skipped_lines, 0);
9012 assert_eq!(terminals.journal_read_errors, 0);
9013 }
9014
9015 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9016 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9017 let registry = Arc::new(Registry::default());
9018 let supervisor_handle = SupervisorHandle::new();
9019 let supervisor =
9020 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9021 .with_handle(supervisor_handle.clone());
9022 let module = supervisor
9023 .spawn(ModuleSpec {
9024 module_id: "terminal-golden".to_string(),
9025 program: fake_aft_stub_path(),
9026 args: Vec::new(),
9027 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9028 reserved: false,
9029 reserved_prefixes: Vec::new(),
9030 protocol: ModuleProtocol::Subc,
9031 overlap: Default::default(),
9032 })
9033 .unwrap();
9034
9035 let deadline = Instant::now() + Duration::from_secs(5);
9036 while module.terminal_history().entries.len() != 2 {
9037 assert!(
9038 Instant::now() < deadline,
9039 "module did not retain two terminal exits: {:?}",
9040 module.terminal_history()
9041 );
9042 sleep(Duration::from_millis(10)).await;
9043 }
9044
9045 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9046 let request = ClientControlRequest::SupervisorTerminals {
9047 module_id: "terminal-golden".to_string(),
9048 };
9049 let frame = Frame::build(
9050 FrameType::Request,
9051 control_flags(),
9052 0,
9053 0,
9054 1,
9055 serde_json::to_vec(&request).unwrap(),
9056 )
9057 .unwrap();
9058 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9059 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9060 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9061 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9062 panic!("expected supervisor.terminals response");
9063 };
9064 assert_eq!(terminals.entries.len(), 2);
9065 assert_eq!(terminals.dropped, 0);
9066
9067 let mut rendered = serde_json::to_value(response).unwrap();
9068 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9071 for (index, entry) in rendered["entries"]
9072 .as_array_mut()
9073 .expect("terminal response entries array")
9074 .iter_mut()
9075 .enumerate()
9076 {
9077 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9078 }
9079
9080 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9081 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9082 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9083 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9084 std::fs::write(&golden_path, &serialized).unwrap();
9085 }
9086 let expected: Value =
9087 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9088 assert_eq!(rendered, expected);
9089 }
9090
9091 #[test]
9092 fn hello_registers_manifest_and_returns_ack() {
9093 let registry = Arc::new(Registry::default());
9094 let handler = ControlHandler::new(Arc::clone(®istry));
9095 let conn = ConnectionId::new(1);
9096
9097 let responses = handler
9098 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9099 .unwrap();
9100
9101 assert_eq!(responses.len(), 1);
9102 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9103 assert_eq!(responses[0].header.channel, 0);
9104 assert_eq!(responses[0].header.corr, 7);
9105 let ack = parse_ack(&responses[0]);
9106 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9107 assert!(ack
9108 .subc_capabilities
9109 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9110 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9111 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9112 assert!(ack
9113 .subc_ops
9114 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9115 assert!(ack
9116 .subc_ops
9117 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9118
9119 let registration = registry.get_module("aft").unwrap().unwrap();
9120 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9121 assert_eq!(registration.state, ChannelState::Active);
9122 assert_eq!(registration.connection_id, conn);
9123 assert_eq!(registration.control_ops, module_baseline_control_ops());
9124 }
9125
9126 #[test]
9127 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9128 let invalid_identifiers = [
9129 ("case_change", "credentials-Provider/v1"),
9130 ("leading_zero", "credentials-provider/v01"),
9131 ("trailing_hyphen", "credentials-provider-/v1"),
9132 ("consecutive_hyphens", "credentials--provider/v1"),
9133 ("uppercase", "Credentials-provider/v1"),
9134 ("missing_v", "credentials-provider/1"),
9135 ("whitespace", "credentials provider/v1"),
9136 ("zero_version", "credentials-provider/v0"),
9137 ("out_of_range_version", "credentials-provider/v4294967296"),
9138 (
9139 "overlength_name",
9140 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9141 ),
9142 ];
9143 let mut cases = invalid_identifiers
9144 .into_iter()
9145 .map(|(name, identifier)| {
9146 (
9147 format!("identifier_{name}"),
9148 "capabilities.provides[0]".to_string(),
9149 identifier.to_string(),
9150 json!({ "provides": [identifier] }),
9151 None,
9152 )
9153 })
9154 .collect::<Vec<_>>();
9155 cases.extend([
9156 (
9157 "unknown_need".to_string(),
9158 "capabilities.requires[0].need".to_string(),
9159 "deferred".to_string(),
9160 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9161 None,
9162 ),
9163 (
9164 "duplicate_provides".to_string(),
9165 "capabilities.provides[1]".to_string(),
9166 "credentials-provider/v1".to_string(),
9167 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9168 None,
9169 ),
9170 (
9171 "duplicate_must_never_reach".to_string(),
9172 "capabilities.must_never_reach[1]".to_string(),
9173 "credentials-provider/v1".to_string(),
9174 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9175 None,
9176 ),
9177 (
9178 "duplicate_requires_same_need".to_string(),
9179 "capabilities.requires[1]".to_string(),
9180 "credentials-provider/v1".to_string(),
9181 json!({ "requires": [
9182 { "capability": "credentials-provider/v1", "need": "required" },
9183 { "capability": "credentials-provider/v1", "need": "required" }
9184 ] }),
9185 None,
9186 ),
9187 (
9188 "duplicate_requires_conflicting_need".to_string(),
9189 "capabilities.requires[1]".to_string(),
9190 "credentials-provider/v1".to_string(),
9191 json!({ "requires": [
9192 { "capability": "credentials-provider/v1", "need": "required" },
9193 { "capability": "credentials-provider/v1", "need": "optional" }
9194 ] }),
9195 None,
9196 ),
9197 (
9198 "capabilities_root_pointer".to_string(),
9199 "runtime_computed[0]".to_string(),
9200 "/capabilities".to_string(),
9201 json!({}),
9202 Some(json!(["/capabilities"])),
9203 ),
9204 (
9205 "capabilities_descendant_pointer".to_string(),
9206 "runtime_computed[0]".to_string(),
9207 "/capabilities/provides".to_string(),
9208 json!({}),
9209 Some(json!(["/capabilities/provides"])),
9210 ),
9211 (
9212 "malformed_pointer_without_leading_slash".to_string(),
9213 "runtime_computed[0]".to_string(),
9214 "capabilities".to_string(),
9215 json!({}),
9216 Some(json!(["capabilities"])),
9217 ),
9218 (
9219 "malformed_pointer_escape".to_string(),
9220 "runtime_computed[0]".to_string(),
9221 "/roles/~2/tools".to_string(),
9222 json!({}),
9223 Some(json!(["/roles/~2/tools"])),
9224 ),
9225 (
9226 "unknown_capabilities_field".to_string(),
9227 "capabilities.future".to_string(),
9228 "<array>".to_string(),
9229 json!({ "future": [] }),
9230 None,
9231 ),
9232 ]);
9233
9234 for (index, (name, field, value, capabilities, runtime_computed)) in
9235 cases.into_iter().enumerate()
9236 {
9237 let registry = Arc::new(Registry::default());
9238 let handler = ControlHandler::new(Arc::clone(®istry));
9239 let response = handler
9240 .handle_control(
9241 ConnectionId::new((index + 1) as u64),
9242 capability_grammar_hello_frame(
9243 capabilities,
9244 runtime_computed,
9245 index as u64 + 1,
9246 ),
9247 )
9248 .expect("invalid HELLO returns a refusal");
9249
9250 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9251 let error = parse_error(&response[0]);
9252 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9253 let message = error["message"]
9254 .as_str()
9255 .expect("error message is a string");
9256 assert!(
9257 message.contains(&field),
9258 "{name}: field missing from {message}"
9259 );
9260 assert!(
9261 message.contains(&value),
9262 "{name}: value missing from {message}"
9263 );
9264 assert_eq!(
9265 registry
9266 .active_registration_count()
9267 .expect("registry reads"),
9268 0,
9269 "{name}: refused HELLO must not create a catalog entry"
9270 );
9271 }
9272 }
9273
9274 #[test]
9275 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9276 let registry = Arc::new(Registry::default());
9277 let handler = ControlHandler::new(Arc::clone(®istry));
9278 let capabilities = json!({
9279 "provides": ["credentials-provider/v1"],
9280 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9281 "must_never_reach": ["federation-transport/v1"]
9282 });
9283 let response = handler
9284 .handle_control(
9285 ConnectionId::new(99),
9286 capability_grammar_hello_frame(
9287 capabilities.clone(),
9288 Some(json!(["/roles/0/tools"])),
9289 99,
9290 ),
9291 )
9292 .expect("valid HELLO registers");
9293 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9294
9295 let request = Frame::build(
9296 FrameType::Request,
9297 control_flags(),
9298 0,
9299 0,
9300 100,
9301 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9302 .expect("catalog request serializes"),
9303 )
9304 .expect("catalog request frame builds");
9305 let response = handler
9306 .handle_catalog_list(request, None)
9307 .expect("catalog list succeeds");
9308 let ClientControlResponse::CatalogList { modules, .. } =
9309 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9310 else {
9311 panic!("catalog request must return catalog.list");
9312 };
9313 assert_eq!(modules.len(), 1);
9314 assert_eq!(
9315 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9316 capabilities
9317 );
9318 }
9319
9320 #[test]
9321 fn catalog_list_mirrors_management_operation_description() {
9322 let registry = Arc::new(Registry::default());
9323 let handler = ControlHandler::new(Arc::clone(®istry));
9324 let description = "List managed records and return their identifiers and metadata.";
9325 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9326 manifest.provides = vec![ProviderRole::ManagementSurface {
9327 operations: vec![ManagementOperation {
9328 name: "records.list".to_string(),
9329 kind: ManagementOperationKind::Query,
9330 description: Some(description.to_string()),
9331 }],
9332 config_schema: json!({"type": "object"}),
9333 observability: vec![ObservabilitySurface {
9334 name: "records.stats".to_string(),
9335 kind: ObservabilityKind::Snapshot,
9336 }],
9337 identity_scope: vec![IdentityScope::Project],
9338 concurrency: Concurrency::ModuleManaged,
9339 }];
9340 registry
9341 .register_with_control_ops(
9342 manifest,
9343 PROTOCOL_VERSION,
9344 ConnectionId::new(99),
9345 Vec::new(),
9346 )
9347 .expect("described management manifest registers");
9348
9349 let request = Frame::build(
9350 FrameType::Request,
9351 control_flags(),
9352 0,
9353 0,
9354 100,
9355 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9356 .expect("catalog request serializes"),
9357 )
9358 .expect("catalog request frame builds");
9359 let response = handler
9360 .handle_catalog_list(request, None)
9361 .expect("catalog list succeeds");
9362 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9363 assert_eq!(
9364 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9365 "catalog.list must preserve the declared operation description verbatim"
9366 );
9367 }
9368
9369 #[test]
9370 fn catalog_list_mirrors_registered_operation_delivery() {
9371 let registry = Arc::new(Registry::default());
9372 let handler = ControlHandler::new(Arc::clone(®istry));
9373 let delivery =
9374 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9375 let mut body = serde_json::to_value(ModuleHelloBody {
9376 manifest: manifest("watch-management", PROTOCOL_VERSION),
9377 protocol_ver: PROTOCOL_VERSION,
9378 control_ops: None,
9379 launch_nonce: None,
9380 })
9381 .unwrap();
9382 body["manifest"]["provides"] = json!([{
9383 "role": "management_surface",
9384 "operations": [
9385 {"name": "status", "kind": "query"},
9386 {"name": "future", "kind": "query"},
9387 {"name": "reset", "kind": "mutate"}
9388 ],
9389 "config_schema": {}, "observability": [], "identity_scope": []
9390 }]);
9391 body["manifest"]["operation_delivery"] = delivery.clone();
9392 let hello = Frame::build(
9393 FrameType::Hello,
9394 control_flags(),
9395 0,
9396 0,
9397 99,
9398 serde_json::to_vec(&body).unwrap(),
9399 )
9400 .unwrap();
9401 let response = handler
9402 .handle_control(ConnectionId::new(99), hello)
9403 .unwrap();
9404 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9405
9406 let (_, registrations) = registry.list_modules().unwrap();
9407 let registered = ®istrations[0].manifest;
9408 assert!(registered.is_watch_operation("status"));
9409 assert!(!registered.is_watch_operation("future"));
9410 assert!(!registered.is_watch_operation("reset"));
9411 assert!(!registered.is_watch_operation("missing"));
9412
9413 let request = Frame::build(
9414 FrameType::Request,
9415 control_flags(),
9416 0,
9417 0,
9418 100,
9419 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9420 )
9421 .unwrap();
9422 let response = handler.handle_catalog_list(request, None).unwrap();
9423 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9424 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9425 let ClientControlResponse::CatalogList { modules, .. } =
9426 serde_json::from_slice(&response[0].body).unwrap()
9427 else {
9428 panic!("expected catalog.list");
9429 };
9430 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9431 }
9432
9433 #[test]
9434 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9435 let registry = Arc::new(Registry::default());
9436 let supervisor = SupervisorHandle::new();
9437 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9438 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9439 let descriptor = json!({
9440 "schema": "ckios.richui.manifest/1",
9441 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9442 "bytes": 4096,
9443 "chunks": 4
9444 });
9445 let connection = ConnectionId::new(201);
9446 let hello = handler
9447 .handle_control(
9448 connection,
9449 hello_frame_with_rich_ui(
9450 "vault",
9451 PROTOCOL_VERSION,
9452 201,
9453 Some("the-real-nonce"),
9454 descriptor.clone(),
9455 ),
9456 )
9457 .unwrap();
9458 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9459
9460 let request =
9461 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9462 let provides = registry
9463 .get_module_by_connection(connection)
9464 .unwrap()
9465 .unwrap()
9466 .manifest
9467 .provides;
9468 handler
9469 .handle_catalog_update(connection, request, provides, None, Some(false))
9470 .unwrap();
9471
9472 let request = Frame::build(
9473 FrameType::Request,
9474 control_flags(),
9475 0,
9476 0,
9477 203,
9478 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9479 )
9480 .unwrap();
9481 let response = handler.handle_catalog_list(request, None).unwrap();
9482 let ClientControlResponse::CatalogList { modules, .. } =
9483 serde_json::from_slice(&response[0].body).unwrap()
9484 else {
9485 panic!("expected catalog.list");
9486 };
9487 assert_eq!(
9488 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9489 descriptor
9490 );
9491 assert!(modules[0].nonce_verified);
9492 }
9493
9494 fn hello_with_role_declarations(declarations: Value) -> Frame {
9495 let mut body = serde_json::to_value(ModuleHelloBody {
9496 manifest: manifest("role-test", PROTOCOL_VERSION),
9497 protocol_ver: PROTOCOL_VERSION,
9498 control_ops: None,
9499 launch_nonce: None,
9500 })
9501 .unwrap();
9502 body["manifest"]["role_declarations"] = declarations;
9503 Frame::build(
9504 FrameType::Hello,
9505 control_flags(),
9506 0,
9507 0,
9508 301,
9509 serde_json::to_vec(&body).unwrap(),
9510 )
9511 .unwrap()
9512 }
9513
9514 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9515 let request = Frame::build(
9516 FrameType::Request,
9517 control_flags(),
9518 0,
9519 0,
9520 302,
9521 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9522 )
9523 .unwrap();
9524 let response = handler.handle_catalog_list(request, None).unwrap();
9525 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9526 let ClientControlResponse::CatalogList { mut modules, .. } =
9527 serde_json::from_slice(&response[0].body).unwrap()
9528 else {
9529 panic!("expected catalog.list");
9530 };
9531 (raw["modules"][0].clone(), modules.remove(0))
9532 }
9533
9534 #[test]
9535 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9536 let registry = Arc::new(Registry::default());
9537 let handler = ControlHandler::new(Arc::clone(®istry));
9538 let declarations = json!({
9539 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9540 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9541 });
9542 let connection = ConnectionId::new(301);
9543 let response = handler
9544 .handle_control(
9545 connection,
9546 hello_with_role_declarations(declarations.clone()),
9547 )
9548 .unwrap();
9549 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9550 let registered = registry
9551 .get_module_by_connection(connection)
9552 .unwrap()
9553 .unwrap()
9554 .manifest;
9555 assert_eq!(
9559 serde_json::to_value(®istered.role_declarations).unwrap(),
9560 declarations
9561 );
9562 let (raw, entry) = role_declarations_catalog(&handler);
9563 assert_eq!(raw["role_declarations"], declarations);
9564 assert_eq!(entry.role_declarations, registered.role_declarations);
9565
9566 let request =
9567 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9568 handler
9569 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9570 .unwrap();
9571 let (raw, entry) = role_declarations_catalog(&handler);
9572 assert_eq!(raw["role_declarations"], declarations);
9573 assert_eq!(entry.role_declarations, registered.role_declarations);
9574 }
9575
9576 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9577 let registry = Arc::new(Registry::default());
9578 let handler = ControlHandler::new(Arc::clone(®istry));
9579 let capture = EventCapture::default();
9580 let _subscriber =
9581 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9582 let response = handler
9583 .handle_control(
9584 ConnectionId::new(301),
9585 hello_with_role_declarations(declarations),
9586 )
9587 .unwrap();
9588 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9589 assert!(registry
9590 .get_module_by_connection(ConnectionId::new(301))
9591 .unwrap()
9592 .unwrap()
9593 .manifest
9594 .role_declarations
9595 .is_empty());
9596 let (raw, entry) = role_declarations_catalog(&handler);
9597 assert!(raw.get("role_declarations").is_none());
9598 assert!(entry.role_declarations.is_empty());
9599 let warnings: Vec<_> = capture
9600 .events()
9601 .into_iter()
9602 .filter(|event| {
9603 event.fields.get("message").map(String::as_str)
9604 == Some("dropping whole role_declarations map from HELLO")
9605 })
9606 .collect();
9607 assert_eq!(warnings.len(), 1);
9608 assert_eq!(warnings[0].level, tracing::Level::WARN);
9609 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9610 assert_eq!(warnings[0].fields["reason"], reason);
9611 }
9612
9613 #[test]
9614 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9615 for value in [
9616 json!(null),
9617 json!(false),
9618 json!(0),
9619 json!("opaque"),
9620 json!([]),
9621 ] {
9622 assert_hello_drops_role_declarations_with_warning(
9623 json!({"routing/v1": {}, "broken/v1": value}),
9624 "role_declarations values must be objects",
9625 );
9626 }
9627 assert_hello_drops_role_declarations_with_warning(
9628 json!([]),
9629 "role_declarations must be an object",
9630 );
9631 }
9632
9633 #[test]
9634 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9635 assert_hello_drops_role_declarations_with_warning(
9636 json!({"routing/v1": {}, "bad key": {}}),
9637 "role_declarations key must use capability-name grammar",
9638 );
9639 }
9640
9641 #[test]
9642 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9643 assert_hello_drops_role_declarations_with_warning(
9644 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9645 "role_declarations exceeds 16384 compact JSON bytes",
9646 );
9647 }
9648
9649 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
9650 let mut body = serde_json::to_value(ModuleHelloBody {
9651 manifest: manifest("flow-ops", PROTOCOL_VERSION),
9652 protocol_ver: PROTOCOL_VERSION,
9653 control_ops: None,
9654 launch_nonce: None,
9655 })
9656 .unwrap();
9657 body["manifest"]["provides"] = json!([{
9658 "role": "management_surface",
9659 "operations": [
9660 {"name": "diagnostics.findings", "kind": "query"},
9661 {"name": "reset", "kind": "mutate"}
9662 ],
9663 "config_schema": {}, "observability": [], "identity_scope": []
9664 }]);
9665 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
9666 if let Some(flow_operations) = flow_operations {
9667 body["manifest"]["flow_operations"] = flow_operations;
9668 }
9669 Frame::build(
9670 FrameType::Hello,
9671 control_flags(),
9672 0,
9673 0,
9674 401,
9675 serde_json::to_vec(&body).unwrap(),
9676 )
9677 .unwrap()
9678 }
9679
9680 #[test]
9681 fn catalog_list_mirrors_flow_operations_unchanged() {
9682 for (capability, flow_operations) in [
9685 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
9686 ("call-modes/v1", json!(["diagnostics.findings"])),
9687 ("flow-scopes/v1", json!([])),
9688 ] {
9689 let registry = Arc::new(Registry::default());
9690 let handler = ControlHandler::new(Arc::clone(®istry));
9691 let connection = ConnectionId::new(401);
9692 let response = handler
9693 .handle_control(
9694 connection,
9695 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
9696 )
9697 .unwrap();
9698 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9699 let registered = registry
9700 .get_module_by_connection(connection)
9701 .unwrap()
9702 .unwrap()
9703 .manifest;
9704 assert_eq!(
9705 serde_json::to_value(®istered.flow_operations).unwrap(),
9706 flow_operations
9707 );
9708 let (raw, entry) = role_declarations_catalog(&handler);
9709 assert_eq!(raw["flow_operations"], flow_operations);
9710 assert_eq!(entry.flow_operations, registered.flow_operations);
9711 }
9712
9713 let registry = Arc::new(Registry::default());
9714 let handler = ControlHandler::new(Arc::clone(®istry));
9715 let response = handler
9716 .handle_control(
9717 ConnectionId::new(401),
9718 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
9719 )
9720 .unwrap();
9721 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9722 let (raw, entry) = role_declarations_catalog(&handler);
9723 assert!(raw.get("flow_operations").is_none());
9724 assert_eq!(entry.flow_operations, None);
9725 }
9726
9727 fn assert_hello_drops_flow_operations_with_warning(
9728 capabilities: Value,
9729 flow_operations: Value,
9730 reason: &str,
9731 ) {
9732 let registry = Arc::new(Registry::default());
9733 let handler = ControlHandler::new(Arc::clone(®istry));
9734 let capture = EventCapture::default();
9735 let _subscriber =
9736 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9737 let response = handler
9738 .handle_control(
9739 ConnectionId::new(401),
9740 hello_with_flow_operations(capabilities, Some(flow_operations)),
9741 )
9742 .unwrap();
9743 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9744 let registered = registry
9745 .get_module_by_connection(ConnectionId::new(401))
9746 .unwrap()
9747 .expect("the module still registers")
9748 .manifest;
9749 assert_eq!(registered.module_id, "flow-ops");
9750 assert_eq!(registered.flow_operations, None);
9751 let (raw, entry) = role_declarations_catalog(&handler);
9752 assert_eq!(raw["module_id"], "flow-ops");
9753 assert!(raw.get("flow_operations").is_none());
9754 assert_eq!(entry.flow_operations, None);
9755 let warnings: Vec<_> = capture
9756 .events()
9757 .into_iter()
9758 .filter(|event| {
9759 event.fields.get("message").map(String::as_str)
9760 == Some("dropping flow_operations from HELLO")
9761 })
9762 .collect();
9763 assert_eq!(warnings.len(), 1);
9764 assert_eq!(warnings[0].level, tracing::Level::WARN);
9765 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
9766 assert_eq!(warnings[0].fields["reason"], reason);
9767 }
9768
9769 #[test]
9770 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
9771 assert_hello_drops_flow_operations_with_warning(
9772 json!(["flow-scopes/v1"]),
9773 json!(["diagnostics.findings", "missing"]),
9774 "flow_operations names 'missing', which is not a declared management operation",
9775 );
9776 }
9777
9778 #[test]
9779 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
9780 assert_hello_drops_flow_operations_with_warning(
9781 json!([]),
9782 json!(["diagnostics.findings"]),
9783 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
9784 );
9785 assert_hello_drops_flow_operations_with_warning(
9787 json!(["agent-run-scopes/v1"]),
9788 json!([]),
9789 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
9790 );
9791 }
9792
9793 #[test]
9794 fn hello_malformed_flow_operations_is_dropped_with_warning() {
9795 assert_hello_drops_flow_operations_with_warning(
9796 json!(["flow-scopes/v1"]),
9797 json!(["reset", "reset"]),
9798 "flow_operations names 'reset' more than once",
9799 );
9800 assert_hello_drops_flow_operations_with_warning(
9801 json!(["flow-scopes/v1"]),
9802 json!(vec!["reset"; 257]),
9803 "flow_operations lists 257 names, more than 256",
9804 );
9805 assert_hello_drops_flow_operations_with_warning(
9806 json!(["flow-scopes/v1"]),
9807 json!("reset"),
9808 "flow_operations must be an array of operation names",
9809 );
9810 }
9811
9812 #[test]
9813 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9814 let registry = Arc::new(Registry::default());
9815 let handler = ControlHandler::new(registry.clone());
9816 let descriptor = json!({
9817 "schema": "ckios.richui.manifest/1",
9818 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9819 "bytes": 128,
9820 "chunks": 2
9821 });
9822 let malformed = [
9823 ("schema", json!("")),
9824 ("schema", json!("a".repeat(65))),
9825 ("schema", json!("Bad schema")),
9826 (
9827 "manifest_digest",
9828 json!(format!("sha256:{}", "A".repeat(64))),
9829 ),
9830 ("manifest_digest", json!("sha256:abc")),
9831 ("bytes", json!(0)),
9832 ("bytes", json!(16_777_217)),
9833 ("bytes", json!("128")),
9834 ("chunks", json!(0)),
9835 ("chunks", json!(1025)),
9836 ("chunks", json!("2")),
9837 ];
9838 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9839 let mut invalid = descriptor.clone();
9840 invalid[field] = invalid_value;
9841 let response = handler
9842 .handle_control(
9843 ConnectionId::new(210 + index as u64),
9844 hello_frame_with_rich_ui(
9845 &format!("bad-rich-ui-{index}"),
9846 PROTOCOL_VERSION,
9847 210 + index as u64,
9848 None,
9849 invalid,
9850 ),
9851 )
9852 .unwrap();
9853 assert_eq!(response[0].header.ty, FrameType::Error);
9854 let error = parse_error(&response[0]);
9855 assert_eq!(error["code"], "invalid_manifest");
9856 assert!(error["message"]
9857 .as_str()
9858 .unwrap()
9859 .contains(&format!("rich_ui.{field}")));
9860 assert!(registry
9861 .get_module(&format!("bad-rich-ui-{index}"))
9862 .unwrap()
9863 .is_none());
9864 }
9865 }
9866
9867 #[test]
9868 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9869 let registry = Arc::new(Registry::default());
9870 let handler = ControlHandler::new(Arc::clone(®istry));
9871 let connection = ConnectionId::new(219);
9872 registry
9873 .register_with_control_ops(
9874 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9875 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9876 "bad schema",
9877 format!("sha256:{}", "a".repeat(64)),
9878 128,
9879 2,
9880 )))
9881 .build(),
9882 PROTOCOL_VERSION,
9883 connection,
9884 Vec::new(),
9885 )
9886 .unwrap();
9887 let request =
9888 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9889 let response = handler
9890 .handle_catalog_update(connection, request, Vec::new(), None, None)
9891 .unwrap();
9892 assert_eq!(response[0].header.ty, FrameType::Error);
9893 let error = parse_error(&response[0]);
9894 assert_eq!(error["code"], "invalid_manifest");
9895 assert!(error["message"]
9896 .as_str()
9897 .unwrap()
9898 .contains("rich_ui.schema"));
9899 }
9900
9901 #[test]
9902 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9903 let registry = Arc::new(Registry::default());
9904 let supervisor = SupervisorHandle::new();
9905 supervisor.open_swap("provider", "candidate-nonce".to_string());
9906 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9907 let connection = ConnectionId::new(218);
9908 let hello = handler
9909 .handle_control(
9910 connection,
9911 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9912 )
9913 .unwrap();
9914 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9915 assert!(registry.list_modules().unwrap().1.is_empty());
9916
9917 registry.promote_candidate("provider").unwrap().unwrap();
9918 let request = Frame::build(
9919 FrameType::Request,
9920 control_flags(),
9921 0,
9922 0,
9923 219,
9924 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9925 )
9926 .unwrap();
9927 let response = handler.handle_catalog_list(request, None).unwrap();
9928 let ClientControlResponse::CatalogList { modules, .. } =
9929 serde_json::from_slice(&response[0].body).unwrap()
9930 else {
9931 panic!("expected catalog.list");
9932 };
9933 assert_eq!(modules.len(), 1);
9934 assert!(modules[0].nonce_verified);
9935 }
9936
9937 #[test]
9938 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9939 let registry = Arc::new(Registry::default());
9940 let handler = ControlHandler::new(Arc::clone(®istry));
9941 let hello = Frame::build(
9942 FrameType::Hello,
9943 control_flags(),
9944 0,
9945 0,
9946 220,
9947 serde_json::to_vec(&ModuleHelloBody {
9948 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9949 protocol_ver: PROTOCOL_VERSION,
9950 control_ops: None,
9951 launch_nonce: Some("unverified".to_string()),
9952 })
9953 .unwrap(),
9954 )
9955 .unwrap();
9956 assert_eq!(
9957 handler
9958 .handle_control(ConnectionId::new(220), hello)
9959 .unwrap()[0]
9960 .header
9961 .ty,
9962 FrameType::HelloAck
9963 );
9964 let request = Frame::build(
9965 FrameType::Request,
9966 control_flags(),
9967 0,
9968 0,
9969 221,
9970 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9971 )
9972 .unwrap();
9973 let response = handler.handle_catalog_list(request, None).unwrap();
9974 let ClientControlResponse::CatalogList { modules, .. } =
9975 serde_json::from_slice(&response[0].body).unwrap()
9976 else {
9977 panic!("expected catalog.list");
9978 };
9979 assert_eq!(
9980 serde_json::to_vec(&modules[0]).unwrap(),
9981 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9982 );
9983 assert!(!modules[0].nonce_verified);
9984 }
9985
9986 #[test]
9987 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9988 let registry = Arc::new(Registry::default());
9989 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9990 [("vault".to_string(), true), ("squatter".to_string(), true)],
9991 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9992 );
9993 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9994 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9995 provides: vec!["credentials-provider/v1".to_string()],
9996 requires: Vec::new(),
9997 must_never_reach: Vec::new(),
9998 });
9999 let frame = Frame::build(
10000 FrameType::Hello,
10001 control_flags(),
10002 0,
10003 0,
10004 77,
10005 serde_json::to_vec(&ModuleHelloBody {
10006 manifest: squatter,
10007 protocol_ver: PROTOCOL_VERSION,
10008 control_ops: None,
10009 launch_nonce: None,
10010 })
10011 .expect("HELLO serializes"),
10012 )
10013 .expect("HELLO frame builds");
10014 let response = handler
10015 .handle_control(ConnectionId::new(77), frame)
10016 .expect("reserved claim receives a typed refusal");
10017 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10018 assert_eq!(
10019 registry
10020 .active_registration_count()
10021 .expect("registry reads"),
10022 0,
10023 "a reserved capability refusal must not leave a catalog entry"
10024 );
10025 }
10026
10027 #[test]
10028 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10029 for settlement in ["timeout", "inconclusive", "drop"] {
10030 let breakers = RouteBindBreakers::default();
10031 let RouteBindAdmission::Admitted {
10032 guard: mut old,
10033 probe: false,
10034 } = breakers.admit("prov")
10035 else {
10036 panic!("ordinary relay admitted")
10037 };
10038 let RouteBindAdmission::Admitted {
10039 guard: mut opener, ..
10040 } = breakers.admit("prov")
10041 else {
10042 panic!("second relay admitted")
10043 };
10044 assert!(
10045 !opener
10046 .record_timeout(1, Duration::ZERO)
10047 .unwrap()
10048 .reopened_after_probe
10049 );
10050 let RouteBindAdmission::Admitted {
10051 guard: mut probe,
10052 probe: true,
10053 } = breakers.admit("prov")
10054 else {
10055 panic!("one cooldown probe admitted")
10056 };
10057 match settlement {
10058 "timeout" => assert!(
10059 !old.record_timeout(1, Duration::ZERO)
10060 .unwrap()
10061 .reopened_after_probe
10062 ),
10063 "inconclusive" => old.record_inconclusive(),
10064 "drop" => drop(old),
10065 _ => unreachable!(),
10066 }
10067 assert!(
10068 matches!(
10069 breakers.admit("prov"),
10070 RouteBindAdmission::Refused {
10071 probe_in_flight: true,
10072 ..
10073 }
10074 ),
10075 "{settlement} of a pre-open relay cannot release the real probe"
10076 );
10077 assert!(
10078 probe
10079 .record_timeout(1, Duration::ZERO)
10080 .unwrap()
10081 .reopened_after_probe
10082 );
10083 assert!(matches!(
10084 breakers.admit("prov"),
10085 RouteBindAdmission::Admitted { probe: true, .. }
10086 ));
10087 }
10088 let breakers = RouteBindBreakers::default();
10089 let admit = || match breakers.admit("prov") {
10090 RouteBindAdmission::Admitted { guard, .. } => guard,
10091 _ => panic!("relay admitted"),
10092 };
10093 admit().record_timeout(1, Duration::ZERO);
10094 let mut old_probe = admit();
10095 breakers.reset_for_new_module_connection("prov");
10096 admit().record_timeout(1, Duration::ZERO);
10097 let _new_probe = admit();
10098 old_probe.record_inconclusive();
10099 assert!(matches!(
10100 breakers.admit("prov"),
10101 RouteBindAdmission::Refused {
10102 probe_in_flight: true,
10103 ..
10104 }
10105 ));
10106 }
10107
10108 #[tokio::test]
10109 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10110 for candidate in [false, true] {
10111 let registry = Arc::new(Registry::default());
10112 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10113 [("vault".to_string(), true), ("squatter".to_string(), true)],
10114 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10115 );
10116 let conn = ConnectionId::new(77);
10117 let (ctx, mut rx) = route_ctx(conn);
10118 let initial = capability_manifest("squatter", &[], &[]);
10119 if candidate {
10120 registry
10121 .register_candidate_with_control_ops(
10122 initial.clone(),
10123 PROTOCOL_VERSION,
10124 conn,
10125 module_baseline_control_ops(),
10126 )
10127 .unwrap();
10128 handler
10129 .forwarding
10130 .register_candidate_module_connection(
10131 conn,
10132 "squatter".to_string(),
10133 PROTOCOL_VERSION,
10134 manifest_concurrency(&initial),
10135 ctx.egress.clone(),
10136 )
10137 .unwrap();
10138 } else {
10139 hello_via_sink(
10140 &handler,
10141 &ctx,
10142 &mut rx,
10143 hello_frame_with_manifest(initial.clone(), 1),
10144 )
10145 .await;
10146 }
10147 let response = handler
10148 .handle_control_frame(
10149 &ctx,
10150 catalog_update_with_capabilities_frame(
10151 2,
10152 capability_manifest("squatter", &["credentials-provider/v1"], &[])
10153 .capabilities
10154 .unwrap(),
10155 ),
10156 )
10157 .await
10158 .unwrap();
10159 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10160 assert_eq!(
10161 registry
10162 .get_module_by_connection(conn)
10163 .unwrap()
10164 .unwrap()
10165 .manifest,
10166 initial
10167 );
10168 }
10169 }
10170
10171 #[test]
10172 fn server_describe_surfaces_required_capability_verdict_fields() {
10173 let registry = Arc::new(Registry::default());
10174 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10175 [
10176 ("consumer".to_string(), true),
10177 ("provider".to_string(), false),
10178 ],
10179 BTreeMap::new(),
10180 );
10181 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10182 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10183 provides: Vec::new(),
10184 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10185 capability: "credentials-provider/v1".to_string(),
10186 need: subc_protocol::manifest::CapabilityNeed::Required,
10187 }],
10188 must_never_reach: Vec::new(),
10189 });
10190 let hello = Frame::build(
10191 FrameType::Hello,
10192 control_flags(),
10193 0,
10194 0,
10195 78,
10196 serde_json::to_vec(&ModuleHelloBody {
10197 manifest: consumer,
10198 protocol_ver: PROTOCOL_VERSION,
10199 control_ops: None,
10200 launch_nonce: None,
10201 })
10202 .expect("HELLO serializes"),
10203 )
10204 .expect("HELLO frame builds");
10205 handler
10206 .handle_control(ConnectionId::new(78), hello)
10207 .expect("consumer registers");
10208 let describe = Frame::build(
10209 FrameType::Request,
10210 control_flags(),
10211 0,
10212 0,
10213 79,
10214 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10215 .expect("request serializes"),
10216 )
10217 .expect("describe frame builds");
10218 let response = handler
10219 .handle_server_describe(describe)
10220 .expect("server.describe succeeds");
10221 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10222 let requirement = &rendered["capability_requirements"][0];
10223 assert_eq!(requirement["consumer"], "consumer");
10224 assert_eq!(requirement["verdict"], "never_provided");
10225 assert_eq!(requirement["episode_seq"], 1);
10226 assert_eq!(requirement["config_satisfiable"], false);
10227 assert_eq!(requirement["runtime_available"], false);
10228 assert!(requirement["detail"]
10229 .as_str()
10230 .expect("detail string")
10231 .contains("credentials-provider/v1"));
10232 }
10233
10234 fn deregister_required_provider(
10240 daemon_shutdown_begun: bool,
10241 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10242 const CAPABILITY: &str = "project-identity/v1";
10243 let registry = Arc::new(Registry::default());
10244 let daemon_shutdown = DaemonShutdownFlag::default();
10245 let handler = ControlHandler::new(Arc::clone(®istry))
10246 .with_capability_config(
10247 [
10248 ("consumer".to_string(), true),
10249 ("provider".to_string(), true),
10250 ],
10251 BTreeMap::new(),
10252 )
10253 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10254 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10255 consumer.capabilities = Some(CapabilityDeclarations {
10256 provides: Vec::new(),
10257 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10258 capability: CAPABILITY.to_string(),
10259 need: subc_protocol::manifest::CapabilityNeed::Required,
10260 }],
10261 must_never_reach: Vec::new(),
10262 });
10263 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10264 let provider_conn = ConnectionId::new(182);
10265 for (conn, manifest, corr) in [
10266 (ConnectionId::new(181), consumer, 181),
10267 (provider_conn, provider, 182),
10268 ] {
10269 let hello = handler
10270 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10271 .expect("module registers");
10272 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10273 }
10274 assert_eq!(
10275 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10276 Some(CapabilityVerdict::Provided),
10277 "the registered provider must satisfy the requirement first"
10278 );
10279 if daemon_shutdown_begun {
10280 daemon_shutdown.set_for_test();
10281 }
10282
10283 let capture = EventCapture::default();
10284 let _subscriber =
10285 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10286 handler
10287 .cleanup_connection(provider_conn)
10288 .expect("provider deregisters");
10289 let events = capture
10290 .events()
10291 .into_iter()
10292 .filter(|event| {
10293 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10294 })
10295 .collect();
10296 (
10297 events,
10298 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10299 )
10300 }
10301
10302 #[test]
10303 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10304 let (events, verdict) = deregister_required_provider(true);
10305 assert_eq!(
10306 verdict,
10307 Some(CapabilityVerdict::NeverProvided),
10308 "shutdown must not change the verdict route.open refusals read"
10309 );
10310 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10311 let event = &events[0];
10312 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10313 assert_eq!(
10314 event.fields.get("cause").map(String::as_str),
10315 Some("\"daemon_shutdown\""),
10316 "{event:?}"
10317 );
10318 assert_eq!(
10319 event.fields.get("verdict").map(String::as_str),
10320 Some("\"never_provided\""),
10321 "{event:?}"
10322 );
10323 }
10324
10325 #[test]
10326 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10327 let (events, verdict) = deregister_required_provider(false);
10328 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10329 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10330 let event = &events[0];
10331 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10332 assert_eq!(event.fields.get("cause"), None, "{event:?}");
10333 assert_eq!(
10334 event.fields.get("verdict").map(String::as_str),
10335 Some("\"never_provided\""),
10336 "{event:?}"
10337 );
10338 }
10339
10340 #[test]
10341 fn catalog_list_omits_capabilities_for_legacy_manifest() {
10342 let registry = Arc::new(Registry::default());
10343 let handler = ControlHandler::new(Arc::clone(®istry));
10344 let hello = handler
10345 .handle_control(
10346 ConnectionId::new(101),
10347 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10348 )
10349 .expect("legacy HELLO registers");
10350 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10351
10352 let request = Frame::build(
10353 FrameType::Request,
10354 control_flags(),
10355 0,
10356 0,
10357 102,
10358 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10359 .expect("catalog request serializes"),
10360 )
10361 .expect("catalog request frame builds");
10362 let response = handler
10363 .handle_catalog_list(request, None)
10364 .expect("catalog list succeeds");
10365 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10366 assert!(
10367 body["modules"][0].get("capabilities").is_none(),
10368 "legacy manifest must retain an absent capabilities field on catalog.list"
10369 );
10370 }
10371
10372 #[test]
10373 fn hello_ack_omits_storage_when_no_storage_config() {
10374 let registry = Arc::new(Registry::default());
10375 let handler = ControlHandler::new(Arc::clone(®istry));
10376 let responses = handler
10377 .handle_control(
10378 ConnectionId::new(1),
10379 hello_frame("aft", PROTOCOL_VERSION, 7),
10380 )
10381 .unwrap();
10382 let ack = parse_ack(&responses[0]);
10383 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10384 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10385 }
10386
10387 #[tokio::test]
10388 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10389 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10390 let registry = Arc::new(Registry::default());
10391 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
10392 let responses = handler
10393 .handle_control(
10394 ConnectionId::new(1),
10395 hello_frame("aft", PROTOCOL_VERSION, 7),
10396 )
10397 .unwrap();
10398 let ack = parse_ack(&responses[0]);
10399 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10400
10401 let described = handler
10402 .handle_control_frame(
10403 &route_ctx(ConnectionId::new(2)).0,
10404 Frame::build(
10405 FrameType::Request,
10406 control_flags(),
10407 0,
10408 0,
10409 9,
10410 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10411 )
10412 .unwrap(),
10413 )
10414 .await
10415 .unwrap();
10416 let ClientControlResponse::ServerDescribe { machine_id, .. } =
10417 serde_json::from_slice(&described[0].body).unwrap()
10418 else {
10419 panic!("server.describe answered with another shape");
10420 };
10421 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10422 }
10423
10424 #[test]
10425 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10426 let registry = Arc::new(Registry::default());
10429 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
10430 crate::daemon_config::StorageConfig::Sqlite {
10431 data_home: std::path::PathBuf::from("/data"),
10432 },
10433 ));
10434
10435 let responses = handler
10436 .handle_control(
10437 ConnectionId::new(1),
10438 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10439 )
10440 .unwrap();
10441 let ack = parse_ack(&responses[0]);
10442 assert_eq!(
10443 ack.storage,
10444 Some(serde_json::json!({
10445 "module_id": "alfonso-routing",
10446 "storage_namespace": "default",
10447 "isolation": { "kind": "module" },
10448 "backend": {
10449 "backend": "sqlite",
10450 "path": "/data/cortexkit/alfonso-routing/store.db"
10451 }
10452 })),
10453 "the delivered descriptor is the module's own sqlite store path"
10454 );
10455 }
10456
10457 #[test]
10458 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10459 let registry = Arc::new(Registry::default());
10460 let handler = ControlHandler::new(Arc::clone(®istry));
10461 let conn = ConnectionId::new(1);
10462 let responses = handler
10463 .handle_control(
10464 conn,
10465 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10466 )
10467 .unwrap();
10468 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10469 let registration = registry.get_module("aft").unwrap().unwrap();
10470 assert_eq!(registration.control_ops, module_baseline_control_ops());
10471
10472 let frame =
10473 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10474 assert!(handler
10475 .guard_module_control_op(&frame, "aft", "route.bind")
10476 .unwrap()
10477 .is_none());
10478 let error = handler
10479 .guard_module_control_op(&frame, "aft", "test.synthetic")
10480 .unwrap()
10481 .expect("synthetic ungranted op should be rejected");
10482 assert_eq!(error.header.ty, FrameType::Error);
10483 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10484 }
10485
10486 #[test]
10487 fn hello_control_ops_some_adds_optional_grants() {
10488 let registry = Arc::new(Registry::default());
10489 let handler = ControlHandler::new(Arc::clone(®istry));
10490 handler
10491 .handle_control(
10492 ConnectionId::new(1),
10493 hello_frame_with_control_ops(
10494 "aft",
10495 PROTOCOL_VERSION,
10496 7,
10497 Some(vec![
10498 "future.synthetic".to_string(),
10499 "route.bind".to_string(),
10500 ]),
10501 ),
10502 )
10503 .unwrap();
10504 let registration = registry.get_module("aft").unwrap().unwrap();
10505 assert_eq!(
10506 registration.control_ops,
10507 vec![
10508 "route.bind".to_string(),
10509 "route.status".to_string(),
10510 "future.synthetic".to_string(),
10511 ]
10512 );
10513 let frame =
10514 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10515 assert!(handler
10516 .guard_module_control_op(&frame, "aft", "future.synthetic")
10517 .unwrap()
10518 .is_none());
10519 }
10520
10521 #[tokio::test]
10522 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10523 let registry = Arc::new(Registry::default());
10524 let forwarding = Arc::new(ForwardingTable::default());
10525 let handler =
10526 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10527 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10528 hello_via_sink(
10529 &handler,
10530 &module_ctx,
10531 &mut module_rx,
10532 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10533 )
10534 .await;
10535
10536 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10537 let responses = handler
10538 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10539 .await
10540 .unwrap();
10541 assert_eq!(responses.len(), 1);
10542 assert_eq!(responses[0].header.ty, FrameType::Error);
10543 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10544 assert!(module_rx.try_recv().is_err());
10545 }
10546
10547 #[tokio::test]
10548 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10549 let registry = Arc::new(Registry::default());
10550 let forwarding = Arc::new(ForwardingTable::default());
10551 let handler =
10552 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10553 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10554 hello_via_sink(
10555 &handler,
10556 &module_ctx,
10557 &mut module_rx,
10558 hello_frame_with_control_ops(
10559 "aft",
10560 PROTOCOL_VERSION,
10561 7,
10562 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10563 ),
10564 )
10565 .await;
10566
10567 let project_root = unique_project_root("demux");
10568 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10569 let route_handler = handler.clone();
10570 let route_task = tokio::spawn(async move {
10571 route_handler
10572 .handle_control_frame(
10573 &route_client_ctx,
10574 route_open_frame(100, "aft", project_root),
10575 )
10576 .await
10577 .unwrap()
10578 });
10579 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10580 .await
10581 .unwrap()
10582 .unwrap();
10583 assert!(matches!(
10584 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10585 ModuleControlRequest::RouteBind { .. }
10586 ));
10587
10588 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10589 let health_handler = handler.clone();
10590 let health_task = tokio::spawn(async move {
10591 health_handler
10592 .handle_control_frame(
10593 &health_client_ctx,
10594 supervisor_health_probe_frame(101, "aft"),
10595 )
10596 .await
10597 .unwrap()
10598 });
10599 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10600 .await
10601 .unwrap()
10602 .unwrap();
10603 assert_eq!(
10604 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10605 ModuleControlRequest::HealthCheck {}
10606 );
10607
10608 handler
10609 .handle_control_frame(
10610 &module_ctx,
10611 health_response(health_frame.header.corr, HealthStatus::Degraded),
10612 )
10613 .await
10614 .unwrap();
10615 let health_response = health_task.await.unwrap();
10616 assert_eq!(health_response.len(), 1);
10617 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10618 ClientControlResponse::SupervisorHealthProbe {
10619 module_id,
10620 status,
10621 detail,
10622 metrics,
10623 } => {
10624 assert_eq!(module_id, "aft");
10625 assert_eq!(status, HealthStatus::Degraded);
10626 assert_eq!(detail.as_deref(), Some("warming"));
10627 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10628 }
10629 other => panic!("unexpected health response: {other:?}"),
10630 }
10631
10632 handler
10633 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10634 .await
10635 .unwrap();
10636 let route_response = route_task.await.unwrap();
10637 assert!(route_response.is_empty());
10638 let published = route_client_rx.recv().await.unwrap();
10639 assert!(matches!(
10640 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10641 ClientControlResponse::RouteOpen { .. }
10642 ));
10643 }
10644
10645 async fn relay_route_open(
10650 handler: &ControlHandler,
10651 client_connection: ConnectionId,
10652 client_egress: &FrameSink,
10653 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10654 corr: u64,
10655 module_id: &str,
10656 project_root_label: &str,
10657 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10658 let ctx = RouteCtx {
10659 connection_id: client_connection,
10660 egress: client_egress.clone(),
10661 };
10662 let handler = handler.clone();
10663 let project_root = unique_project_root(project_root_label);
10664 let module_id = module_id.to_string();
10665 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10666 let task = tokio::spawn(async move {
10667 let _guard = tracing::dispatcher::set_default(&dispatch);
10668 handler
10669 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10670 .await
10671 .unwrap()
10672 });
10673 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10674 .await
10675 .expect("module receives the relayed route.bind")
10676 .expect("module egress is open");
10677 (task, bind.frame)
10678 }
10679
10680 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10681 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10682 ModuleControlRequest::RouteBind {
10683 route_channel,
10684 epoch,
10685 ..
10686 } => (route_channel, epoch),
10687 other => panic!("expected a route.bind request, got {other:?}"),
10688 }
10689 }
10690
10691 fn published_route(frame: &Frame) -> (u16, u32) {
10692 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10693 ClientControlResponse::RouteOpen {
10694 route_channel,
10695 route_epoch,
10696 } => (route_channel, route_epoch),
10697 other => panic!("expected a route.open response, got {other:?}"),
10698 }
10699 }
10700
10701 #[tokio::test]
10721 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10722 let registry = Arc::new(Registry::default());
10723 let forwarding = Arc::new(ForwardingTable::default());
10724 let handler =
10725 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10726
10727 let module_connection = ConnectionId::new(30);
10728 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10729 hello_via_sink(
10730 &handler,
10731 &module_ctx,
10732 &mut module_rx,
10733 hello_frame("aft", PROTOCOL_VERSION, 7),
10734 )
10735 .await;
10736
10737 let dying_client = ConnectionId::new(31);
10738 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10739
10740 let (first_task, first_bind) = relay_route_open(
10743 &handler,
10744 dying_client,
10745 &dying_ctx.egress,
10746 &mut module_rx,
10747 100,
10748 "aft",
10749 "closing-first",
10750 )
10751 .await;
10752 handler
10753 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10754 .await
10755 .unwrap();
10756 assert!(first_task.await.unwrap().is_empty());
10757 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10758
10759 let (second_task, second_bind) = relay_route_open(
10761 &handler,
10762 dying_client,
10763 &dying_ctx.egress,
10764 &mut module_rx,
10765 101,
10766 "aft",
10767 "closing-second",
10768 )
10769 .await;
10770 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10771
10772 assert!(forwarding
10775 .escalate_client_delivery_failure(
10776 dying_client,
10777 first_channel,
10778 first_epoch,
10779 CloseReason::new(
10780 "module_to_client_delivery_failed",
10781 "client egress refused a module frame",
10782 ),
10783 crate::forwarding::UndeliveredFrame {
10784 module_id: None,
10785 sink: &dying_ctx.egress,
10786 },
10787 )
10788 .unwrap());
10789 assert!(!dying_ctx.egress.is_closed());
10790
10791 let ack = handler
10793 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10794 .await;
10795 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10796 if module_loop_error.is_some() {
10797 handler.cleanup_connection(module_connection).unwrap();
10801 }
10802 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10806 .await
10807 .ok()
10808 .flatten();
10809
10810 assert!(
10812 registry
10813 .get_module_by_connection(module_connection)
10814 .unwrap()
10815 .is_some(),
10816 "one client's closing connection ended the shared module connection: \
10817 {module_loop_error:?}"
10818 );
10819 let cotenant = ConnectionId::new(32);
10821 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10822 let (cotenant_task, cotenant_bind) = relay_route_open(
10823 &handler,
10824 cotenant,
10825 &cotenant_ctx.egress,
10826 &mut module_rx,
10827 102,
10828 "aft",
10829 "closing-cotenant",
10830 )
10831 .await;
10832 handler
10833 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10834 .await
10835 .unwrap();
10836 assert!(cotenant_task.await.unwrap().is_empty());
10837 let (cotenant_channel, cotenant_epoch) =
10838 published_route(&cotenant_rx.recv().await.unwrap());
10839 assert!(matches!(
10840 forwarding
10841 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10842 .unwrap(),
10843 DataRoute::Client(DataRouteState::Bound(_))
10844 ));
10845
10846 let goodbye = post_ack_module_frame
10849 .expect("module receives a GOODBYE for the abandoned route channel");
10850 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10851 assert_eq!(goodbye.header.channel, abandoned_channel);
10852 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10853
10854 assert!(dying_rx.try_recv().is_err());
10858 let second_response = second_task.await.unwrap();
10859 assert_eq!(second_response.len(), 1);
10860 assert_eq!(
10861 parse_error(&second_response[0])["code"],
10862 "target_unavailable"
10863 );
10864 }
10865
10866 async fn catalog_snapshot(
10869 handler: &ControlHandler,
10870 client_ctx: &RouteCtx,
10871 corr: u64,
10872 ) -> (u64, Vec<(String, bool)>) {
10873 let body =
10874 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10875 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10876 let reply = handler
10877 .handle_control_frame(client_ctx, request)
10878 .await
10879 .unwrap()
10880 .pop()
10881 .unwrap();
10882 let ClientControlResponse::CatalogList {
10883 generation,
10884 modules,
10885 ..
10886 } = serde_json::from_slice(&reply.body).unwrap()
10887 else {
10888 panic!("not a catalog.list reply");
10889 };
10890 (
10891 generation,
10892 modules
10893 .into_iter()
10894 .map(|entry| (entry.module_id, entry.ready))
10895 .collect(),
10896 )
10897 }
10898
10899 fn hello_held_before_forwarding() -> (
10905 ControlHandler,
10906 Arc<std::sync::Barrier>,
10907 std::thread::JoinHandle<Vec<Frame>>,
10908 RouteCtx,
10909 mpsc::Receiver<crate::router::OutboundFrame>,
10910 ) {
10911 let hold = Arc::new(std::sync::Barrier::new(2));
10912 let handler = ControlHandler::with_forwarding(
10913 Arc::new(Registry::default()),
10914 Arc::new(ForwardingTable::default()),
10915 )
10916 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10917 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10918 let hello = {
10919 let handler = handler.clone();
10920 let module_ctx = module_ctx.clone();
10921 std::thread::spawn(move || {
10922 handler
10923 .handle_hello(
10924 module_ctx.connection_id,
10925 Some(module_ctx.egress.clone()),
10926 hello_frame("aft", PROTOCOL_VERSION, 7),
10927 )
10928 .unwrap()
10929 })
10930 };
10931 (handler, hold, hello, module_ctx, module_rx)
10932 }
10933
10934 #[tokio::test]
10943 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10944 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10945 let client = ConnectionId::new(41);
10946 let (client_ctx, mut client_rx) = route_ctx(client);
10947
10948 hold.wait();
10949 assert!(handler.registry.get_module("aft").unwrap().is_some());
10951 assert!(!handler
10952 .forwarding
10953 .has_live_module_connection("aft")
10954 .unwrap());
10955 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10956 assert!(
10957 listed.is_empty(),
10958 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10959 );
10960 let refused = handler
10961 .handle_control_frame(
10962 &client_ctx,
10963 route_open_frame(301, "aft", unique_project_root("listed-window")),
10964 )
10965 .await
10966 .unwrap();
10967 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10968 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10969 let duplicate = handler
10970 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10971 .await
10972 .unwrap();
10973 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10974
10975 hold.wait();
10976 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10977 let ack = module_rx.try_recv().unwrap().frame;
10978 assert_eq!(ack.header.ty, FrameType::HelloAck);
10979 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10980 assert_eq!(listed, vec![("aft".to_string(), true)]);
10981
10982 let (open_task, bind) = relay_route_open(
10983 &handler,
10984 client,
10985 &client_ctx.egress,
10986 &mut module_rx,
10987 303,
10988 "aft",
10989 "listed-after",
10990 )
10991 .await;
10992 assert!(handler
10993 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10994 .await
10995 .unwrap()
10996 .is_empty());
10997 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10998 published_route(&client_rx.try_recv().unwrap().frame);
10999 }
11000
11001 #[tokio::test]
11008 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11009 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11010 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11011 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11012
11013 hold.wait();
11014 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11015 assert!(listed.is_empty());
11016 assert_eq!(
11017 during, before,
11018 "the generation moved while the module was not listed"
11019 );
11020
11021 hold.wait();
11022 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11023 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11024 assert_eq!(listed, vec![("aft".to_string(), true)]);
11025 assert_eq!(
11026 after,
11027 before + 1,
11028 "the module appeared without exactly one generation bump"
11029 );
11030 }
11031
11032 #[tokio::test]
11037 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11038 let registry = Arc::new(Registry::default());
11039 let handler = ControlHandler::with_forwarding(
11040 Arc::clone(®istry),
11041 Arc::new(ForwardingTable::default()),
11042 );
11043 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11044 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11045 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11046 drop(module_rx);
11047
11048 let replies = handler
11049 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11050 .await
11051 .unwrap();
11052 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11053 assert!(registry.get_module("aft").unwrap().is_none());
11054 assert!(registry
11055 .get_module_by_connection(module_ctx.connection_id)
11056 .unwrap()
11057 .is_none());
11058 assert_eq!(
11059 catalog_snapshot(&handler, &client_ctx, 301).await,
11060 (before, Vec::new())
11061 );
11062 }
11063
11064 async fn route_open_held_after_bind_send() -> (
11068 ControlHandler,
11069 Arc<tokio::sync::Barrier>,
11070 RouteCtx,
11071 mpsc::Receiver<crate::router::OutboundFrame>,
11072 ) {
11073 let hold = Arc::new(tokio::sync::Barrier::new(2));
11074 let handler = ControlHandler::with_forwarding(
11075 Arc::new(Registry::default()),
11076 Arc::new(ForwardingTable::default()),
11077 )
11078 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11079 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11080 hello_via_sink(
11081 &handler,
11082 &module_ctx,
11083 &mut module_rx,
11084 hello_frame("aft", PROTOCOL_VERSION, 7),
11085 )
11086 .await;
11087 (handler, hold, module_ctx, module_rx)
11088 }
11089
11090 #[tokio::test]
11097 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11098 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11099 let client = ConnectionId::new(41);
11100 let (client_ctx, mut client_rx) = route_ctx(client);
11101
11102 let (open_task, bind) = relay_route_open(
11103 &handler,
11104 client,
11105 &client_ctx.egress,
11106 &mut module_rx,
11107 200,
11108 "aft",
11109 "acked-before-mark",
11110 )
11111 .await;
11112 let (module_channel, module_epoch) = route_bind_channel(&bind);
11113 hold.wait().await;
11115
11116 assert!(handler
11118 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11119 .await
11120 .unwrap()
11121 .is_empty());
11122 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11123
11124 hold.wait().await;
11125 assert!(
11126 open_task.await.unwrap().is_empty(),
11127 "route.open answers accepted"
11128 );
11129
11130 match module_rx.try_recv() {
11133 Err(mpsc::error::TryRecvError::Empty) => {}
11134 Ok(outbound) => panic!(
11135 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11136 outbound.frame.header.ty,
11137 outbound.frame.header.channel,
11138 outbound.frame.header.epoch
11139 ),
11140 Err(err) => panic!("module egress closed: {err}"),
11141 }
11142
11143 let router = Router::with_control_handler(Arc::new(handler.clone()));
11146 let request = Frame::build(
11147 FrameType::Request,
11148 Flags::new(true, Priority::Interactive, false),
11149 client_channel,
11150 client_epoch,
11151 201,
11152 b"first request".to_vec(),
11153 )
11154 .unwrap();
11155 router
11156 .route_for_connection(&client_ctx, request)
11157 .await
11158 .unwrap();
11159 let delivered = module_rx.try_recv().unwrap().frame;
11160 assert_eq!(delivered.header.ty, FrameType::Request);
11161 assert_eq!(
11162 (delivered.header.channel, delivered.header.epoch),
11163 (module_channel, module_epoch)
11164 );
11165 assert_eq!(delivered.body, b"first request");
11166 }
11167
11168 #[tokio::test]
11174 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11175 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11176 let client = ConnectionId::new(42);
11177 let (client_ctx, _client_rx) = route_ctx(client);
11178
11179 let (open_task, bind) = relay_route_open(
11180 &handler,
11181 client,
11182 &client_ctx.egress,
11183 &mut module_rx,
11184 300,
11185 "aft",
11186 "released-before-mark",
11187 )
11188 .await;
11189 let (module_channel, module_epoch) = route_bind_channel(&bind);
11190 hold.wait().await;
11191
11192 handler.cleanup_connection(client).unwrap();
11193 assert!(
11194 module_rx.try_recv().is_err(),
11195 "client teardown must not GOODBYE a bind route.open is still queueing"
11196 );
11197
11198 hold.wait().await;
11199 let response = open_task.await.unwrap();
11200 assert_eq!(response.len(), 1);
11201 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11202
11203 let goodbye = module_rx
11204 .try_recv()
11205 .expect("module is told to drop the binding the daemon abandoned")
11206 .frame;
11207 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11208 assert_eq!(
11209 (goodbye.header.channel, goodbye.header.epoch),
11210 (module_channel, module_epoch)
11211 );
11212 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11213 }
11214
11215 #[tokio::test]
11220 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11221 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11222 let client = ConnectionId::new(43);
11223 let (client_ctx, client_rx) = route_ctx(client);
11224
11225 let (open_task, bind) = relay_route_open(
11226 &handler,
11227 client,
11228 &client_ctx.egress,
11229 &mut module_rx,
11230 400,
11231 "aft",
11232 "gone-client-before-mark",
11233 )
11234 .await;
11235 let (module_channel, module_epoch) = route_bind_channel(&bind);
11236 hold.wait().await;
11237
11238 drop(client_rx);
11239 assert!(client_ctx.egress.is_closed());
11240 handler
11241 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11242 .await
11243 .unwrap();
11244
11245 hold.wait().await;
11246 let response = open_task.await.unwrap();
11247 assert_eq!(response.len(), 1);
11248 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11249
11250 let goodbye = module_rx
11251 .try_recv()
11252 .expect("module is told to drop the binding it acked")
11253 .frame;
11254 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11255 assert_eq!(
11256 (goodbye.header.channel, goodbye.header.epoch),
11257 (module_channel, module_epoch)
11258 );
11259 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11260 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11261 }
11262
11263 #[test]
11270 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11271 let handler = ControlHandler::default();
11272 let module_connection = ConnectionId::new(30);
11273 let client_connection = ConnectionId::new(31);
11274
11275 handler
11276 .refuse_to_end_module_connection_for_a_client(
11277 module_connection,
11278 77,
11279 ForwardingError::ConnectionClosing {
11280 connection_id: client_connection,
11281 },
11282 )
11283 .expect("a closing client must never end the module connection");
11284
11285 assert!(matches!(
11286 handler.refuse_to_end_module_connection_for_a_client(
11287 module_connection,
11288 78,
11289 ForwardingError::ConnectionClosing {
11290 connection_id: module_connection,
11291 },
11292 ),
11293 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11294 connection_id
11295 })) if connection_id == module_connection
11296 ));
11297 assert!(matches!(
11298 handler.refuse_to_end_module_connection_for_a_client(
11299 module_connection,
11300 79,
11301 ForwardingError::Poisoned,
11302 ),
11303 Err(RouterError::Forwarding(ForwardingError::Poisoned))
11304 ));
11305 assert!(matches!(
11306 handler.refuse_to_end_module_connection_for_a_client(
11307 module_connection,
11308 80,
11309 ForwardingError::StaleModuleEndpoint,
11310 ),
11311 Err(RouterError::Forwarding(
11312 ForwardingError::StaleModuleEndpoint
11313 ))
11314 ));
11315 }
11316
11317 #[tokio::test]
11330 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11331 let registry = Arc::new(Registry::default());
11332 let forwarding = Arc::new(ForwardingTable::default());
11333 let supervisor = SupervisorHandle::new();
11334 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11335 let handler =
11336 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11337 .with_supervisor(supervisor);
11338
11339 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11340 hello_via_sink(
11341 &handler,
11342 &target_ctx,
11343 &mut target_rx,
11344 hello_frame("target", PROTOCOL_VERSION, 1),
11345 )
11346 .await;
11347
11348 let wrong_nonce = handler
11352 .handle_control_frame(
11353 &route_ctx(ConnectionId::new(91)).0,
11354 route_open_frame_with_admission_facts(
11355 20,
11356 "target",
11357 unique_project_root("admission-facts"),
11358 Some(subc_control::ConsumerIdentity {
11359 module_id: "fed".to_string(),
11360 launch_nonce: "not-the-real-nonce".to_string(),
11361 }),
11362 None,
11363 ),
11364 )
11365 .await
11366 .unwrap();
11367 assert_eq!(
11368 parse_error(&wrong_nonce[0])["code"],
11369 "bad_consumer_identity",
11370 "a mismatched launch nonce must be refused, not stamped Reserved"
11371 );
11372
11373 let never_spawned = handler
11377 .handle_control_frame(
11378 &route_ctx(ConnectionId::new(92)).0,
11379 route_open_frame_with_admission_facts(
11380 21,
11381 "target",
11382 unique_project_root("admission-facts"),
11383 Some(subc_control::ConsumerIdentity {
11384 module_id: "never-spawned".to_string(),
11385 launch_nonce: "any-nonce".to_string(),
11386 }),
11387 None,
11388 ),
11389 )
11390 .await
11391 .unwrap();
11392 assert_eq!(
11393 parse_error(&never_spawned[0])["code"],
11394 "bad_consumer_identity",
11395 "an unspawned module_id must be refused rather than accepted for lack of a record"
11396 );
11397 }
11398
11399 #[tokio::test]
11422 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11423 let registry = Arc::new(Registry::default());
11424 let forwarding = Arc::new(ForwardingTable::default());
11425 let supervisor = SupervisorHandle::new();
11426 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11427 let handler =
11428 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11429 .with_supervisor(supervisor);
11430
11431 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11432 hello_via_sink(
11433 &handler,
11434 &target_ctx,
11435 &mut target_rx,
11436 hello_frame("target", PROTOCOL_VERSION, 1),
11437 )
11438 .await;
11439
11440 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11441 let route_handler = handler.clone();
11442 let route_task = tokio::spawn(async move {
11443 route_handler
11444 .handle_control_frame(
11445 &client_ctx,
11446 route_open_frame_with_admission_facts(
11447 30,
11448 "target",
11449 unique_project_root("admission-facts"),
11450 Some(subc_control::ConsumerIdentity {
11451 module_id: "fed".to_string(),
11452 launch_nonce: "fed-nonce".to_string(),
11453 }),
11454 None,
11455 ),
11456 )
11457 .await
11458 .unwrap()
11459 });
11460
11461 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11468 .await
11469 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11470 .expect("module control channel closed before route.bind");
11471 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11472 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11473 panic!("expected route.bind")
11474 };
11475 assert_eq!(
11476 principal,
11477 Some(Principal::Reserved {
11478 module_id: "fed".to_string()
11479 }),
11480 "a correctly attested consumer must be stamped Reserved for its own id"
11481 );
11482
11483 handler
11484 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11485 .await
11486 .unwrap();
11487 assert!(route_task.await.unwrap().is_empty());
11488 assert!(
11489 matches!(
11490 serde_json::from_slice::<ClientControlResponse>(
11491 &client_rx.recv().await.unwrap().body
11492 )
11493 .unwrap(),
11494 ClientControlResponse::RouteOpen { .. }
11495 ),
11496 "the route must actually open, not merely avoid an error"
11497 );
11498 }
11499
11500 #[tokio::test(start_paused = true)]
11501 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11502 let registry = Arc::new(Registry::default());
11503 let forwarding = Arc::new(ForwardingTable::default());
11504 let supervisor = SupervisorHandle::new();
11505 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11506 let handler =
11507 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11508 .with_supervisor(supervisor);
11509
11510 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11511 hello_via_sink(
11512 &handler,
11513 &target_ctx,
11514 &mut target_rx,
11515 hello_frame("target", PROTOCOL_VERSION, 1),
11516 )
11517 .await;
11518
11519 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11520 let direct_handler = handler.clone();
11521 let direct_open = tokio::spawn(async move {
11522 direct_handler
11523 .handle_control_frame(
11524 &direct_ctx,
11525 route_open_frame(2, "target", unique_project_root("route-census-direct")),
11526 )
11527 .await
11528 .unwrap()
11529 });
11530 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11531 .await
11532 .expect("no direct route.bind within 5s")
11533 .expect("target control channel closed before direct route.bind");
11534 handler
11535 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11536 .await
11537 .unwrap();
11538 assert!(direct_open.await.unwrap().is_empty());
11539 let _ = direct_rx.recv().await.unwrap();
11540
11541 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11542 let reserved_handler = handler.clone();
11543 let reserved_open = tokio::spawn(async move {
11544 reserved_handler
11545 .handle_control_frame(
11546 &reserved_ctx,
11547 route_open_frame_with_admission_facts(
11548 3,
11549 "target",
11550 unique_project_root("admission-facts"),
11551 Some(ConsumerIdentity {
11552 module_id: "fed".to_string(),
11553 launch_nonce: "fed-nonce".to_string(),
11554 }),
11555 None,
11556 ),
11557 )
11558 .await
11559 .unwrap()
11560 });
11561 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11562 .await
11563 .expect("no reserved route.bind within 5s")
11564 .expect("target control channel closed before reserved route.bind");
11565 handler
11566 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11567 .await
11568 .unwrap();
11569 assert!(reserved_open.await.unwrap().is_empty());
11570 let _ = reserved_rx.recv().await.unwrap();
11571
11572 forwarding
11573 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11574 .unwrap();
11575 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11576 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11577 module_id: Some("target".to_string()),
11578 })
11579 .unwrap();
11580 let census_frame =
11581 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11582 let response = handler
11583 .handle_control_frame(&census_ctx, census_frame)
11584 .await
11585 .unwrap()
11586 .pop()
11587 .unwrap();
11588 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11589 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11590 assert!(matches!(
11591 decoded,
11592 ClientControlResponse::SupervisorRoutes { .. }
11593 ));
11594 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11595 assert_eq!(routes.len(), 2);
11596 assert!(routes.iter().all(|route| route["draining"] == true));
11597 assert!(
11600 routes.iter().all(|route| route["drain_reason"] == "reload"),
11601 "draining routes must name the drain's reason: {routes:?}"
11602 );
11603 assert!(routes.iter().any(|route| {
11604 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11605 }));
11606 assert!(routes.iter().any(|route| {
11607 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11608 }));
11609
11610 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11611 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11612 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11613 std::fs::write(
11614 &golden_path,
11615 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11616 )
11617 .unwrap();
11618 }
11619 let expected: Value =
11620 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11621 assert_eq!(actual, expected);
11622 }
11623
11624 async fn query_live_roots(
11625 handler: &ControlHandler,
11626 module_ctx: &RouteCtx,
11627 ) -> ModuleControlResponseToModule {
11628 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11629 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11630 let response = handler
11631 .handle_control_frame(module_ctx, frame)
11632 .await
11633 .unwrap()
11634 .pop()
11635 .unwrap();
11636 serde_json::from_slice(&response.body).unwrap()
11637 }
11638
11639 #[tokio::test(start_paused = true)]
11640 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11641 let registry = Arc::new(Registry::default());
11642 let forwarding = Arc::new(ForwardingTable::default());
11643 let handler = ControlHandler::with_forwarding(registry, forwarding);
11644 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11645 hello_via_sink(
11646 &handler,
11647 &target_ctx,
11648 &mut target_rx,
11649 hello_frame("target", PROTOCOL_VERSION, 1),
11650 )
11651 .await;
11652 let root = unique_project_root("live-roots-known");
11653 let path = ProjectRootId::from_path_allowing_missing(root.path())
11654 .unwrap()
11655 .as_path()
11656 .to_path_buf();
11657 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11658 let open_handler = handler.clone();
11659 let opened = tokio::spawn(async move {
11660 open_handler
11661 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11662 .await
11663 .unwrap()
11664 });
11665 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11666 .await
11667 .unwrap()
11668 .unwrap();
11669 handler
11670 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11671 .await
11672 .unwrap();
11673 assert!(opened.await.unwrap().is_empty());
11674 let _ = client_rx.recv().await.unwrap();
11675
11676 let root = unique_project_root("live-roots-pending");
11677 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11678 .unwrap()
11679 .as_path()
11680 .to_path_buf();
11681 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11682 let open_handler = handler.clone();
11683 let pending = tokio::spawn(async move {
11684 open_handler
11685 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11686 .await
11687 .unwrap()
11688 });
11689 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11690 .await
11691 .unwrap()
11692 .unwrap();
11693 let actual = query_live_roots(&handler, &target_ctx).await;
11694 let ModuleControlResponseToModule::LiveRoots {
11695 roots,
11696 unknown_root_bindings,
11697 total_bindings,
11698 } = actual
11699 else {
11700 panic!("expected live roots")
11701 };
11702 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11703 assert_eq!(unknown_root_bindings, 0);
11704 assert_eq!(
11705 roots.len(),
11706 2,
11707 "root-known arm must retain each canonical root"
11708 );
11709 assert_eq!(
11710 total_bindings,
11711 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11712 );
11713 let counts = roots
11714 .iter()
11715 .map(|root| (root.project_root.clone(), root.bound, root.pending))
11716 .collect::<Vec<_>>();
11717 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11718 expected.sort_by(|a, b| a.0.cmp(&b.0));
11719 assert_eq!(
11720 counts, expected,
11721 "roots must sort by path and count pending separately"
11722 );
11723 handler
11724 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11725 .await
11726 .unwrap();
11727 assert!(pending.await.unwrap().is_empty());
11728 }
11729
11730 #[tokio::test(start_paused = true)]
11731 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11732 let forwarding = Arc::new(ForwardingTable::default());
11733 let handler =
11734 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11735 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11736 hello_via_sink(
11737 &handler,
11738 &target_ctx,
11739 &mut target_rx,
11740 hello_frame("target", PROTOCOL_VERSION, 1),
11741 )
11742 .await;
11743 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11744 let pending = forwarding
11745 .begin_route_bind_relay_for_test(
11746 client_ctx.connection_id,
11747 client_ctx.egress.clone(),
11748 2,
11749 "target",
11750 )
11751 .unwrap();
11752 forwarding
11753 .complete_pending_relay(
11754 target_ctx.connection_id,
11755 pending.corr,
11756 RouteBindRelayOutcome::Accepted,
11757 )
11758 .unwrap();
11759 let actual = query_live_roots(&handler, &target_ctx).await;
11760 let ModuleControlResponseToModule::LiveRoots {
11761 roots,
11762 unknown_root_bindings,
11763 total_bindings,
11764 } = actual
11765 else {
11766 panic!("expected live roots")
11767 };
11768 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11769 assert_eq!(
11770 unknown_root_bindings, 1,
11771 "unknown-root arm must not read as no bindings"
11772 );
11773 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11774 assert_eq!(
11775 total_bindings,
11776 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11777 );
11778 }
11779
11780 #[tokio::test(start_paused = true)]
11788 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11789 let forwarding = Arc::new(ForwardingTable::default());
11790 let handler =
11791 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11792 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11793 let replies = handler
11794 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11795 .await
11796 .unwrap();
11797 let queued_by_hello = module_rx.len();
11798
11799 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11800 let open_handler = handler.clone();
11801 let open = tokio::spawn(async move {
11802 open_handler
11803 .handle_control_frame(
11804 &client_ctx,
11805 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11806 )
11807 .await
11808 .unwrap()
11809 });
11810 let mut spins = 0;
11812 while module_rx.len() == queued_by_hello {
11813 spins += 1;
11814 assert!(spins < 10_000, "route.open never queued a route.bind");
11815 tokio::task::yield_now().await;
11816 }
11817
11818 for reply in replies {
11820 module_ctx.egress.send(reply).await.unwrap();
11821 }
11822
11823 let first = module_rx.recv().await.unwrap().frame;
11824 assert_eq!(
11825 first.header.ty,
11826 FrameType::HelloAck,
11827 "the first frame a registering module reads must be its HELLO_ACK"
11828 );
11829 assert_eq!(first.header.corr, 7);
11830 let second = module_rx.recv().await.unwrap().frame;
11831 assert_eq!(second.header.ty, FrameType::Request);
11832 assert!(
11833 matches!(
11834 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11835 ModuleControlRequest::RouteBind { .. }
11836 ),
11837 "the route.bind follows the ack"
11838 );
11839 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11840
11841 handler
11842 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11843 .await
11844 .unwrap();
11845 assert!(open.await.unwrap().is_empty());
11846 let _ = client_rx.recv().await.unwrap();
11847 }
11848
11849 #[tokio::test(start_paused = true)]
11850 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11851 let handler = ControlHandler::with_forwarding(
11852 Arc::new(Registry::default()),
11853 Arc::new(ForwardingTable::default()),
11854 );
11855 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11856 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11857 hello_via_sink(
11858 &handler,
11859 &first_ctx,
11860 &mut first_rx,
11861 hello_frame("first", PROTOCOL_VERSION, 1),
11862 )
11863 .await;
11864 hello_via_sink(
11865 &handler,
11866 &second_ctx,
11867 &mut second_rx,
11868 hello_frame("second", PROTOCOL_VERSION, 2),
11869 )
11870 .await;
11871 let root = unique_project_root("second-only");
11872 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11873 let cloned = handler.clone();
11874 let open = tokio::spawn(async move {
11875 cloned
11876 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11877 .await
11878 .unwrap()
11879 });
11880 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11881 .await
11882 .unwrap()
11883 .unwrap();
11884 let first = query_live_roots(&handler, &first_ctx).await;
11885 let second = query_live_roots(&handler, &second_ctx).await;
11886 assert!(
11887 matches!(
11888 first,
11889 ModuleControlResponseToModule::LiveRoots {
11890 total_bindings: 0,
11891 ..
11892 }
11893 ),
11894 "cross-module scope must not expose another module's roots"
11895 );
11896 assert!(
11897 matches!(
11898 second,
11899 ModuleControlResponseToModule::LiveRoots {
11900 total_bindings: 1,
11901 ..
11902 }
11903 ),
11904 "second module must see its pending route"
11905 );
11906 handler
11907 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11908 .await
11909 .unwrap();
11910 assert!(open.await.unwrap().is_empty());
11911 }
11912
11913 #[tokio::test(start_paused = true)]
11914 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11915 let handler = ControlHandler::with_forwarding(
11916 Arc::new(Registry::default()),
11917 Arc::new(ForwardingTable::default()),
11918 );
11919 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11920 hello_via_sink(
11921 &handler,
11922 &target_ctx,
11923 &mut target_rx,
11924 hello_frame("target", PROTOCOL_VERSION, 1),
11925 )
11926 .await;
11927 let actual = query_live_roots(&handler, &target_ctx).await;
11928 let ModuleControlResponseToModule::LiveRoots {
11929 roots,
11930 unknown_root_bindings,
11931 total_bindings,
11932 } = actual
11933 else {
11934 panic!("expected live roots")
11935 };
11936 assert!(roots.is_empty());
11937 assert_eq!(unknown_root_bindings, 0);
11938 assert_eq!(total_bindings, 0);
11939 assert_eq!(
11940 total_bindings,
11941 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11942 );
11943 }
11944
11945 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11952 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11953 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11954 let text = std::fs::read_to_string(&path)
11955 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11956 let vectors: Vec<(String, Value)> = text
11957 .lines()
11958 .filter(|line| !line.trim().is_empty())
11959 .map(|line| {
11960 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11961 let id = entry["corpus_id"]
11962 .as_str()
11963 .expect("every vector carries a corpus_id")
11964 .to_string();
11965 (id, entry["package"].clone())
11966 })
11967 .collect();
11968 assert_eq!(
11972 vectors.len(),
11973 3,
11974 "vendored fed corpus changed size; re-sync from subc-federation"
11975 );
11976
11977 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11992 let richest = vectors
11993 .iter()
11994 .filter_map(|(_, package)| package.as_object())
11995 .map(|object| {
11996 object
11997 .keys()
11998 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11999 .count()
12000 })
12001 .max()
12002 .unwrap_or(0);
12003 assert!(
12004 richest >= 2,
12005 "vendored corpus no longer carries a package with unmodelled fields, \
12006 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12007 );
12008
12009 vectors
12010 }
12011
12012 #[tokio::test]
12020 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12021 for (corpus_id, package) in fed_admission_facts_vectors() {
12022 let registry = Arc::new(Registry::default());
12023 let forwarding = Arc::new(ForwardingTable::default());
12024 let supervisor = SupervisorHandle::new();
12025 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12026 let handler =
12027 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12028 .with_supervisor(supervisor)
12029 .with_admission_facts_config(
12030 Some("fed".to_string()),
12031 Some(vec!["target".to_string()]),
12032 );
12033
12034 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12035 hello_via_sink(
12036 &handler,
12037 &target_ctx,
12038 &mut target_rx,
12039 hello_frame("target", PROTOCOL_VERSION, 1),
12040 )
12041 .await;
12042
12043 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12044 let route_handler = handler.clone();
12045 let expected = package.clone();
12046 let route_task = tokio::spawn(async move {
12047 route_handler
12048 .handle_control_frame(
12049 &client_ctx,
12050 route_open_frame_with_admission_facts(
12051 20,
12052 "target",
12053 unique_project_root("admission-facts"),
12054 Some(subc_control::ConsumerIdentity {
12055 module_id: "fed".to_string(),
12056 launch_nonce: "fed-nonce".to_string(),
12057 }),
12058 Some(package),
12059 ),
12060 )
12061 .await
12062 .unwrap()
12063 });
12064
12065 let bind_frame = target_rx.recv().await.unwrap();
12066 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12067 let ModuleControlRequest::RouteBind {
12068 admission_facts, ..
12069 } = bind
12070 else {
12071 panic!("{corpus_id}: expected route.bind")
12072 };
12073 assert_eq!(
12074 admission_facts,
12075 Some(expected),
12076 "{corpus_id}: relay must not add, drop or reshape any field"
12077 );
12078
12079 handler
12080 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12081 .await
12082 .unwrap();
12083 route_task.await.unwrap();
12084 }
12085 }
12086
12087 #[tokio::test]
12088 async fn route_bind_never_relays_the_openers_build_class() {
12089 let registry = Arc::new(Registry::default());
12090 let forwarding = Arc::new(ForwardingTable::default());
12091 let handler =
12092 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12093 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12094 hello_via_sink(
12095 &handler,
12096 &target_ctx,
12097 &mut target_rx,
12098 hello_frame("target", PROTOCOL_VERSION, 1),
12099 )
12100 .await;
12101
12102 let root = unique_project_root("build-class");
12103 let mut identity = BindIdentity::new(
12104 root.path().to_path_buf(),
12105 "unit".to_string(),
12106 "s".to_string(),
12107 );
12108 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12109 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12110 target: RouteTarget::ToolProvider {
12111 module_id: "target".to_string(),
12112 },
12113 identity,
12114 consumer_identity: None,
12115 consumer_capabilities: None,
12116 role_versions: None,
12117 admission_facts: None,
12118 scope: None,
12119 })
12120 .unwrap();
12121 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12122
12123 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12124 let route_handler = handler.clone();
12125 let route_task = tokio::spawn(async move {
12126 route_handler
12127 .handle_control_frame(&client_ctx, open)
12128 .await
12129 .unwrap()
12130 });
12131 let bind_frame = target_rx.recv().await.unwrap();
12132 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12133 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12134 panic!("expected route.bind")
12135 };
12136 assert_eq!(
12137 identity.build_class, None,
12138 "route.bind carried the build class"
12139 );
12140 assert!(
12141 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12142 "the route.bind bytes still name build_class"
12143 );
12144 handler
12145 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12146 .await
12147 .unwrap();
12148 route_task.await.unwrap();
12149 }
12150
12151 #[tokio::test]
12152 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12153 let registry = Arc::new(Registry::default());
12154 let forwarding = Arc::new(ForwardingTable::default());
12155 let supervisor = SupervisorHandle::new();
12156 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12157 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12158 let handler =
12159 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12160 .with_supervisor(supervisor)
12161 .with_admission_facts_config(
12162 Some("fed".to_string()),
12163 Some(vec!["target".to_string()]),
12164 );
12165
12166 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12167 hello_via_sink(
12168 &handler,
12169 &target_ctx,
12170 &mut target_rx,
12171 hello_frame("target", PROTOCOL_VERSION, 1),
12172 )
12173 .await;
12174 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12175 hello_via_sink(
12176 &handler,
12177 &other_ctx,
12178 &mut other_rx,
12179 hello_frame("other", PROTOCOL_VERSION, 2),
12180 )
12181 .await;
12182
12183 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12184 let expected_facts = facts.clone();
12185 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12186 let route_handler = handler.clone();
12187 let route_task = tokio::spawn(async move {
12188 route_handler
12189 .handle_control_frame(
12190 &client_ctx,
12191 route_open_frame_with_admission_facts(
12192 10,
12193 "target",
12194 unique_project_root("admission-facts"),
12195 Some(subc_control::ConsumerIdentity {
12196 module_id: "fed".to_string(),
12197 launch_nonce: "fed-nonce".to_string(),
12198 }),
12199 Some(facts.clone()),
12200 ),
12201 )
12202 .await
12203 .unwrap()
12204 });
12205 let bind_frame = target_rx.recv().await.unwrap();
12206 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12207 let ModuleControlRequest::RouteBind {
12208 admission_facts, ..
12209 } = bind
12210 else {
12211 panic!("expected route.bind")
12212 };
12213 assert_eq!(admission_facts, Some(expected_facts));
12214 handler
12215 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12216 .await
12217 .unwrap();
12218 assert!(route_task.await.unwrap().is_empty());
12219 assert!(matches!(
12220 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12221 .unwrap(),
12222 ClientControlResponse::RouteOpen { .. }
12223 ));
12224
12225 let direct = handler
12226 .handle_control_frame(
12227 &route_ctx(ConnectionId::new(73)).0,
12228 route_open_frame_with_admission_facts(
12229 11,
12230 "target",
12231 unique_project_root("admission-facts"),
12232 None,
12233 Some(json!({"x": 1})),
12234 ),
12235 )
12236 .await
12237 .unwrap();
12238 assert_eq!(
12239 parse_error(&direct[0])["code"],
12240 "admission_facts_not_permitted"
12241 );
12242
12243 let different_reserved = handler
12244 .handle_control_frame(
12245 &route_ctx(ConnectionId::new(77)).0,
12246 route_open_frame_with_admission_facts(
12247 15,
12248 "target",
12249 unique_project_root("admission-facts"),
12250 Some(subc_control::ConsumerIdentity {
12251 module_id: "other".to_string(),
12252 launch_nonce: "other-nonce".to_string(),
12253 }),
12254 Some(json!({"x": 1})),
12255 ),
12256 )
12257 .await
12258 .unwrap();
12259 assert_eq!(
12260 parse_error(&different_reserved[0])["code"],
12261 "admission_facts_not_permitted"
12262 );
12263
12264 let other_target = handler
12265 .handle_control_frame(
12266 &route_ctx(ConnectionId::new(74)).0,
12267 route_open_frame_with_admission_facts(
12268 12,
12269 "other",
12270 unique_project_root("admission-facts"),
12271 Some(subc_control::ConsumerIdentity {
12272 module_id: "fed".to_string(),
12273 launch_nonce: "fed-nonce".to_string(),
12274 }),
12275 Some(json!({"x": 1})),
12276 ),
12277 )
12278 .await
12279 .unwrap();
12280 assert_eq!(
12281 parse_error(&other_target[0])["code"],
12282 "admission_facts_target_not_allowed"
12283 );
12284
12285 let nonexistent = handler
12286 .handle_control_frame(
12287 &route_ctx(ConnectionId::new(75)).0,
12288 route_open_frame_with_admission_facts(
12289 13,
12290 "missing",
12291 unique_project_root("admission-facts"),
12292 None,
12293 Some(json!({"x": 1})),
12294 ),
12295 )
12296 .await
12297 .unwrap();
12298 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12299
12300 let described = handler
12301 .handle_control_frame(
12302 &route_ctx(ConnectionId::new(76)).0,
12303 Frame::build(
12304 FrameType::Request,
12305 control_flags(),
12306 0,
12307 0,
12308 14,
12309 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12310 )
12311 .unwrap(),
12312 )
12313 .await
12314 .unwrap();
12315 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12316 serde_json::from_slice(&described[0].body).unwrap()
12317 else {
12318 panic!("expected server.describe response")
12319 };
12320 assert!(capabilities
12321 .iter()
12322 .any(|cap| cap == "admission_facts_relay_v1"));
12323 }
12324
12325 #[tokio::test]
12326 async fn admission_facts_without_configured_carrier_are_rejected() {
12327 let registry = Arc::new(Registry::default());
12328 let forwarding = Arc::new(ForwardingTable::default());
12329 let handler = ControlHandler::with_forwarding(registry, forwarding);
12330 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12331 hello_via_sink(
12332 &handler,
12333 &target_ctx,
12334 &mut target_rx,
12335 hello_frame("target", PROTOCOL_VERSION, 1),
12336 )
12337 .await;
12338
12339 let responses = handler
12340 .handle_control_frame(
12341 &route_ctx(ConnectionId::new(79)).0,
12342 route_open_frame_with_admission_facts(
12343 16,
12344 "target",
12345 unique_project_root("admission-facts"),
12346 None,
12347 Some(json!({"x": 1})),
12348 ),
12349 )
12350 .await
12351 .unwrap();
12352 assert_eq!(
12353 parse_error(&responses[0])["code"],
12354 "admission_facts_not_permitted"
12355 );
12356 }
12357
12358 #[tokio::test]
12359 async fn route_open_relays_consumer_capabilities_verbatim() {
12360 let registry = Arc::new(Registry::default());
12361 let forwarding = Arc::new(ForwardingTable::default());
12362 let handler =
12363 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12364 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12365 hello_via_sink(
12366 &handler,
12367 &module_ctx,
12368 &mut module_rx,
12369 hello_frame("aft", PROTOCOL_VERSION, 7),
12370 )
12371 .await;
12372
12373 let expected = vec!["elicitation".to_string(), "roots".to_string()];
12374 let expected_for_request = expected.clone();
12375 let project_root = unique_project_root("consumer-capabilities-present");
12376 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12377 let route_handler = handler.clone();
12378 let route_task = tokio::spawn(async move {
12379 route_handler
12380 .handle_control_frame(
12381 &client_ctx,
12382 route_open_frame_with_consumer_capabilities(
12383 401,
12384 "aft",
12385 project_root,
12386 Some(expected_for_request),
12387 ),
12388 )
12389 .await
12390 .unwrap()
12391 });
12392 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12393 .await
12394 .unwrap()
12395 .unwrap();
12396 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12397 let ModuleControlRequest::RouteBind {
12398 consumer_capabilities,
12399 ..
12400 } = bind
12401 else {
12402 panic!("expected route.bind request, got {bind:?}");
12403 };
12404 assert_eq!(consumer_capabilities, Some(expected.clone()));
12405
12406 handler
12407 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12408 .await
12409 .unwrap();
12410 let route_response = route_task.await.unwrap();
12411 assert!(route_response.is_empty());
12412 let published = client_rx.recv().await.unwrap();
12413 assert!(matches!(
12414 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12415 ClientControlResponse::RouteOpen { .. }
12416 ));
12417 }
12418
12419 #[tokio::test]
12420 async fn route_open_without_consumer_capabilities_relays_none() {
12421 let registry = Arc::new(Registry::default());
12422 let forwarding = Arc::new(ForwardingTable::default());
12423 let handler =
12424 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12425 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12426 hello_via_sink(
12427 &handler,
12428 &module_ctx,
12429 &mut module_rx,
12430 hello_frame("aft", PROTOCOL_VERSION, 7),
12431 )
12432 .await;
12433
12434 let project_root = unique_project_root("consumer-capabilities-absent");
12435 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12436 let route_handler = handler.clone();
12437 let route_task = tokio::spawn(async move {
12438 route_handler
12439 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12440 .await
12441 .unwrap()
12442 });
12443 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12444 .await
12445 .unwrap()
12446 .unwrap();
12447 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12448 let ModuleControlRequest::RouteBind {
12449 consumer_capabilities,
12450 ..
12451 } = bind
12452 else {
12453 panic!("expected route.bind request, got {bind:?}");
12454 };
12455 assert_eq!(consumer_capabilities, None);
12456
12457 handler
12458 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12459 .await
12460 .unwrap();
12461 let route_response = route_task.await.unwrap();
12462 assert!(route_response.is_empty());
12463 let published = client_rx.recv().await.unwrap();
12464 assert!(matches!(
12465 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12466 ClientControlResponse::RouteOpen { .. }
12467 ));
12468 }
12469
12470 async fn bind_role_versions_for(
12474 sent: Option<BTreeMap<String, String>>,
12475 connection: u64,
12476 ) -> Option<BTreeMap<String, String>> {
12477 let registry = Arc::new(Registry::default());
12478 let forwarding = Arc::new(ForwardingTable::default());
12479 let handler =
12480 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12481 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12482 hello_via_sink(
12483 &handler,
12484 &module_ctx,
12485 &mut module_rx,
12486 hello_frame("aft", PROTOCOL_VERSION, 7),
12487 )
12488 .await;
12489 let project_root = unique_project_root("role-versions");
12490 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12491 let route_handler = handler.clone();
12492 let route_task = tokio::spawn(async move {
12493 route_handler
12494 .handle_control_frame(
12495 &client_ctx,
12496 route_open_frame_with_role_versions(403, "aft", project_root, sent),
12497 )
12498 .await
12499 .unwrap()
12500 });
12501 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12502 .await
12503 .expect("a well-formed route.open reaches the module as a bind")
12504 .unwrap();
12505 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12506 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12507 panic!("expected route.bind request, got {bind:?}");
12508 };
12509 handler
12510 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12511 .await
12512 .unwrap();
12513 assert!(route_task.await.unwrap().is_empty());
12514 let published = client_rx.recv().await.unwrap();
12515 assert!(matches!(
12516 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12517 ClientControlResponse::RouteOpen { .. }
12518 ));
12519 role_versions
12520 }
12521
12522 #[tokio::test]
12523 async fn route_open_relays_role_versions_verbatim() {
12524 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12525 assert_eq!(
12526 bind_role_versions_for(Some(sent.clone()), 141).await,
12527 Some(sent)
12528 );
12529 }
12530
12531 #[tokio::test]
12534 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12535 assert_eq!(bind_role_versions_for(None, 143).await, None);
12536 assert_eq!(
12537 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12538 None
12539 );
12540 }
12541
12542 #[tokio::test]
12543 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12544 let registry = Arc::new(Registry::default());
12545 let forwarding = Arc::new(ForwardingTable::default());
12546 let handler =
12547 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12548 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12549 hello_via_sink(
12550 &handler,
12551 &module_ctx,
12552 &mut module_rx,
12553 hello_frame("aft", PROTOCOL_VERSION, 7),
12554 )
12555 .await;
12556
12557 let nine: BTreeMap<String, String> = (0..9)
12558 .map(|index| (format!("role-{index}"), "v1".to_string()))
12559 .collect();
12560 for (label, malformed) in [
12561 (
12562 "invalid role name",
12563 role_versions(&[("Tool_Provider", "v1")]),
12564 ),
12565 ("invalid version", role_versions(&[("tool-provider", "v0")])),
12566 ("nine entries", nine),
12567 ] {
12568 let responses = tokio::time::timeout(
12571 Duration::from_secs(1),
12572 handler.handle_control_frame(
12573 &route_ctx(ConnectionId::new(148)).0,
12574 route_open_frame_with_role_versions(
12575 404,
12576 "aft",
12577 unique_project_root("role-versions-malformed"),
12578 Some(malformed),
12579 ),
12580 ),
12581 )
12582 .await
12583 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12584 .unwrap();
12585 assert_eq!(responses.len(), 1, "{label}");
12586 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12587 let error = parse_error(&responses[0]);
12588 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12589 assert_eq!(
12590 error["detail"]["field"], "role_versions",
12591 "{label}: {error}"
12592 );
12593 assert!(
12594 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12595 "{label}: a malformed declaration is terminal"
12596 );
12597 assert!(
12598 module_rx.try_recv().is_err(),
12599 "{label}: the module must never see a bind"
12600 );
12601 }
12602 }
12603
12604 #[tokio::test]
12608 async fn route_role_versions_capability_is_advertised() {
12609 let handler = ControlHandler::new(Arc::new(Registry::default()));
12610 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12611 let ack = hello_via_sink(
12612 &handler,
12613 &ctx,
12614 &mut rx,
12615 hello_frame("m", PROTOCOL_VERSION, 1),
12616 )
12617 .await;
12618 let ack = parse_ack(&ack);
12619 assert!(
12620 ack.subc_capabilities
12621 .iter()
12622 .any(|c| c == "route-role-versions/v1"),
12623 "{:?}",
12624 ack.subc_capabilities
12625 );
12626
12627 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12628 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12629 let reply = handler
12630 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12631 .await
12632 .unwrap()
12633 .pop()
12634 .unwrap();
12635 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12636 serde_json::from_slice(&reply.body).unwrap()
12637 else {
12638 panic!("not a server.describe reply");
12639 };
12640 assert!(
12641 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12642 "{capabilities:?}"
12643 );
12644 }
12645
12646 #[tokio::test]
12650 async fn catalog_events_capability_is_advertised() {
12651 let handler = ControlHandler::new(Arc::new(Registry::default()));
12652 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12653 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12654 let reply = handler
12655 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12656 .await
12657 .unwrap()
12658 .pop()
12659 .unwrap();
12660 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12661 serde_json::from_slice(&reply.body).unwrap()
12662 else {
12663 panic!("not a server.describe reply");
12664 };
12665 assert!(
12666 capabilities
12667 .iter()
12668 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12669 "{capabilities:?}"
12670 );
12671 }
12672
12673 #[tokio::test]
12674 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12675 let handler = ControlHandler::new(Arc::new(Registry::default()));
12676 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12677 let ack = hello_via_sink(
12678 &handler,
12679 &module,
12680 &mut module_rx,
12681 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12682 )
12683 .await;
12684 let ack = parse_ack(&ack);
12685 assert!(
12686 ack.subc_capabilities
12687 .iter()
12688 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12689 "{:?}",
12690 ack.subc_capabilities
12691 );
12692 assert!(ack
12693 .subc_capabilities
12694 .iter()
12695 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12696
12697 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12698 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12699 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12700 let reply = handler
12701 .handle_control_frame(&client, frame)
12702 .await
12703 .unwrap()
12704 .pop()
12705 .unwrap();
12706 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12707 serde_json::from_slice(&reply.body).unwrap()
12708 else {
12709 panic!("not a server.describe reply");
12710 };
12711 assert!(
12712 capabilities
12713 .iter()
12714 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12715 "{capabilities:?}"
12716 );
12717 assert!(capabilities
12718 .iter()
12719 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12720 }
12721
12722 #[tokio::test]
12723 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12724 let handler = ControlHandler::new(Arc::new(Registry::default()));
12725 let (module, mut rx) = route_ctx(ConnectionId::new(156));
12726 let ack = hello_via_sink(
12727 &handler,
12728 &module,
12729 &mut rx,
12730 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12731 )
12732 .await;
12733 assert!(parse_ack(&ack)
12734 .subc_capabilities
12735 .iter()
12736 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12737 assert!(parse_ack(&ack)
12738 .subc_capabilities
12739 .iter()
12740 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12741 let (client, _rx) = route_ctx(ConnectionId::new(157));
12742 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12743 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12744 let reply = handler
12745 .handle_control_frame(&client, frame)
12746 .await
12747 .unwrap()
12748 .pop()
12749 .unwrap();
12750 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12751 serde_json::from_slice(&reply.body).unwrap()
12752 else {
12753 panic!("server.describe reply")
12754 };
12755 assert!(capabilities
12756 .iter()
12757 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12758 assert!(capabilities
12759 .iter()
12760 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12761 }
12762
12763 #[tokio::test]
12764 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12765 let handler = ControlHandler::new(Arc::new(Registry::default()));
12766 let (module, mut rx) = route_ctx(ConnectionId::new(158));
12767 let ack = hello_via_sink(
12768 &handler,
12769 &module,
12770 &mut rx,
12771 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12772 )
12773 .await;
12774 assert!(parse_ack(&ack)
12775 .subc_capabilities
12776 .iter()
12777 .any(|cap| cap == "scope-workdir/v1"));
12778 let (client, _rx) = route_ctx(ConnectionId::new(159));
12779 let frame = Frame::build(
12780 FrameType::Request,
12781 control_flags(),
12782 0,
12783 0,
12784 5,
12785 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12786 )
12787 .unwrap();
12788 let reply = handler
12789 .handle_control_frame(&client, frame)
12790 .await
12791 .unwrap()
12792 .remove(0);
12793 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12794 serde_json::from_slice(&reply.body).unwrap()
12795 else {
12796 panic!("server.describe reply")
12797 };
12798 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12799 }
12800
12801 #[tokio::test]
12802 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
12803 let handler = ControlHandler::new(Arc::new(Registry::default()));
12804 let (module, mut rx) = route_ctx(ConnectionId::new(160));
12805 let ack = hello_via_sink(
12806 &handler,
12807 &module,
12808 &mut rx,
12809 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
12810 )
12811 .await;
12812 assert!(parse_ack(&ack)
12813 .subc_capabilities
12814 .iter()
12815 .any(|cap| cap == "worker-runs/v1"));
12816 let (client, _rx) = route_ctx(ConnectionId::new(161));
12817 let frame = Frame::build(
12818 FrameType::Request,
12819 control_flags(),
12820 0,
12821 0,
12822 5,
12823 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12824 )
12825 .unwrap();
12826 let reply = handler
12827 .handle_control_frame(&client, frame)
12828 .await
12829 .unwrap()
12830 .remove(0);
12831 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12832 serde_json::from_slice(&reply.body).unwrap()
12833 else {
12834 panic!("server.describe reply")
12835 };
12836 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
12837 }
12838
12839 #[tokio::test]
12840 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12841 let handler = ControlHandler::new(Arc::new(Registry::default()));
12842 let (module, mut rx) = route_ctx(ConnectionId::new(154));
12843 let ack = hello_via_sink(
12844 &handler,
12845 &module,
12846 &mut rx,
12847 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12848 )
12849 .await;
12850 assert!(parse_ack(&ack)
12851 .subc_capabilities
12852 .iter()
12853 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12854 let (client, _rx) = route_ctx(ConnectionId::new(155));
12855 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12856 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12857 let reply = handler
12858 .handle_control_frame(&client, frame)
12859 .await
12860 .unwrap()
12861 .pop()
12862 .unwrap();
12863 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12864 serde_json::from_slice(&reply.body).unwrap()
12865 else {
12866 panic!("not a server.describe reply")
12867 };
12868 assert!(capabilities
12869 .iter()
12870 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12871 }
12872
12873 #[tokio::test]
12874 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12875 let registry = Arc::new(Registry::default());
12876 let forwarding = Arc::new(ForwardingTable::default());
12877 let handler =
12878 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12879 .with_health_probe_timeout(Duration::from_secs(5));
12880 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12881 hello_via_sink(
12882 &handler,
12883 &module_ctx,
12884 &mut module_rx,
12885 non_routable_hello_frame_with_control_ops(
12886 "mcp",
12887 300,
12888 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12889 ),
12890 )
12891 .await;
12892 assert!(registry
12893 .get_module("mcp")
12894 .unwrap()
12895 .unwrap()
12896 .manifest
12897 .provides
12898 .is_empty());
12899
12900 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12901 let route_response = handler
12902 .handle_control_frame(
12903 &route_client_ctx,
12904 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12905 )
12906 .await
12907 .unwrap();
12908 assert_eq!(route_response[0].header.ty, FrameType::Error);
12909 assert_eq!(
12910 parse_error(&route_response[0])["code"],
12911 "target_unavailable"
12912 );
12913 assert!(parse_error(&route_response[0])["message"]
12914 .as_str()
12915 .unwrap()
12916 .contains("does not provide the requested target"));
12917 assert!(module_rx.try_recv().is_err());
12918
12919 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12920 let health_handler = handler.clone();
12921 let health_task = tokio::spawn(async move {
12922 health_handler
12923 .handle_control_frame(
12924 &health_client_ctx,
12925 supervisor_health_probe_frame(302, "mcp"),
12926 )
12927 .await
12928 .unwrap()
12929 });
12930 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12931 .await
12932 .unwrap()
12933 .unwrap();
12934 assert_eq!(
12935 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12936 ModuleControlRequest::HealthCheck {}
12937 );
12938 handler
12939 .handle_control_frame(
12940 &module_ctx,
12941 health_response(health_frame.header.corr, HealthStatus::Ok),
12942 )
12943 .await
12944 .unwrap();
12945 let health_response = health_task.await.unwrap();
12946 assert_eq!(health_response[0].header.ty, FrameType::Response);
12947 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12948 ClientControlResponse::SupervisorHealthProbe {
12949 module_id, status, ..
12950 } => {
12951 assert_eq!(module_id, "mcp");
12952 assert_eq!(status, HealthStatus::Ok);
12953 }
12954 other => panic!("unexpected health response: {other:?}"),
12955 }
12956
12957 forwarding
12962 .cleanup_connection(module_ctx.connection_id)
12963 .unwrap();
12964 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12965 let cleanup_response = tokio::time::timeout(
12966 Duration::from_millis(200),
12967 handler.handle_control_frame(
12968 &cleanup_probe_ctx,
12969 supervisor_health_probe_frame(303, "mcp"),
12970 ),
12971 )
12972 .await
12973 .expect("probe should fail immediately when the control lane is gone")
12974 .unwrap();
12975 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12976 assert_eq!(
12977 parse_error(&cleanup_response[0])["code"],
12978 "target_unavailable"
12979 );
12980 assert!(parse_error(&cleanup_response[0])["message"]
12981 .as_str()
12982 .unwrap()
12983 .contains("no module connection"));
12984
12985 handler
12986 .cleanup_connection(module_ctx.connection_id)
12987 .unwrap();
12988 }
12989
12990 #[tokio::test]
12991 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12992 let registry = Arc::new(Registry::default());
12993 let supervisor_handle = SupervisorHandle::new();
12994 let supervisor =
12995 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12996 .with_handle(supervisor_handle.clone())
12997 .with_connection_file_path(
12998 std::env::temp_dir()
12999 .join(format!("subc-route-open-warming-{}", std::process::id())),
13000 );
13001 let module = supervisor
13002 .supervise_configured(
13003 ModuleSpec {
13004 module_id: "warming".to_string(),
13005 program: fake_aft_stub_path(),
13006 args: Vec::new(),
13007 env: Vec::new(),
13008 reserved: false,
13009 reserved_prefixes: Vec::new(),
13010 protocol: ModuleProtocol::Subc,
13011 overlap: Default::default(),
13012 },
13013 true,
13014 )
13015 .unwrap();
13016 assert_eq!(module.state().unwrap(), ModuleState::Running);
13017
13018 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13019 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13020 let response = handler
13021 .handle_control_frame(
13022 &ctx,
13023 route_open_frame(304, "warming", unique_project_root("warming")),
13024 )
13025 .await
13026 .unwrap();
13027 module.stop().await.unwrap();
13028
13029 assert_eq!(response[0].header.ty, FrameType::Error);
13030 let error = parse_error(&response[0]);
13031 assert_eq!(error["code"], "module_warming");
13032 assert!(error["message"]
13033 .as_str()
13034 .unwrap()
13035 .contains("state=running, enabled=true, live=false"));
13036 }
13037
13038 #[test]
13039 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13040 let handler = ControlHandler::new(Arc::new(Registry::default()));
13041 let capture = EventCapture::default();
13042 let _subscriber =
13043 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13044 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13045 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13046 let pending = (0..limit).collect::<Vec<_>>();
13047 let response = handler
13048 .route_open_capacity_refusal(
13049 &ctx,
13050 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13051 "busy",
13052 pending.len(),
13053 limit,
13054 )
13055 .unwrap();
13056 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13057 let event = capture
13058 .events()
13059 .into_iter()
13060 .find(|event| {
13061 event.target == "control"
13062 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13063 })
13064 .expect("connection admission refusal event");
13065 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13066 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13067 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13070 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13071 }
13072
13073 #[test]
13074 fn route_open_target_cap_logs_admission_reason_and_capacity() {
13075 let handler = ControlHandler::new(Arc::new(Registry::default()));
13076 let capture = EventCapture::default();
13077 let _subscriber =
13078 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13079 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13080 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13081 let guards = (0..limit)
13082 .map(|_| {
13083 handler
13084 .route_bind_concurrency
13085 .try_admit("busy", limit)
13086 .unwrap()
13087 })
13088 .collect::<Vec<_>>();
13089 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13090 Err(in_flight) => in_flight,
13091 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13092 };
13093 let response = handler
13094 .route_open_target_capacity_refusal(
13095 &ctx,
13096 &RouteOpenCaller::unknown(),
13097 &route_open_frame(397, "busy", unique_project_root("target-cap")),
13098 "busy",
13099 in_flight,
13100 )
13101 .unwrap();
13102 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13103 let event = capture
13104 .events()
13105 .into_iter()
13106 .find(|event| {
13107 event.target == "control"
13108 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13109 })
13110 .expect("target admission refusal event");
13111 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13112 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13113 drop(guards);
13114 }
13115
13116 #[cfg(unix)]
13117 #[tokio::test(start_paused = true)]
13118 async fn supervisor_list_shows_a_failed_modules_next_retry() {
13119 let registry = Arc::new(Registry::default());
13120 let handle = SupervisorHandle::new();
13121 let supervisor =
13122 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13123 .with_handle(handle.clone());
13124 let module = supervisor
13125 .spawn(crate::ModuleSpec {
13126 module_id: "retry-list".into(),
13127 program: PathBuf::from("/bin/sh"),
13128 args: vec!["-c".into(), "exit 17".into()],
13129 env: vec![],
13130 reserved: false,
13131 reserved_prefixes: vec![],
13132 protocol: ModuleProtocol::None,
13133 overlap: Default::default(),
13134 })
13135 .unwrap();
13136 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13137 while module.state().unwrap() != crate::ModuleState::Failed {
13138 assert!(
13139 std::time::Instant::now() < deadline,
13140 "waiting for budget failure"
13141 );
13142 tokio::task::yield_now().await;
13143 }
13144 let handler = ControlHandler::new(registry).with_supervisor(handle);
13145 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13146 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13147 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13148 assert_eq!(body["modules"][0]["state"], "failed");
13149 let retry = body["modules"][0]["next_retry_at_ms"]
13150 .as_u64()
13151 .expect("scheduled retry on the wire");
13152 let now = std::time::SystemTime::now()
13153 .duration_since(std::time::UNIX_EPOCH)
13154 .unwrap()
13155 .as_millis() as u64;
13156 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13157 module.set_enabled(false).await.unwrap();
13158 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13159 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13160 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13161 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13162 }
13163
13164 #[tokio::test]
13165 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
13166 let registry = Arc::new(Registry::default());
13167 let handle = SupervisorHandle::new();
13168 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13169 .with_handle(handle.clone());
13170 supervisor
13171 .supervise_configured(
13172 crate::ModuleSpec {
13173 module_id: "drift".into(),
13174 program: PathBuf::from("disabled-module"),
13175 args: Vec::new(),
13176 env: Vec::new(),
13177 reserved: false,
13178 reserved_prefixes: Vec::new(),
13179 protocol: ModuleProtocol::Subc,
13180 overlap: Default::default(),
13181 },
13182 false,
13183 )
13184 .unwrap();
13185 let handler = ControlHandler::new(registry).with_supervisor(handle);
13186 for configured in [None, Some(false), Some(true)] {
13187 handler.capability_evaluator.configure(
13188 configured.map(|enabled| ("drift".to_string(), enabled)),
13189 BTreeMap::new(),
13190 );
13191 let frame =
13192 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13193 let replies = handler
13194 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
13195 .await
13196 .unwrap();
13197 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13198 assert!(body.get("configured_enabled").is_none());
13199 assert_eq!(body["op"], "supervisor.ack");
13200 let frame =
13201 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13202 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13203 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13204 assert_eq!(
13205 body["modules"][0].get("configured_enabled"),
13206 configured.map(Value::Bool).as_ref()
13207 );
13208 }
13209 }
13210
13211 #[tokio::test(flavor = "current_thread")]
13212 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
13213 let handler = ControlHandler::new(Arc::new(Registry::default()));
13214 let capture = EventCapture::default();
13215 let _subscriber =
13216 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13217 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
13218 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13219
13220 let _ = handler
13221 .handle_client_control_request(
13222 &ctx,
13223 frame,
13224 ClientControlRequest::SupervisorSetEnabled {
13225 module_id: "broca".to_string(),
13226 enabled: false,
13227 },
13228 &mut None,
13229 )
13230 .await
13231 .unwrap();
13232
13233 let events = capture
13234 .events()
13235 .into_iter()
13236 .filter(|event| {
13237 event.target == "control"
13238 && event.fields.get("message").map(String::as_str)
13239 == Some("supervisor request received")
13240 })
13241 .collect::<Vec<_>>();
13242 assert_eq!(events.len(), 1, "one supervisor request log line");
13243 let event = &events[0];
13244 assert_eq!(event.level, tracing::Level::INFO);
13245 assert_eq!(
13246 event.fields.get("op"),
13247 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
13248 );
13249 assert_eq!(
13250 event.fields.get("module_id"),
13251 Some(&"\"broca\"".to_string())
13252 );
13253 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
13254 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
13255 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
13256 }
13257
13258 #[tokio::test(flavor = "current_thread")]
13259 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
13260 let registry = Arc::new(Registry::default());
13261 let module_connection = ConnectionId::new(303);
13262 registry
13263 .register_with_control_ops(
13264 manifest("aft", PROTOCOL_VERSION),
13265 PROTOCOL_VERSION,
13266 module_connection,
13267 module_baseline_control_ops(),
13268 )
13269 .unwrap();
13270 let handler = ControlHandler::new(registry);
13271 let capture = EventCapture::default();
13272 let _subscriber =
13273 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13274 let (ctx, _rx) = route_ctx(module_connection);
13275 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13276
13277 let _ = handler
13278 .handle_client_control_request(
13279 &ctx,
13280 frame,
13281 ClientControlRequest::SupervisorSetEnabled {
13282 module_id: "broca".to_string(),
13283 enabled: false,
13284 },
13285 &mut None,
13286 )
13287 .await
13288 .unwrap();
13289
13290 let callers = capture
13291 .events()
13292 .into_iter()
13293 .filter(|event| {
13294 event.fields.get("message").map(String::as_str)
13295 == Some("supervisor request received")
13296 })
13297 .map(|event| event.fields.get("caller").cloned())
13298 .collect::<Vec<_>>();
13299 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13300 }
13301
13302 #[tokio::test(flavor = "current_thread")]
13303 async fn supervisor_list_does_not_log_request_received() {
13304 let handler = ControlHandler::new(Arc::new(Registry::default()));
13305 let capture = EventCapture::default();
13306 let _subscriber =
13307 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13308 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13309 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13310
13311 handler
13312 .handle_client_control_request(
13313 &ctx,
13314 frame,
13315 ClientControlRequest::SupervisorList {},
13316 &mut None,
13317 )
13318 .await
13319 .unwrap();
13320
13321 assert!(capture.events().into_iter().all(|event| {
13322 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13323 }));
13324 }
13325
13326 #[tokio::test(flavor = "current_thread")]
13327 async fn supervisor_rescan_preview_does_not_log_request_received() {
13328 let handler = ControlHandler::new(Arc::new(Registry::default()));
13329 let capture = EventCapture::default();
13330 let _subscriber =
13331 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13332 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13333 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13334
13335 handler
13336 .handle_client_control_request(
13337 &ctx,
13338 frame,
13339 ClientControlRequest::SupervisorRescan { preview: true },
13340 &mut None,
13341 )
13342 .await
13343 .unwrap();
13344
13345 assert!(capture.events().into_iter().all(|event| {
13346 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13347 }));
13348 }
13349
13350 struct ApproveOperator;
13351 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13352 fn prompt(
13353 &self,
13354 _: &str,
13355 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13356 ) -> crate::operator_confirm::ProviderResult {
13357 crate::operator_confirm::ProviderResult::Approved
13358 }
13359 }
13360
13361 #[tokio::test]
13362 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13363 assert_eq!(
13364 ControlHandler::default().privileged_operator_prompt,
13365 PrivilegedOperatorPrompt::Log
13366 );
13367 for (initial, changed, running) in [
13368 ("log", "enforce", PrivilegedOperatorPrompt::Log),
13369 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13370 ] {
13371 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13372 let path = dir.join("subc.jsonc");
13373 std::fs::write(
13374 &path,
13375 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13376 )
13377 .unwrap();
13378 let config = crate::daemon_config::load(&path).unwrap().unwrap();
13379 let registry = Arc::new(Registry::default());
13380 let handle = SupervisorHandle::new();
13381 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13382 .with_handle(handle.clone());
13383 let handler = ControlHandler::new(registry)
13384 .with_supervisor(handle)
13385 .with_privileged_operator_prompt(config.privileged_operator_prompt)
13386 .with_supervisor_rescan(supervisor, &path, config.port);
13387 handler.forwarding.operator_confirms().configure(
13388 Arc::new(ApproveOperator),
13389 crate::operator_confirm::OperatorLimits::default(),
13390 );
13391 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13392
13393 for (disk_mode, needs_restart) in [
13394 (Some(initial), false),
13395 (Some(changed), true),
13396 (Some(changed), true),
13397 (None, running == PrivilegedOperatorPrompt::Enforce),
13398 (Some(initial), false),
13399 ] {
13400 let mut doc = json!({"version":1});
13401 if let Some(mode) = disk_mode {
13402 doc["privileged_operator_prompt"] = json!(mode);
13403 }
13404 std::fs::write(&path, doc.to_string()).unwrap();
13405 for preview in [true, false] {
13406 let request = ClientControlRequest::SupervisorRescan { preview };
13407 let frame = Frame::build(
13408 FrameType::Request,
13409 control_flags(),
13410 0,
13411 0,
13412 1,
13413 serde_json::to_vec(&request).unwrap(),
13414 )
13415 .unwrap();
13416 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13417 assert_eq!(frames.len(), 1);
13418 let ClientControlResponse::SupervisorRescan { result } =
13419 serde_json::from_slice(&frames[0].body).unwrap()
13420 else {
13421 panic!("expected a rescan response");
13422 };
13423 let expected = if needs_restart {
13424 vec!["privileged_operator_prompt"]
13425 } else {
13426 Vec::new()
13427 };
13428 assert_eq!(result.restart_required, expected);
13429 assert_eq!(handler.privileged_operator_prompt, running);
13430 assert_eq!(
13431 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13432 running
13433 );
13434 }
13435 }
13436 }
13437 }
13438
13439 #[tokio::test]
13440 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13441 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13442 let path = dir.join("subc.jsonc");
13443 let registry = Arc::new(Registry::default());
13444 let handle = SupervisorHandle::new();
13445 let supervisor =
13446 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13447 let handler = ControlHandler::new(registry)
13448 .with_supervisor(handle)
13449 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13450 .with_supervisor_rescan(supervisor, &path, None);
13451 handler.forwarding.operator_confirms().configure(
13452 Arc::new(ApproveOperator),
13453 crate::operator_confirm::OperatorLimits::default(),
13454 );
13455 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13456 for preview in [true, false] {
13457 std::fs::write(
13458 &path,
13459 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13460 )
13461 .unwrap();
13462 let request = ClientControlRequest::SupervisorRescan { preview };
13463 let frame = Frame::build(
13464 FrameType::Request,
13465 control_flags(),
13466 0,
13467 0,
13468 1,
13469 serde_json::to_vec(&request).unwrap(),
13470 )
13471 .unwrap();
13472 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13473 assert_eq!(frames.len(), 1);
13474 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13475 assert_eq!(
13476 handler.privileged_operator_prompt,
13477 PrivilegedOperatorPrompt::Enforce
13478 );
13479 }
13480 }
13481
13482 #[tokio::test]
13487 async fn route_open_refusal_names_the_check_that_refused() {
13488 let handler = ControlHandler::new(Arc::new(Registry::default()));
13489 let capture = EventCapture::default();
13490 let _subscriber =
13491 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13492 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13493 let response = handler
13494 .handle_control_frame(
13495 &ctx,
13496 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13497 )
13498 .await
13499 .unwrap();
13500
13501 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13502 let event = capture
13503 .events()
13504 .into_iter()
13505 .find(|event| {
13506 event.target == "control"
13507 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13508 })
13509 .expect("route.open refusal event");
13510 assert_eq!(
13511 event.fields.get("reason"),
13512 Some(&"\"not_registered\"".to_string())
13513 );
13514 }
13515
13516 #[tokio::test]
13517 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13518 let registry = Arc::new(Registry::default());
13519 let supervisor_handle = SupervisorHandle::new();
13520 let supervisor =
13521 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13522 .with_handle(supervisor_handle.clone())
13523 .with_connection_file_path(std::env::temp_dir().join(format!(
13524 "subc-route-open-refusal-info-{}",
13525 std::process::id()
13526 )));
13527 let module = supervisor
13528 .supervise_configured(
13529 ModuleSpec {
13530 module_id: "warming".to_string(),
13531 program: fake_aft_stub_path(),
13532 args: Vec::new(),
13533 env: Vec::new(),
13534 reserved: false,
13535 reserved_prefixes: Vec::new(),
13536 protocol: ModuleProtocol::Subc,
13537 overlap: Default::default(),
13538 },
13539 true,
13540 )
13541 .unwrap();
13542 assert_eq!(module.state().unwrap(), ModuleState::Running);
13543
13544 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13545 assert!(handler
13546 .counters()
13547 .snapshot()
13548 .get("route_open_refused_by_code")
13549 .is_none());
13550 let capture = EventCapture::default();
13551 let _subscriber =
13552 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13553 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13554 let response = handler
13555 .handle_control_frame(
13556 &ctx,
13557 route_open_frame(394, "warming", unique_project_root("refusal-info")),
13558 )
13559 .await
13560 .unwrap();
13561 module.stop().await.unwrap();
13562
13563 assert_eq!(parse_error(&response[0])["code"], "module_warming");
13564 let event = capture
13565 .events()
13566 .into_iter()
13567 .find(|event| {
13568 event.target == "control"
13569 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13570 })
13571 .expect("route.open refusal event");
13572 assert_eq!(
13573 event.fields.get("module_id"),
13574 Some(&"\"warming\"".to_string())
13575 );
13576 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13577 assert_eq!(
13578 event.fields.get("reason"),
13579 Some(&"\"supervised_not_registered\"".to_string())
13580 );
13581 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13582 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13583 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13584 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13585 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13586 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13587 assert_eq!(
13588 handler.counters().snapshot()["route_open_refused_by_code"],
13589 json!({ "module_warming": 1 })
13590 );
13591 }
13592
13593 const OUTAGE_START: &str = "route.open refusing module: not serving";
13594 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13595
13596 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13597 capture
13598 .events()
13599 .into_iter()
13600 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13601 .collect()
13602 }
13603
13604 fn supervise_stub(
13605 registry: &Arc<Registry>,
13606 module_id: &str,
13607 enabled: bool,
13608 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13609 let supervisor_handle = SupervisorHandle::new();
13610 let supervisor =
13611 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13612 .with_handle(supervisor_handle.clone())
13613 .with_connection_file_path(std::env::temp_dir().join(format!(
13614 "subc-route-outage-{module_id}-{}",
13615 std::process::id()
13616 )));
13617 let module = supervisor
13618 .supervise_configured(
13619 ModuleSpec {
13620 module_id: module_id.to_string(),
13621 program: fake_aft_stub_path(),
13622 args: Vec::new(),
13623 env: Vec::new(),
13624 reserved: false,
13625 reserved_prefixes: Vec::new(),
13626 protocol: ModuleProtocol::Subc,
13627 overlap: Default::default(),
13628 },
13629 enabled,
13630 )
13631 .unwrap();
13632 (supervisor_handle, module)
13633 }
13634
13635 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13636 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13637 module_id: module_id.to_string(),
13638 drain_timeout_ms: Some(50),
13639 })
13640 .unwrap();
13641 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13642 }
13643
13644 #[test]
13648 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13649 let registry = Arc::new(Registry::default());
13650 let forwarding = Arc::new(ForwardingTable::default());
13651 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13652 let second = ControlHandler::with_forwarding(registry, forwarding);
13653 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13654 }
13655
13656 #[tokio::test(flavor = "current_thread")]
13660 async fn route_open_unknown_module_refusals_add_no_outage_state() {
13661 let handler = ControlHandler::new(Arc::new(Registry::default()));
13662 let capture = EventCapture::default();
13663 let _subscriber =
13664 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13665 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13666 for corr in 0..8 {
13667 let response = handler
13668 .handle_control_frame(
13669 &ctx,
13670 route_open_frame(
13671 380 + corr,
13672 &format!("nobody-{corr}"),
13673 unique_project_root("outage-unknown"),
13674 ),
13675 )
13676 .await
13677 .unwrap();
13678 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13679 }
13680
13681 assert_eq!(handler.route_outages.tracked_module_count(), 0);
13682 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13683 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13684 }
13685
13686 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13691 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13692 let registry = Arc::new(Registry::default());
13693 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
13694 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13695 let capture = EventCapture::default();
13696 let _subscriber =
13697 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13698 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13699 handler.route_outages.record_accepted("outage-restart");
13702
13703 let response = handler
13704 .handle_control_frame(
13705 &ctx,
13706 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13707 )
13708 .await
13709 .unwrap();
13710 assert_eq!(response[0].header.ty, FrameType::Error);
13711 let starts = outage_lines(&capture, OUTAGE_START);
13712 assert_eq!(starts.len(), 1, "{starts:?}");
13713 assert_eq!(starts[0].level, tracing::Level::WARN);
13714 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13715 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13716 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13717 handler.route_outages.record_accepted("outage-restart");
13718 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13719
13720 let restart = handler
13721 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13722 .await
13723 .unwrap();
13724 assert_eq!(
13725 restart[0].header.ty,
13726 FrameType::Response,
13727 "{:?}",
13728 parse_error(&restart[0])
13729 );
13730 handler
13731 .handle_control_frame(
13732 &ctx,
13733 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13734 )
13735 .await
13736 .unwrap();
13737 module.stop().await.unwrap();
13738
13739 let starts = outage_lines(&capture, OUTAGE_START);
13740 assert_eq!(starts.len(), 2, "{starts:?}");
13741 assert_eq!(starts[1].level, tracing::Level::INFO);
13742 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13743 }
13744
13745 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13749 async fn failed_operator_restart_leaves_no_operator_mark() {
13750 let registry = Arc::new(Registry::default());
13751 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
13752 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13753 let capture = EventCapture::default();
13754 let _subscriber =
13755 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13756 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13757 handler.route_outages.record_accepted("outage-disabled");
13758
13759 let restart = handler
13760 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13761 .await
13762 .unwrap();
13763 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13764 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13765
13766 handler
13767 .handle_control_frame(
13768 &ctx,
13769 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13770 )
13771 .await
13772 .unwrap();
13773 let starts = outage_lines(&capture, OUTAGE_START);
13774 assert_eq!(starts.len(), 1, "{starts:?}");
13775 assert_eq!(starts[0].level, tracing::Level::WARN);
13776 }
13777
13778 #[tokio::test(flavor = "current_thread")]
13779 async fn route_open_unknown_module_escapes_target_module_id() {
13780 let handler = ControlHandler::new(Arc::new(Registry::default()));
13781 let capture = EventCapture::default();
13782 let _subscriber =
13783 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13784 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13785 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13786 let response = handler
13787 .handle_control_frame(
13788 &ctx,
13789 route_open_frame(
13790 395,
13791 hostile_module_id,
13792 unique_project_root("hostile-target-module-id"),
13793 ),
13794 )
13795 .await
13796 .unwrap();
13797
13798 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13799 let event = capture
13800 .events()
13801 .into_iter()
13802 .find(|event| {
13803 event.target == "control"
13804 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13805 })
13806 .expect("route.open unknown-module refusal event");
13807 let logged = event.fields.get("module_id").expect("module_id field");
13808 assert!(!logged.bytes().any(|byte| byte < 0x20));
13809 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13810 }
13811
13812 #[tokio::test(flavor = "current_thread")]
13813 async fn route_open_module_rejection_uses_daemon_counter_key() {
13814 let registry = Arc::new(Registry::default());
13815 let forwarding = Arc::new(ForwardingTable::default());
13816 let handler =
13817 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13818 let module_connection = ConnectionId::new(95);
13819 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13820 hello_via_sink(
13821 &handler,
13822 &module_ctx,
13823 &mut module_rx,
13824 hello_frame("aft", PROTOCOL_VERSION, 395),
13825 )
13826 .await;
13827
13828 let client_connection = ConnectionId::new(96);
13829 let (client_ctx, _client_rx) = route_ctx(client_connection);
13830 let capture = EventCapture::default();
13831 let _subscriber =
13832 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13833 let (route_task, bind) = relay_route_open(
13834 &handler,
13835 client_connection,
13836 &client_ctx.egress,
13837 &mut module_rx,
13838 396,
13839 "aft",
13840 "hostile-module-code",
13841 )
13842 .await;
13843 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13844 let rejection = Frame::build(
13845 FrameType::Error,
13846 control_flags(),
13847 0,
13848 0,
13849 bind.header.corr,
13850 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13851 )
13852 .unwrap();
13853 handler
13854 .handle_control_frame(&module_ctx, rejection)
13855 .await
13856 .unwrap();
13857
13858 let response = route_task.await.unwrap();
13859 assert_eq!(parse_error(&response[0])["code"], hostile_code);
13860 let counters = handler.counters().snapshot();
13861 assert_eq!(
13862 counters["route_open_refused_by_code"],
13863 json!({ "module_rejected": 1 })
13864 );
13865 assert!(counters["route_open_refused_by_code"]
13866 .get(hostile_code)
13867 .is_none());
13868
13869 let event = capture
13870 .events()
13871 .into_iter()
13872 .find(|event| {
13873 event.target == "control"
13874 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13875 })
13876 .expect("route.open module-rejection refusal event");
13877 let logged = event.fields.get("module_code").expect("module_code field");
13878 assert!(!logged.bytes().any(|byte| byte < 0x20));
13879 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13880 }
13881
13882 #[tokio::test]
13883 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13884 let registry = Arc::new(Registry::default());
13885 let supervisor_handle = SupervisorHandle::new();
13886 let missing_program = std::env::temp_dir().join(format!(
13887 "subc-route-open-missing-program-{}",
13888 std::process::id()
13889 ));
13890 let supervisor =
13891 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13892 .with_handle(supervisor_handle.clone());
13893 let module = supervisor
13894 .supervise_configured(
13895 ModuleSpec {
13896 module_id: "failed".to_string(),
13897 program: missing_program,
13898 args: Vec::new(),
13899 env: Vec::new(),
13900 reserved: false,
13901 reserved_prefixes: Vec::new(),
13902 protocol: ModuleProtocol::Subc,
13903 overlap: Default::default(),
13904 },
13905 true,
13906 )
13907 .unwrap();
13908 assert_eq!(module.state().unwrap(), ModuleState::Failed);
13909
13910 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13911 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13912 let response = handler
13913 .handle_control_frame(
13914 &ctx,
13915 route_open_frame(305, "failed", unique_project_root("failed")),
13916 )
13917 .await
13918 .unwrap();
13919
13920 assert_eq!(response[0].header.ty, FrameType::Error);
13921 let error = parse_error(&response[0]);
13922 assert_eq!(error["code"], "target_unavailable");
13923 assert!(error["message"]
13924 .as_str()
13925 .unwrap()
13926 .contains("state=failed, enabled=true, live=false"));
13927 }
13928
13929 #[tokio::test]
13930 async fn route_open_role_mismatch_remains_target_unavailable() {
13931 let registry = Arc::new(Registry::default());
13932 let handler = ControlHandler::new(Arc::clone(®istry));
13933 handler
13934 .handle_control(
13935 ConnectionId::new(41),
13936 non_routable_hello_frame_with_control_ops("health-only", 306, None),
13937 )
13938 .unwrap();
13939
13940 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13941 let response = handler
13942 .handle_control_frame(
13943 &ctx,
13944 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13945 )
13946 .await
13947 .unwrap();
13948
13949 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13950 assert!(parse_error(&response[0])["message"]
13951 .as_str()
13952 .unwrap()
13953 .contains("does not provide the requested target"));
13954 }
13955
13956 #[tokio::test]
13957 async fn route_open_inactive_registration_remains_target_unavailable() {
13958 let registry = Arc::new(Registry::default());
13959 let handler = ControlHandler::new(Arc::clone(®istry));
13960 handler
13961 .handle_control(
13962 ConnectionId::new(43),
13963 hello_frame("inactive", PROTOCOL_VERSION, 308),
13964 )
13965 .unwrap();
13966 assert!(registry
13967 .set_module_state_for_test("inactive", ChannelState::Closed)
13968 .unwrap());
13969
13970 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13971 let response = handler
13972 .handle_control_frame(
13973 &ctx,
13974 route_open_frame(309, "inactive", unique_project_root("inactive")),
13975 )
13976 .await
13977 .unwrap();
13978
13979 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13980 assert!(parse_error(&response[0])["message"]
13981 .as_str()
13982 .unwrap()
13983 .contains("is not active"));
13984 }
13985
13986 #[tokio::test]
13987 async fn late_health_reply_is_recorded_through_the_module_response_path() {
13988 let registry = Arc::new(Registry::default());
13989 let forwarding = Arc::new(ForwardingTable::default());
13990 let supervisor_handle = SupervisorHandle::new();
13991 let supervisor =
13992 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
13993 .with_forwarding(Arc::clone(&forwarding))
13994 .with_handle(supervisor_handle.clone());
13995 let module = supervisor
13996 .supervise_configured(
13997 crate::ModuleSpec {
13998 module_id: "late-health-response".to_string(),
13999 program: PathBuf::from("disabled-module"),
14000 args: Vec::new(),
14001 env: Vec::new(),
14002 reserved: false,
14003 reserved_prefixes: Vec::new(),
14004 protocol: ModuleProtocol::Subc,
14005 overlap: Default::default(),
14006 },
14007 false,
14008 )
14009 .unwrap();
14010 let handler =
14011 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14012 .with_supervisor(supervisor_handle);
14013 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
14014 handler
14015 .handle_control_frame(
14016 &module_ctx,
14017 hello_frame_with_control_ops(
14018 "late-health-response",
14019 PROTOCOL_VERSION,
14020 7,
14021 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14022 ),
14023 )
14024 .await
14025 .unwrap();
14026 let probe_started_at = Instant::now() - Duration::from_millis(80);
14027 let pending = forwarding
14028 .begin_health_probe_rpc_for(
14029 "late-health-response",
14030 MODULE_CONTROL_OP_HEALTH_CHECK,
14031 probe_started_at,
14032 Instant::now() - Duration::from_millis(1),
14033 )
14034 .unwrap();
14035 assert!(forwarding
14036 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
14037 .unwrap());
14038
14039 let responses = handler
14040 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
14041 .await
14042 .unwrap();
14043
14044 assert!(responses.is_empty());
14045 let health = module.status().unwrap().health;
14046 assert_eq!(health.late_answer_count, 1);
14047 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
14048 }
14049
14050 #[tokio::test]
14051 async fn health_probe_timeout_and_module_death_are_typed() {
14052 let registry = Arc::new(Registry::default());
14053 let forwarding = Arc::new(ForwardingTable::default());
14054 let handler =
14055 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14056 .with_health_probe_timeout(Duration::from_millis(50));
14057 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14058 hello_via_sink(
14059 &handler,
14060 &module_ctx,
14061 &mut module_rx,
14062 hello_frame_with_control_ops(
14063 "aft",
14064 PROTOCOL_VERSION,
14065 7,
14066 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14067 ),
14068 )
14069 .await;
14070
14071 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14072 let responses = handler
14073 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
14074 .await
14075 .unwrap();
14076 assert_eq!(responses[0].header.ty, FrameType::Error);
14077 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
14078 let _ = module_rx.try_recv();
14079
14080 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
14081 let health_handler = handler.clone();
14082 let death_task = tokio::spawn(async move {
14083 health_handler
14084 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
14085 .await
14086 .unwrap()
14087 });
14088 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14089 .await
14090 .unwrap()
14091 .unwrap();
14092 handler
14093 .cleanup_connection(module_ctx.connection_id)
14094 .unwrap();
14095 let responses = death_task.await.unwrap();
14096 assert_eq!(responses[0].header.ty, FrameType::Error);
14097 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
14098 }
14099
14100 #[test]
14101 fn hello_requires_exact_protocol_version() {
14102 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
14103 let registry = Arc::new(Registry::default());
14104 let handler = ControlHandler::new(Arc::clone(®istry));
14105 let responses = handler
14106 .handle_control(
14107 ConnectionId::new(connection),
14108 hello_frame("aft", offered, 9),
14109 )
14110 .unwrap();
14111
14112 assert_eq!(responses.len(), 1);
14113 assert_eq!(responses[0].header.ty, FrameType::Error);
14114 let error = parse_error(&responses[0]);
14115 assert_eq!(error["code"], "version_unsupported");
14116 assert!(registry.get_module("aft").unwrap().is_none());
14117 assert_eq!(registry.active_registration_count().unwrap(), 0);
14118 }
14119 }
14120
14121 #[test]
14122 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
14123 let registry = Arc::new(Registry::default());
14124 let forwarding = Arc::new(ForwardingTable::default());
14125 let handler =
14126 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14127 let module_connection = ConnectionId::new(301);
14128 let registration = registry
14129 .register_with_control_ops(
14130 manifest("aft-push", PROTOCOL_VERSION),
14131 PROTOCOL_VERSION,
14132 module_connection,
14133 module_baseline_control_ops(),
14134 )
14135 .unwrap();
14136 let (module_tx, _module_rx) = mpsc::channel(8);
14137 let endpoint = forwarding
14138 .register_module_connection(
14139 module_connection,
14140 "aft-push".to_string(),
14141 PROTOCOL_VERSION,
14142 manifest_concurrency(®istration.manifest),
14143 FrameSink::new(module_tx),
14144 )
14145 .unwrap();
14146
14147 let unknown = Frame::build(
14149 FrameType::Push,
14150 control_flags(),
14151 0,
14152 0,
14153 5,
14154 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
14155 )
14156 .unwrap();
14157 let out = handler.handle_status_update(endpoint, unknown).unwrap();
14158 assert!(
14159 out.is_empty(),
14160 "unknown push op must be ignored, got {out:?}"
14161 );
14162
14163 let malformed = Frame::build(
14165 FrameType::Push,
14166 control_flags(),
14167 0,
14168 0,
14169 6,
14170 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
14171 )
14172 .unwrap();
14173 let out = handler.handle_status_update(endpoint, malformed).unwrap();
14174 assert_eq!(out.len(), 1);
14175 assert_eq!(out[0].header.ty, FrameType::Error);
14176 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
14177 }
14178
14179 #[test]
14180 fn hello_rejected_when_connection_already_owns_client_routes() {
14181 let registry = Arc::new(Registry::default());
14182 let forwarding = Arc::new(ForwardingTable::default());
14183 let handler =
14184 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14185 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
14187 let client_connection = ConnectionId::new(202);
14188
14189 let responses = handler
14192 .handle_control(
14193 client_connection,
14194 hello_frame("aft-second", PROTOCOL_VERSION, 9),
14195 )
14196 .unwrap();
14197 assert_eq!(responses[0].header.ty, FrameType::Error);
14198 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
14199 assert!(registry.get_module("aft-second").unwrap().is_none());
14200 }
14201
14202 #[tokio::test]
14203 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
14204 let registry = Arc::new(Registry::default());
14205 let forwarding = Arc::new(ForwardingTable::default());
14206 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
14207 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
14208 hello_via_sink(
14209 &handler,
14210 &module_ctx,
14211 &mut module_rx,
14212 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
14213 )
14214 .await;
14215 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
14216 let pending = forwarding
14217 .begin_route_bind_relay_for_test(
14218 client_ctx.connection_id,
14219 client_ctx.egress.clone(),
14220 2,
14221 "alpha",
14222 )
14223 .unwrap();
14224 forwarding
14225 .complete_pending_relay(
14226 module_ctx.connection_id,
14227 pending.corr,
14228 RouteBindRelayOutcome::Accepted,
14229 )
14230 .unwrap();
14231 client_rx.try_recv().unwrap();
14232 for module_id in ["beta", "alpha"] {
14233 let replies = handler
14234 .handle_control_frame(
14235 &module_ctx,
14236 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
14237 )
14238 .await
14239 .unwrap();
14240 assert_eq!(replies.len(), 1, "second HELLO must be refused");
14241 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
14242 }
14243 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
14244 assert!(registry.get_module("beta").unwrap().is_none());
14245 assert!(matches!(
14246 forwarding
14247 .lookup_data_route(
14248 client_ctx.connection_id,
14249 pending.client_channel,
14250 pending.client_epoch,
14251 )
14252 .unwrap(),
14253 DataRoute::Client(DataRouteState::Bound(_))
14254 ));
14255 assert!(handler
14256 .hello_launch_nonces
14257 .lock()
14258 .unwrap()
14259 .presented(module_ctx.connection_id, Some("alpha-nonce")));
14260 assert!(module_rx.try_recv().is_err());
14261 }
14262
14263 #[test]
14264 fn reserved_module_hello_requires_matching_launch_nonce() {
14265 let registry = Arc::new(Registry::default());
14266 let supervisor = SupervisorHandle::new();
14267 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
14270 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14271
14272 let no_nonce = handler
14274 .handle_control(
14275 ConnectionId::new(1),
14276 hello_frame("vault", PROTOCOL_VERSION, 1),
14277 )
14278 .unwrap();
14279 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
14280 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
14281 assert!(registry.get_module("vault").unwrap().is_none());
14282
14283 let wrong = handler
14285 .handle_control(
14286 ConnectionId::new(2),
14287 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
14288 )
14289 .unwrap();
14290 assert_eq!(wrong[0].header.ty, FrameType::Error);
14291 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14292 assert!(registry.get_module("vault").unwrap().is_none());
14293
14294 let ok = handler
14296 .handle_control(
14297 ConnectionId::new(3),
14298 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14299 )
14300 .unwrap();
14301 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14302 assert!(registry.get_module("vault").unwrap().is_some());
14303 }
14304
14305 #[test]
14306 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14307 let registry = Arc::new(Registry::default());
14308 let supervisor = SupervisorHandle::new();
14309 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14310 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14311 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14312
14313 let squat = handler
14314 .handle_control(
14315 ConnectionId::new(1),
14316 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14317 )
14318 .unwrap();
14319 assert_eq!(squat[0].header.ty, FrameType::Error);
14320 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14321 assert!(parse_error(&squat[0])["message"]
14322 .as_str()
14323 .unwrap()
14324 .contains("fed:"));
14325
14326 let accepted_peer = handler
14327 .handle_control(
14328 ConnectionId::new(2),
14329 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14330 )
14331 .unwrap();
14332 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14333
14334 let accepted_short = handler
14335 .handle_control(
14336 ConnectionId::new(3),
14337 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14338 )
14339 .unwrap();
14340 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14341
14342 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14343 let response = handler
14344 .handle_control(
14345 ConnectionId::new(conn),
14346 hello_frame(module_id, PROTOCOL_VERSION, conn),
14347 )
14348 .unwrap();
14349 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14350 }
14351 }
14352
14353 #[test]
14354 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14355 let registry = Arc::new(Registry::default());
14356 let supervisor = SupervisorHandle::new();
14357 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14358 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14359 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14360 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14361
14362 let owner_nonce = handler
14363 .handle_control(
14364 ConnectionId::new(1),
14365 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14366 )
14367 .unwrap();
14368 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14369 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14370 assert!(registry.get_module("fed:special").unwrap().is_none());
14371
14372 let exact_nonce = handler
14373 .handle_control(
14374 ConnectionId::new(2),
14375 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14376 )
14377 .unwrap();
14378 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14379 assert!(registry.get_module("fed:special").unwrap().is_some());
14380 }
14381
14382 #[test]
14383 fn non_reserved_module_ignores_launch_nonce() {
14384 let registry = Arc::new(Registry::default());
14385 let handler = ControlHandler::new(Arc::clone(®istry));
14388 let no_nonce = handler
14389 .handle_control(
14390 ConnectionId::new(1),
14391 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14392 )
14393 .unwrap();
14394 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14395 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14396
14397 let echoed_nonce = handler
14398 .handle_control(
14399 ConnectionId::new(2),
14400 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14401 )
14402 .unwrap();
14403 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14404 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14405 }
14406
14407 #[test]
14408 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14409 let handler = ControlHandler::default();
14410 let conn = ConnectionId::new(1);
14411 let malformed = Frame::build(
14412 FrameType::Hello,
14413 control_flags(),
14414 0,
14415 0,
14416 3,
14417 b"{not json".to_vec(),
14418 )
14419 .unwrap();
14420
14421 let error = handler.handle_control(conn, malformed).unwrap();
14422 assert_eq!(error[0].header.ty, FrameType::Error);
14423 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14424
14425 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14426 let pong = handler.handle_control(conn, ping).unwrap();
14427 assert_eq!(pong[0].header.ty, FrameType::Pong);
14428 assert_eq!(pong[0].header.corr, 4);
14429 }
14430
14431 #[test]
14432 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14433 let registry = Arc::new(Registry::default());
14434 let handler = ControlHandler::new(Arc::clone(®istry));
14435
14436 handler
14437 .handle_control(
14438 ConnectionId::new(1),
14439 hello_frame("aft", PROTOCOL_VERSION, 1),
14440 )
14441 .unwrap();
14442 let duplicate = handler
14443 .handle_control(
14444 ConnectionId::new(2),
14445 hello_frame("aft", PROTOCOL_VERSION, 2),
14446 )
14447 .unwrap();
14448
14449 assert_eq!(duplicate[0].header.ty, FrameType::Error);
14450 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14451 let registration = registry.get_module("aft").unwrap().unwrap();
14452 assert_eq!(registration.connection_id, ConnectionId::new(1));
14453 }
14454
14455 #[test]
14456 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14457 let registry = Arc::new(Registry::default());
14458 let forwarding = Arc::new(ForwardingTable::default());
14459 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14460 let handler =
14461 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14462 .with_process_liveness(process_liveness);
14463 let (ctx, route_channel, route_epoch) =
14464 bind_liveness_route(®istry, &forwarding, "aft-dead");
14465 let responses = handler
14466 .handle_route_poll(
14467 &ctx,
14468 route_poll_frame(41, PollKind::Liveness, route_channel),
14469 route_channel,
14470 route_epoch,
14471 PollKind::Liveness,
14472 )
14473 .unwrap();
14474
14475 assert_eq!(responses.len(), 1);
14476 assert_eq!(responses[0].header.ty, FrameType::Response);
14477 assert_route_poll_liveness(&responses[0], false);
14478 }
14479
14480 #[test]
14481 fn liveness_poll_without_process_source_uses_bound_route() {
14482 let registry = Arc::new(Registry::default());
14483 let forwarding = Arc::new(ForwardingTable::default());
14484 let handler =
14485 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14486 let (ctx, route_channel, route_epoch) =
14487 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
14488 let responses = handler
14489 .handle_route_poll(
14490 &ctx,
14491 route_poll_frame(42, PollKind::Liveness, route_channel),
14492 route_channel,
14493 route_epoch,
14494 PollKind::Liveness,
14495 )
14496 .unwrap();
14497
14498 assert_route_poll_liveness(&responses[0], true);
14499 }
14500
14501 #[test]
14502 fn liveness_poll_untracked_process_source_uses_bound_route() {
14503 let registry = Arc::new(Registry::default());
14504 let forwarding = Arc::new(ForwardingTable::default());
14505 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14506 let handler =
14507 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14508 .with_process_liveness(process_liveness);
14509 let (ctx, route_channel, route_epoch) =
14510 bind_liveness_route(®istry, &forwarding, "aft-untracked");
14511 let responses = handler
14512 .handle_route_poll(
14513 &ctx,
14514 route_poll_frame(43, PollKind::Liveness, route_channel),
14515 route_channel,
14516 route_epoch,
14517 PollKind::Liveness,
14518 )
14519 .unwrap();
14520
14521 assert_route_poll_liveness(&responses[0], true);
14522 }
14523
14524 #[tokio::test]
14525 async fn unknown_op_returns_unknown_control_op() {
14526 let handler = ControlHandler::default();
14527 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14528 let request = Frame::build(
14529 FrameType::Request,
14530 control_flags(),
14531 0,
14532 0,
14533 55,
14534 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14535 )
14536 .unwrap();
14537
14538 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14539
14540 assert_eq!(response.len(), 1);
14541 assert_eq!(response[0].header.ty, FrameType::Error);
14542 assert_eq!(response[0].header.corr, 55);
14543 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14544 }
14545
14546 #[tokio::test]
14547 async fn supervisor_provenance_rejects_unknown_exact_module() {
14548 let handler = ControlHandler::default();
14549 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14550 let request = Frame::build(
14551 FrameType::Request,
14552 control_flags(),
14553 0,
14554 0,
14555 57,
14556 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14557 )
14558 .unwrap();
14559
14560 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14561
14562 assert_eq!(response.len(), 1);
14563 assert_eq!(response[0].header.ty, FrameType::Error);
14564 assert_eq!(response[0].header.corr, 57);
14565 let error = parse_error(&response[0]);
14566 assert_eq!(error["code"], "unknown_module");
14567 assert_eq!(error["message"], "module_id 'missing' is not supervised");
14568 }
14569
14570 #[test]
14571 fn provenance_probe_override_keeps_handler_tests_deterministic() {
14572 let expected = subc_control::RunningImageAgreement::Unavailable {
14573 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14574 };
14575 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14576 assert_eq!(handler.provenance_probe_override, Some(expected));
14577 }
14578
14579 #[test]
14580 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14581 let verdict = reload_verdict(
14582 std::path::Path::new("/bin/new"),
14583 Some(std::path::Path::new("/bin/old")),
14584 subc_control::RunningImageAgreement::Unavailable {
14585 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14586 },
14587 );
14588 assert!(matches!(
14589 verdict.path,
14590 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14591 if configured == std::path::Path::new("/bin/new")
14592 && spawned_from == std::path::Path::new("/bin/old")
14593 ));
14594 }
14595
14596 #[test]
14597 fn launch_spec_without_running_digest_is_unavailable_not_match() {
14598 use subc_control::{
14599 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14600 };
14601 let configured = crate::supervise::ModuleSpec {
14602 module_id: "legacy".into(),
14603 program: "/module".into(),
14604 args: Vec::new(),
14605 env: Vec::new(),
14606 reserved: false,
14607 reserved_prefixes: Vec::new(),
14608 protocol: subc_control::ModuleProtocol::Subc,
14609 overlap: Default::default(),
14610 };
14611 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14612 assert_eq!(
14613 launch_spec_agreement(&configured, false, Some(&digest)),
14614 Agreement::Unavailable {
14615 reason: Reason::NotRunning
14616 }
14617 );
14618 assert_eq!(
14619 launch_spec_agreement(&configured, true, None),
14620 Agreement::Unavailable {
14621 reason: Reason::SpawnedSpecUnavailable
14622 }
14623 );
14624 assert_eq!(
14625 launch_spec_agreement(&configured, true, Some(&digest)),
14626 Agreement::Match
14627 );
14628 }
14629
14630 #[cfg(windows)]
14631 #[test]
14632 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14633 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14634 let configured = dir.join("module.exe");
14635 std::fs::write(&configured, b"executable path fixture").unwrap();
14636 let spawned = std::fs::canonicalize(&configured).unwrap();
14637 assert_ne!(
14638 configured, spawned,
14639 "fixture must exercise different path spellings"
14640 );
14641 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14642 let lower_case = configured.to_string_lossy().to_lowercase();
14643 for spelling in [
14644 &configured,
14645 Path::new(&forward_slashes),
14646 Path::new(&lower_case),
14647 ] {
14648 let verdict = reload_verdict(
14649 spelling,
14650 Some(&spawned),
14651 subc_control::RunningImageAgreement::Unavailable {
14652 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14653 },
14654 );
14655 assert_eq!(
14656 verdict.path,
14657 ReloadPathAgreement::Match,
14658 "configured {spelling:?}, spawned {spawned:?}"
14659 );
14660 }
14661 }
14662
14663 #[cfg(windows)]
14664 #[test]
14665 fn windows_reload_path_keeps_different_existing_program_pending() {
14666 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14667 let old = dir.join("old.exe");
14668 let new = dir.join("new.exe");
14669 std::fs::write(&old, b"same executable content").unwrap();
14670 std::fs::copy(&old, &new).unwrap();
14671 let spawned = std::fs::canonicalize(&old).unwrap();
14672 let image = subc_control::RunningImageAgreement::Unavailable {
14673 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14674 };
14675 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14676 assert!(
14677 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14678 if configured == new && spawned_from == spawned)
14679 );
14680 let missing = dir.join("missing.exe");
14681 assert!(matches!(
14682 reload_verdict(&missing, Some(&spawned), image).path,
14683 ReloadPathAgreement::Mismatch { .. }
14684 ));
14685 }
14686
14687 #[test]
14688 fn reload_verdict_detects_replaced_image_at_same_path() {
14689 let image = subc_control::RunningImageAgreement::Mismatch {
14690 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14691 digest: "old".into(),
14692 },
14693 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14694 digest: "new".into(),
14695 },
14696 };
14697 let verdict = reload_verdict(
14698 std::path::Path::new("/bin/same"),
14699 Some(std::path::Path::new("/bin/same")),
14700 image.clone(),
14701 );
14702 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14703 assert_eq!(verdict.image, image);
14704 }
14705
14706 #[test]
14707 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14708 let image = subc_control::RunningImageAgreement::Unavailable {
14709 reason: subc_control::RunningImageUnavailableReason::NotRunning,
14710 };
14711 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14712 assert_eq!(
14713 verdict.path,
14714 subc_control::ReloadPathAgreement::Unavailable {
14715 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14716 }
14717 );
14718 assert_eq!(verdict.image, image);
14719
14720 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14721 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14722 };
14723 let verdict = reload_verdict(
14724 std::path::Path::new("/bin/same"),
14725 Some(std::path::Path::new("/bin/same")),
14726 unconfirmed.clone(),
14727 );
14728 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14729 assert_eq!(verdict.image, unconfirmed);
14730 }
14731
14732 #[test]
14733 fn reload_verdict_preserves_each_image_unavailability_reason() {
14734 use subc_control::RunningImageUnavailableReason as Reason;
14735
14736 for reason in [
14737 Reason::NotRunning,
14738 Reason::UnsupportedPlatform,
14739 Reason::RunningExecutableUnreadable,
14740 Reason::SpawnedPathUnreadable,
14741 Reason::HashFailed,
14742 Reason::ProcessIdentityUnconfirmed,
14743 Reason::Unknown("future_probe_reason".to_string()),
14744 ] {
14745 let image = subc_control::RunningImageAgreement::Unavailable {
14746 reason: reason.clone(),
14747 };
14748 let verdict = reload_verdict(
14749 std::path::Path::new("/bin/same"),
14750 Some(std::path::Path::new("/bin/same")),
14751 image.clone(),
14752 );
14753 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14754 assert_eq!(verdict.image, image, "{reason:?}");
14755 }
14756 }
14757
14758 #[tokio::test]
14759 async fn malformed_control_bodies_return_invalid_control_body() {
14760 let handler = ControlHandler::default();
14761 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14762
14763 for (corr, body) in [
14764 (56, br#"{"route_channel":1}"#.as_slice()),
14765 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14766 (
14767 58,
14768 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14769 ),
14770 ] {
14771 let request = Frame::build(
14772 FrameType::Request,
14773 control_flags(),
14774 0,
14775 0,
14776 corr,
14777 body.to_vec(),
14778 )
14779 .unwrap();
14780 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14781
14782 assert_eq!(response.len(), 1);
14783 assert_eq!(response[0].header.ty, FrameType::Error);
14784 assert_eq!(response[0].header.corr, corr);
14785 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14786 }
14787 }
14788
14789 #[tokio::test]
14790 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14791 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14792 let registry = Arc::new(Registry::default());
14793 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14794 let router = Router::with_control_handler(Arc::clone(&control));
14795 let connection = router.begin_connection();
14796 let (ctx, mut rx) = route_ctx(connection.id());
14797
14798 router
14799 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14800 .await
14801 .unwrap();
14802 let response = rx.recv().await.unwrap();
14803 let ack = parse_ack(&response);
14804 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14805 let channel = 1;
14806
14807 let goodbye =
14808 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14809 router.route_for_connection(&ctx, goodbye).await.unwrap();
14810 assert!(rx.try_recv().is_err());
14811 assert!(registry.get_module("aft").unwrap().is_none());
14812
14813 router
14814 .route_for_connection(&ctx, channel_request(channel, 13))
14815 .await
14816 .unwrap();
14817 let error_frame = rx.recv().await.unwrap();
14818 assert_eq!(error_frame.header.ty, FrameType::Error);
14819 assert_eq!(error_frame.header.channel, channel);
14820 let captured = crate::router::test_log::captured_logs(&logs);
14821 assert_eq!(
14822 captured
14823 .lines()
14824 .filter(|line| {
14825 line.contains("module_id=aft")
14826 && line.contains("reason=explicit_goodbye")
14827 && line.contains("module registration ended")
14828 })
14829 .count(),
14830 1,
14831 "unexpected GOODBYE registry log: {captured}"
14832 );
14833 }
14834
14835 #[tokio::test]
14836 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14837 let registry = Arc::new(Registry::default());
14838 let handler = ControlHandler::new(registry).with_capability_config(
14839 [("prov".to_string(), true), ("cons".to_string(), true)],
14840 BTreeMap::new(),
14841 );
14842 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14843 register_capability_manifest(
14844 &handler,
14845 &provider_ctx,
14846 &mut provider_rx,
14847 capability_manifest("prov", &["thing/v1"], &[]),
14848 1,
14849 )
14850 .await;
14851 let mut consumer = capability_manifest("cons", &[], &[]);
14852 consumer.capabilities.as_mut().unwrap().requires.push(
14853 subc_protocol::manifest::CapabilityRequirement {
14854 capability: "thing/v1".to_string(),
14855 need: subc_protocol::manifest::CapabilityNeed::Required,
14856 },
14857 );
14858 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14859 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14860 assert_eq!(
14861 handler.capability_evaluator.verdict("cons", "thing/v1"),
14862 Some(CapabilityVerdict::Provided)
14863 );
14864 let (mut client_rx, _) = open_route_for_capability_test(
14865 &handler,
14866 &provider_ctx,
14867 &mut provider_rx,
14868 703,
14869 3,
14870 "prov",
14871 None,
14872 )
14873 .await;
14874 let goodbye =
14875 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14876 handler
14877 .handle_control_frame(&provider_ctx, goodbye)
14878 .await
14879 .unwrap();
14880 assert_eq!(
14881 handler.capability_evaluator.verdict("cons", "thing/v1"),
14882 Some(CapabilityVerdict::NeverProvided)
14883 );
14884 let closed = client_rx
14885 .try_recv()
14886 .expect("GOODBYE pushes route.closed before route GOODBYE");
14887 assert!(
14888 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14889 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14890 );
14891 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14892 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14893 }
14894
14895 struct RouteClosedFixture {
14896 handler: ControlHandler,
14897 #[cfg(unix)]
14898 supervisor: Supervisor,
14899 module_ctx: RouteCtx,
14900 #[cfg(unix)]
14901 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14902 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14903 route: Arc<crate::forwarding::RouteBinding>,
14904 }
14905
14906 async fn route_closed_fixture() -> RouteClosedFixture {
14907 let registry = Arc::new(Registry::default());
14908 let handler = ControlHandler::new(Arc::clone(®istry));
14909 #[cfg(unix)]
14910 let supervisor = Supervisor::new(registry, RestartPolicy::default())
14911 .with_forwarding(Arc::clone(&handler.forwarding));
14912 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14913 register_capability_manifest(
14914 &handler,
14915 &module_ctx,
14916 &mut module_rx,
14917 capability_manifest("prov", &[], &[]),
14918 1,
14919 )
14920 .await;
14921 let (client_rx, bind) = open_route_for_capability_test(
14922 &handler,
14923 &module_ctx,
14924 &mut module_rx,
14925 702,
14926 2,
14927 "prov",
14928 None,
14929 )
14930 .await;
14931 let ModuleControlRequest::RouteBind {
14932 route_channel: channel,
14933 epoch: route_epoch,
14934 ..
14935 } = bind
14936 else {
14937 panic!("expected route.bind");
14938 };
14939 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14940 handler
14941 .forwarding
14942 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14943 .unwrap()
14944 else {
14945 panic!("the acknowledged route must be bound");
14946 };
14947 RouteClosedFixture {
14948 handler,
14949 #[cfg(unix)]
14950 supervisor,
14951 module_ctx,
14952 #[cfg(unix)]
14953 module_rx,
14954 client_rx,
14955 route,
14956 }
14957 }
14958
14959 impl RouteClosedFixture {
14960 #[cfg(unix)]
14961 async fn cut(&mut self) {
14962 let notices = async {
14963 let mut draining = self.module_rx.recv().await.unwrap();
14964 assert!(matches!(
14965 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14966 &draining.body
14967 )
14968 .unwrap(),
14969 subc_protocol::session::ModuleControlCommand::Draining {
14970 reason: RouteCloseReason::Restart,
14971 ..
14972 }
14973 ));
14974 draining.flushed.take().unwrap().send(()).unwrap();
14975 let mut closing = self.client_rx.recv().await.unwrap();
14976 assert!(matches!(
14977 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14978 ClientControlPush::RouteClosing {
14979 reason: RouteCloseReason::Restart,
14980 ..
14981 }
14982 ));
14983 closing.flushed.take().unwrap().send(()).unwrap();
14984 };
14985 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14986 result.unwrap();
14987 }
14988
14989 fn disconnect(&mut self) -> ClientControlPush {
14990 self.handler
14991 .cleanup_connection(self.module_ctx.connection_id)
14992 .unwrap();
14993 let closed = self.client_rx.try_recv().unwrap();
14994 assert_eq!(closed.header.ty, FrameType::Push);
14995 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14996 assert!(matches!(
14997 &push,
14998 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14999 if channels == &[self.route.client_channel]
15000 ));
15001 assert_eq!(
15002 self.client_rx.try_recv().unwrap().header.ty,
15003 FrameType::Goodbye
15004 );
15005 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
15006 push
15007 }
15008 }
15009
15010 #[cfg(unix)]
15011 #[tokio::test(start_paused = true)]
15012 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
15013 let mut fixture = route_closed_fixture().await;
15014 fixture.cut().await;
15015 assert!(matches!(
15016 fixture.disconnect(),
15017 ClientControlPush::RouteClosed {
15018 reason: RouteCloseReason::Restart,
15019 drained: true,
15020 ..
15021 }
15022 ));
15023 }
15024
15025 #[cfg(unix)]
15026 #[tokio::test(start_paused = true)]
15027 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
15028 let mut fixture = route_closed_fixture().await;
15029 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
15030 let started = tokio::time::Instant::now();
15031 fixture.cut().await;
15032 assert!(started.elapsed() >= Duration::from_secs(2));
15033 assert_eq!(fixture.route.flow.in_flight(), 1);
15034 assert!(matches!(
15035 fixture.disconnect(),
15036 ClientControlPush::RouteClosed {
15037 reason: RouteCloseReason::Restart,
15038 drained: false,
15039 ..
15040 }
15041 ));
15042 }
15043
15044 #[tokio::test]
15045 async fn module_connection_crash_reports_crash_and_not_drained() {
15046 let mut fixture = route_closed_fixture().await;
15047 assert!(matches!(
15051 fixture.disconnect(),
15052 ClientControlPush::RouteClosed {
15053 reason: RouteCloseReason::Crash,
15054 drained: false,
15055 ..
15056 }
15057 ));
15058 }
15059
15060 #[tokio::test]
15061 async fn dropping_router_connection_releases_registration() {
15062 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15063 let registry = Arc::new(Registry::default());
15064 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15065 let router = Router::with_control_handler(Arc::clone(&control));
15066 let connection = router.begin_connection();
15067 let connection_id = connection.id();
15068 let (ctx, mut rx) = route_ctx(connection_id);
15069
15070 router
15071 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
15072 .await
15073 .unwrap();
15074 let response = rx.recv().await.unwrap();
15075 let ack = parse_ack(&response);
15076 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15077 assert!(registry.get_module("aft").unwrap().is_some());
15078
15079 drop(connection);
15080
15081 assert!(registry.get_module("aft").unwrap().is_none());
15082 assert_eq!(registry.active_registration_count().unwrap(), 0);
15083
15084 control
15085 .cleanup_connection(ConnectionId::new(u64::MAX))
15086 .unwrap();
15087 let captured = crate::router::test_log::captured_logs(&logs);
15088 println!("captured registry lifecycle logs:\n{captured}");
15089 let events: Vec<_> = captured
15090 .lines()
15091 .filter(|line| {
15092 line.contains("module registered module_id=aft ")
15093 || line.contains("module registration ended")
15094 })
15095 .collect();
15096 assert_eq!(
15097 events.len(),
15098 2,
15099 "unexpected registry lifecycle logs: {captured}"
15100 );
15101 assert!(events[0].contains("module registered module_id=aft "));
15102 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
15103 assert!(events[1].contains(&format!(
15104 "module_id=aft connection_id={}",
15105 connection_id.get()
15106 )));
15107 assert!(events[1].contains("reason=connection_closed"));
15108 assert!(events[1].contains("module registration ended"));
15109 assert_eq!(
15110 captured
15111 .lines()
15112 .filter(|line| line.contains("module registered module_id=aft "))
15113 .count(),
15114 1,
15115 "legacy registration admission line must appear once: {captured}"
15116 );
15117 }
15118
15119 #[tokio::test]
15120 async fn hello_registration_keeps_legacy_module_registered_line_once() {
15121 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15122 let registry = Arc::new(Registry::default());
15123 let control = ControlHandler::new(Arc::clone(®istry));
15124 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
15125 hello_via_sink(
15126 &control,
15127 &ctx,
15128 &mut rx,
15129 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
15130 )
15131 .await;
15132
15133 let captured = crate::router::test_log::captured_logs(&logs);
15134 let admissions: Vec<_> = captured
15135 .lines()
15136 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
15137 .collect();
15138 assert_eq!(
15139 admissions.len(),
15140 1,
15141 "legacy admission line must remain exactly once: {captured}"
15142 );
15143 assert!(admissions[0].contains("routable_provider=true"));
15144 assert!(admissions[0].contains("connection_id=777"));
15145 }
15146
15147 fn capability_manifest(
15148 module_id: &str,
15149 provides: &[&str],
15150 must_never_reach: &[&str],
15151 ) -> ModuleManifest {
15152 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
15153 manifest.capabilities = Some(CapabilityDeclarations {
15154 provides: provides
15155 .iter()
15156 .map(|capability| (*capability).to_string())
15157 .collect(),
15158 requires: Vec::new(),
15159 must_never_reach: must_never_reach
15160 .iter()
15161 .map(|capability| (*capability).to_string())
15162 .collect(),
15163 });
15164 manifest
15165 }
15166
15167 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
15168 Frame::build(
15169 FrameType::Hello,
15170 control_flags(),
15171 0,
15172 0,
15173 corr,
15174 serde_json::to_vec(&ModuleHelloBody {
15175 protocol_ver: manifest.protocol_ver,
15176 manifest,
15177 control_ops: None,
15178 launch_nonce: None,
15179 })
15180 .expect("capability test HELLO serializes"),
15181 )
15182 .expect("capability test HELLO frame builds")
15183 }
15184
15185 fn catalog_update_with_capabilities_frame(
15186 corr: u64,
15187 capabilities: CapabilityDeclarations,
15188 ) -> Frame {
15189 Frame::build(
15190 FrameType::Request,
15191 control_flags(),
15192 0,
15193 0,
15194 corr,
15195 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15196 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
15197 capabilities: Some(capabilities),
15198 ready: None,
15199 })
15200 .expect("capability catalog.update serializes"),
15201 )
15202 .expect("capability catalog.update frame builds")
15203 }
15204
15205 async fn register_capability_manifest(
15206 handler: &ControlHandler,
15207 ctx: &RouteCtx,
15208 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15209 manifest: ModuleManifest,
15210 corr: u64,
15211 ) {
15212 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
15213 }
15214
15215 async fn open_route_for_capability_test(
15216 handler: &ControlHandler,
15217 target_ctx: &RouteCtx,
15218 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15219 client_connection_id: u64,
15220 corr: u64,
15221 target_module_id: &str,
15222 consumer_identity: Option<ConsumerIdentity>,
15223 ) -> (
15224 mpsc::Receiver<crate::router::OutboundFrame>,
15225 ModuleControlRequest,
15226 ) {
15227 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
15228 let route_handler = handler.clone();
15229 let target_module_id = target_module_id.to_string();
15230 let route_task = tokio::spawn(async move {
15231 route_handler
15232 .handle_control_frame(
15233 &client_ctx,
15234 route_open_frame_with_admission_facts(
15235 corr,
15236 &target_module_id,
15237 unique_project_root("admission-facts"),
15238 consumer_identity,
15239 None,
15240 ),
15241 )
15242 .await
15243 .expect("capability test route.open succeeds")
15244 });
15245 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
15246 .await
15247 .expect("capability test route.open must reach route.bind")
15248 .expect("target control receiver stays open");
15249 let bind_request: ModuleControlRequest =
15250 serde_json::from_slice(&bind.body).expect("route.bind decodes");
15251 handler
15252 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
15253 .await
15254 .expect("capability test route.bind ACK succeeds");
15255 assert!(route_task.await.expect("route.open task joins").is_empty());
15256 let opened = client_rx
15257 .recv()
15258 .await
15259 .expect("successful route.open publishes a response");
15260 assert!(matches!(
15261 serde_json::from_slice::<ClientControlResponse>(&opened.body),
15262 Ok(ClientControlResponse::RouteOpen { .. })
15263 ));
15264 (client_rx, bind_request)
15265 }
15266
15267 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
15268 assert_eq!(frame.header.ty, FrameType::Push);
15269 assert_eq!(frame.header.channel, 0);
15270 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
15271 .expect("route.closed control push decodes");
15272 let ClientControlPush::RouteClosed { channels, .. } = &push else {
15273 panic!("expected route.closed");
15274 };
15275 assert_eq!(channels.len(), 1, "exactly one violating route closed");
15276 let closed_channel = channels[0];
15277 let channels = channels.clone();
15278 assert_eq!(
15279 push,
15280 ClientControlPush::RouteClosed {
15281 module_id: target_module_id.to_string(),
15282 channels,
15283 reason: RouteCloseReason::CapabilityDenied,
15284 drained: false,
15285 abandoned: 0,
15286 excluded_subscriptions: 0,
15287 terminal: Some(false),
15288 }
15289 );
15290 closed_channel
15291 }
15292
15293 #[tokio::test]
15294 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15295 let registry = Arc::new(Registry::default());
15296 let forwarding = Arc::new(ForwardingTable::default());
15297 let supervisor = SupervisorHandle::new();
15298 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15299 let handler =
15300 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15301 .with_supervisor(supervisor);
15302 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15303 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15304 register_capability_manifest(
15305 &handler,
15306 &target_ctx,
15307 &mut target_rx,
15308 capability_manifest("target", &["credentials-provider/v1"], &[]),
15309 1,
15310 )
15311 .await;
15312 register_capability_manifest(
15313 &handler,
15314 &opener_ctx,
15315 &mut opener_rx,
15316 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15317 2,
15318 )
15319 .await;
15320
15321 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15322 let replies = handler
15323 .handle_control_frame(
15324 &client_ctx,
15325 route_open_frame_with_admission_facts(
15326 3,
15327 "target",
15328 unique_project_root("admission-facts"),
15329 Some(ConsumerIdentity {
15330 module_id: "opener".to_string(),
15331 launch_nonce: "opener-nonce".to_string(),
15332 }),
15333 None,
15334 ),
15335 )
15336 .await
15337 .expect("denied route.open returns a typed frame");
15338 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15339 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15340 assert!(
15341 target_rx.try_recv().is_err(),
15342 "forbidden route.open must not relay route.bind"
15343 );
15344 }
15345
15346 #[tokio::test]
15347 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15348 let registry = Arc::new(Registry::default());
15349 let forwarding = Arc::new(ForwardingTable::default());
15350 let supervisor = SupervisorHandle::new();
15351 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15352 let handler =
15353 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15354 .with_supervisor(supervisor);
15355 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15356 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15357 register_capability_manifest(
15358 &handler,
15359 &target_ctx,
15360 &mut target_rx,
15361 capability_manifest("target", &["credentials-provider/v1"], &[]),
15362 1,
15363 )
15364 .await;
15365 register_capability_manifest(
15366 &handler,
15367 &old_opener_ctx,
15368 &mut old_opener_rx,
15369 capability_manifest("opener", &[], &[]),
15370 2,
15371 )
15372 .await;
15373 let (mut client_rx, _) = open_route_for_capability_test(
15374 &handler,
15375 &target_ctx,
15376 &mut target_rx,
15377 712,
15378 3,
15379 "target",
15380 Some(ConsumerIdentity {
15381 module_id: "opener".to_string(),
15382 launch_nonce: "opener-nonce".to_string(),
15383 }),
15384 )
15385 .await;
15386 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15387
15388 handler
15389 .cleanup_connection(old_opener_ctx.connection_id)
15390 .expect("old opener registration cleans up");
15391 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15392 register_capability_manifest(
15393 &handler,
15394 &new_opener_ctx,
15395 &mut new_opener_rx,
15396 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15397 4,
15398 )
15399 .await;
15400
15401 let closed_channel = assert_capability_denied_push(
15402 client_rx
15403 .try_recv()
15404 .expect("HELLO deny addition must emit route.closed")
15405 .frame,
15406 "target",
15407 );
15408 let goodbye = client_rx
15409 .try_recv()
15410 .expect("HELLO deny addition must send the client a route GOODBYE")
15411 .frame;
15412 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15413 assert_eq!(goodbye.header.channel, closed_channel);
15414 assert_ne!(goodbye.header.channel, 0);
15415 assert_ne!(goodbye.header.epoch, 0);
15416 assert!(client_rx.try_recv().is_err());
15417 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15418 assert!(matches!(
15419 target_rx.try_recv(),
15420 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15421 ));
15422 }
15423
15424 #[tokio::test]
15425 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15426 let registry = Arc::new(Registry::default());
15427 let forwarding = Arc::new(ForwardingTable::default());
15428 let supervisor = SupervisorHandle::new();
15429 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15430 let handler =
15431 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15432 .with_supervisor(supervisor);
15433 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15434 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15435 register_capability_manifest(
15436 &handler,
15437 &target_ctx,
15438 &mut target_rx,
15439 capability_manifest("target", &[], &[]),
15440 1,
15441 )
15442 .await;
15443 register_capability_manifest(
15444 &handler,
15445 &opener_ctx,
15446 &mut opener_rx,
15447 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15448 2,
15449 )
15450 .await;
15451 let (mut client_rx, _) = open_route_for_capability_test(
15452 &handler,
15453 &target_ctx,
15454 &mut target_rx,
15455 722,
15456 3,
15457 "target",
15458 Some(ConsumerIdentity {
15459 module_id: "opener".to_string(),
15460 launch_nonce: "opener-nonce".to_string(),
15461 }),
15462 )
15463 .await;
15464 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15465
15466 let replies = handler
15467 .handle_control_frame(
15468 &target_ctx,
15469 catalog_update_with_capabilities_frame(
15470 4,
15471 CapabilityDeclarations {
15472 provides: vec!["credentials-provider/v1".to_string()],
15473 requires: Vec::new(),
15474 must_never_reach: Vec::new(),
15475 },
15476 ),
15477 )
15478 .await
15479 .expect("claim catalog.update succeeds");
15480 assert!(matches!(
15481 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15482 Ok(ModuleControlResponseToModule::CatalogUpdate {})
15483 ));
15484 let closed_channel = assert_capability_denied_push(
15485 client_rx
15486 .try_recv()
15487 .expect("claim addition must emit route.closed")
15488 .frame,
15489 "target",
15490 );
15491 let goodbye = client_rx
15492 .try_recv()
15493 .expect("claim addition must send the client a route GOODBYE")
15494 .frame;
15495 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15496 assert_eq!(goodbye.header.channel, closed_channel);
15497 assert_ne!(goodbye.header.channel, 0);
15498 assert_ne!(goodbye.header.epoch, 0);
15499 assert!(client_rx.try_recv().is_err());
15500 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15501 assert!(matches!(
15502 target_rx.try_recv(),
15503 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15504 ));
15505 }
15506
15507 #[tokio::test]
15508 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15509 let registry = Arc::new(Registry::default());
15510 let forwarding = Arc::new(ForwardingTable::default());
15511 let supervisor = SupervisorHandle::new();
15512 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15513 let handler =
15514 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15515 .with_supervisor(supervisor);
15516 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15517 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15518 register_capability_manifest(
15519 &handler,
15520 &target_ctx,
15521 &mut target_rx,
15522 capability_manifest("target", &["credentials-provider/v1"], &[]),
15523 1,
15524 )
15525 .await;
15526 register_capability_manifest(
15527 &handler,
15528 &opener_ctx,
15529 &mut opener_rx,
15530 capability_manifest("opener", &[], &[]),
15531 2,
15532 )
15533 .await;
15534 let (mut client_rx, _) = open_route_for_capability_test(
15535 &handler,
15536 &target_ctx,
15537 &mut target_rx,
15538 732,
15539 3,
15540 "target",
15541 Some(ConsumerIdentity {
15542 module_id: "opener".to_string(),
15543 launch_nonce: "opener-nonce".to_string(),
15544 }),
15545 )
15546 .await;
15547 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15548
15549 handler
15550 .handle_control_frame(
15551 &target_ctx,
15552 catalog_update_with_capabilities_frame(
15553 4,
15554 CapabilityDeclarations {
15555 provides: Vec::new(),
15556 requires: Vec::new(),
15557 must_never_reach: Vec::new(),
15558 },
15559 ),
15560 )
15561 .await
15562 .expect("claim removal catalog.update succeeds");
15563 assert_eq!(
15564 forwarding.active_binding_count().unwrap(),
15565 1,
15566 "removing an attested target claim must leave the route census unchanged"
15567 );
15568 assert!(
15569 client_rx.try_recv().is_err(),
15570 "claim removal must not emit route.closed or a client route GOODBYE"
15571 );
15572 assert!(
15573 target_rx.try_recv().is_err(),
15574 "claim removal must not send the target a route GOODBYE"
15575 );
15576 }
15577
15578 #[tokio::test]
15581 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15582 let registry = Arc::new(Registry::default());
15583 let forwarding = Arc::new(ForwardingTable::default());
15584 let supervisor = SupervisorHandle::new();
15585 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15586 let handler =
15587 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15588 .with_supervisor(supervisor);
15589 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15590 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15591 register_capability_manifest(
15592 &handler,
15593 &target_ctx,
15594 &mut target_rx,
15595 capability_manifest("target", &["credentials-provider/v1"], &[]),
15596 1,
15597 )
15598 .await;
15599 register_capability_manifest(
15600 &handler,
15601 &opener_ctx,
15602 &mut opener_rx,
15603 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15604 2,
15605 )
15606 .await;
15607
15608 let (_client_rx, bind) = open_route_for_capability_test(
15609 &handler,
15610 &target_ctx,
15611 &mut target_rx,
15612 742,
15613 3,
15614 "target",
15615 None,
15616 )
15617 .await;
15618 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15619 panic!("direct scope-honesty route must bind");
15620 };
15621 assert_eq!(principal, Some(Principal::Direct));
15622 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15623 }
15624
15625 #[tokio::test]
15628 async fn must_never_reach_self_route_is_capability_forbidden() {
15629 let registry = Arc::new(Registry::default());
15630 let forwarding = Arc::new(ForwardingTable::default());
15631 let supervisor = SupervisorHandle::new();
15632 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15633 let handler =
15634 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15635 .with_supervisor(supervisor);
15636 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15637 register_capability_manifest(
15638 &handler,
15639 &self_ctx,
15640 &mut self_rx,
15641 capability_manifest(
15642 "self-provider",
15643 &["credentials-provider/v1"],
15644 &["credentials-provider/v1"],
15645 ),
15646 1,
15647 )
15648 .await;
15649
15650 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15651 let replies = handler
15652 .handle_control_frame(
15653 &client_ctx,
15654 route_open_frame_with_admission_facts(
15655 2,
15656 "self-provider",
15657 unique_project_root("admission-facts"),
15658 Some(ConsumerIdentity {
15659 module_id: "self-provider".to_string(),
15660 launch_nonce: "self-nonce".to_string(),
15661 }),
15662 None,
15663 ),
15664 )
15665 .await
15666 .expect("self-route refusal returns a typed frame");
15667 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15668 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15669 assert!(
15670 self_rx.try_recv().is_err(),
15671 "self denial must not relay route.bind"
15672 );
15673 }
15674
15675 #[test]
15676 fn unsupported_channel_zero_frame_returns_error() {
15677 let handler = ControlHandler::default();
15678 let request = Frame::build(
15679 FrameType::Request,
15680 control_flags(),
15681 0,
15682 0,
15683 21,
15684 b"opaque".to_vec(),
15685 )
15686 .unwrap();
15687
15688 let response = handler
15689 .handle_control(ConnectionId::new(1), request)
15690 .unwrap();
15691
15692 assert_eq!(response[0].header.ty, FrameType::Error);
15693 assert_eq!(
15694 parse_error(&response[0])["code"],
15695 "unsupported_control_frame"
15696 );
15697 }
15698
15699 mod swap {
15704 use super::*;
15705
15706 const INCUMBENT: ConnectionId = ConnectionId::new(30);
15707 const CANDIDATE: ConnectionId = ConnectionId::new(40);
15708
15709 struct Swap {
15710 registry: Arc<Registry>,
15711 forwarding: Arc<ForwardingTable>,
15712 handler: ControlHandler,
15713 incumbent_ctx: RouteCtx,
15714 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15715 candidate_ctx: RouteCtx,
15716 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15717 }
15718
15719 async fn swap_with_incumbent() -> Swap {
15720 let registry = Arc::new(Registry::default());
15721 let forwarding = Arc::new(ForwardingTable::default());
15722 let handler =
15723 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15724 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15725 hello_via_sink(
15726 &handler,
15727 &incumbent_ctx,
15728 &mut incumbent_rx,
15729 hello_frame("aft", PROTOCOL_VERSION, 7),
15730 )
15731 .await;
15732 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15733 Swap {
15734 registry,
15735 forwarding,
15736 handler,
15737 incumbent_ctx,
15738 incumbent_rx,
15739 candidate_ctx,
15740 candidate_rx,
15741 }
15742 }
15743
15744 fn register_candidate(swap: &Swap, ready: Option<bool>) {
15745 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15746 candidate_manifest.ready = ready;
15747 let registration = swap
15748 .registry
15749 .register_candidate_with_control_ops(
15750 candidate_manifest,
15751 PROTOCOL_VERSION,
15752 CANDIDATE,
15753 module_baseline_control_ops(),
15754 )
15755 .unwrap();
15756 swap.forwarding
15757 .register_candidate_module_connection(
15758 CANDIDATE,
15759 "aft".to_string(),
15760 PROTOCOL_VERSION,
15761 manifest_concurrency(®istration.manifest),
15762 swap.candidate_ctx.egress.clone(),
15763 )
15764 .unwrap();
15765 }
15766
15767 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15768 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15769 swap.registry.promote_candidate("aft").unwrap().unwrap();
15770 cutover.incumbent.unwrap()
15771 }
15772
15773 #[tokio::test]
15782 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15783 let mut swap = swap_with_incumbent().await;
15784 let hold = Arc::new(std::sync::Barrier::new(2));
15785 let handler = swap
15786 .handler
15787 .clone()
15788 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15789 handler
15790 .supervisor
15791 .open_swap("aft", "candidate-nonce".to_string());
15792 let hello = {
15793 let handler = handler.clone();
15794 let candidate_ctx = swap.candidate_ctx.clone();
15795 std::thread::spawn(move || {
15796 handler
15797 .handle_hello(
15798 candidate_ctx.connection_id,
15799 Some(candidate_ctx.egress.clone()),
15800 hello_frame_with_nonce(
15801 "aft",
15802 PROTOCOL_VERSION,
15803 8,
15804 Some("candidate-nonce"),
15805 ),
15806 )
15807 .unwrap()
15808 })
15809 };
15810
15811 hold.wait();
15812 assert!(swap
15815 .registry
15816 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15817 .unwrap()
15818 .is_some());
15819 assert!(swap
15820 .registry
15821 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15822 .unwrap()
15823 .is_none());
15824 assert!(swap
15825 .forwarding
15826 .module_endpoint_for_connection(CANDIDATE)
15827 .unwrap()
15828 .is_none());
15829
15830 hold.wait();
15831 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15832 assert_eq!(
15833 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15834 FrameType::HelloAck
15835 );
15836 let published = swap
15837 .registry
15838 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15839 .unwrap()
15840 .expect("candidate is published once its HELLO completes");
15841 assert_eq!(published.connection_id, CANDIDATE);
15842 assert!(swap
15843 .forwarding
15844 .module_endpoint_for_connection(CANDIDATE)
15845 .unwrap()
15846 .is_some());
15847 }
15848
15849 fn keyed_total(counters: &Value, key: &str) -> u64 {
15850 counters[key]
15851 .as_object()
15852 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15853 .unwrap_or(0)
15854 }
15855
15856 #[tokio::test]
15857 async fn replacement_logs_old_end_and_new_admission_once() {
15858 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15859 let mut swap = swap_with_incumbent().await;
15860 swap.handler
15861 .supervisor
15862 .open_swap("aft", "candidate-nonce".to_string());
15863 hello_via_sink(
15864 &swap.handler,
15865 &swap.candidate_ctx,
15866 &mut swap.candidate_rx,
15867 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15868 )
15869 .await;
15870 cutover(&swap);
15871 swap.handler.cleanup_connection(INCUMBENT).unwrap();
15872
15873 let captured = crate::router::test_log::captured_logs(&logs);
15874 println!("captured replacement registry logs:\n{captured}");
15875 let new_admissions: Vec<_> = captured
15876 .lines()
15877 .filter(|line| {
15878 line.contains("connection_id=40")
15879 && line.contains("swap candidate registered; not routable until cutover")
15880 })
15881 .collect();
15882 assert_eq!(
15883 new_admissions.len(),
15884 1,
15885 "unexpected admission logs: {captured}"
15886 );
15887 assert!(new_admissions[0].contains("module_id=aft"));
15888 assert!(new_admissions[0].contains("connection_id=40"));
15889 assert!(
15890 new_admissions[0].contains("swap candidate registered; not routable until cutover")
15891 );
15892 assert!(new_admissions[0].contains("ready=true"));
15893
15894 let old_admissions: Vec<_> = captured
15895 .lines()
15896 .filter(|line| {
15897 line.contains("module registered module_id=aft ")
15898 && line.contains("connection_id=30")
15899 })
15900 .collect();
15901 assert_eq!(
15902 old_admissions.len(),
15903 1,
15904 "unexpected incumbent admission: {captured}"
15905 );
15906
15907 let old_ends: Vec<_> = captured
15908 .lines()
15909 .filter(|line| {
15910 line.contains("connection_id=30") && line.contains("module registration ended")
15911 })
15912 .collect();
15913 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15914 assert!(old_ends[0].contains("module_id=aft"));
15915 assert!(old_ends[0].contains("connection_id=30"));
15916 assert!(old_ends[0].contains("reason=replaced"));
15917 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15918 assert_eq!(
15919 captured
15920 .lines()
15921 .filter(|line| line.contains("module registration promoted"))
15922 .count(),
15923 1,
15924 "unexpected promotion logs: {captured}"
15925 );
15926 }
15927
15928 #[tokio::test]
15934 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15935 let mut swap = swap_with_incumbent().await;
15936 let handler = swap.handler.clone();
15937
15938 let cotenant = ConnectionId::new(31);
15940 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15941 let (cotenant_task, cotenant_bind) = relay_route_open(
15942 &handler,
15943 cotenant,
15944 &cotenant_ctx.egress,
15945 &mut swap.incumbent_rx,
15946 100,
15947 "aft",
15948 "swap-cotenant",
15949 )
15950 .await;
15951 handler
15952 .handle_control_frame(
15953 &swap.incumbent_ctx,
15954 route_bind_ack(cotenant_bind.header.corr),
15955 )
15956 .await
15957 .unwrap();
15958 assert!(cotenant_task.await.unwrap().is_empty());
15959 let (cotenant_channel, cotenant_epoch) =
15960 published_route(&cotenant_rx.recv().await.unwrap());
15961
15962 let caller = ConnectionId::new(32);
15964 let (caller_ctx, mut caller_rx) = route_ctx(caller);
15965 let (caller_task, caller_bind) = relay_route_open(
15966 &handler,
15967 caller,
15968 &caller_ctx.egress,
15969 &mut swap.incumbent_rx,
15970 101,
15971 "aft",
15972 "swap-caller",
15973 )
15974 .await;
15975 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15976
15977 register_candidate(&swap, None);
15978 cutover(&swap);
15979
15980 let ack = handler
15982 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15983 .await;
15984 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15985 if module_loop_error.is_some() {
15986 handler.cleanup_connection(INCUMBENT).unwrap();
15989 }
15990
15991 assert!(
15993 cotenant_rx.try_recv().is_err(),
15994 "the co-tenant route on the incumbent was torn down by one late ack: \
15995 {module_loop_error:?}"
15996 );
15997 assert!(matches!(
15998 swap.forwarding
15999 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16000 .unwrap(),
16001 DataRoute::Client(DataRouteState::Bound(_))
16002 ));
16003 assert_eq!(module_loop_error, None);
16004 assert!(swap
16005 .registry
16006 .get_module_by_connection(INCUMBENT)
16007 .unwrap()
16008 .is_some());
16009
16010 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
16012 .await
16013 .expect("the incumbent is told to drop the abandoned binding")
16014 .unwrap()
16015 .frame;
16016 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16017 assert_eq!(goodbye.header.channel, abandoned_channel);
16018 assert_eq!(goodbye.header.epoch, abandoned_epoch);
16019 assert!(swap.incumbent_rx.try_recv().is_err());
16020
16021 let response = caller_task.await.unwrap();
16023 assert_eq!(response.len(), 1);
16024 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
16025 assert!(caller_rx.try_recv().is_err());
16026
16027 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
16031 let counters = handler.counters().snapshot();
16032 assert_eq!(
16033 keyed_total(&counters, "route_open_accepted_by_principal"),
16034 1
16035 );
16036 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
16037 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
16038 }
16039
16040 #[tokio::test]
16044 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
16045 let mut swap = swap_with_incumbent().await;
16046 register_candidate(&swap, None);
16047 let incumbent = cutover(&swap);
16048 swap.forwarding
16049 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
16050 .unwrap()
16051 .expect("the incumbent is still registered");
16052
16053 let client = ConnectionId::new(33);
16054 let (client_ctx, mut client_rx) = route_ctx(client);
16055 let route_handler = swap.handler.clone();
16056 let open_ctx = RouteCtx {
16057 connection_id: client,
16058 egress: client_ctx.egress.clone(),
16059 };
16060 let mut route_task = tokio::spawn(async move {
16061 route_handler
16062 .handle_control_frame(
16063 &open_ctx,
16064 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
16065 )
16066 .await
16067 .unwrap()
16068 });
16069 let bind = tokio::select! {
16070 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
16071 response = &mut route_task => {
16072 let response = response.unwrap();
16073 panic!(
16074 "post-cutover route.open was refused instead of relayed to the candidate: {}",
16075 parse_error(&response[0])["code"]
16076 );
16077 }
16078 };
16079 swap.handler
16080 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
16081 .await
16082 .unwrap();
16083 assert!(route_task.await.unwrap().is_empty());
16084 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
16085 match swap
16086 .forwarding
16087 .lookup_data_route(client, channel, epoch)
16088 .unwrap()
16089 {
16090 DataRoute::Client(DataRouteState::Bound(route)) => {
16091 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
16092 }
16093 other => panic!("expected a bound route on the candidate, got {other:?}"),
16094 }
16095 assert!(swap.incumbent_rx.try_recv().is_err());
16096 }
16097
16098 #[tokio::test]
16103 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
16104 let swap = swap_with_incumbent().await;
16105 register_candidate(&swap, Some(false));
16106 let update = Frame::build(
16107 FrameType::Request,
16108 control_flags(),
16109 0,
16110 0,
16111 55,
16112 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16113 provides: manifest("aft", PROTOCOL_VERSION).provides,
16114 capabilities: None,
16115 ready: Some(true),
16116 })
16117 .unwrap(),
16118 )
16119 .unwrap();
16120
16121 let replies = swap
16122 .handler
16123 .handle_control_frame(&swap.candidate_ctx, update)
16124 .await
16125 .unwrap();
16126
16127 assert_eq!(replies.len(), 1);
16128 assert_eq!(
16129 replies[0].header.ty,
16130 FrameType::Response,
16131 "candidate catalog.update was refused: {:?}",
16132 serde_json::from_slice::<Value>(&replies[0].body).ok()
16133 );
16134 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
16135 assert_eq!(
16136 swap.registry
16137 .get_module("aft")
16138 .unwrap()
16139 .unwrap()
16140 .connection_id,
16141 INCUMBENT
16142 );
16143 }
16144 }
16145
16146 mod swap_admission {
16150 use super::*;
16151
16152 const INCUMBENT_NONCE: &str = "incumbent-nonce";
16153 const CANDIDATE_NONCE: &str = "candidate-nonce";
16154
16155 fn handler_with_incumbent(
16156 module_id: &str,
16157 reserved: bool,
16158 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
16159 let registry = Arc::new(Registry::default());
16160 let supervisor = SupervisorHandle::new();
16161 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
16162 if reserved {
16163 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
16164 }
16165 let handler =
16166 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
16167 let incumbent = handler
16168 .handle_control(
16169 ConnectionId::new(1),
16170 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
16171 )
16172 .unwrap();
16173 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
16174 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
16175 (registry, supervisor, handler)
16176 }
16177
16178 #[test]
16184 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
16185 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
16186
16187 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
16188 let replies = handler
16189 .handle_control(
16190 ConnectionId::new(connection),
16191 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
16192 )
16193 .unwrap();
16194 assert_eq!(replies[0].header.ty, FrameType::Error);
16195 assert_eq!(
16196 parse_error(&replies[0])["code"],
16197 "swap_token_invalid",
16198 "nonce {nonce:?}"
16199 );
16200 }
16201 assert!(registry.get_candidate("aft").unwrap().is_none());
16202 assert_eq!(
16203 registry.get_module("aft").unwrap().unwrap().connection_id,
16204 ConnectionId::new(1)
16205 );
16206
16207 let admitted = handler
16210 .handle_control(
16211 ConnectionId::new(4),
16212 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
16213 )
16214 .unwrap();
16215 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
16216 assert_eq!(
16217 registry
16218 .get_candidate("aft")
16219 .unwrap()
16220 .unwrap()
16221 .connection_id,
16222 ConnectionId::new(4)
16223 );
16224 assert_eq!(
16225 registry.get_module("aft").unwrap().unwrap().connection_id,
16226 ConnectionId::new(1),
16227 "the candidate must not take the active slot"
16228 );
16229 let replayed = handler
16230 .handle_control(
16231 ConnectionId::new(5),
16232 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
16233 )
16234 .unwrap();
16235 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
16236
16237 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
16241 let squatter = handler
16242 .handle_control(
16243 ConnectionId::new(6),
16244 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
16245 )
16246 .unwrap();
16247 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
16248 assert!(
16249 registry.get_module("aft").unwrap().is_none(),
16250 "a squatter took the active slot of an id being swapped"
16251 );
16252 }
16253
16254 #[test]
16259 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
16260 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
16261
16262 let replies = handler
16263 .handle_control(
16264 ConnectionId::new(2),
16265 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16266 )
16267 .unwrap();
16268
16269 assert_eq!(
16270 replies[0].header.ty,
16271 FrameType::HelloAck,
16272 "reserved candidate refused: {:?}",
16273 serde_json::from_slice::<Value>(&replies[0].body).ok()
16274 );
16275 assert_eq!(
16276 registry
16277 .get_candidate("vault")
16278 .unwrap()
16279 .unwrap()
16280 .connection_id,
16281 ConnectionId::new(2)
16282 );
16283 }
16284
16285 #[test]
16288 fn without_an_open_swap_the_ordinary_gates_decide() {
16289 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16290 supervisor.close_swap("vault");
16291
16292 let candidate = handler
16293 .handle_control(
16294 ConnectionId::new(2),
16295 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16296 )
16297 .unwrap();
16298 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16299 let duplicate = handler
16300 .handle_control(
16301 ConnectionId::new(3),
16302 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16303 )
16304 .unwrap();
16305 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16306 assert!(registry.get_candidate("vault").unwrap().is_none());
16307 }
16308 }
16309
16310 mod scopes {
16314 mod idle_tests {
16315 use super::*;
16316 include!("scope_idle_tests.rs");
16317 }
16318 mod route_open_timings {
16319 use super::*;
16320 include!("route_open_timing_tests.rs");
16321 }
16322 mod route_open_refusal_log {
16323 use super::*;
16324 include!("route_open_refusal_log_tests.rs");
16325 }
16326 mod activity {
16327 include!("activity_scope_tests.rs");
16328 }
16329 mod delta {
16330 include!("scope_handler_delta_tests.rs");
16331 }
16332 mod call_modes {
16333 include!("scope_call_mode_tests.rs");
16334 }
16335 #[tokio::test]
16336 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16337 let rig = rig().await;
16338 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16339 "s".into(),
16340 1,
16341 "tool".into(),
16342 ))
16343 .unwrap();
16344 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16345 let (direct, _rx) = wide_ctx(999);
16346 for (ctx, expected) in [
16347 (&rig.owner, FrameType::Response),
16348 (&direct, FrameType::Error),
16349 ] {
16350 let frame =
16351 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16352 .unwrap();
16353 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16354 assert_eq!(replies[0].header.ty, expected);
16355 if expected == FrameType::Response {
16356 let reply: subc_control::ScopeServedByReply =
16357 serde_json::from_slice(&replies[0].body).unwrap();
16358 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16359 } else {
16360 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16361 }
16362 }
16363 let reply = call(
16364 &rig.handler,
16365 &rig.owner,
16366 &ModuleControlRequestFromModule::ScopeDescribe {
16367 owner: Principal::Reserved {
16368 module_id: OWNER.into(),
16369 },
16370 scope_ref: "s".into(),
16371 },
16372 )
16373 .await;
16374 assert!(matches!(
16375 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16376 ModuleControlResponseToModule::ScopeDescribe { .. }
16377 ));
16378 }
16379
16380 use subc_protocol::scope::{
16381 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16382 ScopeStatus,
16383 };
16384
16385 use super::*;
16386
16387 const OWNER: &str = "prefrontal-core";
16388
16389 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16390 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16391 }
16392
16393 async fn call(
16394 handler: &ControlHandler,
16395 ctx: &RouteCtx,
16396 request: &ModuleControlRequestFromModule,
16397 ) -> Frame {
16398 let body = serde_json::to_vec(request).unwrap();
16399 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16400 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16401 assert_eq!(replies.len(), 1, "{replies:?}");
16402 replies.pop().unwrap()
16403 }
16404
16405 async fn sync(
16406 handler: &ControlHandler,
16407 ctx: &RouteCtx,
16408 generation: u64,
16409 scopes: Vec<ScopeRecord>,
16410 ) -> Result<ModuleControlResponseToModule, String> {
16411 let reply = call(
16412 handler,
16413 ctx,
16414 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16415 )
16416 .await;
16417 match reply.header.ty {
16418 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16419 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16420 }
16421 }
16422
16423 async fn describe(
16424 handler: &ControlHandler,
16425 ctx: &RouteCtx,
16426 owner: &str,
16427 scope_ref: &str,
16428 ) -> ModuleControlResponseToModule {
16429 let reply = call(
16430 handler,
16431 ctx,
16432 &ModuleControlRequestFromModule::ScopeDescribe {
16433 owner: Principal::Reserved {
16434 module_id: owner.to_string(),
16435 },
16436 scope_ref: scope_ref.to_string(),
16437 },
16438 )
16439 .await;
16440 assert_eq!(
16441 reply.header.ty,
16442 FrameType::Response,
16443 "{:?}",
16444 parse_error(&reply)
16445 );
16446 serde_json::from_slice(&reply.body).unwrap()
16447 }
16448
16449 #[tokio::test]
16450 async fn scope_describe_never_echoes_bind_proof_or_hash() {
16451 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16452 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16453 let mut rig = rig().await;
16454 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16455 .await;
16456 let _bound = rig
16457 .bound(
16458 Some(AFT),
16459 PLEXUS,
16460 Some(
16461 rig.selector("s", Some(1))
16462 .with_bind_proof(Some(PROOF.into())),
16463 ),
16464 )
16465 .await;
16466 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16467 let output = serde_json::to_string(&reply).unwrap();
16468 assert!(output.contains("\"status\":\"live\""));
16469 assert!(!output.contains(PROOF));
16470 assert!(!output.contains(HASH));
16471 }
16472
16473 async fn module(
16475 handler: &ControlHandler,
16476 connection: u64,
16477 module_id: &str,
16478 nonce: Option<&str>,
16479 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16480 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16481 hello_via_sink(
16482 handler,
16483 &ctx,
16484 &mut rx,
16485 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16486 )
16487 .await;
16488 (ctx, rx)
16489 }
16490
16491 #[tokio::test]
16494 async fn a_client_connection_cannot_sync_or_describe() {
16495 let handler = ControlHandler::new(Arc::new(Registry::default()));
16496 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16497 for request in [
16498 ModuleControlRequestFromModule::ScopeSync {
16499 generation: 1,
16500 scopes: vec![head("s", 1)],
16501 },
16502 ModuleControlRequestFromModule::ScopeDescribe {
16503 owner: Principal::Direct,
16504 scope_ref: "s".to_string(),
16505 },
16506 ] {
16507 let reply = call(&handler, &ctx, &request).await;
16508 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16509 }
16510 assert!(
16511 !handler
16512 .scopes
16513 .read()
16514 .unwrap()
16515 .describe(
16516 &Principal::Reserved {
16517 module_id: OWNER.to_string()
16518 },
16519 "s"
16520 )
16521 .owner_synced
16522 );
16523 }
16524
16525 #[tokio::test]
16528 async fn a_module_without_a_supervised_launch_cannot_sync() {
16529 let handler = ControlHandler::new(Arc::new(Registry::default()));
16530 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16531 assert_eq!(
16532 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16533 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16534 );
16535 }
16536
16537 #[tokio::test]
16538 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16539 let supervisor = SupervisorHandle::new();
16540 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16541 let handler = ControlHandler::new(Arc::new(Registry::default()))
16542 .with_supervisor(supervisor.clone());
16543 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16544 sync(&handler, &incumbent, 1, vec![head("s", 1)])
16545 .await
16546 .expect("the current launch syncs");
16547
16548 supervisor.open_swap(OWNER, "n2".to_string());
16551 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16552 assert_eq!(
16553 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16554 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16555 );
16556 sync(&handler, &incumbent, 2, vec![head("s", 1)])
16557 .await
16558 .expect("the serving owner syncs during the swap");
16559
16560 supervisor.close_swap(OWNER);
16563 assert_eq!(
16564 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16565 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16566 );
16567 sync(&handler, &incumbent, 3, vec![head("s", 1)])
16568 .await
16569 .expect("the serving owner syncs after the rollback");
16570 handler.cleanup_connection(candidate.connection_id).unwrap();
16571
16572 supervisor.open_swap(OWNER, "n3".to_string());
16577 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16578 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16579 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16580 .await
16581 .expect("the promoted launch takes authority");
16582 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16583 panic!("unexpected reply {reply:?}");
16584 };
16585 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16586 assert_eq!(
16587 sync(&handler, &incumbent, 4, Vec::new()).await,
16588 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16589 );
16590 }
16591
16592 #[tokio::test]
16595 async fn closing_the_authority_connection_frees_sync_authority() {
16596 let supervisor = SupervisorHandle::new();
16597 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16598 let handler = ControlHandler::new(Arc::new(Registry::default()))
16599 .with_supervisor(supervisor.clone());
16600 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16601 sync(&handler, &first, 10, vec![head("s", 1)])
16602 .await
16603 .unwrap();
16604 handler.cleanup_connection(first.connection_id).unwrap();
16605
16606 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16607 sync(&handler, &second, 1, vec![head("s", 1)])
16608 .await
16609 .expect("the next connection takes the released authority");
16610 }
16611
16612 #[tokio::test]
16613 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16614 let supervisor = SupervisorHandle::new();
16615 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16616 let handler =
16617 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16618 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16619 sync(&handler, &first, 10, vec![head("s", 1)])
16620 .await
16621 .unwrap();
16622 handler
16623 .handle_control_frame(
16624 &first,
16625 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16626 )
16627 .await
16628 .unwrap();
16629 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16630 sync(&handler, &second, 1, vec![head("s", 1)])
16631 .await
16632 .expect("GOODBYE releases authority even if the old socket remains open");
16633 }
16634
16635 #[tokio::test]
16636 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16637 let registry = Arc::new(Registry::default());
16638 let supervisor_handle = SupervisorHandle::new();
16639 let supervisor =
16640 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16641 .with_handle(supervisor_handle.clone())
16642 .with_daemon_incarnation("incarnation-7".to_string());
16643 supervisor
16646 .supervise_configured(
16647 ModuleSpec {
16648 module_id: OWNER.to_string(),
16649 program: PathBuf::from("/nonexistent/prefrontal-core"),
16650 args: Vec::new(),
16651 env: Vec::new(),
16652 reserved: false,
16653 reserved_prefixes: Vec::new(),
16654 protocol: ModuleProtocol::Subc,
16655 overlap: Default::default(),
16656 },
16657 false,
16658 )
16659 .unwrap();
16660 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16661 let handler =
16662 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16663 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16664
16665 let ModuleControlResponseToModule::ScopeDescribe {
16667 status,
16668 daemon_incarnation,
16669 owner_synced,
16670 owner_configured,
16671 scope,
16672 ..
16673 } = describe(&handler, &reader, OWNER, "s").await
16674 else {
16675 panic!("not a describe reply");
16676 };
16677 assert_eq!(status, ScopeStatus::NotLive);
16678 assert_eq!(daemon_incarnation, "incarnation-7");
16679 assert!(!owner_synced);
16680 assert!(owner_configured);
16681 assert!(scope.is_none());
16682
16683 let ModuleControlResponseToModule::ScopeDescribe {
16686 status,
16687 owner_configured,
16688 ..
16689 } = describe(&handler, &reader, "ghost", "s").await
16690 else {
16691 panic!("not a describe reply");
16692 };
16693 assert_eq!(status, ScopeStatus::NotLive);
16694 assert!(!owner_configured);
16695
16696 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16698 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16699 let ModuleControlResponseToModule::ScopeDescribe {
16700 status,
16701 scope_epoch,
16702 owner_synced,
16703 scope,
16704 ..
16705 } = describe(&handler, &reader, OWNER, "s").await
16706 else {
16707 panic!("not a describe reply");
16708 };
16709 assert_eq!(status, ScopeStatus::Live);
16710 assert_eq!(scope_epoch, Some(4));
16711 assert!(owner_synced);
16712 let stamp = scope.expect("a live scope carries its stamp");
16713 assert!(
16714 stamp.owner_authorized,
16715 "prefrontal-core is the default authority"
16716 );
16717 assert_eq!(stamp.kind, ScopeKind::Head);
16718 }
16719
16720 #[tokio::test]
16721 async fn scope_authority_owners_decides_owner_authorized() {
16722 let supervisor = SupervisorHandle::new();
16723 supervisor.set_spawn_nonce("broca", "b1".to_string());
16724 let handler = ControlHandler::new(Arc::new(Registry::default()))
16725 .with_supervisor(supervisor)
16726 .with_scope_authority_owners(vec!["broca".to_string()]);
16727 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16728 let mut gated = head("s", 1);
16729 gated.attributes.agent_id = Some("agent".to_string());
16730 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16731 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16732 describe(&handler, &broca, "broca", "s").await
16733 else {
16734 panic!("not a describe reply");
16735 };
16736 assert!(scope.unwrap().owner_authorized);
16737 }
16738
16739 #[tokio::test]
16743 async fn scope_ops_and_the_scopes_capability_are_advertised() {
16744 let handler = ControlHandler::new(Arc::new(Registry::default()));
16745 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16746 let ack = hello_via_sink(
16747 &handler,
16748 &ctx,
16749 &mut rx,
16750 hello_frame("m", PROTOCOL_VERSION, 1),
16751 )
16752 .await;
16753 let ack = parse_ack(&ack);
16754 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16755 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16756 }
16757 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16758
16759 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16760 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16761 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16762 let reply = handler
16763 .handle_control_frame(&client, frame)
16764 .await
16765 .unwrap()
16766 .pop()
16767 .unwrap();
16768 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16769 serde_json::from_slice(&reply.body).unwrap()
16770 else {
16771 panic!("not a server.describe reply");
16772 };
16773 assert!(
16774 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16775 "{capabilities:?}"
16776 );
16777 }
16778
16779 const PLEXUS: &str = "plexus";
16782 const OTHER: &str = "other";
16783 const AFT: &str = "aft";
16784 const BROCA: &str = "broca";
16785 const MAGIC: &str = "magic-context";
16786
16787 fn nonce(module_id: &str) -> String {
16788 format!("nonce-{module_id}")
16789 }
16790
16791 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16792 let (tx, rx) = mpsc::channel(64);
16793 (
16794 RouteCtx {
16795 connection_id: ConnectionId::new(connection),
16796 egress: FrameSink::new(tx),
16797 },
16798 rx,
16799 )
16800 }
16801
16802 struct Rig {
16806 handler: ControlHandler,
16807 forwarding: Arc<ForwardingTable>,
16808 owner: RouteCtx,
16809 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16810 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16811 generation: u64,
16812 next_connection: u64,
16813 _supervisor: Supervisor,
16814 }
16815
16816 async fn rig() -> Rig {
16817 rig_with_flow_support(true).await
16818 }
16819
16820 async fn rig_with_flow_support(flow_support: bool) -> Rig {
16821 rig_with_scope_capabilities(flow_support, false).await
16822 }
16823
16824 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16825 let mut provides = Vec::new();
16826 if flow_support {
16827 provides.push("flow-scopes/v1");
16828 }
16829 if run_support {
16830 provides.push("agent-run-scopes/v1");
16831 }
16832 rig_with_target_capabilities(&provides).await
16833 }
16834
16835 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16836 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16837 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16838 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16841 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16842 Frame::build(
16843 FrameType::Hello,
16844 control_flags(),
16845 0,
16846 0,
16847 connection,
16848 serde_json::to_vec(&body).unwrap(),
16849 )
16850 .unwrap()
16851 }
16852
16853 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16854 let registry = Arc::new(Registry::default());
16855 let forwarding = Arc::new(ForwardingTable::default());
16856 let supervisor_handle = SupervisorHandle::new();
16857 let supervisor =
16858 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16859 .with_handle(supervisor_handle.clone());
16860 supervisor
16861 .supervise_configured(
16862 ModuleSpec {
16863 module_id: OWNER.to_string(),
16864 program: PathBuf::from("/nonexistent/prefrontal-core"),
16865 args: Vec::new(),
16866 env: Vec::new(),
16867 reserved: false,
16868 reserved_prefixes: Vec::new(),
16869 protocol: ModuleProtocol::Subc,
16870 overlap: Default::default(),
16871 },
16872 false,
16873 )
16874 .unwrap();
16875 for module_id in [OWNER, AFT, BROCA, MAGIC] {
16876 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16877 }
16878 let handler =
16879 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16880 .with_supervisor(supervisor_handle);
16881 let (owner, mut owner_rx) = wide_ctx(1);
16882 hello_via_sink(
16883 &handler,
16884 &owner,
16885 &mut owner_rx,
16886 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16887 )
16888 .await;
16889 let mut modules = BTreeMap::new();
16890 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16891 let (ctx, mut rx) = wide_ctx(connection);
16892 let hello = hello_with_capabilities(module_id, connection, provides);
16893 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16894 modules.insert(module_id.to_string(), (ctx, rx));
16895 }
16896 Rig {
16897 handler,
16898 forwarding,
16899 owner,
16900 _owner_rx: owner_rx,
16901 modules,
16902 generation: 0,
16903 next_connection: 100,
16904 _supervisor: supervisor,
16905 }
16906 }
16907
16908 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16909 ScopeCarrier::new(Principal::Reserved {
16910 module_id: module_id.to_string(),
16911 })
16912 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16913 }
16914
16915 fn session(scope_epoch: u64) -> ScopeRecord {
16918 let mut record = head("s", scope_epoch);
16919 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16920 record.attributes.agent_id = Some("agent-1".to_string());
16921 record.attributes.delegates = true;
16922 record
16923 }
16924
16925 impl Rig {
16926 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16927 self.generation += 1;
16928 sync(&self.handler, &self.owner, self.generation, scopes)
16929 .await
16930 .expect("the owner's sync is accepted");
16931 }
16932
16933 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16934 ScopeSelector::new(
16935 Principal::Reserved {
16936 module_id: OWNER.to_string(),
16937 },
16938 scope_ref.to_string(),
16939 scope_epoch,
16940 )
16941 }
16942
16943 fn open_frame(
16944 &mut self,
16945 opener: Option<&str>,
16946 target: &str,
16947 scope: Option<ScopeSelector>,
16948 ) -> (
16949 RouteCtx,
16950 mpsc::Receiver<crate::router::OutboundFrame>,
16951 Frame,
16952 ) {
16953 self.next_connection += 1;
16954 let (ctx, rx) = wide_ctx(self.next_connection);
16955 let root = unique_project_root("scoped-open");
16956 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16957 target: RouteTarget::ToolProvider {
16958 module_id: target.to_string(),
16959 },
16960 identity: BindIdentity::new(
16961 root.path().to_path_buf(),
16962 "unit".to_string(),
16963 "session".to_string(),
16964 ),
16965 consumer_identity: opener.map(|module_id| ConsumerIdentity {
16966 module_id: module_id.to_string(),
16967 launch_nonce: nonce(module_id),
16968 }),
16969 consumer_capabilities: None,
16970 role_versions: None,
16971 admission_facts: None,
16972 scope,
16973 })
16974 .unwrap();
16975 let frame = Frame::build(
16976 FrameType::Request,
16977 control_flags(),
16978 0,
16979 0,
16980 self.next_connection,
16981 body,
16982 )
16983 .unwrap();
16984 (ctx, rx, frame)
16985 }
16986
16987 async fn refused(
16989 &mut self,
16990 opener: Option<&str>,
16991 target: &str,
16992 scope: Option<ScopeSelector>,
16993 ) -> String {
16994 self.refusal_body(opener, target, scope).await["code"]
16995 .as_str()
16996 .unwrap()
16997 .to_string()
16998 }
16999
17000 async fn refusal_body(
17001 &mut self,
17002 opener: Option<&str>,
17003 target: &str,
17004 scope: Option<ScopeSelector>,
17005 ) -> Value {
17006 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
17007 let replies = tokio::time::timeout(
17008 Duration::from_secs(2),
17009 self.handler.handle_control_frame(&ctx, frame),
17010 )
17011 .await
17012 .expect("the open must be refused before waiting for a bind ack")
17013 .unwrap();
17014 assert_eq!(replies.len(), 1, "{replies:?}");
17015 assert_eq!(replies[0].header.ty, FrameType::Error);
17016 let (_, module_rx) = self.modules.get_mut(target).unwrap();
17017 assert!(
17018 module_rx.try_recv().is_err(),
17019 "a refused open relays nothing"
17020 );
17021 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
17022 parse_error(&replies[0])
17023 }
17024
17025 async fn relayed(
17027 &mut self,
17028 opener: Option<&str>,
17029 target: &str,
17030 scope: Option<ScopeSelector>,
17031 ) -> Relayed {
17032 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
17033 let handler = self.handler.clone();
17034 let task_ctx = ctx.clone();
17035 let task = tokio::spawn(async move {
17036 handler
17037 .handle_control_frame(&task_ctx, frame)
17038 .await
17039 .unwrap()
17040 });
17041 let (_, module_rx) = self.modules.get_mut(target).unwrap();
17042 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
17043 .await
17044 .expect("the target receives the relayed route.bind")
17045 .unwrap()
17046 .frame;
17047 Relayed {
17048 target: target.to_string(),
17049 client: ctx,
17050 client_rx: rx,
17051 task,
17052 bind,
17053 }
17054 }
17055
17056 async fn ack(&self, relayed: &Relayed) {
17057 let (module, _) = &self.modules[&relayed.target];
17058 self.handler
17059 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
17060 .await
17061 .unwrap();
17062 }
17063
17064 async fn bound(
17066 &mut self,
17067 opener: Option<&str>,
17068 target: &str,
17069 scope: Option<ScopeSelector>,
17070 ) -> Bound {
17071 let relayed = self.relayed(opener, target, scope).await;
17072 self.ack(&relayed).await;
17073 let Relayed {
17074 target,
17075 client,
17076 mut client_rx,
17077 task,
17078 bind,
17079 } = relayed;
17080 assert!(
17081 task.await.unwrap().is_empty(),
17082 "the open is answered by commit"
17083 );
17084 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
17085 Bound {
17086 target,
17087 client,
17088 client_rx,
17089 channel,
17090 epoch,
17091 bind,
17092 }
17093 }
17094
17095 fn live(&self, route: &Bound) -> bool {
17096 matches!(
17097 self.forwarding
17098 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17099 .unwrap(),
17100 DataRoute::Client(DataRouteState::Bound(_))
17101 )
17102 }
17103 }
17104
17105 struct Relayed {
17106 target: String,
17107 client: RouteCtx,
17108 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17109 task: tokio::task::JoinHandle<Vec<Frame>>,
17110 bind: Frame,
17111 }
17112
17113 struct Bound {
17114 target: String,
17115 client: RouteCtx,
17116 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17117 channel: u16,
17118 epoch: u32,
17119 bind: Frame,
17120 }
17121
17122 impl Bound {
17123 fn closed_reason(&mut self) -> RouteCloseReason {
17126 let mut reason = None;
17127 let mut goodbye = false;
17128 while let Ok(outbound) = self.client_rx.try_recv() {
17129 let frame = outbound.frame;
17130 match frame.header.ty {
17131 FrameType::Goodbye => {
17132 assert_eq!(
17133 (frame.header.channel, frame.header.epoch),
17134 (self.channel, self.epoch)
17135 );
17136 goodbye = true;
17137 }
17138 FrameType::Push => {
17139 let ClientControlPush::RouteClosed {
17140 reason: r,
17141 module_id,
17142 ..
17143 } = serde_json::from_slice(&frame.body).unwrap()
17144 else {
17145 panic!("unexpected push");
17146 };
17147 assert_eq!(module_id, self.target);
17148 reason = Some(r);
17149 }
17150 other => panic!("unexpected frame {other:?}"),
17151 }
17152 }
17153 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
17154 reason.expect("the client is told why the route closed")
17155 }
17156
17157 fn untouched(&mut self) -> bool {
17158 self.client_rx.try_recv().is_err()
17159 }
17160
17161 fn stamp(&self) -> Option<ScopeStamp> {
17162 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
17163 ModuleControlRequest::RouteBind { scope, .. } => scope,
17164 other => panic!("expected a route.bind, got {other:?}"),
17165 }
17166 }
17167 }
17168
17169 #[tokio::test]
17170 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
17171 ) {
17172 let mut rig = rig().await;
17173 let mut record = session(1);
17174 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
17175 record.child_owners = vec![Principal::Reserved {
17176 module_id: MAGIC.to_string(),
17177 }];
17178 rig.sync(vec![record]).await;
17179 let scope = || Some(rig_selector("s", Some(1)));
17180
17181 rig.bound(Some(OWNER), PLEXUS, scope()).await;
17184 rig.bound(Some(AFT), OTHER, scope()).await;
17185 rig.bound(Some(BROCA), PLEXUS, scope()).await;
17186
17187 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
17191 assert_eq!(
17192 rig.refused(opener, target, scope()).await,
17193 error_codes::SCOPE_NOT_CARRIER,
17194 "{opener:?} -> {target}"
17195 );
17196 }
17197 }
17198
17199 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17200 ScopeSelector::new(
17201 Principal::Reserved {
17202 module_id: OWNER.to_string(),
17203 },
17204 scope_ref.to_string(),
17205 scope_epoch,
17206 )
17207 }
17208
17209 #[tokio::test]
17210 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
17211 let mut rig = rig().await;
17212 rig.sync(vec![session(1)]).await;
17213 for opener in [OWNER, AFT] {
17214 assert_eq!(
17215 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
17216 .await,
17217 error_codes::SCOPE_EPOCH_REQUIRED,
17218 "{opener}"
17219 );
17220 }
17221 }
17222
17223 #[tokio::test]
17224 async fn admission_separates_not_synced_not_live_and_ended() {
17225 let mut rig = rig().await;
17226 let code = rig
17228 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17229 .await;
17230 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
17231 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
17232
17233 let ghost = ScopeSelector::new(
17235 Principal::Reserved {
17236 module_id: "ghost".to_string(),
17237 },
17238 "s".to_string(),
17239 Some(1),
17240 );
17241 assert_eq!(
17242 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
17243 error_codes::SCOPE_NOT_LIVE
17244 );
17245
17246 rig.sync(vec![session(2)]).await;
17247 assert_eq!(
17248 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
17249 .await,
17250 error_codes::SCOPE_NOT_LIVE
17251 );
17252 for epoch in [1, 3] {
17253 assert_eq!(
17254 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
17255 .await,
17256 error_codes::SCOPE_ENDED,
17257 "epoch {epoch}"
17258 );
17259 }
17260 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
17262 .await;
17263 }
17264
17265 #[tokio::test]
17266 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
17267 let mut rig = rig().await;
17268 rig.sync(vec![session(1)]).await;
17269 let route = rig
17270 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17271 .await;
17272 let stamp = route.stamp().expect("a scoped bind carries the stamp");
17273 assert_eq!(stamp.scope_ref, "s");
17274 assert_eq!(stamp.scope_epoch, 1);
17275 assert_eq!(stamp.kind, ScopeKind::Head);
17276 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
17277 assert!(stamp.attributes.delegates);
17278 assert!(stamp.owner_authorized);
17279 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17280 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
17281
17282 let (broca, mut broca_rx) = wide_ctx(50);
17285 hello_via_sink(
17286 &rig.handler,
17287 &broca,
17288 &mut broca_rx,
17289 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17290 )
17291 .await;
17292 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17293 .await
17294 .unwrap();
17295 let own = ScopeSelector::new(
17296 Principal::Reserved {
17297 module_id: BROCA.to_string(),
17298 },
17299 "b".to_string(),
17300 Some(1),
17301 );
17302 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17303 assert!(!route.stamp().unwrap().owner_authorized);
17304 }
17305
17306 #[tokio::test]
17307 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17308 let mut rig = rig().await;
17309 let mut record = head("s", 1);
17310 record.carriers = vec![carrier(AFT, None)];
17311 let flow_id = "Flow:run-7/step_2!~";
17312 record.attributes.flow_id = Some(flow_id.to_string());
17313 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17314 .await
17315 .unwrap();
17316 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17317 panic!("not a sync reply");
17318 };
17319 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17320 let route = rig
17321 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17322 .await;
17323 assert!(rig.live(&route), "the stamped bind committed");
17324 let stamp = route.stamp().expect("a flow scope carries a stamp");
17325 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17326 assert_eq!(stamp.attributes.agent_id, None);
17327 assert!(!stamp.attributes.delegates);
17328 assert!(stamp.owner_authorized);
17329 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17330 assert_eq!(unscoped.stamp(), None);
17331 }
17332
17333 #[tokio::test]
17334 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17335 let mut rig = rig_with_flow_support(false).await;
17336 let mut record = session(1);
17337 record.attributes.flow_id = Some("flow:7".to_string());
17338 rig.sync(vec![record]).await;
17339 for opener in [OWNER, AFT] {
17340 let body = rig
17341 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17342 .await;
17343 assert_eq!(body["code"], "target_flow_unsupported");
17344 let message = body["message"].as_str().unwrap();
17345 for required in [PLEXUS, "flow-scopes/v1"] {
17346 assert!(message.contains(required), "{message}");
17347 }
17348 }
17349 }
17350
17351 #[tokio::test]
17352 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17353 let mut rig = rig_with_flow_support(true).await;
17354 let mut record = session(1);
17355 record.attributes.flow_id = Some("flow:7".to_string());
17356 rig.sync(vec![record]).await;
17357 for opener in [OWNER, AFT] {
17358 let route = rig
17359 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17360 .await;
17361 assert!(rig.live(&route));
17362 assert_eq!(
17363 route.stamp().unwrap().attributes.flow_id.as_deref(),
17364 Some("flow:7")
17365 );
17366 }
17367 }
17368
17369 #[tokio::test]
17370 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17371 use std::future::Future;
17372
17373 let mut rig = rig_with_flow_support(true).await;
17374 let mut record = session(1);
17375 record.attributes.flow_id = Some("flow:7".to_string());
17376 rig.sync(vec![record]).await;
17377 let (client, mut client_rx, frame) =
17378 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17379 for _ in 0..64 {
17382 client.egress.try_send(route_bind_ack(1)).unwrap();
17383 }
17384 let handler = rig.handler.clone();
17385 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17386 std::future::poll_fn(|cx| {
17387 assert!(open.as_mut().poll(cx).is_pending());
17388 std::task::Poll::Ready(())
17389 })
17390 .await;
17391 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17392
17393 let old_connection = rig.modules[PLEXUS].0.connection_id;
17394 rig.handler.cleanup_connection(old_connection).unwrap();
17395 let (replacement, mut replacement_rx) = wide_ctx(200);
17396 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17397 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17398 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17399 let hello = Frame::build(
17400 FrameType::Hello,
17401 control_flags(),
17402 0,
17403 0,
17404 200,
17405 serde_json::to_vec(&body).unwrap(),
17406 )
17407 .unwrap();
17408 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17409
17410 client_rx.try_recv().unwrap();
17411 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17412 .await
17413 .expect("the replacement is refused without waiting for a bind ack")
17414 .unwrap();
17415 assert_eq!(replies.len(), 1);
17416 let body = parse_error(&replies[0]);
17417 assert_eq!(body["code"], "target_flow_unsupported");
17418 for required in [PLEXUS, "flow-scopes/v1"] {
17419 assert!(body["message"].as_str().unwrap().contains(required));
17420 }
17421 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17422 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17423 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17424 assert!(rig
17425 .handler
17426 .registry
17427 .get_module_by_connection(replacement.connection_id)
17428 .unwrap()
17429 .is_some());
17430 }
17431
17432 #[tokio::test]
17433 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17434 use std::future::Future;
17435 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17436 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17437 const NEXT_HASH: &str =
17438 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17439 let mut rig = rig().await;
17440 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17441 record.carriers.push(ScopeCarrier::new(Principal::Direct));
17442 rig.sync(vec![record.clone()]).await;
17443 let (client, mut rx, frame) = rig.open_frame(
17444 None,
17445 PLEXUS,
17446 Some(
17447 rig.selector("s", Some(1))
17448 .with_bind_proof(Some(PROOF.into())),
17449 ),
17450 );
17451 for _ in 0..64 {
17454 client.egress.try_send(route_bind_ack(1)).unwrap();
17455 }
17456 let handler = rig.handler.clone();
17457 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17458 std::future::poll_fn(|cx| {
17459 assert!(open.as_mut().poll(cx).is_pending());
17460 std::task::Poll::Ready(())
17461 })
17462 .await;
17463 record.bind_proof_sha256 = Some(NEXT_HASH.into());
17464 rig.sync(vec![record]).await;
17465 rx.try_recv().unwrap();
17466 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17467 .await
17468 .unwrap()
17469 .unwrap();
17470 assert_eq!(
17471 parse_error(&replies[0])["code"],
17472 error_codes::SCOPE_BIND_PROOF_MISMATCH
17473 );
17474 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17475 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17476 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17477 }
17478
17479 #[tokio::test]
17480 async fn scope_bind_proof_reserved_openers_ignore_proof() {
17481 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17482 let mut rig = rig().await;
17483 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17484 .await;
17485 for opener in [OWNER, AFT] {
17486 for proof in [None, Some("malformed+proof=")] {
17487 let route = rig
17488 .bound(
17489 Some(opener),
17490 PLEXUS,
17491 Some(
17492 rig.selector("s", Some(1))
17493 .with_bind_proof(proof.map(str::to_string)),
17494 ),
17495 )
17496 .await;
17497 assert!(rig.live(&route));
17498 }
17499 }
17500 }
17501
17502 #[tokio::test]
17503 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17504 use std::future::Future;
17505 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17506 const NEXT_HASH: &str =
17507 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17508 let mut rig = rig().await;
17509 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17510 .await;
17511 let (client, mut rx, frame) =
17512 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17513 for _ in 0..64 {
17514 client.egress.try_send(route_bind_ack(1)).unwrap();
17515 }
17516 let handler = rig.handler.clone();
17517 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17518 std::future::poll_fn(|cx| {
17519 assert!(open.as_mut().poll(cx).is_pending());
17520 std::task::Poll::Ready(())
17521 })
17522 .await;
17523 rig.sync(vec![
17524 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17525 ])
17526 .await;
17527 rx.try_recv().unwrap();
17528 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17529 let bind = std::future::poll_fn(|cx| {
17530 assert!(open.as_mut().poll(cx).is_pending());
17531 module_rx.poll_recv(cx)
17532 })
17533 .await
17534 .unwrap()
17535 .frame;
17536 handler
17537 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17538 .await
17539 .unwrap();
17540 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17541 .await
17542 .unwrap()
17543 .unwrap();
17544 assert_eq!(replies.len(), 1);
17545 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17548 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17549 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17550 let route = rig
17551 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17552 .await;
17553 assert!(rig.live(&route));
17554 }
17555
17556 #[tokio::test]
17557 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17558 {
17559 let mut rig = rig_with_flow_support(false).await;
17560 rig.sync(vec![session(1)]).await;
17561 let route = rig
17562 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17563 .await;
17564 assert!(rig.live(&route));
17565 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17566 let mut record = session(1);
17567 record.attributes.flow_id = Some("flow:7".to_string());
17568 rig.sync(vec![record]).await;
17569 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17570 assert!(rig.live(&unscoped));
17571 assert_eq!(unscoped.stamp(), None);
17572 }
17573
17574 #[tokio::test]
17575 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17576 let mut rig = rig().await;
17577 let mut record = session(1);
17578 record.attributes.flow_id = Some("flow:7".to_string());
17579 rig.sync(vec![record.clone()]).await;
17580 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17581 let mut owner_route = rig
17582 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17583 .await;
17584 let mut carrier_route = rig
17585 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17586 .await;
17587 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17588 rig.sync(vec![record.clone()]).await;
17589 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17590 assert!(rig.live(&owner_route) && owner_route.untouched());
17591 assert!(rig.live(&carrier_route) && carrier_route.untouched());
17592
17593 record.attributes.flow_id = Some("flow:8".to_string());
17594 rig.sync(vec![record]).await;
17595 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17596 let before = before.unwrap();
17597 assert_eq!(after.scope_epoch, before.scope_epoch);
17598 assert!(after.version > before.version);
17599 for route in [&mut owner_route, &mut carrier_route] {
17600 assert!(!rig.live(route));
17601 assert_eq!(
17602 route.closed_reason(),
17603 RouteCloseReason::ScopeDelegationChanged
17604 );
17605 }
17606 assert!(rig.live(&unscoped) && unscoped.untouched());
17607 for target in [PLEXUS, OTHER] {
17610 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17611 let goodbye = module_rx
17612 .try_recv()
17613 .expect("the provider sees the drain")
17614 .frame;
17615 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17616 assert!(module_rx.try_recv().is_err());
17617 }
17618 let rebound = rig
17619 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17620 .await;
17621 assert_eq!(
17622 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17623 Some("flow:8")
17624 );
17625 }
17626
17627 #[tokio::test]
17632 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17633 let mut rig = rig().await;
17634 rig.sync(vec![session(1)]).await;
17635 let mut cotenant = rig
17636 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17637 .await;
17638
17639 let mut changed = session(1);
17640 changed.child_owners.push(Principal::Reserved {
17641 module_id: MAGIC.to_string(),
17642 });
17643 let mut ended = None;
17644 for (code, next) in [
17645 (error_codes::SCOPE_CHANGED, vec![changed]),
17646 (error_codes::SCOPE_ENDED, Vec::new()),
17647 ] {
17648 let relayed = rig
17649 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17650 .await;
17651 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17652 ended = Some(next.is_empty());
17653 rig.sync(next).await;
17654 rig.ack(&relayed).await;
17655 let replies = relayed.task.await.unwrap();
17656 assert_eq!(replies.len(), 1, "{replies:?}");
17657 assert_eq!(parse_error(&replies[0])["code"], code);
17658 assert_eq!(
17659 subc_protocol::error_codes::is_retryable_route_open(code),
17660 code == error_codes::SCOPE_CHANGED
17661 );
17662 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17663 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17665 let mut goodbyes = Vec::new();
17668 while let Ok(outbound) = plexus_rx.try_recv() {
17669 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17670 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17671 }
17672 assert!(
17673 goodbyes.contains(&(bind_channel, bind_epoch)),
17674 "{goodbyes:?}"
17675 );
17676 assert!(rig
17677 .handler
17678 .registry
17679 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17680 .unwrap()
17681 .is_some());
17682 }
17683 assert_eq!(ended, Some(true));
17684 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17687 }
17688
17689 #[tokio::test]
17692 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17693 struct Case {
17694 name: &'static str,
17695 change: fn(&mut ScopeRecord),
17696 closed: [Option<RouteCloseReason>; 4],
17699 }
17700 use RouteCloseReason::*;
17701 let cases = [
17702 Case {
17703 name: "a carrier entry removed",
17704 change: |r| {
17705 r.carriers.retain(|c| {
17706 c.principal
17707 != Principal::Reserved {
17708 module_id: AFT.to_string(),
17709 }
17710 })
17711 },
17712 closed: [None, Some(ScopeCarrierRemoved), None, None],
17713 },
17714 Case {
17715 name: "a target removed from a carrier",
17716 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17717 closed: [None, None, None, Some(ScopeCarrierRemoved)],
17718 },
17719 Case {
17720 name: "a bare carrier narrowed to targets",
17721 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17722 closed: [None, Some(ScopeCarrierRemoved), None, None],
17723 },
17724 Case {
17725 name: "delegates turned off",
17726 change: |r| r.attributes.delegates = false,
17727 closed: [Some(ScopeDelegationChanged); 4],
17728 },
17729 Case {
17730 name: "agent_id changed",
17731 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17732 closed: [Some(ScopeDelegationChanged); 4],
17733 },
17734 Case {
17735 name: "a carrier added, child owners changed, the record re-sent",
17736 change: |r| {
17737 r.carriers.push(carrier(MAGIC, None));
17738 r.child_owners.push(Principal::Reserved {
17739 module_id: MAGIC.to_string(),
17740 });
17741 },
17742 closed: [None; 4],
17743 },
17744 Case {
17745 name: "a target added",
17746 change: |r| {
17747 r.carriers[1]
17748 .targets
17749 .as_mut()
17750 .unwrap()
17751 .push("third".to_string())
17752 },
17753 closed: [None; 4],
17754 },
17755 Case {
17756 name: "delegates turned on",
17757 change: |r| r.attributes.delegates = true,
17758 closed: [None; 4],
17759 },
17760 ];
17761 for case in cases {
17762 let mut rig = rig().await;
17763 rig.sync(vec![session(1)]).await;
17764 let scope = || Some(rig_selector("s", Some(1)));
17765 let mut routes = [
17766 rig.bound(Some(OWNER), PLEXUS, scope()).await,
17767 rig.bound(Some(AFT), PLEXUS, scope()).await,
17768 rig.bound(Some(BROCA), PLEXUS, scope()).await,
17769 rig.bound(Some(BROCA), OTHER, scope()).await,
17770 ];
17771 let mut record = session(1);
17772 (case.change)(&mut record);
17773 rig.sync(vec![record]).await;
17774 for (index, expected) in case.closed.iter().enumerate() {
17775 let route = &mut routes[index];
17776 match expected {
17777 Some(reason) => {
17778 assert!(!rig.live(route), "{}: route {index} still live", case.name);
17779 assert_eq!(
17780 route.closed_reason(),
17781 *reason,
17782 "{}: route {index}",
17783 case.name
17784 );
17785 }
17786 None => {
17787 assert!(rig.live(route), "{}: route {index} closed", case.name);
17788 assert!(
17789 route.untouched(),
17790 "{}: route {index} was told something",
17791 case.name
17792 );
17793 }
17794 }
17795 }
17796 }
17797 }
17798
17799 #[tokio::test]
17800 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17801 for next in [Vec::new(), vec![session(2)]] {
17803 let mut rig = rig().await;
17804 rig.sync(vec![session(1)]).await;
17805 let mut route = rig
17806 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17807 .await;
17808 rig.sync(next).await;
17809 assert!(!rig.live(&route));
17810 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17811 }
17812
17813 let mut rig = rig().await;
17816 let mut child = session(1);
17817 child.scope_ref = "child".to_string();
17818 child.kind = ScopeKind::Worker;
17819 child.parent = Some(ScopeParent::new(
17820 Principal::Reserved {
17821 module_id: OWNER.to_string(),
17822 },
17823 "s".to_string(),
17824 1,
17825 ));
17826 rig.sync(vec![session(1), child.clone()]).await;
17827 let mut child_route = rig
17828 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17829 .await;
17830 assert_eq!(
17831 child_route.stamp().unwrap().parent_state,
17832 Some(ParentState::Linked)
17833 );
17834 rig.sync(vec![child]).await;
17835 assert!(!rig.live(&child_route));
17836 assert_eq!(
17837 child_route.closed_reason(),
17838 RouteCloseReason::ScopeParentEnded
17839 );
17840 }
17841
17842 #[tokio::test]
17843 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17844 ) {
17845 let mut rig = rig().await;
17846 rig.sync(vec![session(1)]).await;
17847 let mut route = rig
17848 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17849 .await;
17850 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17851 rig.sync(vec![session(1)]).await;
17852 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17853 assert!(rig.live(&route) && route.untouched());
17854
17855 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17859 .forwarding
17860 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17861 .unwrap()
17862 else {
17863 panic!("the route is bound");
17864 };
17865 binding.flow.acquire_tagged(9, false).await.unwrap();
17866 let mut widened = session(1);
17867 widened.carriers.push(carrier(MAGIC, None));
17868 rig.sync(vec![widened]).await;
17869 assert!(rig.live(&route) && route.untouched());
17870 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17871 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17872 assert_eq!(binding.flow.in_flight(), 1);
17875 binding
17876 .flow
17877 .acquire_tagged(10, false)
17878 .await
17879 .expect("the flow is still open");
17880 }
17881
17882 #[tokio::test]
17885 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17886 let mut rig = rig().await;
17887 rig.sync(vec![session(1)]).await;
17888 let mut on_incumbent = rig
17889 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17890 .await;
17891
17892 let (candidate, _candidate_rx) = wide_ctx(9);
17895 let registration = rig
17896 .handler
17897 .registry
17898 .register_candidate_with_control_ops(
17899 manifest(PLEXUS, PROTOCOL_VERSION),
17900 PROTOCOL_VERSION,
17901 candidate.connection_id,
17902 module_baseline_control_ops(),
17903 )
17904 .unwrap();
17905 rig.forwarding
17906 .register_candidate_module_connection(
17907 candidate.connection_id,
17908 PLEXUS.to_string(),
17909 PROTOCOL_VERSION,
17910 manifest_concurrency(®istration.manifest),
17911 candidate.egress.clone(),
17912 )
17913 .unwrap();
17914 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17915 rig.handler
17916 .registry
17917 .promote_candidate(PLEXUS)
17918 .unwrap()
17919 .unwrap();
17920 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17921
17922 rig.sync(Vec::new()).await;
17923 assert!(!rig.live(&on_incumbent));
17924 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17925 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17926 let goodbye = incumbent_rx
17927 .try_recv()
17928 .expect("the superseded endpoint is told")
17929 .frame;
17930 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17931 }
17932 }
17933}
17934
17935#[cfg(test)]
17936mod concurrency_default_exposure_tests {
17937 use super::*;
17938
17939 fn hello_body(role_json: &str) -> Vec<u8> {
17940 format!(
17941 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17942 )
17943 .into_bytes()
17944 }
17945
17946 fn manifest_from(body: &[u8]) -> ModuleManifest {
17947 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17948 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17949 .expect("manifest parses")
17950 }
17951
17952 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17953
17954 #[test]
17955 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17956 let body = hello_body(&format!(
17957 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17958 ));
17959 let manifest = manifest_from(&body);
17960 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17963 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17964 }
17965
17966 #[test]
17967 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17968 let body = hello_body(&format!(
17969 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17970 ));
17971 let manifest = manifest_from(&body);
17972 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17973 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17974 }
17975
17976 #[test]
17977 fn non_management_roles_are_never_reported() {
17978 let body = hello_body(
17979 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17980 );
17981 let manifest = manifest_from(&body);
17982 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17983 }
17984}