1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_flow_operations, validate_hello_rich_ui_descriptor,
26 validate_hello_role_declarations, validate_hello_self_signal_declarations,
27 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
28 ProviderRole,
29 },
30 scope::{
31 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
32 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
33 SCOPE_SYNC_OP,
34 },
35 session::{
36 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
37 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
38 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
39 ROLE_VERSIONS_FIELD,
40 },
41 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
42 Priority, RouteTarget, PROTOCOL_VERSION,
43};
44use tokio::time::{timeout_at, Instant};
45use tracing::{debug, info, warn};
46
47use crate::{
48 caller_identity::CallerIdentity,
49 capability_requirements::{
50 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
51 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
52 RuntimeModule,
53 },
54 child_roster::DaemonShutdownFlag,
55 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
56 forwarding::{
57 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
58 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
59 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
60 },
61 observability::{
62 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
63 },
64 privileged_prompt::{prompt_text, PrivilegedOperation},
65 provenance::{
66 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
67 },
68 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
69 router::{RouteCtx, RouterError},
70 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
71 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
72 stderr_tail::{CaptureState, TailEntry},
73 supervise::{
74 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
75 SupervisorHandle, SwapHelloAdmission,
76 },
77 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
78};
79
80pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
86
87const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
88const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
89const CAP_PING_PONG: &str = "ping_pong_v1";
90const CAP_SESSION_ATTACH: &str = "session_attach_v1";
91const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
92
93const SUBC_CONTROL_OPS: &[&str] = &[
94 ops::ACTIVITY_RECORD,
95 ops::ACTIVITY_QUERY,
96 ops::ACTIVITY_SUBSCRIBE,
97 ops::SERVER_DESCRIBE,
98 ops::CATALOG_LIST,
99 ops::ROUTE_OPEN,
100 ops::ROUTE_POLL,
101 ops::ROUTE_CLOSING,
102 ops::ROUTE_CLOSED,
103 ops::SUPERVISOR_LIST,
104 ops::SUPERVISOR_RESTART,
105 ops::SUPERVISOR_SWAP,
106 ops::SUPERVISOR_RELOAD,
107 ops::SUPERVISOR_RESCAN,
108 ops::SUPERVISOR_RELEASE_RESERVED,
109 ops::SUPERVISOR_SET_ENABLED,
110 ops::SUPERVISOR_HEALTH_PROBE,
111 ops::SUPERVISOR_HEALTH,
112 ops::SUPERVISOR_STDERR_TAIL,
113 ops::SUPERVISOR_TERMINALS,
114 ops::SUPERVISOR_ROUTES,
115 ops::SUPERVISOR_PROVENANCE,
116 ops::SUPERVISOR_SPAWN_SNAPSHOT,
117 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
118];
119
120const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
127
128mod rescan_checked;
129use rescan_checked::{RescanMode, RescanPlan};
130
131const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
132 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
133 "supervisor.live_roots",
134 SCOPE_SYNC_OP,
135 SCOPE_APPLY_OP,
136 SCOPE_DESCRIBE_OP,
137 "operator.confirm",
138];
139
140const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
144
145const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
146
147pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
155
156pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
171
172pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
190
191const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
192const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
193
194fn reload_verdict(
195 configured: &Path,
196 spawned_from: Option<&Path>,
197 image: subc_control::RunningImageAgreement,
198) -> PendingReloadVerdict {
199 let path = match spawned_from {
200 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
201 ReloadPathAgreement::Match
202 }
203 Some(spawned_from) => ReloadPathAgreement::Mismatch {
204 configured: configured.to_path_buf(),
205 spawned_from: spawned_from.to_path_buf(),
206 },
207 None => ReloadPathAgreement::Unavailable {
208 reason: if matches!(
209 image,
210 subc_control::RunningImageAgreement::Unavailable {
211 reason: subc_control::RunningImageUnavailableReason::NotRunning
212 }
213 ) {
214 ReloadPathUnavailableReason::NotRunning
215 } else {
216 ReloadPathUnavailableReason::SpawnedPathUnavailable
217 },
218 },
219 };
220 PendingReloadVerdict {
221 path,
222 image,
223 spec: None,
224 }
225}
226
227fn launch_spec_agreement(
228 configured: &crate::supervise::ModuleSpec,
229 running: bool,
230 spawned: Option<&crate::supervise::LaunchSpecDigest>,
231) -> subc_control::LaunchSpecAgreement {
232 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
233 if !running {
234 return Agreement::Unavailable {
235 reason: Reason::NotRunning,
236 };
237 }
238 match spawned {
239 None => Agreement::Unavailable {
240 reason: Reason::SpawnedSpecUnavailable,
241 },
242 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
243 Agreement::Match
244 }
245 Some(_) => Agreement::Mismatch,
246 }
247}
248
249fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
250 if configured == spawned_from {
251 return true;
252 }
253 #[cfg(windows)]
254 {
255 match (
261 std::fs::canonicalize(configured),
262 std::fs::canonicalize(spawned_from),
263 ) {
264 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
265 _ => false,
266 }
267 }
268 #[cfg(not(windows))]
269 {
270 false
271 }
272}
273
274#[derive(Clone)]
275struct DaemonProvenanceFacts {
276 build: DaemonBuildProvenance,
277 pid: Option<u32>,
278 started_at_ms: Option<u64>,
279 start_clock: Option<crate::clock::StartClock>,
280 executable_path: Option<PathBuf>,
281 executable_identity: Option<SpawnedFileIdentity>,
282 process_start_time: Option<u64>,
283 probe: ExecutableIdentityProbe,
284}
285
286impl Default for DaemonProvenanceFacts {
287 fn default() -> Self {
288 Self {
289 build: DaemonBuildProvenance {
290 build_git_sha: None,
291 build_lock_digest: None,
292 },
293 pid: None,
294 started_at_ms: None,
295 start_clock: None,
296 executable_path: None,
297 executable_identity: None,
298 process_start_time: None,
299 probe: ExecutableIdentityProbe::default(),
300 }
301 }
302}
303
304#[derive(Debug, Clone)]
305struct SupervisorRescanContext {
306 supervisor: Supervisor,
307 config_path: PathBuf,
308 configured_port: Option<u16>,
309 storage_config: Option<crate::daemon_config::StorageConfig>,
310 admission_facts_carrier_module_id: Option<String>,
311 admission_facts_targets: Option<Vec<String>>,
312 scope_authority_owners: Vec<String>,
313 privileged_operator_prompt: PrivilegedOperatorPrompt,
314 generation: Arc<std::sync::atomic::AtomicU64>,
320}
321
322const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
343 "reloading",
344 "supervisor_not_live",
345 "registration_not_active",
346 "no_forwarding_connection",
347 "relay_send_failed",
348];
349
350#[derive(Clone)]
352pub struct ControlHandler {
353 registry: Arc<Registry>,
354 activity: Option<crate::activity::ActivityLedger>,
355 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
356 forwarding: Arc<ForwardingTable>,
357 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
358 supervisor: SupervisorHandle,
359 subc_capabilities: Arc<[String]>,
360 route_bind_relay_timeout: Duration,
364 route_bind_relay_timeouts: BTreeMap<String, Duration>,
368 route_bind_breakers: RouteBindBreakers,
371 route_bind_concurrency: RouteBindConcurrency,
374 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
379 route_bind_breaker_threshold: u32,
381 route_bind_breaker_cooldown: Duration,
383 health_probe_timeout: Duration,
384 storage_config: Option<crate::daemon_config::StorageConfig>,
387 machine_id: Option<crate::machine_id::MachineId>,
391 admission_facts_carrier_module_id: Option<String>,
392 admission_facts_targets: Option<Vec<String>>,
393 scopes: Arc<RwLock<ScopeTable>>,
397 scope_deadline_added: Arc<tokio::sync::Notify>,
401 scope_authority_owners: Vec<String>,
404 privileged_operator_prompt: PrivilegedOperatorPrompt,
406 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
409 #[cfg(test)]
410 caller_lookup: Option<TestCallerLookup>,
411 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
414 rescan: Option<SupervisorRescanContext>,
415 connected_clients: ConnectedClients,
416 counters: DaemonCounters,
417 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
418 daemon_shutdown: DaemonShutdownFlag,
423 daemon_provenance: DaemonProvenanceFacts,
424 #[cfg(test)]
425 control_dispatch_delay: Option<Duration>,
426 #[cfg(test)]
427 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
428 #[cfg(test)]
432 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
433 #[cfg(test)]
439 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
440}
441
442impl fmt::Debug for ControlHandler {
443 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
444 f.debug_struct("ControlHandler")
445 .field("registry", &self.registry)
446 .field("forwarding", &self.forwarding)
447 .field("process_liveness", &self.process_liveness.is_some())
448 .field("supervisor", &self.supervisor)
449 .field("subc_capabilities", &self.subc_capabilities)
450 .finish()
451 }
452}
453
454struct RouteOpenRequest {
455 target: RouteTarget,
456 identity: BindIdentity,
457 consumer_identity: Option<ConsumerIdentity>,
458 consumer_capabilities: Option<Vec<String>>,
459 role_versions: Option<BTreeMap<String, String>>,
460 admission_facts: Option<serde_json::Value>,
461 scope: Option<ScopeSelector>,
462}
463
464#[derive(Clone, Copy)]
465enum RouteOpenPhase {
466 Admission,
467 ProjectRoot,
468 EgressReservation,
469 BindSend,
470 BindAckWait,
471}
472
473struct RouteOpenTimings {
477 module_id: String,
478 principal: String,
479 admission: Duration,
480 project_root: Duration,
481 egress_reservation: Duration,
482 bind_send: Duration,
483 bind_ack_wait: Duration,
484 phase: Option<RouteOpenPhase>,
485 phase_started_at: Instant,
486}
487
488impl RouteOpenTimings {
489 fn new() -> Self {
490 Self {
491 module_id: String::new(),
492 principal: "unresolved".to_string(),
494 admission: Duration::ZERO,
495 project_root: Duration::ZERO,
496 egress_reservation: Duration::ZERO,
497 bind_send: Duration::ZERO,
498 bind_ack_wait: Duration::ZERO,
499 phase: Some(RouteOpenPhase::Admission),
500 phase_started_at: Instant::now(),
501 }
502 }
503
504 fn finish(&mut self) {
505 let elapsed = self.phase_started_at.elapsed();
506 match self.phase.take() {
507 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
508 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
509 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
510 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
511 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
512 None => {}
513 }
514 }
515
516 fn enter(&mut self, phase: RouteOpenPhase) {
517 self.finish();
518 self.phase = Some(phase);
519 self.phase_started_at = Instant::now();
520 }
521}
522
523struct RouteBindReservationGuard {
524 forwarding: Arc<ForwardingTable>,
525 endpoint: ModuleEndpointId,
526 relay_corr: u64,
527 armed: bool,
528}
529
530struct ModuleControlRpcGuard {
531 forwarding: Arc<ForwardingTable>,
532 endpoint: ModuleEndpointId,
533 corr: u64,
534 armed: bool,
535}
536
537impl ModuleControlRpcGuard {
538 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
539 Self {
540 forwarding,
541 endpoint,
542 corr,
543 armed: true,
544 }
545 }
546
547 fn disarm(&mut self) {
548 self.armed = false;
549 }
550}
551
552impl Drop for ModuleControlRpcGuard {
553 fn drop(&mut self) {
554 if self.armed {
555 let _ = self
556 .forwarding
557 .cancel_module_control_rpc(self.endpoint, self.corr);
558 }
559 }
560}
561
562impl RouteBindReservationGuard {
563 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
564 Self {
565 forwarding,
566 endpoint,
567 relay_corr,
568 armed: true,
569 }
570 }
571
572 fn release_and_disarm(&mut self) {
573 if !self.armed {
574 return;
575 }
576 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
577 self.endpoint,
578 self.relay_corr,
579 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
580 ) {
581 send_goodbye_target_best_effort(
582 &self.forwarding.counters(),
583 &target,
584 "canceled route.bind",
585 );
586 }
587 self.armed = false;
588 }
589
590 fn disarm(&mut self) {
591 self.armed = false;
592 }
593}
594
595impl Drop for RouteBindReservationGuard {
596 fn drop(&mut self) {
597 self.release_and_disarm();
598 }
599}
600
601#[derive(Debug, Clone, Default)]
631pub(crate) struct RouteBindBreakers {
632 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
633}
634
635#[derive(Debug, Clone, Default)]
636pub(crate) struct RouteBindConcurrency {
637 modules: Arc<Mutex<HashMap<String, usize>>>,
638}
639
640struct RouteBindConcurrencyGuard {
641 concurrency: RouteBindConcurrency,
642 module_id: String,
643}
644
645impl RouteBindConcurrency {
646 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
649 let mut modules = self
650 .modules
651 .lock()
652 .expect("route.bind concurrency mutex poisoned");
653 let in_flight = modules.entry(module_id.to_string()).or_default();
654 if *in_flight >= limit {
655 return Err(*in_flight);
656 }
657 *in_flight += 1;
658 Ok(RouteBindConcurrencyGuard {
659 concurrency: self.clone(),
660 module_id: module_id.to_string(),
661 })
662 }
663}
664
665impl Drop for RouteBindConcurrencyGuard {
666 fn drop(&mut self) {
667 let mut modules = self
668 .concurrency
669 .modules
670 .lock()
671 .expect("route.bind concurrency mutex poisoned");
672 let remove = {
673 let in_flight = modules
674 .get_mut(&self.module_id)
675 .expect("admitted route.bind has a concurrency entry");
676 *in_flight -= 1;
677 *in_flight == 0
678 };
679 if remove {
680 modules.remove(&self.module_id);
681 }
682 }
683}
684
685#[derive(Debug, Default)]
686struct ModuleBreakerState {
687 consecutive_timeouts: u32,
689 cooldown_until: Option<Instant>,
692 probe_in_flight: Option<Arc<()>>,
697}
698
699enum RouteBindAdmission<'a> {
701 Admitted {
702 guard: RouteBindBreakerGuard<'a>,
703 probe: bool,
706 },
707 Refused {
708 consecutive_timeouts: u32,
709 retry_in: Duration,
713 probe_in_flight: bool,
714 },
715}
716
717struct RouteBindBreakerGuard<'a> {
724 breakers: RouteBindBreakers,
725 module_id: &'a str,
726 probe_token: Option<Arc<()>>,
727 settled: bool,
728}
729
730impl RouteBindBreakerGuard<'_> {
731 fn record_accepted(&mut self) -> bool {
735 self.settled = true;
736 self.breakers.record_accepted(self.module_id)
737 }
738
739 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
742 self.settled = true;
743 self.breakers.record_timeout(
744 self.module_id,
745 self.probe_token.as_ref(),
746 threshold,
747 cooldown,
748 )
749 }
750
751 fn record_inconclusive(&mut self) {
760 self.settled = true;
761 self.breakers
762 .record_inconclusive(self.module_id, self.probe_token.as_ref());
763 }
764}
765
766impl Drop for RouteBindBreakerGuard<'_> {
767 fn drop(&mut self) {
768 if !self.settled {
769 self.breakers
770 .record_inconclusive(self.module_id, self.probe_token.as_ref());
771 }
772 }
773}
774
775struct BreakerOpened {
779 consecutive_timeouts: u32,
780 reopened_after_probe: bool,
783}
784
785impl RouteBindBreakers {
786 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
787 self.modules
788 .lock()
789 .expect("route.bind breaker mutex poisoned")
790 }
791
792 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
795 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
796 probe: probe_token.is_some(),
797 guard: RouteBindBreakerGuard {
798 breakers: self.clone(),
799 module_id,
800 probe_token,
801 settled: false,
802 },
803 };
804
805 let mut modules = self.lock();
806 let Some(state) = modules.get_mut(module_id) else {
807 return admitted(None);
808 };
809 let Some(cooldown_until) = state.cooldown_until else {
810 return admitted(None);
811 };
812 if state.probe_in_flight.is_some() {
813 return RouteBindAdmission::Refused {
814 consecutive_timeouts: state.consecutive_timeouts,
815 retry_in: Duration::ZERO,
816 probe_in_flight: true,
817 };
818 }
819 let now = Instant::now();
820 if now < cooldown_until {
821 return RouteBindAdmission::Refused {
822 consecutive_timeouts: state.consecutive_timeouts,
823 retry_in: cooldown_until - now,
824 probe_in_flight: false,
825 };
826 }
827 let token = Arc::new(());
828 state.probe_in_flight = Some(Arc::clone(&token));
829 admitted(Some(token))
830 }
831
832 fn record_accepted(&self, module_id: &str) -> bool {
833 self.lock()
834 .remove(module_id)
835 .is_some_and(|state| state.cooldown_until.is_some())
836 }
837
838 fn record_timeout(
839 &self,
840 module_id: &str,
841 probe_token: Option<&Arc<()>>,
842 threshold: u32,
843 cooldown: Duration,
844 ) -> Option<BreakerOpened> {
845 let mut modules = self.lock();
846 let state = modules.entry(module_id.to_string()).or_default();
847 let was_open = state.cooldown_until.is_some();
848 let was_probe = Self::owns_probe(state, probe_token);
849 if was_probe {
850 state.probe_in_flight = None;
851 }
852 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
853 if state.consecutive_timeouts < threshold {
854 return None;
855 }
856 state.cooldown_until = Some(Instant::now() + cooldown);
857 Some(BreakerOpened {
858 consecutive_timeouts: state.consecutive_timeouts,
859 reopened_after_probe: was_open && was_probe,
860 })
861 }
862
863 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
864 state
868 .probe_in_flight
869 .as_ref()
870 .zip(token)
871 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
872 }
873
874 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
875 if let Some(state) = self.lock().get_mut(module_id) {
876 if Self::owns_probe(state, probe_token) {
877 state.probe_in_flight = None;
878 }
879 }
880 }
881
882 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
897 self.lock()
898 .remove(module_id)
899 .map(|state| state.consecutive_timeouts)
900 .filter(|discarded| *discarded > 0)
901 }
902
903 fn open_snapshot(&self) -> Option<serde_json::Value> {
910 let now = Instant::now();
911 let modules = self.lock();
912 let open = modules
913 .iter()
914 .filter_map(|(module_id, state)| {
915 let cooldown_until = state.cooldown_until?;
916 Some((
917 module_id.clone(),
918 serde_json::json!({
919 "consecutive_timeouts": state.consecutive_timeouts,
920 "cooldown_remaining_ms":
921 cooldown_until.saturating_duration_since(now).as_millis() as u64,
922 "probe_in_flight": state.probe_in_flight.is_some(),
923 }),
924 ))
925 })
926 .collect::<serde_json::Map<String, serde_json::Value>>();
927 (!open.is_empty()).then_some(serde_json::Value::Object(open))
928 }
929}
930
931impl ControlHandler {
932 pub fn new(registry: Arc<Registry>) -> Self {
933 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
934 }
935
936 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
937 let counters = forwarding.counters();
938 let route_bind_breakers = forwarding.route_bind_breakers();
942 let route_bind_concurrency = forwarding.route_bind_concurrency();
943 let route_outages = forwarding.route_outages();
944 Self {
945 registry,
946 activity: None,
947 activity_streams: Arc::new(Mutex::new(HashMap::new())),
948 forwarding,
949 process_liveness: None,
950 supervisor: SupervisorHandle::new(),
951 subc_capabilities: Arc::from([
952 CAP_MANIFEST_REGISTRATION.to_string(),
953 CAP_CHANNEL_LIFECYCLE.to_string(),
954 CAP_PING_PONG.to_string(),
955 CAP_SESSION_ATTACH.to_string(),
956 CAP_ADMISSION_FACTS_RELAY.to_string(),
957 CAP_SCOPES_V1.to_string(),
958 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
959 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
960 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
961 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
962 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
963 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
964 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
965 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
966 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
967 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
968 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
969 ]),
970 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
971 route_bind_relay_timeouts: BTreeMap::new(),
972 route_bind_breakers,
973 route_bind_concurrency,
974 route_outages,
975 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
976 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
977 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
978 storage_config: None,
979 machine_id: None,
980 admission_facts_carrier_module_id: None,
981 admission_facts_targets: None,
982 scopes: Arc::new(RwLock::new(ScopeTable::new(
983 crate::daemon_config::default_scope_authority_owners(),
984 ))),
985 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
986 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
987 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
988 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
989 #[cfg(test)]
990 caller_lookup: None,
991 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
992 rescan: None,
993 connected_clients: ConnectedClients::new(),
994 counters,
995 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
996 daemon_shutdown: DaemonShutdownFlag::default(),
997 daemon_provenance: DaemonProvenanceFacts::default(),
998 #[cfg(test)]
999 control_dispatch_delay: None,
1000 #[cfg(test)]
1001 provenance_probe_override: None,
1002 #[cfg(test)]
1003 route_bind_relay_sent_hold: None,
1004 #[cfg(test)]
1005 hello_registry_to_forwarding_hold: None,
1006 }
1007 }
1008
1009 pub fn with_storage_config(
1012 mut self,
1013 storage_config: Option<crate::daemon_config::StorageConfig>,
1014 ) -> Self {
1015 self.storage_config = storage_config;
1016 self
1017 }
1018
1019 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1022 self.activity = Some(crate::activity::ActivityLedger::open(
1023 run_dir.join("activity.jsonl"),
1024 incarnation,
1025 ));
1026 self.supervisor.set_activity(self.activity.clone());
1027 self
1028 }
1029
1030 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1033 self.machine_id = machine_id;
1034 self
1035 }
1036
1037 pub fn with_admission_facts_config(
1041 mut self,
1042 carrier_module_id: Option<String>,
1043 targets: Option<Vec<String>>,
1044 ) -> Self {
1045 self.admission_facts_carrier_module_id = carrier_module_id;
1046 self.admission_facts_targets = targets;
1047 self
1048 }
1049
1050 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1054 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1055 self.scope_authority_owners = owners;
1056 self
1057 }
1058
1059 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1061 self.privileged_operator_prompt = mode;
1062 self
1063 }
1064
1065 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1068 self.route_bind_relay_timeout = timeout;
1069 self
1070 }
1071
1072 pub fn with_route_bind_relay_timeouts(
1078 mut self,
1079 timeouts: impl IntoIterator<Item = (String, Duration)>,
1080 ) -> Self {
1081 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1082 self
1083 }
1084
1085 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1091 self.route_bind_relay_timeouts
1092 .get(module_id)
1093 .copied()
1094 .unwrap_or(self.route_bind_relay_timeout)
1095 }
1096
1097 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1105 self.route_bind_breaker_threshold = threshold.max(1);
1106 self.route_bind_breaker_cooldown = cooldown;
1107 self
1108 }
1109
1110 #[cfg(test)]
1111 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1112 self.health_probe_timeout = timeout;
1113 self
1114 }
1115
1116 #[cfg(test)]
1117 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1118 self.control_dispatch_delay = Some(delay);
1119 self
1120 }
1121
1122 #[cfg(test)]
1126 pub(crate) fn with_route_bind_relay_sent_hold(
1127 mut self,
1128 hold: Arc<tokio::sync::Barrier>,
1129 ) -> Self {
1130 self.route_bind_relay_sent_hold = Some(hold);
1131 self
1132 }
1133
1134 #[cfg(test)]
1139 pub(crate) fn with_hello_registry_to_forwarding_hold(
1140 mut self,
1141 hold: Arc<std::sync::Barrier>,
1142 ) -> Self {
1143 self.hello_registry_to_forwarding_hold = Some(hold);
1144 self
1145 }
1146
1147 #[cfg(test)]
1148 fn hold_between_registry_and_forwarding(&self) {
1149 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1150 hold.wait();
1151 hold.wait();
1152 }
1153 }
1154
1155 pub fn with_process_liveness(
1156 mut self,
1157 process_liveness: Arc<dyn ModuleProcessLiveness>,
1158 ) -> Self {
1159 self.process_liveness = Some(process_liveness);
1160 self
1161 }
1162
1163 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1164 self.supervisor = supervisor;
1165 self.supervisor.set_activity(self.activity.clone());
1166 self
1167 }
1168
1169 pub fn with_daemon_provenance(
1170 mut self,
1171 pid: u32,
1172 started_at_ms: u64,
1173 executable_path: Option<PathBuf>,
1174 build_git_sha: Option<String>,
1175 build_lock_digest: Option<String>,
1176 ) -> Self {
1177 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1178 let process_start_time = process_start_time(pid);
1179 self.daemon_provenance = DaemonProvenanceFacts {
1180 build: DaemonBuildProvenance {
1181 build_git_sha,
1182 build_lock_digest,
1183 },
1184 pid: Some(pid),
1185 started_at_ms: Some(started_at_ms),
1186 start_clock: None,
1187 executable_path,
1188 executable_identity,
1189 process_start_time,
1190 probe: ExecutableIdentityProbe::default(),
1191 };
1192 self
1193 }
1194
1195 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1196 self.daemon_provenance.start_clock = Some(clock);
1197 self
1198 }
1199
1200 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1202 self.scopes
1203 .write()
1204 .unwrap_or_else(|poisoned| poisoned.into_inner())
1205 .set_wall_clock(Arc::new(clock));
1206 self
1207 }
1208
1209 #[cfg(test)]
1210 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1211 self.provenance_probe_override = Some(result);
1212 self
1213 }
1214
1215 pub fn with_capability_config(
1219 self,
1220 modules: impl IntoIterator<Item = (String, bool)>,
1221 reserved_capabilities: BTreeMap<String, String>,
1222 ) -> Self {
1223 self.capability_evaluator
1224 .configure(modules, reserved_capabilities);
1225 self
1226 }
1227
1228 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1232 self.daemon_shutdown = flag;
1233 self
1234 }
1235
1236 pub fn with_supervisor_rescan(
1237 mut self,
1238 supervisor: Supervisor,
1239 config_path: impl Into<PathBuf>,
1240 configured_port: Option<u16>,
1241 ) -> Self {
1242 self.rescan = Some(SupervisorRescanContext {
1243 supervisor,
1244 config_path: config_path.into(),
1245 configured_port,
1246 storage_config: self.storage_config.clone(),
1247 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1248 admission_facts_targets: self.admission_facts_targets.clone(),
1249 scope_authority_owners: self.scope_authority_owners.clone(),
1250 privileged_operator_prompt: self.privileged_operator_prompt,
1251 generation: Arc::default(),
1252 });
1253 self
1254 }
1255
1256 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1257 self.connected_clients = connected_clients;
1258 self
1259 }
1260
1261 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1262 Arc::clone(&self.forwarding)
1263 }
1264
1265 pub(crate) fn counters(&self) -> DaemonCounters {
1266 self.counters.clone()
1267 }
1268
1269 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1272 tokio::spawn(async move {
1273 loop {
1274 self.capability_evaluator
1275 .wait_for_change_or_deadline()
1276 .await;
1277 self.refresh_capability_requirements();
1278 }
1279 });
1280 }
1281
1282 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1288 tokio::spawn(async move {
1289 loop {
1290 let has_deadlines = self
1291 .scopes
1292 .read()
1293 .unwrap_or_else(|poisoned| poisoned.into_inner())
1294 .has_deadlines();
1295 if !has_deadlines {
1296 self.scope_deadline_added.notified().await;
1297 continue;
1298 }
1299 tokio::time::sleep(Duration::from_secs(1)).await;
1300 if let Err(error) = self.sweep_expired_scopes() {
1301 warn!(%error, "scope expiry sweep failed");
1302 }
1303 }
1304 });
1305 }
1306
1307 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1308 let mut table = self
1309 .scopes
1310 .write()
1311 .unwrap_or_else(|poisoned| poisoned.into_inner());
1312 let swept = table.sweep_expired();
1313 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1314 drop(table);
1315 self.close_scope_drained_routes(drained);
1316 Ok(())
1317 }
1318
1319 fn publish_scope_changes(
1320 &self,
1321 changes: &[crate::scopes::ScopeTagChange],
1322 expired: &[crate::scopes::ScopeExpired],
1323 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1324 if changes.is_empty() {
1325 return Ok(Vec::new());
1326 }
1327 let drained = self
1328 .forwarding
1329 .publish_scope_changes(changes)
1330 .map_err(RouterError::Forwarding)?;
1331 let mut counts = HashMap::new();
1332 for route in &drained {
1333 *counts
1334 .entry((
1335 route.scope.owner.as_str(),
1336 route.scope.scope_ref.as_str(),
1337 route.scope.tag.scope_epoch,
1338 ))
1339 .or_insert(0usize) += 1;
1340 }
1341 for scope in expired {
1342 let routes_closed = counts
1343 .get(&(
1344 scope.owner.as_str(),
1345 scope.scope_ref.as_str(),
1346 scope.scope_epoch,
1347 ))
1348 .copied()
1349 .unwrap_or(0);
1350 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1351 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1352 routes_closed, "scope expired");
1353 }
1354 Ok(drained)
1355 }
1356
1357 fn runtime_capability_snapshot(
1358 &self,
1359 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1360 let runtime = self
1361 .supervisor
1362 .list()
1363 .into_iter()
1364 .map(|module| {
1365 let status = module.status().map_err(|err| {
1366 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1367 })?;
1368 Ok(RuntimeModule {
1369 module_id: status.module_id,
1370 state: status.state,
1371 enabled: status.enabled,
1372 })
1373 })
1374 .collect::<Result<Vec<_>, RouterError>>()?;
1375 let registrations = self.registry.list_registered_modules().map_err(|err| {
1381 RouterError::backend(
1382 0,
1383 0,
1384 format!("failed to list capability registrations: {err}"),
1385 )
1386 })?;
1387 let registrations = registrations
1388 .into_iter()
1389 .map(|registration| RegisteredModule {
1390 module_id: registration.manifest.module_id,
1391 module_version: registration.manifest.module_version,
1392 capabilities: registration.manifest.capabilities,
1393 })
1394 .collect();
1395 Ok((runtime, registrations))
1396 }
1397
1398 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1405 let cached_registration = RegisteredModule {
1406 module_id: registration.manifest.module_id.clone(),
1407 module_version: registration.manifest.module_version.clone(),
1408 capabilities: registration.manifest.capabilities.clone(),
1409 };
1410 if self.capability_evaluator.record_hello(&cached_registration) {
1411 warn!(
1412 module_id = %cached_registration.module_id,
1413 "capability claims drifted from the cached manifest"
1414 );
1415 }
1416 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1417 self.enforce_capability_denies();
1418 }
1419 self.refresh_capability_requirements();
1420 }
1421
1422 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1426 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1427 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1428 self.supervisor.set_swap_promotion_observer(observer);
1429 }
1430
1431 pub fn refresh_capability_requirements(&self) {
1432 match self.runtime_capability_snapshot() {
1433 Ok((runtime, registrations)) => {
1434 log_requirement_events(
1435 self.capability_evaluator
1436 .evaluate_now(&runtime, ®istrations),
1437 &self.daemon_shutdown,
1438 );
1439 }
1440 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1441 }
1442 }
1443
1444 fn enforce_capability_denies(&self) {
1448 let registrations = match self.registry.list_registered_modules() {
1449 Ok(snapshot) => snapshot,
1450 Err(err) => {
1451 warn!(error = %err, "failed to read registrations for capability deny census");
1452 return;
1453 }
1454 };
1455 let manifests = registrations
1456 .into_iter()
1457 .map(|registration| {
1458 (
1459 registration.manifest.module_id.clone(),
1460 registration.manifest,
1461 )
1462 })
1463 .collect::<BTreeMap<_, _>>();
1464 let census = match self.forwarding.route_census(None) {
1465 Ok(census) => census,
1466 Err(err) => {
1467 warn!(error = %err, "failed to read route census for capability deny enforcement");
1468 return;
1469 }
1470 };
1471
1472 for (target_module_id, routes) in census {
1473 let Some(target_manifest) = manifests.get(&target_module_id) else {
1474 continue;
1475 };
1476 let mut closed_routes = Vec::new();
1477 let mut goodbyes = Vec::new();
1478 for route in routes {
1479 let Principal::Reserved {
1480 module_id: opening_module_id,
1481 } = &route.principal
1482 else {
1483 continue;
1484 };
1485 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1486 continue;
1487 };
1488 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1489 continue;
1490 };
1491
1492 match self.forwarding.release_client_route(
1493 route.goodbye_target.connection_id,
1494 route.goodbye_target.channel,
1495 route.goodbye_target.epoch,
1496 ) {
1497 Ok(RouteRelease::Removed(module_goodbye)) => {
1498 warn!(
1499 opening_module_id,
1500 target_module_id,
1501 capability,
1502 "force-closing route because an attested capability deny edge now matches"
1503 );
1504 goodbyes.push(module_goodbye);
1505 goodbyes.push(route.goodbye_target.clone());
1506 closed_routes.push(route);
1507 }
1508 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1509 Err(err) => warn!(
1510 opening_module_id,
1511 target_module_id,
1512 capability,
1513 error = %err,
1514 "failed to force-close capability-denied route"
1515 ),
1516 }
1517 }
1518
1519 if closed_routes.is_empty() {
1520 continue;
1521 }
1522 send_route_control_pushes(
1523 &self.forwarding,
1524 closed_routes,
1525 ClientControlPush::RouteClosed {
1526 module_id: target_module_id,
1527 channels: Vec::new(),
1528 reason: RouteCloseReason::CapabilityDenied,
1529 drained: false,
1530 abandoned: 0,
1531 excluded_subscriptions: 0,
1532 terminal: Some(false),
1533 },
1534 );
1535 self.emit_route_goodbyes(goodbyes);
1538 }
1539 }
1540
1541 fn not_ready_reason(
1547 &self,
1548 registration: &crate::registry::ModuleRegistration,
1549 ) -> Option<NotReadyReason> {
1550 if !registration.ready {
1551 return Some(NotReadyReason {
1552 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1553 capability: None,
1554 });
1555 }
1556 self.first_unprovided_required_capability(registration)
1557 .map(|capability| NotReadyReason {
1558 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1559 capability: Some(capability),
1560 })
1561 }
1562
1563 fn first_unprovided_required_capability(
1581 &self,
1582 registration: &crate::registry::ModuleRegistration,
1583 ) -> Option<String> {
1584 let required = registration
1585 .manifest
1586 .capabilities
1587 .iter()
1588 .flat_map(|declarations| declarations.requires.iter())
1589 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1590 .map(|requirement| requirement.capability.as_str())
1591 .collect::<BTreeSet<_>>();
1592 if required.is_empty() {
1593 return None;
1594 }
1595 let module_id = registration.manifest.module_id.as_str();
1596 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1597 if required
1598 .iter()
1599 .any(|capability| verdict(capability).is_none())
1600 {
1601 self.refresh_capability_requirements();
1602 }
1603 required
1604 .into_iter()
1605 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1606 .map(str::to_string)
1607 }
1608
1609 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1610 self.capability_evaluator
1611 .statuses()
1612 .into_iter()
1613 .map(capability_requirement_status)
1614 .collect()
1615 }
1616
1617 fn deregister_connection(
1624 &self,
1625 connection_id: ConnectionId,
1626 reason: RegistrationEndReason,
1627 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1628 self.registry
1629 .deregister_connection_with_reason(connection_id, reason)
1630 }
1631
1632 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1633 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1634 return None;
1635 }
1636 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1637 parse_client_control_request(&frame.body)
1638 else {
1639 return None;
1640 };
1641 Some(target_module_id(&target).to_string())
1642 }
1643
1644 pub(crate) fn route_open_capacity_refusal(
1645 &self,
1646 ctx: &RouteCtx,
1647 frame: &Frame,
1648 target_module_id: &str,
1649 in_flight: usize,
1650 limit: usize,
1651 ) -> Result<Frame, RouterError> {
1652 let caller = self.route_open_caller_from_frame(frame);
1655 self.route_open_admission_refusal_frame(
1656 ctx,
1657 &caller,
1658 frame,
1659 target_module_id,
1660 "open_admission_full",
1661 (in_flight, limit),
1662 format!(
1663 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1664 ),
1665 )
1666 }
1667
1668 fn route_open_target_capacity_refusal(
1669 &self,
1670 ctx: &RouteCtx,
1671 caller: &RouteOpenCaller,
1672 frame: &Frame,
1673 target_module_id: &str,
1674 in_flight: usize,
1675 ) -> Result<Frame, RouterError> {
1676 self.route_open_admission_refusal_frame(
1677 ctx,
1678 caller,
1679 frame,
1680 target_module_id,
1681 "target_binds_full",
1682 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1683 format!(
1684 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1685 ),
1686 )
1687 }
1688
1689 #[allow(clippy::too_many_arguments)]
1697 fn route_open_admission_refusal_frame(
1698 &self,
1699 ctx: &RouteCtx,
1700 caller: &RouteOpenCaller,
1701 frame: &Frame,
1702 target_module_id: &str,
1703 reason: &'static str,
1704 (in_flight, limit): (usize, usize),
1705 message: impl Into<String>,
1706 ) -> Result<Frame, RouterError> {
1707 let code = error_codes::TARGET_UNAVAILABLE;
1708 self.counters.increment_route_open_refused(code);
1709 info!(
1710 target: "control",
1711 code,
1712 reason,
1713 module_id = ?target_module_id,
1714 connection_id = ctx.connection_id.get(),
1715 principal = %caller.principal,
1716 scope = %caller.scope,
1717 harness = %caller.harness,
1718 in_flight,
1719 limit,
1720 "route.open refused"
1721 );
1722 control_error_frame(frame, code, message.into())
1723 }
1724
1725 #[cfg(test)]
1731 pub fn handle_control(
1732 &self,
1733 connection_id: ConnectionId,
1734 frame: Frame,
1735 ) -> Result<Vec<Frame>, RouterError> {
1736 match frame.header.ty {
1737 FrameType::Ping => Ok(vec![pong(&frame)?]),
1738 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1739 FrameType::Goodbye => self.handle_goodbye(connection_id),
1740 ty => Ok(vec![control_error_frame(
1741 &frame,
1742 "unsupported_control_frame",
1743 format!("unsupported channel-0 frame {ty:?}"),
1744 )?]),
1745 }
1746 }
1747
1748 pub async fn handle_control_frame(
1749 &self,
1750 ctx: &RouteCtx,
1751 frame: Frame,
1752 ) -> Result<Vec<Frame>, RouterError> {
1753 self.handle_control_frame_timed(ctx, frame, None).await
1754 }
1755
1756 pub(crate) fn record_caller_endpoints(
1757 &self,
1758 connection: ConnectionId,
1759 endpoints: Option<(SocketAddr, SocketAddr)>,
1760 ) {
1761 let mut callers = self
1762 .caller_endpoints
1763 .lock()
1764 .unwrap_or_else(|p| p.into_inner());
1765 callers.remove(&connection);
1766 if let Some(endpoints) = endpoints {
1767 callers.insert(connection, endpoints);
1768 }
1769 }
1770
1771 #[cfg(test)]
1772 pub(crate) fn with_caller_lookup_for_test(
1773 mut self,
1774 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1775 ) -> Self {
1776 self.caller_lookup = Some(Arc::new(lookup));
1777 self
1778 }
1779
1780 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1781 let endpoints = self
1782 .caller_endpoints
1783 .lock()
1784 .unwrap_or_else(|p| p.into_inner())
1785 .get(&connection)
1786 .copied();
1787 #[cfg(test)]
1788 let lookup = self.caller_lookup.clone();
1789 tokio::task::spawn_blocking(move || {
1790 #[cfg(test)]
1791 if let Some(lookup) = lookup {
1792 return lookup(endpoints);
1793 }
1794 endpoints
1795 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1796 .unwrap_or_default()
1797 })
1798 .await
1799 .unwrap_or_default()
1800 }
1801
1802 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1803 frame.header.channel == 0
1804 && frame.header.ty == FrameType::Request
1805 && self
1806 .forwarding
1807 .module_endpoint_for_connection(ctx.connection_id)
1808 .is_ok_and(|endpoint| endpoint.is_none())
1809 && parse_client_control_request(&frame.body)
1810 .ok()
1811 .is_some_and(|request| privileged_operation(&request).is_some())
1812 }
1813
1814 pub(crate) async fn handle_control_frame_timed(
1815 &self,
1816 ctx: &RouteCtx,
1817 frame: Frame,
1818 dispatch_started_at: Option<StdInstant>,
1819 ) -> Result<Vec<Frame>, RouterError> {
1820 match frame.header.ty {
1821 FrameType::Ping => Ok(vec![pong(&frame)?]),
1822 FrameType::Hello => {
1823 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1824 }
1825 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1826 FrameType::Cancel => {
1827 if self
1830 .forwarding
1831 .operator_confirms()
1832 .cancel(ctx.connection_id, frame.header.corr)
1833 || self
1834 .supervisor
1835 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1836 || self.cancel_activity_subscription(ctx, &frame)?
1837 {
1838 Ok(Vec::new())
1839 } else {
1840 Ok(vec![control_error_frame(
1841 &frame,
1842 "unknown_subscription",
1843 "no subscription has this correlation id",
1844 )?])
1845 }
1846 }
1847 FrameType::Request => {
1848 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1851 if op
1852 .as_ref()
1853 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1854 {
1855 return self.handle_activity_record(ctx, frame).await;
1856 }
1857 if op.as_ref().is_some_and(|probe| {
1858 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1859 }) {
1860 return self.handle_activity_read(ctx, frame).await;
1861 }
1862 if op
1863 .as_ref()
1864 .is_some_and(|probe| probe.op == "operator.confirm")
1865 {
1866 return self.handle_operator_confirm(ctx, frame);
1867 }
1868 if op
1869 .as_ref()
1870 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1871 {
1872 return self.handle_scope_served_by(ctx.connection_id, frame);
1873 }
1874 if self
1875 .forwarding
1876 .module_endpoint_for_connection(ctx.connection_id)
1877 .map_err(RouterError::Forwarding)?
1878 .is_some()
1879 {
1880 if !is_known_module_request_op(&frame.body) {
1881 return Ok(vec![control_error_frame(
1882 &frame,
1883 "unsupported_control_frame",
1884 "module-originated channel-0 REQUEST is not supported",
1885 )?]);
1886 }
1887 let request = match parse_module_control_request_from_module(&frame.body) {
1888 Ok(request) => request,
1889 Err((err, ControlRequestBodyError::UnknownOp)) => {
1890 return Ok(vec![control_error_frame(
1891 &frame,
1892 "unsupported_control_frame",
1893 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1894 )?])
1895 }
1896 Err((err, ControlRequestBodyError::InvalidBody)) => {
1897 return Ok(vec![control_error_frame(
1898 &frame,
1899 "invalid_control_body",
1900 format!("malformed module control body: {err}"),
1901 )?])
1902 }
1903 };
1904 let op = module_control_request_op(&request);
1905 let corr = frame.header.corr;
1906 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1907 let result =
1908 self.handle_module_control_request(ctx.connection_id, frame, request);
1909 log_slow_control_dispatch(
1910 dispatch_started_at,
1911 op,
1912 ctx.connection_id,
1913 corr,
1914 None,
1915 );
1916 return result;
1917 }
1918
1919 if is_known_module_request_op(&frame.body) {
1920 return Ok(vec![control_error_frame(
1921 &frame,
1922 "not_registered",
1923 "catalog.update requires an active module registration owned by this connection",
1924 )?]);
1925 }
1926
1927 let request = match parse_client_control_request(&frame.body) {
1928 Ok(request) => request,
1929 Err((err, ControlRequestBodyError::UnknownOp)) => {
1930 return Ok(vec![control_error_frame(
1931 &frame,
1932 "unknown_control_op",
1933 format!("unknown client control op: {err}"),
1934 )?])
1935 }
1936 Err((err, ControlRequestBodyError::InvalidBody)) => {
1937 return Ok(vec![control_error_frame(
1938 &frame,
1939 "invalid_control_body",
1940 format!("malformed client control body: {err}"),
1941 )?])
1942 }
1943 };
1944 let op = client_control_request_op(&request);
1945 let corr = frame.header.corr;
1946 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1947 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1948 #[cfg(test)]
1949 if let Some(delay) = self.control_dispatch_delay {
1950 tokio::time::sleep(delay).await;
1951 }
1952 let result = self
1953 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1954 .await;
1955 if let Some(timings) = route_open_timings.as_mut() {
1956 timings.finish();
1957 }
1958 log_slow_control_dispatch(
1959 dispatch_started_at,
1960 op,
1961 ctx.connection_id,
1962 corr,
1963 route_open_timings.as_ref(),
1964 );
1965 result
1966 }
1967 FrameType::Push => {
1968 let Some(endpoint) = self
1969 .forwarding
1970 .module_endpoint_for_connection(ctx.connection_id)
1971 .map_err(RouterError::Forwarding)?
1972 else {
1973 return Ok(vec![control_error_frame(
1974 &frame,
1975 "unsupported_control_frame",
1976 "client-originated channel-0 PUSH is not supported",
1977 )?]);
1978 };
1979 self.handle_status_update(endpoint, frame)
1980 }
1981 FrameType::Response | FrameType::Error
1982 if self
1983 .forwarding
1984 .module_endpoint_for_connection(ctx.connection_id)
1985 .map_err(RouterError::Forwarding)?
1986 .is_some() =>
1987 {
1988 self.handle_module_relay_response(ctx.connection_id, frame)
1989 }
1990 ty => Ok(vec![control_error_frame(
1991 &frame,
1992 "unsupported_control_frame",
1993 format!("unsupported channel-0 frame {ty:?}"),
1994 )?]),
1995 }
1996 }
1997
1998 pub fn cleanup_connection(
1999 &self,
2000 connection_id: ConnectionId,
2001 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2002 self.caller_endpoints
2003 .lock()
2004 .unwrap_or_else(|p| p.into_inner())
2005 .remove(&connection_id);
2006 self.forwarding
2007 .operator_confirms()
2008 .connection_closed(connection_id);
2009 self.cleanup_connection_with_end_reason(
2010 connection_id,
2011 RegistrationEndReason::ConnectionClosed,
2012 )
2013 }
2014
2015 fn cleanup_connection_with_end_reason(
2016 &self,
2017 connection_id: ConnectionId,
2018 requested_reason: RegistrationEndReason,
2019 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2020 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2021 self.registry
2022 .get_module_by_connection(connection_id)?
2023 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2024 .and_then(|module| module.registration_end_reason().ok().flatten())
2025 .unwrap_or(requested_reason)
2026 } else {
2027 requested_reason
2028 };
2029 let crash_closed = self
2030 .registry
2031 .get_module_by_connection(connection_id)?
2032 .and_then(|registration| {
2033 self.forwarding
2034 .module_endpoint_for_connection(connection_id)
2035 .ok()
2036 .flatten()
2037 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2038 .map(|routes| (registration.manifest.module_id, routes))
2039 });
2040 let crash_closed = crash_closed.map(|(module_id, routes)| {
2041 let terminal = match self.supervisor.get(&module_id) {
2042 None => false,
2043 Some(module) => match module.will_recover_after_connection_loss() {
2044 Ok(will_recover) => !will_recover,
2045 Err(err) => {
2046 warn!(
2047 %module_id,
2048 error = %err,
2049 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2050 );
2051 false
2052 }
2053 },
2054 };
2055 let (reason, drained) = match self
2059 .forwarding
2060 .daemon_drain_result_for_connection(connection_id)
2061 {
2062 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2063 Ok(None) => (RouteCloseReason::Crash, false),
2064 Err(err) => {
2065 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2066 (RouteCloseReason::Crash, false)
2067 }
2068 };
2069 (module_id, routes, reason, drained, terminal)
2070 });
2071 let registrations = self.deregister_connection(connection_id, end_reason);
2072 let cleanup = if crash_closed.is_some() {
2073 self.forwarding.cleanup_connection_counted(connection_id)
2074 } else {
2075 self.forwarding
2081 .cleanup_connection(connection_id)
2082 .map(|released| crate::forwarding::ConnectionCleanup {
2083 released,
2084 abandoned_relays: 0,
2085 })
2086 };
2087 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2092 let abandoned = cleanup
2093 .as_ref()
2094 .map_or(0, |cleanup| cleanup.abandoned_relays);
2095 send_route_control_pushes(
2096 &self.forwarding,
2097 routes,
2098 ClientControlPush::RouteClosed {
2099 module_id,
2100 channels: Vec::new(),
2101 reason,
2102 drained,
2103 abandoned,
2104 excluded_subscriptions: 0,
2105 terminal: Some(terminal),
2106 },
2107 );
2108 }
2109 if let Ok(cleanup) = cleanup {
2110 self.emit_route_goodbyes(cleanup.released);
2111 }
2112 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2116 crate::supervise::notify_registration_release();
2117 self.capability_evaluator.wake_deadline_loop();
2118 self.refresh_capability_requirements();
2119 }
2120 self.supervisor.remove_spawn_subscribers(connection_id);
2121 self.activity_streams
2122 .lock()
2123 .unwrap_or_else(|p| p.into_inner())
2124 .retain(|(id, _), task| {
2125 if *id == connection_id {
2126 task.abort();
2127 false
2128 } else {
2129 true
2130 }
2131 });
2132 self.hello_launch_nonces
2135 .lock()
2136 .unwrap_or_else(|poisoned| poisoned.into_inner())
2137 .forget(connection_id);
2138 self.scopes
2139 .write()
2140 .unwrap_or_else(|poisoned| poisoned.into_inner())
2141 .release_connection(connection_id);
2142 registrations
2143 }
2144
2145 pub(crate) fn handle_route_goodbye(
2146 &self,
2147 connection_id: ConnectionId,
2148 route_channel: u16,
2149 route_epoch: u32,
2150 ) -> Result<bool, RouterError> {
2151 debug!(
2152 connection_id = connection_id.get(),
2153 route_channel, route_epoch, "handling route GOODBYE"
2154 );
2155 let RouteRelease::Removed(released_route) = self
2156 .forwarding
2157 .release_client_route(connection_id, route_channel, route_epoch)
2158 .map_err(RouterError::Forwarding)?
2159 else {
2160 return Ok(false);
2161 };
2162 self.emit_route_goodbyes(vec![released_route]);
2163 Ok(true)
2164 }
2165
2166 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2167 for released in released_routes {
2168 let frame = match Frame::build_with_version(
2169 released.negotiated_ver,
2170 FrameType::Goodbye,
2171 control_flags(),
2172 released.channel,
2173 released.epoch,
2174 0,
2175 Vec::new(),
2176 ) {
2177 Ok(frame) => frame,
2178 Err(err) => {
2179 warn!(
2180 route_channel = released.channel,
2181 error = %err,
2182 "failed to build route GOODBYE frame"
2183 );
2184 continue;
2185 }
2186 };
2187 if !released.close_on_delivery_failure() {
2188 crate::forwarding::send_module_route_goodbye(
2189 &self.counters,
2190 &released.sink,
2191 frame,
2192 released.module_id.as_deref(),
2193 "client route released",
2194 );
2195 continue;
2196 }
2197 if let Err(err) = released.sink.try_send(frame) {
2198 warn!(
2199 target_connection_id = released.connection_id.get(),
2200 route_channel = released.channel,
2201 error = %err,
2202 "route GOODBYE was not delivered to client; closing target connection"
2203 );
2204 if self
2205 .forwarding
2206 .escalate_client_delivery_failure(
2207 released.connection_id,
2208 released.channel,
2209 released.epoch,
2210 CloseReason::new(
2211 "route_goodbye_delivery_failed",
2212 format!(
2213 "failed to enqueue route GOODBYE for channel {}: {err}",
2214 released.channel
2215 ),
2216 ),
2217 crate::forwarding::UndeliveredFrame {
2218 module_id: released.module_id.as_deref(),
2219 sink: &released.sink,
2220 },
2221 )
2222 .unwrap_or(false)
2223 {
2224 self.counters.increment_goodbye_relay_client_failed();
2225 }
2226 }
2227 }
2228 }
2229
2230 fn send_abandoned_route_bind_goodbye(
2242 &self,
2243 module_sink: &crate::FrameSink,
2244 negotiated_ver: u8,
2245 module_channel: u16,
2246 module_epoch: u32,
2247 ) {
2248 let frame = match Frame::build_with_version(
2249 negotiated_ver,
2250 FrameType::Goodbye,
2251 control_flags(),
2252 module_channel,
2253 module_epoch,
2254 0,
2255 Vec::new(),
2256 ) {
2257 Ok(frame) => frame,
2258 Err(err) => {
2259 warn!(
2260 route_channel = module_channel,
2261 error = %err,
2262 "failed to build GOODBYE for abandoned route.bind"
2263 );
2264 return;
2265 }
2266 };
2267 crate::forwarding::send_module_route_goodbye(
2268 &self.counters,
2269 module_sink,
2270 frame,
2271 None,
2272 "abandoned route.bind",
2273 );
2274 }
2275
2276 fn handle_hello(
2277 &self,
2278 connection_id: ConnectionId,
2279 sink: Option<crate::FrameSink>,
2280 frame: Frame,
2281 ) -> Result<Vec<Frame>, RouterError> {
2282 debug!(
2283 connection_id = connection_id.get(),
2284 corr = frame.header.corr,
2285 "handling HELLO"
2286 );
2287 if self
2291 .registry
2292 .get_module_by_connection(connection_id)
2293 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2294 .is_some()
2295 {
2296 return Ok(vec![control_error_frame(
2297 &frame,
2298 "invalid_hello",
2299 "connection is already registered as a module",
2300 )?]);
2301 }
2302 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2303 Ok(value) => value,
2304 Err(err) => {
2305 return Ok(vec![control_error_frame(
2306 &frame,
2307 "invalid_hello",
2308 format!("malformed HELLO body: {err}"),
2309 )?])
2310 }
2311 };
2312 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2313 return Ok(vec![control_error_frame(
2314 &frame,
2315 "invalid_capability_grammar",
2316 err.to_string(),
2317 )?]);
2318 }
2319 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2320 return Ok(vec![control_error_frame(
2321 &frame,
2322 "invalid_manifest",
2323 err.to_string(),
2324 )?]);
2325 }
2326 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2327 return Ok(vec![control_error_frame(
2328 &frame,
2329 "invalid_event_declaration",
2330 err.to_string(),
2331 )?]);
2332 }
2333 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2334 return Ok(vec![control_error_frame(
2335 &frame,
2336 "invalid_manifest",
2337 err.to_string(),
2338 )?]);
2339 }
2340 if let Some(provenance) = hello_value
2341 .get("manifest")
2342 .and_then(|manifest| manifest.get("provenance"))
2343 {
2344 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2345 return Ok(vec![control_error_frame(
2346 &frame,
2347 "invalid_manifest",
2348 format!("malformed manifest provenance: {err}"),
2349 )?]);
2350 }
2351 }
2352 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2353 let module_id = hello_value["manifest"]["module_id"]
2354 .as_str()
2355 .unwrap_or("<unknown>");
2356 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2357 }
2358 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2359 let module_id = hello_value["manifest"]["module_id"]
2360 .as_str()
2361 .unwrap_or("<unknown>");
2362 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2363 }
2364 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2365 Ok(hello) => hello,
2366 Err(err) => {
2367 return Ok(vec![control_error_frame(
2368 &frame,
2369 "invalid_hello",
2370 format!("malformed HELLO body: {err}"),
2371 )?])
2372 }
2373 };
2374
2375 if hello.protocol_ver != hello.manifest.protocol_ver {
2376 return Ok(vec![control_error_frame(
2377 &frame,
2378 "invalid_manifest",
2379 format!(
2380 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2381 hello.protocol_ver, hello.manifest.protocol_ver
2382 ),
2383 )?]);
2384 }
2385
2386 if hello.manifest.module_id.trim().is_empty() {
2387 return Ok(vec![control_error_frame(
2388 &frame,
2389 "invalid_manifest",
2390 "manifest module_id must not be empty",
2391 )?]);
2392 }
2393
2394 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2395 Ok(negotiated_ver) => negotiated_ver,
2396 Err(message) => {
2397 return Ok(vec![control_error_frame(
2398 &frame,
2399 "version_unsupported",
2400 message,
2401 )?])
2402 }
2403 };
2404
2405 let swap_admission = self
2413 .supervisor
2414 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2415 if swap_admission == SwapHelloAdmission::Refused {
2416 warn!(
2417 module_id = %hello.manifest.module_id,
2418 connection_id = connection_id.get(),
2419 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2420 );
2421 return Ok(vec![control_error_frame(
2422 &frame,
2423 "swap_token_invalid",
2424 format!(
2425 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2426 hello.manifest.module_id
2427 ),
2428 )?]);
2429 }
2430 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2431
2432 let nonce_verified = if swap_candidate {
2437 true
2438 } else {
2439 match self.supervisor.reserved_hello_verification(
2440 &hello.manifest.module_id,
2441 hello.launch_nonce.as_deref(),
2442 ) {
2443 Ok(verified) => verified,
2444 Err(rejection) => {
2445 let message = match rejection {
2446 ReservedHelloRejection::Exact { module_id } => format!(
2447 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2448 ),
2449 ReservedHelloRejection::Prefix {
2450 prefix,
2451 owner_module_id,
2452 } => format!(
2453 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2454 hello.manifest.module_id
2455 ),
2456 };
2457 return Ok(vec![control_error_frame(
2458 &frame,
2459 "reserved_module",
2460 message,
2461 )?]);
2462 }
2463 }
2464 };
2465
2466 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2467 &hello.manifest.module_id,
2468 hello.manifest.capabilities.as_ref(),
2469 );
2470 if let Some(refusal) = reserved_capability_refusals.first() {
2471 let capability = refusal.capability.clone();
2472 let bound_module = refusal.claimants[0].clone();
2473 log_duplicate_claim_events(reserved_capability_refusals);
2474 return Ok(vec![control_error_frame(
2475 &frame,
2476 "reserved_capability",
2477 format!(
2478 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2479 capability, bound_module, hello.manifest.module_id
2480 ),
2481 )?]);
2482 }
2483
2484 if self
2487 .forwarding
2488 .connection_has_client_routes(connection_id)
2489 .map_err(RouterError::Forwarding)?
2490 {
2491 return Ok(vec![control_error_frame(
2492 &frame,
2493 "invalid_hello",
2494 "connection has open client routes and cannot also register as a module",
2495 )?]);
2496 }
2497
2498 self.hello_launch_nonces
2503 .lock()
2504 .unwrap_or_else(|poisoned| poisoned.into_inner())
2505 .record(connection_id, hello.launch_nonce.as_deref());
2506 let control_ops = effective_module_control_ops(hello.control_ops);
2507 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2510 if swap_candidate {
2511 return self.register_swap_candidate(
2512 connection_id,
2513 sink,
2514 &frame,
2515 hello.manifest,
2516 negotiated_ver,
2517 control_ops,
2518 hello_ack,
2519 );
2520 }
2521 let registration = match self
2528 .registry
2529 .register_unpublished_verified_with_control_ops(
2530 hello.manifest,
2531 negotiated_ver,
2532 connection_id,
2533 control_ops,
2534 nonce_verified,
2535 ) {
2536 Ok(registration) => registration,
2537 Err(RegistryError::DuplicateModuleId { module_id }) => {
2538 return Ok(vec![control_error_frame(
2539 &frame,
2540 "duplicate_module_id",
2541 format!(
2542 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2543 ),
2544 )?])
2545 }
2546 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2547 return Ok(vec![control_error_frame(
2548 &frame,
2549 "invalid_module_id",
2550 err.to_string(),
2551 )?])
2552 }
2553 Err(err) => {
2554 return Ok(vec![control_error_frame(
2555 &frame,
2556 "registry_error",
2557 err.to_string(),
2558 )?])
2559 }
2560 };
2561 #[cfg(test)]
2562 self.hold_between_registry_and_forwarding();
2563
2564 let reply = if let Some(sink) = sink {
2565 let concurrency = manifest_concurrency(®istration.manifest);
2581 if let Err(err) = self.forwarding.register_module_connection_acked(
2582 connection_id,
2583 registration.manifest.module_id.clone(),
2584 negotiated_ver,
2585 concurrency,
2586 sink,
2587 hello_ack,
2588 ) {
2589 if matches!(
2594 self.deregister_connection(
2595 connection_id,
2596 RegistrationEndReason::RegistrationFailed,
2597 ),
2598 Ok(r) if !r.is_empty()
2599 ) {
2600 crate::supervise::notify_registration_release();
2601 }
2602 return Ok(vec![control_error_frame(
2603 &frame,
2604 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2605 "invalid_hello"
2606 } else {
2607 forwarding_error_code(&err)
2608 },
2609 err.to_string(),
2610 )?]);
2611 }
2612 Vec::new()
2613 } else {
2614 vec![hello_ack]
2617 };
2618
2619 if !self
2624 .registry
2625 .publish_registration(connection_id)
2626 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2627 {
2628 debug!(
2629 module_id = %registration.manifest.module_id,
2630 connection_id = connection_id.get(),
2631 "registration was removed before it could be published"
2632 );
2633 }
2634
2635 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2644 info!(
2645 module_id = %registration.manifest.module_id,
2646 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2647 );
2648 }
2649
2650 self.apply_registration_capabilities(®istration);
2651
2652 self.supervisor
2653 .record_activity_registration(®istration.manifest.module_id);
2654
2655 info!(
2656 module_id = %registration.manifest.module_id,
2657 module_version = %registration.manifest.module_version,
2658 negotiated_ver,
2659 routable_provider = manifest_provides_routable_role(®istration.manifest),
2660 connection_id = connection_id.get(),
2661 "module registered"
2662 );
2663
2664 Ok(reply)
2665 }
2666
2667 #[allow(clippy::too_many_arguments)]
2684 fn register_swap_candidate(
2685 &self,
2686 connection_id: ConnectionId,
2687 sink: Option<crate::FrameSink>,
2688 frame: &Frame,
2689 manifest: ModuleManifest,
2690 negotiated_ver: u8,
2691 control_ops: Vec<String>,
2692 hello_ack: Frame,
2693 ) -> Result<Vec<Frame>, RouterError> {
2694 let module_id = manifest.module_id.clone();
2695 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2696 manifest,
2697 negotiated_ver,
2698 connection_id,
2699 control_ops,
2700 ) {
2701 Ok(registration) => registration,
2702 Err(RegistryError::DuplicateModuleId { module_id }) => {
2703 return Ok(vec![control_error_frame(
2704 frame,
2705 "duplicate_module_id",
2706 format!(
2707 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2708 ),
2709 )?])
2710 }
2711 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2712 return Ok(vec![control_error_frame(
2713 frame,
2714 "invalid_module_id",
2715 err.to_string(),
2716 )?])
2717 }
2718 Err(err) => {
2719 return Ok(vec![control_error_frame(
2720 frame,
2721 "registry_error",
2722 err.to_string(),
2723 )?])
2724 }
2725 };
2726 #[cfg(test)]
2727 self.hold_between_registry_and_forwarding();
2728 let reply = if let Some(sink) = sink {
2729 let concurrency = manifest_concurrency(®istration.manifest);
2733 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2734 connection_id,
2735 module_id.clone(),
2736 negotiated_ver,
2737 concurrency,
2738 sink,
2739 hello_ack,
2740 ) {
2741 if matches!(
2742 self.deregister_connection(
2743 connection_id,
2744 RegistrationEndReason::RegistrationFailed,
2745 ),
2746 Ok(r) if !r.is_empty()
2747 ) {
2748 crate::supervise::notify_registration_release();
2749 }
2750 return Ok(vec![control_error_frame(
2751 frame,
2752 forwarding_error_code(&err),
2753 err.to_string(),
2754 )?]);
2755 }
2756 Vec::new()
2757 } else {
2758 vec![hello_ack]
2759 };
2760 if !self
2761 .registry
2762 .publish_registration(connection_id)
2763 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2764 {
2765 debug!(
2766 module_id = %module_id,
2767 connection_id = connection_id.get(),
2768 "swap candidate registration was removed before it could be published"
2769 );
2770 }
2771 self.supervisor.mark_swap_candidate_admitted(&module_id);
2772 info!(
2773 module_id = %module_id,
2774 module_version = %registration.manifest.module_version,
2775 negotiated_ver,
2776 ready = registration.ready,
2777 connection_id = connection_id.get(),
2778 "swap candidate registered; not routable until cutover"
2779 );
2780 Ok(reply)
2781 }
2782
2783 fn build_hello_ack(
2784 &self,
2785 frame: &Frame,
2786 negotiated_ver: u8,
2787 module_id: &str,
2788 ) -> Result<Frame, RouterError> {
2789 let ack = ModuleHelloAckBody {
2790 negotiated_ver,
2791 subc_ops: module_subc_ops(),
2792 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2793 storage: self
2794 .storage_config
2795 .as_ref()
2796 .map(|cfg| cfg.descriptor_for(module_id)),
2797 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2798 };
2799 let body = serde_json::to_vec(&ack).map_err(|err| {
2800 RouterError::backend(
2801 0,
2802 frame.header.corr,
2803 format!("failed to encode HELLO_ACK: {err}"),
2804 )
2805 })?;
2806
2807 Frame::build_with_version(
2808 negotiated_ver,
2809 FrameType::HelloAck,
2810 control_flags(),
2811 0,
2812 0,
2813 frame.header.corr,
2814 body,
2815 )
2816 .map_err(RouterError::FrameBuild)
2817 }
2818
2819 async fn handle_client_control_request(
2820 &self,
2821 ctx: &RouteCtx,
2822 frame: Frame,
2823 request: ClientControlRequest,
2824 route_open_timings: &mut Option<RouteOpenTimings>,
2825 ) -> Result<Vec<Frame>, RouterError> {
2826 if let Some(operation) = privileged_operation(&request) {
2827 let caller = self.resolve_caller(ctx.connection_id).await;
2828 let op = client_control_request_op(&request);
2829 let text = prompt_text(operation, &caller);
2830 match self.privileged_operator_prompt {
2831 PrivilegedOperatorPrompt::Log => {
2832 info!(target: "subc_daemon::privileged_prompt", operation = op,
2833 target = ?operation, caller = %caller.description(), summary = %text,
2834 "privileged request would prompt the operator");
2835 }
2836 PrivilegedOperatorPrompt::Enforce => {
2837 let key =
2838 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2839 let outcome = self
2840 .forwarding
2841 .operator_confirms()
2842 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2843 .await;
2844 if outcome.code != "confirmed" {
2845 return Ok(vec![outcome.frame(&frame)]);
2846 }
2847 }
2848 }
2849 }
2850 match request {
2851 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2852 ClientControlRequest::CatalogList { module_id } => {
2853 self.handle_catalog_list(frame, module_id)
2854 }
2855 ClientControlRequest::RouteOpen {
2856 target,
2857 identity,
2858 consumer_identity,
2859 consumer_capabilities,
2860 role_versions,
2861 admission_facts,
2862 scope,
2863 } => {
2864 self.handle_route_open(
2865 ctx,
2866 frame,
2867 RouteOpenRequest {
2868 target,
2869 identity,
2870 consumer_identity,
2871 consumer_capabilities,
2872 role_versions,
2873 admission_facts,
2874 scope,
2875 },
2876 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2877 )
2878 .await
2879 }
2880 ClientControlRequest::RoutePoll {
2881 route_channel,
2882 route_epoch,
2883 kind,
2884 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2885 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2886 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2887 self.handle_supervisor_spawn_snapshot(frame)
2888 }
2889 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2890 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2891 }
2892 ClientControlRequest::SupervisorRestart {
2893 module_id,
2894 drain_timeout_ms,
2895 } => {
2896 self.log_supervisor_request_received(
2897 ctx,
2898 frame.header.corr,
2899 ops::SUPERVISOR_RESTART,
2900 Some(&module_id),
2901 None,
2902 )?;
2903 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2904 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2905 .await
2906 }
2907 ClientControlRequest::SupervisorSwap {
2908 module_id,
2909 ready_timeout_ms,
2910 } => {
2911 self.log_supervisor_request_received(
2912 ctx,
2913 frame.header.corr,
2914 ops::SUPERVISOR_SWAP,
2915 Some(&module_id),
2916 None,
2917 )?;
2918 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2919 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2920 .await
2921 }
2922 ClientControlRequest::SupervisorReload { module_id } => {
2923 self.log_supervisor_request_received(
2924 ctx,
2925 frame.header.corr,
2926 ops::SUPERVISOR_RELOAD,
2927 Some(&module_id),
2928 None,
2929 )?;
2930 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2931 self.handle_supervisor_reload(frame, module_id, requester)
2932 .await
2933 }
2934 ClientControlRequest::SupervisorRescan { preview } => {
2935 if !preview {
2936 self.log_supervisor_request_received(
2937 ctx,
2938 frame.header.corr,
2939 ops::SUPERVISOR_RESCAN,
2940 None,
2941 None,
2942 )?;
2943 }
2944 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2945 self.handle_supervisor_rescan(
2946 frame,
2947 if preview {
2948 RescanMode::Preview
2949 } else {
2950 RescanMode::Execute
2951 },
2952 requester,
2953 )
2954 .await
2955 }
2956 ClientControlRequest::SupervisorRescanChecked {
2957 preview,
2958 candidate,
2959 expect,
2960 } => {
2961 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2962 .await
2963 }
2964 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2965 self.log_supervisor_request_received(
2966 ctx,
2967 frame.header.corr,
2968 ops::SUPERVISOR_RELEASE_RESERVED,
2969 Some(&module_id),
2970 None,
2971 )?;
2972 self.handle_supervisor_release_reserved(frame, module_id)
2973 .await
2974 }
2975 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2976 self.log_supervisor_request_received(
2977 ctx,
2978 frame.header.corr,
2979 ops::SUPERVISOR_SET_ENABLED,
2980 Some(&module_id),
2981 Some(enabled),
2982 )?;
2983 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2984 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2985 .await
2986 }
2987 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2988 self.handle_supervisor_health_probe(frame, module_id).await
2989 }
2990 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2991 ClientControlRequest::SupervisorRoutes { module_id } => {
2992 self.handle_supervisor_routes(frame, module_id)
2993 }
2994 ClientControlRequest::SupervisorProvenance { module_id } => {
2995 self.handle_supervisor_provenance(frame, module_id).await
2996 }
2997 ClientControlRequest::SupervisorStderrTail {
2998 module_id,
2999 max_lines,
3000 max_bytes,
3001 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3002 ClientControlRequest::SupervisorTerminals { module_id } => {
3003 self.handle_supervisor_terminals(frame, module_id).await
3004 }
3005 }
3006 }
3007
3008 fn log_supervisor_request_received(
3009 &self,
3010 ctx: &RouteCtx,
3011 corr: u64,
3012 op: &'static str,
3013 module_id: Option<&str>,
3014 enabled: Option<bool>,
3015 ) -> Result<(), RouterError> {
3016 let caller = self.supervisor_requester(ctx, corr)?;
3017 let caller = principal_label(&caller);
3018 info!(
3021 target: "control",
3022 op,
3023 module_id,
3024 enabled,
3025 connection_id = ctx.connection_id.get(),
3026 caller = %caller,
3027 "supervisor request received"
3028 );
3029 Ok(())
3030 }
3031
3032 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3033 Ok(self
3034 .registry
3035 .get_module_by_connection(ctx.connection_id)
3036 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3037 .map(|registration| Principal::Reserved {
3038 module_id: registration.manifest.module_id,
3039 })
3040 .unwrap_or(Principal::Direct))
3041 }
3042
3043 fn handle_module_control_request(
3044 &self,
3045 connection_id: ConnectionId,
3046 frame: Frame,
3047 request: ModuleControlRequestFromModule,
3048 ) -> Result<Vec<Frame>, RouterError> {
3049 match request {
3050 ModuleControlRequestFromModule::CatalogUpdate {
3051 provides,
3052 capabilities,
3053 ready,
3054 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3055 ModuleControlRequestFromModule::LiveRoots {} => {
3056 let registered = self
3057 .registry
3058 .get_module_by_connection(connection_id)
3059 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3060 let Some(registration) = registered else {
3061 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3062 };
3063 let response = self
3064 .forwarding
3065 .live_roots(®istration.manifest.module_id)
3066 .map_err(RouterError::Forwarding)?;
3067 Ok(vec![control_response_body_frame(
3068 &frame,
3069 &response,
3070 "ModuleControlResponseToModule::LiveRoots",
3071 )?])
3072 }
3073 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3074 self.handle_scope_sync(connection_id, frame, generation, scopes)
3075 }
3076 ModuleControlRequestFromModule::ScopeApply {
3077 generation,
3078 upsert,
3079 end,
3080 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3081 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3082 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3083 }
3084 }
3085 }
3086
3087 fn handle_scope_served_by(
3088 &self,
3089 connection_id: ConnectionId,
3090 frame: Frame,
3091 ) -> Result<Vec<Frame>, RouterError> {
3092 let registered = self
3093 .registry
3094 .get_module_by_connection(connection_id)
3095 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3096 let Some(registered) = registered else {
3097 return Ok(vec![control_error_frame(
3098 &frame,
3099 "not_registered",
3100 "scope.served_by requires an active module registration owned by this connection",
3101 )?]);
3102 };
3103 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3104 {
3105 Ok(request) => request,
3106 Err(err) => {
3107 return Ok(vec![control_error_frame(
3108 &frame,
3109 "invalid_control_body",
3110 format!("malformed scope.served_by body: {err}"),
3111 )?])
3112 }
3113 };
3114 let answer = self
3115 .scopes
3116 .read()
3117 .unwrap_or_else(|poisoned| poisoned.into_inner())
3118 .served_by(
3119 ®istered.manifest.module_id,
3120 &request.scope_ref,
3121 request.scope_epoch,
3122 &request.module_id,
3123 );
3124 let reply = subc_control::ScopeServedByReply::new(
3125 answer,
3126 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3127 );
3128 Ok(vec![control_response_body_frame(
3129 &frame,
3130 &reply,
3131 "ScopeServedByReply",
3132 )?])
3133 }
3134
3135 fn handle_operator_confirm(
3136 &self,
3137 ctx: &RouteCtx,
3138 frame: Frame,
3139 ) -> Result<Vec<Frame>, RouterError> {
3140 use crate::operator_confirm::{audit, Outcome};
3141 let registration = self
3142 .registry
3143 .get_module_by_connection(ctx.connection_id)
3144 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3145 let Some(registration) = registration else {
3146 let outcome = Outcome::refusal("not_registered");
3147 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3148 return Ok(vec![outcome.frame(&frame)]);
3149 };
3150 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3151 Ok(request) => request,
3152 Err(_) => {
3153 let outcome = Outcome::refusal("invalid_control_body");
3154 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3155 return Ok(vec![outcome.frame(&frame)]);
3156 }
3157 };
3158 let module_id = registration.manifest.module_id;
3159 let nonce = self
3163 .hello_launch_nonces
3164 .lock()
3165 .unwrap_or_else(|p| p.into_inner())
3166 .nonce(ctx.connection_id)
3167 .map(str::to_owned);
3168 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3169 self.supervisor
3170 .spawned_consumer_authorized(&module_id, nonce)
3171 });
3172 let confirms = self.forwarding.operator_confirms();
3173 self.forwarding
3174 .with_operator_route(
3175 ctx.connection_id,
3176 request.route_channel,
3177 request.route_epoch,
3178 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3179 )
3180 .map_err(RouterError::Forwarding)
3181 }
3182
3183 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3186 let mut scopes = ScopeRecords::from(scopes);
3187 for (index, record) in scopes.records.iter_mut().enumerate() {
3188 if let Some(refusal) = Self::canonicalize_workdir(record) {
3189 scopes.refusals.insert(index, refusal);
3190 }
3191 }
3192 scopes
3193 }
3194
3195 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3196 let workdir = record.workdir.as_mut()?;
3197 let path = std::path::Path::new(workdir);
3198 if !path.is_absolute() {
3199 return Some((
3200 error_codes::INVALID_WORKDIR,
3201 "workdir must be absolute".to_string(),
3202 ));
3203 }
3204 match ProjectRootId::from_path_allowing_missing(path) {
3205 Ok(root) => match root.as_path().to_str() {
3206 Some(canonical) => {
3207 *workdir = canonical.to_string();
3208 None
3209 }
3210 None => Some((
3211 error_codes::INVALID_WORKDIR,
3212 "canonical workdir is not UTF-8".to_string(),
3213 )),
3214 },
3215 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3216 }
3217 }
3218
3219 fn handle_scope_sync(
3223 &self,
3224 connection_id: ConnectionId,
3225 frame: Frame,
3226 generation: u64,
3227 scopes: Vec<ScopeRecord>,
3228 ) -> Result<Vec<Frame>, RouterError> {
3229 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3230 }
3231
3232 fn handle_scope_change(
3233 &self,
3234 connection_id: ConnectionId,
3235 frame: Frame,
3236 generation: u64,
3237 scopes: Vec<ScopeRecord>,
3238 end: Option<Vec<ScopeEnd>>,
3239 ) -> Result<Vec<Frame>, RouterError> {
3240 let is_apply = end.is_some();
3241 let Some(registration) = self
3242 .registry
3243 .get_module_by_connection(connection_id)
3244 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3245 else {
3246 return Ok(vec![control_error_frame(
3247 &frame,
3248 "not_registered",
3249 "scope.sync and scope.apply require an active module registration owned by this connection",
3250 )?]);
3251 };
3252 let owner = registration.manifest.module_id;
3253 let scopes = Self::prepare_scope_records(scopes);
3254 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3255 let is_current_launch = |connection: ConnectionId| {
3256 self.hello_launch_nonces
3257 .lock()
3258 .unwrap_or_else(|poisoned| poisoned.into_inner())
3259 .presented(connection, current_nonce.as_deref())
3260 };
3261 let mut table = self
3266 .scopes
3267 .write()
3268 .unwrap_or_else(|poisoned| poisoned.into_inner());
3269 let outcome = match end {
3270 Some(end) => table.apply(
3271 &owner,
3272 connection_id,
3273 is_current_launch,
3274 generation,
3275 scopes,
3276 end,
3277 ),
3278 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3279 };
3280 let drained = match &outcome {
3281 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3282 Err(_) => Vec::new(),
3283 };
3284 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3285 drop(table);
3286 if has_deadlines {
3287 self.scope_deadline_added.notify_one();
3288 }
3289 match outcome {
3290 Ok(applied) => {
3291 let counts = ScopeOutcomeCounts::of(&applied.results);
3292 info!(
3293 owner = %owner,
3294 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3295 generation,
3296 records = applied.results.len(),
3297 created = counts.created,
3298 replaced = counts.replaced,
3299 updated = counts.updated,
3300 unchanged = counts.unchanged,
3301 refused = counts.refused,
3302 ended = applied.ended.len(),
3303 tag_changes = applied.tag_changes.len(),
3304 routes_closed = drained.len(),
3305 "scope change accepted"
3306 );
3307 for refused in applied
3313 .results
3314 .iter()
3315 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3316 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3317 {
3318 warn!(
3319 owner = %owner,
3320 generation,
3321 scope_ref = %refused.scope_ref,
3322 scope_epoch = refused.scope_epoch,
3323 code = refused.code.as_deref().unwrap_or(""),
3324 "scope record refused"
3325 );
3326 }
3327 self.close_scope_drained_routes(drained);
3328 let response = if is_apply {
3329 ModuleControlResponseToModule::ScopeApply {
3330 generation,
3331 results: applied.results,
3332 end_results: applied.end_results,
3333 ended: applied.ended,
3334 }
3335 } else {
3336 ModuleControlResponseToModule::ScopeSync {
3337 generation,
3338 results: applied.results,
3339 ended: applied.ended,
3340 }
3341 };
3342 Ok(vec![control_response_body_frame(
3343 &frame,
3344 &response,
3345 "scope change response",
3346 )?])
3347 }
3348 Err(refusal) => {
3349 info!(
3350 owner = %owner,
3351 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3352 generation,
3353 code = refusal.code,
3354 "scope change refused"
3355 );
3356 Ok(vec![control_error_frame(
3357 &frame,
3358 refusal.code,
3359 refusal.message,
3360 )?])
3361 }
3362 }
3363 }
3364
3365 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3371 if drained.is_empty() {
3372 return;
3373 }
3374 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3375 BTreeMap::new();
3376 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3377 for route in drained {
3378 warn!(
3379 module_id = %route.module_id,
3380 reason = ?route.reason,
3381 client_connection_id = route.client.connection_id.get(),
3382 route_channel = route.client.channel,
3383 "closing route because its scope changed"
3384 );
3385 pushes
3386 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3387 .or_insert_with(|| (route.reason, Vec::new()))
3388 .1
3389 .push(EndpointRoute {
3390 goodbye_target: route.client.clone(),
3391 principal: Principal::Unverified,
3392 bound_at: Instant::now(),
3393 draining: false,
3394 drain_reason: None,
3395 });
3396 goodbyes.push(route.module);
3397 goodbyes.push(route.client);
3398 }
3399 for ((module_id, _), (reason, routes)) in pushes {
3400 send_route_control_pushes(
3401 &self.forwarding,
3402 routes,
3403 ClientControlPush::RouteClosed {
3404 module_id,
3405 channels: Vec::new(),
3406 reason,
3407 drained: false,
3408 abandoned: 0,
3409 excluded_subscriptions: 0,
3410 terminal: Some(false),
3411 },
3412 );
3413 }
3414 self.emit_route_goodbyes(goodbyes);
3415 }
3416
3417 fn handle_scope_describe(
3420 &self,
3421 connection_id: ConnectionId,
3422 frame: Frame,
3423 owner: Principal,
3424 scope_ref: String,
3425 ) -> Result<Vec<Frame>, RouterError> {
3426 let registered = self
3427 .registry
3428 .get_module_by_connection(connection_id)
3429 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3430 let Some(registered) = registered else {
3431 return Ok(vec![control_error_frame(
3432 &frame,
3433 "not_registered",
3434 "scope.describe requires an active module registration owned by this connection",
3435 )?]);
3436 };
3437 let mut description = self
3438 .scopes
3439 .read()
3440 .unwrap_or_else(|poisoned| poisoned.into_inner())
3441 .describe(&owner, &scope_ref);
3442 if !registered
3445 .manifest
3446 .capabilities
3447 .as_ref()
3448 .is_some_and(|caps| {
3449 caps.provides
3450 .iter()
3451 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3452 })
3453 {
3454 if let Some(stamp) = description.stamp.as_mut() {
3455 stamp.attributes.on_behalf_of_agent = None;
3456 }
3457 }
3458 if !registered
3459 .manifest
3460 .capabilities
3461 .as_ref()
3462 .is_some_and(|caps| {
3463 caps.provides
3464 .iter()
3465 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3466 })
3467 {
3468 if let Some(stamp) = description.stamp.as_mut() {
3469 stamp.attributes.flow_version = None;
3470 stamp.attributes.flow_code_hash = None;
3471 stamp.attributes.flow_install_id = None;
3472 }
3473 }
3474 let owner_configured = match &owner {
3475 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3480 _ => false,
3481 };
3482 let response = ModuleControlResponseToModule::ScopeDescribe {
3483 status: description.status,
3484 scope_epoch: description.scope_epoch,
3485 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3486 owner_synced: description.owner_synced,
3487 owner_configured,
3488 scope: description.stamp,
3489 };
3490 Ok(vec![control_response_body_frame(
3491 &frame,
3492 &response,
3493 "ModuleControlResponseToModule::ScopeDescribe",
3494 )?])
3495 }
3496
3497 fn handle_catalog_update(
3498 &self,
3499 connection_id: ConnectionId,
3500 frame: Frame,
3501 provides: Vec<ProviderRole>,
3502 capabilities: Option<CapabilityDeclarations>,
3503 ready: Option<bool>,
3504 ) -> Result<Vec<Frame>, RouterError> {
3505 self.refresh_capability_requirements();
3506 let Some(registration) = self
3507 .registry
3508 .get_module_by_connection(connection_id)
3509 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3510 else {
3511 return Ok(vec![control_error_frame(
3512 &frame,
3513 "not_registered",
3514 "catalog.update requires an active module registration owned by this connection",
3515 )?]);
3516 };
3517
3518 if let Some(message) =
3519 catalog_update_frozen_field_message(®istration.manifest, &provides)
3520 {
3521 return Ok(vec![control_error_frame(
3522 &frame,
3523 "catalog_update_frozen_field",
3524 message,
3525 )?]);
3526 }
3527
3528 let mut candidate = registration.manifest.clone();
3529 candidate.provides = provides.clone();
3530 candidate.capabilities = capabilities
3531 .clone()
3532 .or_else(|| registration.manifest.capabilities.clone());
3533 if let Some(rich_ui) = &candidate.rich_ui {
3534 if let Err(err) = rich_ui.validate() {
3535 return Ok(vec![control_error_frame(
3536 &frame,
3537 "invalid_manifest",
3538 err.to_string(),
3539 )?]);
3540 }
3541 }
3542 if let Err(err) = candidate.validate_capability_grammar() {
3543 return Ok(vec![control_error_frame(
3544 &frame,
3545 "invalid_capability_grammar",
3546 err.to_string(),
3547 )?]);
3548 }
3549
3550 let mut conflicts = self
3553 .capability_evaluator
3554 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3555 if let Some(conflict) = conflicts.first() {
3556 let message = format!(
3557 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3558 conflict.capability, conflict.claimants[0], candidate.module_id
3559 );
3560 for conflict in &mut conflicts {
3561 conflict.source = DuplicateClaimSource::CatalogUpdate;
3562 }
3563 log_duplicate_claim_events(conflicts);
3564 return Ok(vec![control_error_frame(
3565 &frame,
3566 "reserved_capability",
3567 message,
3568 )?]);
3569 }
3570
3571 let updated = self
3572 .registry
3573 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3574 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3575 if updated.is_none() {
3576 return Ok(vec![control_error_frame(
3577 &frame,
3578 "not_registered",
3579 "catalog.update requires an active module registration owned by this connection",
3580 )?]);
3581 }
3582 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3583 log_duplicate_claim_events(
3584 self.capability_evaluator
3585 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3586 );
3587 }
3588 if capability_census_trigger(
3589 registration.manifest.capabilities.as_ref(),
3590 updated
3591 .as_ref()
3592 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3593 ) {
3594 self.enforce_capability_denies();
3595 }
3596 self.refresh_capability_requirements();
3597
3598 let response = ModuleControlResponseToModule::CatalogUpdate {};
3599 control_response_body_frame(
3600 &frame,
3601 &response,
3602 "ModuleControlResponseToModule::CatalogUpdate",
3603 )
3604 .map(|frame| vec![frame])
3605 }
3606
3607 pub(crate) fn with_activity_ledger_config(
3608 mut self,
3609 run_dir: &Path,
3610 incarnation: String,
3611 config: crate::activity::ActivityConfig,
3612 ) -> Self {
3613 self.activity = Some(crate::activity::ActivityLedger::open_config(
3614 run_dir.join("activity.jsonl"),
3615 incarnation,
3616 config,
3617 ));
3618 self.supervisor.set_activity(self.activity.clone());
3619 self
3620 }
3621
3622 pub(crate) async fn configure_activity(
3623 &self,
3624 config: crate::activity::ActivityConfig,
3625 ) -> Result<(), &'static str> {
3626 match &self.activity {
3627 Some(ledger) => ledger.configure(config).await,
3628 None => Ok(()),
3629 }
3630 }
3631
3632 fn cancel_activity_subscription(
3633 &self,
3634 ctx: &RouteCtx,
3635 frame: &Frame,
3636 ) -> Result<bool, RouterError> {
3637 let task = self
3638 .activity_streams
3639 .lock()
3640 .unwrap_or_else(|p| p.into_inner())
3641 .remove(&(ctx.connection_id, frame.header.corr));
3642 let Some(task) = task else {
3643 return Ok(false);
3644 };
3645 task.abort();
3646 let end = Frame::build_with_version(
3647 response_version(frame),
3648 FrameType::StreamEnd,
3649 control_flags(),
3650 0,
3651 0,
3652 frame.header.corr,
3653 Vec::new(),
3654 )
3655 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3656 let sink = ctx.egress.clone();
3657 tokio::spawn(async move {
3658 let _ = sink.send(end).await;
3659 });
3660 Ok(true)
3661 }
3662
3663 async fn handle_activity_read(
3664 &self,
3665 ctx: &RouteCtx,
3666 frame: Frame,
3667 ) -> Result<Vec<Frame>, RouterError> {
3668 use subc_control::{
3669 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3670 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3671 };
3672 #[derive(Deserialize)]
3673 struct Identity {
3674 #[serde(default)]
3675 consumer_identity: Option<ConsumerIdentity>,
3676 }
3677 let principal = match self
3681 .forwarding
3682 .module_id_for_connection(ctx.connection_id)
3683 .map_err(RouterError::Forwarding)?
3684 {
3685 Some(module_id) => Ok(Principal::Reserved { module_id }),
3686 None => {
3687 let identity: Identity = match serde_json::from_slice(&frame.body) {
3688 Ok(identity) => identity,
3689 Err(_) => {
3690 return Ok(vec![control_error_frame(
3691 &frame,
3692 ACTIVITY_MALFORMED,
3693 "malformed activity read",
3694 )?])
3695 }
3696 };
3697 self.route_open_principal(&frame, identity.consumer_identity)?
3698 }
3699 };
3700 let admitted = matches!(principal, Ok(Principal::Direct))
3701 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3702 if !admitted {
3703 return Ok(vec![control_error_frame(
3704 &frame,
3705 ACTIVITY_READ_FORBIDDEN,
3706 "activity reads require an operator or trusted mirror principal",
3707 )?]);
3708 }
3709 let Some(ledger) = &self.activity else {
3710 return Ok(vec![control_error_frame(
3711 &frame,
3712 ACTIVITY_UNAVAILABLE,
3713 "activity ledger unavailable",
3714 )?]);
3715 };
3716 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3717 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3718 {
3719 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3720 Ok(request) => ledger.query(request).await,
3721 Err(_) => Err(ACTIVITY_MALFORMED),
3722 };
3723 return match result {
3724 Ok(reply) => Ok(vec![control_response_body_frame(
3725 &frame,
3726 &reply,
3727 ops::ACTIVITY_QUERY,
3728 )?]),
3729 Err(code) => Ok(vec![control_error_frame(
3730 &frame,
3731 code,
3732 "activity query refused",
3733 )?]),
3734 };
3735 }
3736 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3737 Ok(request) => request,
3738 Err(_) => {
3739 return Ok(vec![control_error_frame(
3740 &frame,
3741 ACTIVITY_MALFORMED,
3742 "malformed activity subscription",
3743 )?])
3744 }
3745 };
3746 let mut subscription = match ledger.subscribe(request.since).await {
3747 Ok(subscription) => subscription,
3748 Err(code) => {
3749 return Ok(vec![control_error_frame(
3750 &frame,
3751 code,
3752 "activity subscription refused",
3753 )?])
3754 }
3755 };
3756 let key = (ctx.connection_id, frame.header.corr);
3757 let sink = ctx.egress.clone();
3758 let mut streams = self
3759 .activity_streams
3760 .lock()
3761 .unwrap_or_else(|p| p.into_inner());
3762 streams.retain(|_, task| !task.is_finished());
3763 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3764 return Ok(vec![control_error_frame(
3765 &frame,
3766 "duplicate_subscription",
3767 "activity subscription correlation id already in use",
3768 )?]);
3769 }
3770 let task = tokio::spawn(async move {
3771 loop {
3772 let entry = if let Some(entry) = subscription.replay.pop_front() {
3773 Some(entry)
3774 } else {
3775 subscription.receiver.recv().await
3776 };
3777 let Some(entry) = entry else {
3778 break;
3779 };
3780 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3781 return;
3782 };
3783 let Ok(next) = Frame::build_with_version(
3784 response_version(&frame),
3785 FrameType::StreamData,
3786 control_flags(),
3787 0,
3788 0,
3789 frame.header.corr,
3790 body,
3791 ) else {
3792 return;
3793 };
3794 if sink.send(next).await.is_err() {
3795 return;
3796 }
3797 }
3798 if let Some(code) = subscription.terminal_code() {
3801 if let Ok(end) =
3802 control_error_frame(&frame, code, "activity subscriber fell behind")
3803 {
3804 let _ = sink.send(end).await;
3805 }
3806 }
3807 });
3808 streams.insert(key, task.abort_handle());
3809 Ok(Vec::new())
3810 }
3811
3812 async fn handle_activity_record(
3813 &self,
3814 ctx: &RouteCtx,
3815 frame: Frame,
3816 ) -> Result<Vec<Frame>, RouterError> {
3817 use subc_control::{
3818 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3819 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3820 };
3821 let result = async {
3822 let module = self
3823 .forwarding
3824 .module_id_for_connection(ctx.connection_id)
3825 .ok()
3826 .flatten()
3827 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3828 let request: ActivityRecordRequest =
3829 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3830 crate::activity::validate(&module, &request)?;
3831 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3832 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3833 let at_ms = std::time::SystemTime::now()
3834 .duration_since(std::time::UNIX_EPOCH)
3835 .unwrap_or_default()
3836 .as_millis() as u64;
3837 ledger.record(module, request, actor, at_ms).await
3838 }
3839 .await;
3840 match result {
3841 Ok(entry) => Ok(vec![control_response_body_frame(
3842 &frame,
3843 &ActivityRecordResponse { seq: entry.seq },
3844 "activity.record",
3845 )?]),
3846 Err(code) => Ok(vec![control_error_frame(
3847 &frame,
3848 code,
3849 "activity recording refused",
3850 )?]),
3851 }
3852 }
3853
3854 fn activity_actor(
3855 &self,
3856 connection: ConnectionId,
3857 module: &str,
3858 on: Option<&subc_control::ActivityOn>,
3859 ) -> Result<subc_control::ActivityActor, &'static str> {
3860 use crate::forwarding::{DataRoute, DataRouteState};
3861 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3862 let principal = |id: &str| Principal::Reserved {
3863 module_id: id.to_owned(),
3864 };
3865 let mut actor = ActivityActor {
3866 principal: principal(module),
3867 scope_owner: None,
3868 agent_id: None,
3869 run_id: None,
3870 flow_id: None,
3871 project_root: None,
3872 workdir: None,
3873 project_id: None,
3874 };
3875 let scope = match on {
3876 None => return Ok(actor),
3877 Some(ActivityOn::Scope(scope)) => {
3878 let selector = subc_protocol::scope::ScopeSelector::new(
3879 principal(module),
3880 scope.scope_ref.clone(),
3881 Some(scope.epoch),
3882 );
3883 self.scopes
3884 .read()
3885 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3886 .attribute(&principal(module), module, &selector, true)
3887 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3888 }
3889 Some(ActivityOn::Route(route)) => {
3890 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3891 .forwarding
3892 .lookup_data_route(connection, route.channel, route.epoch)
3893 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3894 else {
3895 return Err(ACTIVITY_ACTOR_UNKNOWN);
3896 };
3897 if binding.module_id != module {
3898 return Err(ACTIVITY_ACTOR_UNKNOWN);
3899 }
3900 actor.principal = binding.principal.clone();
3901 actor.project_root = binding
3902 .project_root
3903 .as_ref()
3904 .map(|root| root.as_path().to_path_buf());
3905 let Some(bound) = &binding.scope else {
3906 return Ok(actor);
3907 };
3908 let owner = principal(&bound.owner);
3909 let selector = subc_protocol::scope::ScopeSelector::new(
3910 owner.clone(),
3911 bound.scope_ref.clone(),
3912 Some(bound.tag.scope_epoch),
3913 );
3914 let admitted = self
3915 .scopes
3916 .read()
3917 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3918 .attribute(&owner, module, &selector, true)
3919 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3920 if admitted.tag != bound.tag {
3921 return Err(ACTIVITY_ACTOR_UNKNOWN);
3922 }
3923 admitted
3924 }
3925 };
3926 actor.scope_owner = Some(scope.stamp.owner);
3927 actor.agent_id = scope.stamp.attributes.agent_id;
3928 actor.run_id = scope.stamp.attributes.run_id;
3929 actor.flow_id = scope.stamp.attributes.flow_id;
3930 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3931 actor.project_id = scope.stamp.project_id;
3932 Ok(actor)
3933 }
3934
3935 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3936 self.refresh_capability_requirements();
3937 let mut counters = self.counters.snapshot();
3943 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3944 obj.insert(
3945 "activity_unparseable_lines".into(),
3946 ledger.read_errors().into(),
3947 );
3948 obj.insert(
3949 "activity_lifecycle_append_failures".into(),
3950 ledger.lifecycle_failures().into(),
3951 );
3952 obj.insert(
3953 "activity_started_digest_skips".into(),
3954 ledger.lifecycle_digest_skips().into(),
3955 );
3956 }
3957 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3958 self.forwarding.client_route_concentration(),
3959 counters.as_object_mut(),
3960 ) {
3961 obj.insert(
3962 "client_connections_with_routes".into(),
3963 connections_with_routes.into(),
3964 );
3965 obj.insert("max_routes_on_one_connection".into(), max.into());
3966 }
3967 if let (Some(open_breakers), Some(obj)) = (
3973 self.route_bind_breakers.open_snapshot(),
3974 counters.as_object_mut(),
3975 ) {
3976 obj.insert("route_bind_breakers_open".into(), open_breakers);
3977 }
3978 let response = ClientControlResponse::ServerDescribe {
3979 protocol_ver: PROTOCOL_VERSION,
3980 subc_ops: subc_ops(),
3981 capabilities: self.describe_capabilities(),
3982 connected_clients: self.connected_clients.count(),
3983 counters: Some(counters),
3984 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3985 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3986 capability_requirements: self.capability_requirement_statuses(),
3987 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3988 config_path: self
3989 .rescan
3990 .as_ref()
3991 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3992 };
3993 Ok(vec![control_response_body_frame(
3994 &frame,
3995 &response,
3996 "ClientControlResponse::ServerDescribe",
3997 )?])
3998 }
3999
4000 fn handle_catalog_list(
4001 &self,
4002 frame: Frame,
4003 module_id: Option<String>,
4004 ) -> Result<Vec<Frame>, RouterError> {
4005 let (generation, modules) = self
4006 .registry
4007 .list_modules_with_nonce_verification()
4008 .map_err(|err| {
4009 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4010 })?;
4011 let entries = modules
4012 .into_iter()
4013 .filter(|(registration, _)| {
4014 module_id
4015 .as_deref()
4016 .map(|wanted| registration.manifest.module_id == wanted)
4017 .unwrap_or(true)
4018 })
4019 .map(|(registration, nonce_verified)| {
4020 let not_ready = self.not_ready_reason(®istration);
4021 let roles = registration.manifest.provides;
4022 CatalogEntry::new(
4023 registration.manifest.module_id,
4024 roles,
4025 registration.control_ops,
4026 )
4027 .with_ready(not_ready.is_none())
4028 .with_not_ready(not_ready)
4029 .with_module_version(Some(registration.manifest.module_version))
4030 .with_capabilities(registration.manifest.capabilities)
4031 .with_self_signals(registration.manifest.self_signals)
4032 .with_events(registration.manifest.events)
4033 .with_operation_delivery(registration.manifest.operation_delivery)
4034 .with_operation_payload(registration.manifest.operation_payload)
4035 .with_flow_operations(registration.manifest.flow_operations)
4036 .with_rich_ui(registration.manifest.rich_ui)
4037 .with_role_declarations(registration.manifest.role_declarations)
4038 .with_nonce_verified(nonce_verified)
4039 })
4040 .collect();
4041 let response = ClientControlResponse::CatalogList {
4042 generation,
4043 modules: entries,
4044 subc_ops: subc_ops(),
4045 };
4046 Ok(vec![control_response_body_frame(
4047 &frame,
4048 &response,
4049 "ClientControlResponse::CatalogList",
4050 )?])
4051 }
4052
4053 fn attest_route_open_consumer(
4058 &self,
4059 consumer_identity: Option<&ConsumerIdentity>,
4060 ) -> Option<Principal> {
4061 let Some(consumer_identity) = consumer_identity else {
4062 return Some(Principal::Direct);
4063 };
4064 self.supervisor
4065 .spawned_consumer_authorized(
4066 &consumer_identity.module_id,
4067 &consumer_identity.launch_nonce,
4068 )
4069 .then(|| Principal::Reserved {
4070 module_id: consumer_identity.module_id.clone(),
4071 })
4072 }
4073
4074 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4078 let Ok(ClientControlRequest::RouteOpen {
4079 identity,
4080 consumer_identity,
4081 scope,
4082 ..
4083 }) = parse_client_control_request(&frame.body)
4084 else {
4085 return RouteOpenCaller::unknown();
4086 };
4087 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4088 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4089 }
4090
4091 fn route_open_principal(
4092 &self,
4093 frame: &Frame,
4094 consumer_identity: Option<ConsumerIdentity>,
4095 ) -> Result<Result<Principal, Frame>, RouterError> {
4096 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4097 self.route_open_principal_attested(frame, consumer_identity, attested)
4098 }
4099
4100 fn route_open_principal_attested(
4104 &self,
4105 frame: &Frame,
4106 consumer_identity: Option<ConsumerIdentity>,
4107 attested: Option<Principal>,
4108 ) -> Result<Result<Principal, Frame>, RouterError> {
4109 if let Some(principal) = attested {
4110 return Ok(Ok(principal));
4111 }
4112 let Some(consumer_identity) = consumer_identity else {
4113 return Ok(Ok(Principal::Direct));
4114 };
4115
4116 Ok(Err(control_error_frame(
4117 frame,
4118 "bad_consumer_identity",
4119 format!(
4120 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4121 consumer_identity.module_id
4122 ),
4123 )?))
4124 }
4125
4126 #[allow(clippy::too_many_arguments)]
4133 fn route_open_refusal_frame(
4134 &self,
4135 ctx: &RouteCtx,
4136 caller: &RouteOpenCaller,
4137 frame: &Frame,
4138 module_id: &str,
4139 reason: &'static str,
4140 code: &'static str,
4141 message: impl Into<String>,
4142 ) -> Result<Frame, RouterError> {
4143 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4144 control_error_frame(frame, code, message.into())
4145 }
4146
4147 #[allow(clippy::too_many_arguments)]
4165 fn route_open_breaker_refusal_frame(
4166 &self,
4167 ctx: &RouteCtx,
4168 caller: &RouteOpenCaller,
4169 frame: &Frame,
4170 module_id: &str,
4171 consecutive_timeouts: u32,
4172 retry_in: Duration,
4173 probe_in_flight: bool,
4174 ) -> Result<Frame, RouterError> {
4175 self.counters
4176 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4177 debug!(
4178 target: "control",
4179 code = "module_timeout",
4180 module_id = ?module_id,
4181 connection_id = ctx.connection_id.get(),
4182 principal = %caller.principal,
4183 scope = %caller.scope,
4184 harness = %caller.harness,
4185 consecutive_timeouts,
4186 retry_in_ms = retry_in.as_millis() as u64,
4187 probe_in_flight,
4188 "route.open refused by open bind-relay breaker"
4189 );
4190 let detail = if probe_in_flight {
4197 "one test route is already being tried; retry once it settles".to_string()
4198 } else {
4199 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4200 };
4201 control_error_frame(
4202 frame,
4203 "module_timeout",
4204 format!(
4205 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4206 timed out in a row); {detail}; its established routes are unaffected"
4207 ),
4208 )
4209 }
4210
4211 fn observe_route_open_refusal(
4224 &self,
4225 ctx: &RouteCtx,
4226 caller: &RouteOpenCaller,
4227 module_id: &str,
4228 reason: &'static str,
4229 code: &'static str,
4230 ) {
4231 self.counters.increment_route_open_refused(code);
4232 info!(
4233 target: "control",
4234 code,
4235 reason,
4236 module_id = ?module_id,
4237 connection_id = ctx.connection_id.get(),
4238 principal = %caller.principal,
4239 scope = %caller.scope,
4240 harness = %caller.harness,
4241 "route.open refused"
4242 );
4243 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4244 self.route_outages.record_not_serving(module_id, reason);
4245 }
4246 }
4247
4248 fn observe_route_open_accept(
4293 &self,
4294 ctx: &RouteCtx,
4295 module_id: &str,
4296 principal: &str,
4297 scope: &str,
4298 ) {
4299 self.route_outages.record_accepted(module_id);
4300 self.counters.increment_route_open_accepted(principal);
4301 info!(
4302 target: "control",
4303 principal,
4304 module_id,
4305 connection_id = ctx.connection_id.get(),
4306 scope = %scope,
4307 "route.open accepted"
4308 );
4309 }
4310
4311 fn supervised_absent_route_open_refusal_frame(
4312 &self,
4313 ctx: &RouteCtx,
4314 caller: &RouteOpenCaller,
4315 frame: &Frame,
4316 module_id: &str,
4317 code: &'static str,
4318 status: &crate::supervise::ModuleStatus,
4319 ) -> Result<Frame, RouterError> {
4320 self.counters.increment_route_open_refused(code);
4321 info!(
4322 target: "control",
4323 code,
4324 reason = "supervised_not_registered",
4325 module_id = ?module_id,
4326 connection_id = ctx.connection_id.get(),
4327 principal = %caller.principal,
4328 scope = %caller.scope,
4329 harness = %caller.harness,
4330 state = %status.state,
4331 enabled = status.enabled,
4332 live = status.live,
4333 "route.open refused"
4334 );
4335 self.route_outages
4339 .record_not_serving(module_id, "supervised_not_registered");
4340 let message = if let Some(pid) = status.waiting_for_previous_pid {
4341 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4342 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4343 })
4344 } else {
4345 format!(
4346 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4347 status.state, status.enabled, status.live
4348 )
4349 };
4350 control_error_frame(frame, code, message)
4351 }
4352
4353 async fn handle_route_open(
4354 &self,
4355 ctx: &RouteCtx,
4356 frame: Frame,
4357 request: RouteOpenRequest,
4358 timings: &mut RouteOpenTimings,
4359 ) -> Result<Vec<Frame>, RouterError> {
4360 let RouteOpenRequest {
4361 target,
4362 mut identity,
4363 consumer_identity,
4364 consumer_capabilities,
4365 role_versions,
4366 admission_facts,
4367 scope,
4368 } = request;
4369 let target_module_id = target_module_id(&target).to_string();
4370 timings.module_id.clone_from(&target_module_id);
4371 if self
4372 .registry
4373 .get_module_by_connection(ctx.connection_id)
4374 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4375 .is_some()
4376 {
4377 return Ok(vec![control_error_frame(
4378 &frame,
4379 "invalid_request",
4380 "module connections cannot open client routes",
4381 )?]);
4382 }
4383 debug!(
4384 connection_id = ctx.connection_id.get(),
4385 corr = frame.header.corr,
4386 module_id = %target_module_id,
4387 "handling route.open"
4388 );
4389
4390 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4395 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4396
4397 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4403 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4404 self.observe_route_open_refusal(
4405 ctx,
4406 &caller,
4407 &target_module_id,
4408 "invalid_role_versions",
4409 error_codes::INVALID_REQUEST,
4410 );
4411 return Ok(vec![control_error_body_frame(
4412 &frame,
4413 ErrorBody {
4414 code: error_codes::INVALID_REQUEST.to_string(),
4415 message: error.to_string(),
4416 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4417 },
4418 )?]);
4419 }
4420
4421 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4422 if status.waiting_for_previous_pid.is_some() {
4423 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4424 ctx,
4425 &caller,
4426 &frame,
4427 &target_module_id,
4428 "module_warming",
4429 &status,
4430 )?]);
4431 }
4432 }
4433
4434 let Some(registration) = self
4453 .registry
4454 .get_module(&target_module_id)
4455 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4456 else {
4457 if let Some((status, warming)) =
4458 self.supervisor_status(&target_module_id, frame.header.corr)?
4459 {
4460 if status.protocol == ModuleProtocol::None {
4467 return Ok(vec![self.route_open_refusal_frame(
4468 ctx,
4469 &caller,
4470 &frame,
4471 &target_module_id,
4472 "protocol_none",
4473 error_codes::MODULE_NO_PROTOCOL,
4474 format!(
4475 "module_id '{target_module_id}' is declared protocol: none; \
4476 it speaks no subc wire and serves no routes"
4477 ),
4478 )?]);
4479 }
4480 let code = if warming {
4481 "module_warming"
4482 } else {
4483 "target_unavailable"
4484 };
4485 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4486 ctx,
4487 &caller,
4488 &frame,
4489 &target_module_id,
4490 code,
4491 &status,
4492 )?]);
4493 }
4494 if let Some(removed_ago_ms) =
4495 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4496 {
4497 return Ok(vec![self.route_open_refusal_frame(
4498 ctx,
4499 &caller,
4500 &frame,
4501 &target_module_id,
4502 "removed",
4503 error_codes::MODULE_REMOVED,
4504 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4505 )?]);
4506 }
4507 return Ok(vec![self.route_open_refusal_frame(
4508 ctx,
4509 &caller,
4510 &frame,
4511 &target_module_id,
4512 "not_registered",
4513 error_codes::UNKNOWN_MODULE,
4514 format!("module_id '{target_module_id}' is not registered"),
4515 )?]);
4516 };
4517
4518 if !registration.ready {
4522 self.counters
4523 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4524 info!(
4525 target: "control",
4526 code = error_codes::MODULE_WARMING,
4527 module_id = ?target_module_id,
4528 connection_id = ctx.connection_id.get(),
4529 principal = %caller.principal,
4530 scope = %caller.scope,
4531 harness = %caller.harness,
4532 reason = "declared_not_ready",
4533 "route.open refused"
4534 );
4535 self.route_outages
4538 .record_not_serving(&target_module_id, "declared_not_ready");
4539 return Ok(vec![control_error_body_frame(
4540 &frame,
4541 ErrorBody {
4542 code: error_codes::MODULE_WARMING.to_string(),
4543 message: format!(
4544 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4545 ),
4546 detail: Some(serde_json::json!({
4547 "reason": "declared_not_ready"
4548 })),
4549 },
4550 )?]);
4551 }
4552
4553 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4572 self.counters
4573 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4574 info!(
4575 target: "control",
4576 code = error_codes::MODULE_WARMING,
4577 module_id = ?target_module_id,
4578 connection_id = ctx.connection_id.get(),
4579 principal = %caller.principal,
4580 scope = %caller.scope,
4581 harness = %caller.harness,
4582 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4583 capability = %capability,
4584 "route.open refused"
4585 );
4586 return Ok(vec![control_error_body_frame(
4587 &frame,
4588 ErrorBody {
4589 code: error_codes::MODULE_WARMING.to_string(),
4590 message: format!(
4591 "module_id '{target_module_id}' requires capability '{capability}', \
4592 which no registered module provides; retry"
4593 ),
4594 detail: Some(serde_json::json!({
4595 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4596 "capability": capability,
4597 })),
4598 },
4599 )?]);
4600 }
4601
4602 if !target_has_required_role(&target, ®istration.manifest.provides) {
4603 return Ok(vec![self.route_open_refusal_frame(
4604 ctx,
4605 &caller,
4606 &frame,
4607 &target_module_id,
4608 "role_not_provided",
4609 "target_unavailable",
4610 format!("module_id '{target_module_id}' does not provide the requested target"),
4611 )?]);
4612 }
4613
4614 if registration.state != ChannelState::Active {
4615 return Ok(vec![self.route_open_refusal_frame(
4616 ctx,
4617 &caller,
4618 &frame,
4619 &target_module_id,
4620 "registration_not_active",
4621 "target_unavailable",
4622 format!("module_id '{target_module_id}' is not active"),
4623 )?]);
4624 }
4625
4626 if self
4627 .forwarding
4628 .module_is_draining(&target_module_id)
4629 .map_err(RouterError::Forwarding)?
4630 {
4631 return Ok(vec![self.route_open_refusal_frame(
4632 ctx,
4633 &caller,
4634 &frame,
4635 &target_module_id,
4636 "reloading",
4637 "module_reloading",
4638 format!("module_id '{target_module_id}' is reloading"),
4639 )?]);
4640 }
4641
4642 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4643 process_liveness.process_live(&target_module_id) == Some(false)
4644 }) {
4645 if process_liveness.process_replacing(&target_module_id) {
4652 return Ok(vec![self.route_open_refusal_frame(
4653 ctx,
4654 &caller,
4655 &frame,
4656 &target_module_id,
4657 "reloading",
4658 "module_reloading",
4659 format!("module_id '{target_module_id}' is reloading"),
4660 )?]);
4661 }
4662 return Ok(vec![self.route_open_refusal_frame(
4663 ctx,
4664 &caller,
4665 &frame,
4666 &target_module_id,
4667 "supervisor_not_live",
4668 "target_unavailable",
4669 format!("module_id '{target_module_id}' is not live"),
4670 )?]);
4671 }
4672
4673 if !self
4674 .forwarding
4675 .has_live_module_connection(&target_module_id)
4676 .map_err(RouterError::Forwarding)?
4677 {
4678 return Ok(vec![self.route_open_refusal_frame(
4679 ctx,
4680 &caller,
4681 &frame,
4682 &target_module_id,
4683 "no_forwarding_connection",
4684 "target_unavailable",
4685 format!("module_id '{target_module_id}' has no live forwarding connection"),
4686 )?]);
4687 }
4688
4689 if let Some(error) =
4690 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4691 {
4692 self.observe_route_open_refusal(
4693 ctx,
4694 &caller,
4695 &target_module_id,
4696 "op_not_allowed",
4697 "op_not_allowed",
4698 );
4699 return Ok(vec![error]);
4700 }
4701
4702 let principal =
4703 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4704 Ok(principal) => principal,
4705 Err(error) => {
4706 self.observe_route_open_refusal(
4707 ctx,
4708 &caller,
4709 &target_module_id,
4710 "bad_consumer_identity",
4711 "bad_consumer_identity",
4712 );
4713 return Ok(vec![error]);
4714 }
4715 };
4716 timings.principal = principal_label(&principal);
4717
4718 if let Principal::Reserved {
4722 module_id: opening_module_id,
4723 } = &principal
4724 {
4725 if let Some(opening_registration) = self
4726 .registry
4727 .get_module(opening_module_id)
4728 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4729 {
4730 if let Some(capability) =
4731 denied_capability(&opening_registration.manifest, ®istration.manifest)
4732 {
4733 warn!(
4734 opening_module_id,
4735 target_module_id,
4736 capability,
4737 "refusing route.open because an attested capability deny edge matches"
4738 );
4739 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4740 &frame,
4741 &target_module_id,
4742 "capability_deny_edge",
4743 "capability_forbidden",
4744 format!(
4745 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4746 ),
4747 )?]);
4748 }
4749 }
4750 }
4751
4752 if admission_facts.is_some() {
4753 let carrier_matches = matches!(
4754 &principal,
4755 Principal::Reserved { module_id }
4756 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4757 );
4758 if !carrier_matches {
4759 return Ok(vec![self.route_open_refusal_frame(
4760 ctx,
4761 &caller,
4762 &frame,
4763 &target_module_id,
4764 "admission_facts_carrier_not_permitted",
4765 "admission_facts_not_permitted",
4766 "admission facts may only be carried by the configured reserved module",
4767 )?]);
4768 }
4769
4770 let target_allowed = self
4771 .admission_facts_targets
4772 .as_ref()
4773 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4774 if !target_allowed {
4775 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4776 &frame,
4777 &target_module_id,
4778 "admission_facts_target_not_listed",
4779 "admission_facts_target_not_allowed",
4780 format!(
4781 "admission facts are not permitted for target module_id '{target_module_id}'"
4782 ),
4783 )?]);
4784 }
4785
4786 }
4790
4791 let (bound_scope, mut scope_stamp) = match &scope {
4797 None => (None, None),
4798 Some(selector) => {
4799 let owner_configured = match &selector.owner {
4800 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4806 _ => false,
4807 };
4808 let mut scopes = self
4809 .scopes
4810 .write()
4811 .unwrap_or_else(|poisoned| poisoned.into_inner());
4812 let admitted = scopes
4813 .admit(&principal, &target_module_id, selector, owner_configured)
4814 .and_then(|admission| {
4815 crate::scopes::check_target_flow_support(
4816 &admission.stamp,
4817 &target_module_id,
4818 registration.manifest.capabilities.as_ref(),
4819 )?;
4820 crate::scopes::check_target_agent_run_support(
4821 &admission.stamp,
4822 &target_module_id,
4823 registration.manifest.capabilities.as_ref(),
4824 )?;
4825 let bound = scopes.reserve_bind(
4826 &admission,
4827 &principal,
4828 ctx.connection_id,
4829 selector.hold_presence,
4830 );
4831 Ok((admission, bound))
4832 });
4833 match admitted {
4834 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4835 Err(refusal) => {
4836 return Ok(vec![self.route_open_refusal_frame(
4837 ctx,
4838 &caller,
4839 &frame,
4840 &target_module_id,
4841 refusal.code,
4842 refusal.code,
4843 refusal.message,
4844 )?]);
4845 }
4846 }
4847 }
4848 };
4849 if let Some(stamp) = &scope_stamp {
4852 caller.scope = route_open_scope_label(stamp);
4853 }
4854
4855 timings.enter(RouteOpenPhase::ProjectRoot);
4877 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4878 Ok(project_root) => project_root,
4879 Err(err) => {
4880 return Ok(vec![control_error_frame(
4881 &frame,
4882 "invalid_project_root",
4883 err.to_string(),
4884 )?])
4885 }
4886 };
4887 timings.enter(RouteOpenPhase::Admission);
4888 identity.project_root = project_root.as_path().to_path_buf();
4889
4890 let _concurrency_guard = match self
4901 .route_bind_concurrency
4902 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4903 {
4904 Ok(guard) => guard,
4905 Err(in_flight) => {
4906 return Ok(vec![self.route_open_target_capacity_refusal(
4907 ctx,
4908 &caller,
4909 &frame,
4910 &target_module_id,
4911 in_flight,
4912 )?]);
4913 }
4914 };
4915
4916 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4919 RouteBindAdmission::Admitted { guard, probe } => {
4920 if probe {
4921 info!(
4922 module_id = %target_module_id,
4923 connection_id = ctx.connection_id.get(),
4924 "route.bind breaker half-open: admitting one probe"
4925 );
4926 }
4927 guard
4928 }
4929 RouteBindAdmission::Refused {
4930 consecutive_timeouts,
4931 retry_in,
4932 probe_in_flight,
4933 } => {
4934 return Ok(vec![self.route_open_breaker_refusal_frame(
4935 ctx,
4936 &caller,
4937 &frame,
4938 &target_module_id,
4939 consecutive_timeouts,
4940 retry_in,
4941 probe_in_flight,
4942 )?]);
4943 }
4944 };
4945
4946 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4951 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4952 timings.enter(RouteOpenPhase::EgressReservation);
4953 let pending = match self
4954 .forwarding
4955 .begin_route_bind_relay_for(
4956 ctx.connection_id,
4957 ctx.egress.clone(),
4958 response_version(&frame),
4959 frame.header.corr,
4960 &target_module_id,
4961 principal.clone(),
4962 bound_scope,
4963 Some(project_root),
4964 relay_deadline,
4965 )
4966 .await
4967 {
4968 Ok(pending) => pending,
4969 Err(err) => {
4970 return Ok(vec![self.route_open_refusal_frame(
4971 ctx,
4972 &caller,
4973 &frame,
4974 &target_module_id,
4975 "relay_reservation_failed",
4976 forwarding_error_code(&err),
4977 err.to_string(),
4978 )?])
4979 }
4980 };
4981 timings.enter(RouteOpenPhase::Admission);
4982 let crate::forwarding::PendingRouteBindRelay {
4983 endpoint,
4984 module_sink,
4985 negotiated_ver,
4986 client_channel,
4987 client_epoch,
4988 module_channel,
4989 module_epoch,
4990 corr: relay_corr,
4991 receiver,
4992 } = pending;
4993 let mut reservation =
4994 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4995
4996 let relay_registration = self
5001 .registry
5002 .get_module_by_connection(endpoint.connection_id)
5003 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5004 let relay_capabilities = relay_registration
5005 .as_ref()
5006 .and_then(|registration| registration.manifest.capabilities.as_ref());
5007 if let Some(selector) = &scope {
5008 if let Err(refusal) = self
5011 .scopes
5012 .read()
5013 .unwrap_or_else(|poisoned| poisoned.into_inner())
5014 .check_bind_proof(&principal, selector)
5015 {
5016 reservation.release_and_disarm();
5017 return Ok(vec![self.route_open_refusal_frame(
5018 ctx,
5019 &caller,
5020 &frame,
5021 &target_module_id,
5022 refusal.code,
5023 refusal.code,
5024 refusal.message,
5025 )?]);
5026 }
5027 }
5028 if let Some(stamp) = scope_stamp
5029 .as_ref()
5030 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5031 {
5032 if let Err(refusal) = crate::scopes::check_target_flow_support(
5033 stamp,
5034 &target_module_id,
5035 relay_capabilities,
5036 )
5037 .and_then(|()| {
5038 crate::scopes::check_target_agent_run_support(
5039 stamp,
5040 &target_module_id,
5041 relay_capabilities,
5042 )
5043 }) {
5044 reservation.release_and_disarm();
5045 return Ok(vec![self.route_open_refusal_frame(
5046 ctx,
5047 &caller,
5048 &frame,
5049 &target_module_id,
5050 refusal.code,
5051 refusal.code,
5052 refusal.message,
5053 )?]);
5054 }
5055 }
5056
5057 if !relay_capabilities.is_some_and(|caps| {
5060 caps.provides
5061 .iter()
5062 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5063 }) {
5064 if let Some(stamp) = scope_stamp.as_mut() {
5065 stamp.workdir = None;
5066 stamp.project_id = None;
5067 }
5068 }
5069 let worker_run = scope_stamp
5074 .as_ref()
5075 .is_some_and(crate::scopes::is_worker_run)
5076 && relay_capabilities.is_some_and(|caps| {
5077 caps.provides
5078 .iter()
5079 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5080 });
5081 if !worker_run
5082 && !relay_capabilities.is_some_and(|caps| {
5083 caps.provides
5084 .iter()
5085 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5086 })
5087 {
5088 if let Some(stamp) = scope_stamp.as_mut() {
5089 stamp.attributes.on_behalf_of_agent = None;
5090 }
5091 }
5092 if !relay_capabilities.is_some_and(|caps| {
5093 caps.provides
5094 .iter()
5095 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5096 }) {
5097 if let Some(stamp) = scope_stamp.as_mut() {
5098 stamp.attributes.flow_version = None;
5099 stamp.attributes.flow_code_hash = None;
5100 stamp.attributes.flow_install_id = None;
5101 }
5102 }
5103 let call_mode = if worker_run
5104 || relay_capabilities
5105 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5106 {
5107 Some(derive_call_mode(scope_stamp.as_ref()))
5108 } else {
5109 if let Some(stamp) = scope_stamp.as_mut() {
5110 stamp.expires_at_ms = None;
5111 }
5112 None
5113 };
5114
5115 debug!(
5116 connection_id = ctx.connection_id.get(),
5117 client_channel,
5118 client_epoch,
5119 module_channel,
5120 module_epoch,
5121 "reserved route handle pair"
5122 );
5123 let route_principal_label = principal_label(&principal);
5126 let relay = ModuleControlRequest::RouteBind {
5127 route_channel: module_channel,
5128 epoch: module_epoch,
5129 target,
5130 identity,
5131 principal: Some(principal),
5132 consumer_capabilities,
5133 role_versions,
5134 admission_facts,
5135 scope: scope_stamp,
5136 call_mode,
5137 };
5138 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5139 RouterError::backend(
5140 0,
5141 frame.header.corr,
5142 format!("failed to encode route.bind request: {err}"),
5143 )
5144 })?;
5145 let relay_frame = Frame::build_with_version(
5146 negotiated_ver,
5147 FrameType::Request,
5148 control_flags(),
5149 0,
5150 0,
5151 relay_corr,
5152 relay_body,
5153 )
5154 .map_err(RouterError::FrameBuild)?;
5155
5156 timings.enter(RouteOpenPhase::BindSend);
5157 let sent = module_sink.send(relay_frame).await;
5158 timings.enter(RouteOpenPhase::Admission);
5159 if let Err(err) = sent {
5160 reservation.release_and_disarm();
5161 return Ok(vec![self.route_open_refusal_frame(
5162 ctx,
5163 &caller,
5164 &frame,
5165 &target_module_id,
5166 "relay_send_failed",
5167 "target_unavailable",
5168 err.to_string(),
5169 )?]);
5170 }
5171
5172 #[cfg(test)]
5173 if let Some(hold) = &self.route_bind_relay_sent_hold {
5174 hold.wait().await;
5178 hold.wait().await;
5179 }
5180
5181 if self
5186 .forwarding
5187 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5188 .map_err(RouterError::Forwarding)?
5189 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5190 {
5191 self.send_abandoned_route_bind_goodbye(
5192 &module_sink,
5193 negotiated_ver,
5194 module_channel,
5195 module_epoch,
5196 );
5197 }
5198
5199 timings.enter(RouteOpenPhase::BindAckWait);
5200 let outcome = timeout_at(relay_deadline, receiver).await;
5201 timings.enter(RouteOpenPhase::Admission);
5202 match outcome {
5203 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5204 reservation.disarm();
5205 if breaker.record_accepted() {
5206 info!(
5207 module_id = %target_module_id,
5208 "route.bind breaker closed: the probe was accepted"
5209 );
5210 }
5211 self.observe_route_open_accept(
5212 ctx,
5213 &target_module_id,
5214 &route_principal_label,
5215 &caller.scope,
5216 );
5217 Ok(Vec::new())
5218 }
5219 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5220 reservation.release_and_disarm();
5221 breaker.record_inconclusive();
5225 let scope_code = match body.code.as_str() {
5229 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5230 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5231 _ => None,
5232 };
5233 if let Some(code) = scope_code {
5234 self.observe_route_open_refusal(
5235 ctx,
5236 &caller,
5237 &target_module_id,
5238 "scope_changed_before_commit",
5239 code,
5240 );
5241 return Ok(vec![control_error_body_frame(&frame, body)?]);
5242 }
5243 self.counters
5244 .increment_route_open_refused("module_rejected");
5245 info!(
5246 target: "control",
5247 code = "module_rejected",
5248 module_code = ?body.code,
5249 module_id = ?target_module_id,
5250 connection_id = ctx.connection_id.get(),
5251 principal = %caller.principal,
5252 scope = %caller.scope,
5253 harness = %caller.harness,
5254 "route.open refused"
5255 );
5256 Ok(vec![control_error_body_frame(&frame, body)?])
5257 }
5258 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5259 reservation.release_and_disarm();
5260 breaker.record_inconclusive();
5261 tracing::warn!(
5268 module_id = %target_module_id,
5269 "route.bind relay abandoned: {message}"
5270 );
5271 Ok(vec![self.route_open_refusal_frame(
5272 ctx,
5273 &caller,
5274 &frame,
5275 &target_module_id,
5276 "relay_abandoned",
5277 "target_unavailable",
5278 message,
5279 )?])
5280 }
5281 Ok(Err(_)) => {
5282 reservation.release_and_disarm();
5283 breaker.record_inconclusive();
5284 Ok(vec![self.route_open_refusal_frame(
5285 ctx,
5286 &caller,
5287 &frame,
5288 &target_module_id,
5289 "relay_waiter_canceled",
5290 "target_unavailable",
5291 "route.bind relay waiter was canceled before the module responded",
5292 )?])
5293 }
5294 Err(_) => {
5295 reservation.release_and_disarm();
5296 if let Some(opened) = breaker.record_timeout(
5300 self.route_bind_breaker_threshold,
5301 self.route_bind_breaker_cooldown,
5302 ) {
5303 warn!(
5304 module_id = %target_module_id,
5305 consecutive_timeouts = opened.consecutive_timeouts,
5306 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5307 reopened_after_probe = opened.reopened_after_probe,
5308 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5309 );
5310 }
5311 tracing::warn!(
5319 module_id = %target_module_id,
5320 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5321 "route.bind relay timed out: module did not ack within budget"
5322 );
5323 Ok(vec![self.route_open_refusal_frame(
5324 ctx,
5325 &caller,
5326 &frame,
5327 &target_module_id,
5328 "relay_timed_out",
5329 "module_timeout",
5330 format!(
5331 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5332 route_bind_relay_timeout
5333 ),
5334 )?])
5335 }
5336 }
5337 }
5338
5339 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5340 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5341 snapshot: self.supervisor.spawn_snapshot(),
5342 };
5343 Ok(vec![control_response_body_frame(
5344 &frame,
5345 &response,
5346 "ClientControlResponse::SupervisorSpawnSnapshot",
5347 )?])
5348 }
5349
5350 fn handle_supervisor_spawn_subscribe(
5351 &self,
5352 ctx: &RouteCtx,
5353 frame: Frame,
5354 since: Option<SpawnCursor>,
5355 ) -> Result<Vec<Frame>, RouterError> {
5356 match self.supervisor.subscribe_spawns(
5357 ctx.connection_id,
5358 frame.header.corr,
5359 response_version(&frame),
5360 since,
5361 ctx.egress.clone(),
5362 ) {
5363 Ok(()) => Ok(Vec::new()),
5364 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5365 Ok(vec![control_error_body_frame(
5366 &frame,
5367 ErrorBody {
5368 code: "spawn_cursor_incarnation_mismatch".to_string(),
5369 message: "spawn cursor belongs to a different daemon incarnation"
5370 .to_string(),
5371 detail: Some(serde_json::json!({
5372 "current_daemon_incarnation": current
5373 })),
5374 },
5375 )?])
5376 }
5377 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5378 &frame,
5379 ErrorBody {
5380 code: "spawn_cursor_too_old".to_string(),
5381 message: "spawn cursor predates the retained event ring".to_string(),
5382 detail: Some(serde_json::json!({
5383 "oldest_retained_cursor": oldest
5384 })),
5385 },
5386 )?]),
5387 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5388 0,
5389 frame.header.corr,
5390 format!("failed to open supervisor spawn subscription: {error}"),
5391 )),
5392 }
5393 }
5394
5395 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5396 let generation = self
5397 .registry
5398 .generation()
5399 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5400 let mut modules = Vec::new();
5401 for module in self.supervisor.list() {
5402 let status = module.status_for_control("list").map_err(|err| {
5403 RouterError::backend(
5404 0,
5405 frame.header.corr,
5406 format!("failed to read supervisor status: {err}"),
5407 )
5408 })?;
5409 let (configured, _) = module.configuration().map_err(|err| {
5410 RouterError::backend(
5411 0,
5412 frame.header.corr,
5413 format!("failed to read module configuration: {err}"),
5414 )
5415 })?;
5416 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5417 RouterError::backend(
5418 0,
5419 frame.header.corr,
5420 format!("failed to read retry time: {err}"),
5421 )
5422 })?;
5423 let image = module.running_image_agreement().await;
5425 let resources = Some(module.child_resource_usage());
5428 let mut verdict =
5429 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5430 verdict.spec = Some(launch_spec_agreement(
5431 &configured,
5432 status.process_alive,
5433 status.launch_spec_digest.as_ref(),
5434 ));
5435 let pending_reload = Some(verdict);
5436 let configured_enabled = self
5437 .capability_evaluator
5438 .configured_enabled(&status.module_id);
5439 modules.push(
5442 SupervisorEntry::new(
5443 status.module_id,
5444 status.state.to_string(),
5445 status.enabled,
5446 status.live,
5447 status.health.status,
5448 )
5449 .with_configured_enabled(configured_enabled)
5450 .with_next_retry_at_ms(next_retry_at_ms)
5451 .with_launch_nonce_env(Some(!cfg!(unix)))
5452 .with_launch_nonce_source(status.launch_nonce_source)
5453 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5454 .with_protocol(status.protocol)
5455 .with_pending_reload(pending_reload)
5456 .with_last_probe_ms(status.health.last_probe_ms)
5457 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5458 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5459 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5460 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5461 .with_restart_count(Some(status.restart_count))
5462 .with_max_restarts(Some(status.max_restarts))
5463 .with_lifetime_restarts(Some(status.lifetime_restarts))
5464 .with_spawn_generation(Some(status.spawn_generation))
5465 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5466 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5467 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5468 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5469 .with_resources(resources),
5470 );
5471 }
5472 let response = ClientControlResponse::SupervisorList {
5473 generation,
5474 modules,
5475 };
5476 Ok(vec![control_response_body_frame(
5477 &frame,
5478 &response,
5479 "ClientControlResponse::SupervisorList",
5480 )?])
5481 }
5482
5483 fn handle_supervisor_stderr_tail(
5484 &self,
5485 frame: Frame,
5486 module_id: String,
5487 max_lines: Option<u32>,
5488 max_bytes: Option<u32>,
5489 ) -> Result<Vec<Frame>, RouterError> {
5490 let Some(module) = self.supervisor.get(&module_id) else {
5491 return Ok(vec![control_error_frame(
5492 &frame,
5493 "unknown_module",
5494 format!("module_id '{module_id}' is not supervised"),
5495 )?]);
5496 };
5497
5498 let snapshot = module.stderr_tail(
5499 max_lines.map(|value| value as usize),
5500 max_bytes.map(|value| value as usize),
5501 );
5502
5503 let response = ClientControlResponse::SupervisorStderrTail {
5504 module_id,
5505 tail: StderrTail {
5506 capture: match snapshot.capture {
5507 CaptureState::Captured => StderrCaptureState::Captured,
5508 CaptureState::Incomplete { reason } => {
5509 StderrCaptureState::Incomplete { reason }
5510 }
5511 CaptureState::NotCaptured { reason } => {
5512 StderrCaptureState::NotCaptured { reason }
5513 }
5514 },
5515 entries: snapshot
5516 .entries
5517 .into_iter()
5518 .map(|entry| match entry {
5519 TailEntry::Line {
5520 text,
5521 truncated,
5522 at_ms,
5523 } => StderrTailEntry::Line {
5524 text,
5525 truncated,
5526 at_ms,
5527 },
5528 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5529 })
5530 .collect(),
5531 dropped_lines: snapshot.dropped_lines,
5532 },
5533 };
5534 Ok(vec![control_response_body_frame(
5535 &frame,
5536 &response,
5537 "ClientControlResponse::SupervisorStderrTail",
5538 )?])
5539 }
5540
5541 async fn handle_supervisor_terminals(
5542 &self,
5543 frame: Frame,
5544 module_id: String,
5545 ) -> Result<Vec<Frame>, RouterError> {
5546 let Some(module) = self.supervisor.get(&module_id) else {
5547 return Ok(vec![control_error_frame(
5548 &frame,
5549 "unknown_module",
5550 format!("module_id '{module_id}' is not supervised"),
5551 )?]);
5552 };
5553
5554 let terminals = module
5557 .read_durable_terminal_history()
5558 .await
5559 .map_err(|error| {
5560 RouterError::backend(
5561 0,
5562 frame.header.corr,
5563 format!("failed to read terminal history: {error}"),
5564 )
5565 })?;
5566 let response = ClientControlResponse::SupervisorTerminals {
5567 module_id,
5568 terminals,
5569 };
5570 Ok(vec![control_response_body_frame(
5571 &frame,
5572 &response,
5573 "ClientControlResponse::SupervisorTerminals",
5574 )?])
5575 }
5576
5577 fn handle_supervisor_routes(
5578 &self,
5579 frame: Frame,
5580 module_id: Option<String>,
5581 ) -> Result<Vec<Frame>, RouterError> {
5582 let modules = self
5583 .forwarding
5584 .route_census(module_id.as_deref())
5585 .map_err(RouterError::Forwarding)?
5586 .into_iter()
5587 .map(|(module_id, routes)| SupervisorRouteModule {
5588 module_id,
5589 routes: routes
5590 .into_iter()
5591 .map(|route| SupervisorRoute {
5592 consumer: match route.principal {
5593 Principal::Reserved { module_id } => {
5594 SupervisorRouteConsumer::Reserved { module_id }
5595 }
5596 Principal::Direct | Principal::Unverified => {
5597 SupervisorRouteConsumer::Direct {
5598 connection_id: route.goodbye_target.connection_id.get(),
5599 }
5600 }
5601 },
5602 age_ms: Instant::now()
5603 .saturating_duration_since(route.bound_at)
5604 .as_millis()
5605 .try_into()
5606 .unwrap_or(u64::MAX),
5607 draining: route.draining,
5608 drain_reason: route.drain_reason,
5609 })
5610 .collect(),
5611 })
5612 .collect();
5613 let response = ClientControlResponse::SupervisorRoutes { modules };
5614 Ok(vec![control_response_body_frame(
5615 &frame,
5616 &response,
5617 "ClientControlResponse::SupervisorRoutes",
5618 )?])
5619 }
5620
5621 async fn handle_supervisor_provenance(
5622 &self,
5623 frame: Frame,
5624 module_id: Option<String>,
5625 ) -> Result<Vec<Frame>, RouterError> {
5626 let mut selected = if let Some(module_id) = module_id {
5627 let Some(module) = self.supervisor.get(&module_id) else {
5628 return Ok(vec![control_error_frame(
5629 &frame,
5630 "unknown_module",
5631 format!("module_id '{module_id}' is not supervised"),
5632 )?]);
5633 };
5634 vec![module]
5635 } else {
5636 self.supervisor.list()
5637 };
5638
5639 let mut modules = Vec::with_capacity(selected.len());
5640 for module in selected.drain(..) {
5641 let (status, observed_image) = module
5642 .status_and_running_image_agreement()
5643 .await
5644 .map_err(|err| {
5645 RouterError::backend(
5646 0,
5647 frame.header.corr,
5648 format!("failed to read supervisor status: {err}"),
5649 )
5650 })?;
5651 let module_declared = self
5652 .registry
5653 .get_module(&status.module_id)
5654 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5655 .and_then(|registration| registration.manifest.provenance)
5656 .map(|build| ModuleDeclaredProvenance::Reported { build })
5657 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5658 #[cfg(test)]
5659 let running_image = match &self.provenance_probe_override {
5660 Some(result) => result.clone(),
5661 None => observed_image,
5662 };
5663 #[cfg(not(test))]
5664 let running_image = observed_image;
5665 modules.push(SupervisorModuleProvenance {
5666 module_id: status.module_id,
5667 module_declared,
5668 daemon_observed: SupervisorObservedProcess {
5669 launch_nonce_source: status.launch_nonce_source,
5670 pid: status.pid,
5671 spawned_at_ms: status.spawned_at_ms,
5672 spawned_from: status.spawned_from,
5673 running_image,
5674 },
5675 });
5676 }
5677 let daemon = SupervisorDaemonProvenance {
5678 daemon_build: self.daemon_provenance.build.clone(),
5679 daemon_observed: DaemonObservedProcess {
5680 pid: self.daemon_provenance.pid,
5681 started_at_ms: self
5682 .daemon_provenance
5683 .start_clock
5684 .map(|clock| clock.started_at_ms())
5685 .or(self.daemon_provenance.started_at_ms),
5686 running_image: self
5687 .daemon_provenance
5688 .probe
5689 .observe(
5690 self.daemon_provenance.pid,
5691 self.daemon_provenance.executable_path.as_deref(),
5692 self.daemon_provenance.executable_identity.clone(),
5693 self.daemon_provenance.process_start_time,
5694 )
5695 .await,
5696 },
5697 };
5698 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5699 Ok(vec![control_response_body_frame(
5700 &frame,
5701 &response,
5702 "ClientControlResponse::SupervisorProvenance",
5703 )?])
5704 }
5705
5706 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5707 self.refresh_capability_requirements();
5708 let generation = self
5709 .registry
5710 .generation()
5711 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5712 let modules = self
5713 .supervisor
5714 .list()
5715 .into_iter()
5716 .map(|module| {
5717 let status = module.status_for_control("health").map_err(|err| {
5718 RouterError::backend(
5719 0,
5720 frame.header.corr,
5721 format!("failed to read supervisor health: {err}"),
5722 )
5723 })?;
5724 let module_id = status.module_id;
5725 let capability_detail = self
5726 .capability_evaluator
5727 .required_problem_detail(&module_id);
5728 Ok(SupervisorHealthEntry {
5729 module_id,
5730 status: status.health.status,
5731 detail: append_capability_problem_detail(
5732 status.health.detail,
5733 capability_detail,
5734 ),
5735 metrics: status.health.metrics,
5736 consecutive_failures: status.health.consecutive_failures,
5737 late_answer_count: status.health.late_answer_count,
5738 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5739 last_action: status.health.last_action,
5740 last_action_ms: status.health.last_action_ms,
5741 last_probe_ms: status.health.last_probe_ms,
5742 })
5743 })
5744 .collect::<Result<Vec<_>, RouterError>>()?;
5745 let response = ClientControlResponse::SupervisorHealth {
5746 generation,
5747 modules,
5748 };
5749 Ok(vec![control_response_body_frame(
5750 &frame,
5751 &response,
5752 "ClientControlResponse::SupervisorHealth",
5753 )?])
5754 }
5755
5756 async fn handle_supervisor_restart(
5757 &self,
5758 frame: Frame,
5759 module_id: String,
5760 drain_timeout_ms: Option<u64>,
5761 requester: Principal,
5762 ) -> Result<Vec<Frame>, RouterError> {
5763 let operation_lock = self.supervisor.operation_lock();
5764 let _operation_guard = operation_lock.lock().await;
5765 let Some(module) = self.supervisor.get(&module_id) else {
5766 return Ok(vec![control_error_frame(
5767 &frame,
5768 "unknown_module",
5769 format!("module_id '{module_id}' is not supervised"),
5770 )?]);
5771 };
5772
5773 self.route_outages.mark_operator_action(&module_id);
5774 if let Err(err) = module
5775 .restart_requested(drain_timeout_ms, Some(requester))
5776 .await
5777 {
5778 self.route_outages
5779 .operator_action_ended_unrefused(&module_id);
5780 let (code, message) = match err {
5781 crate::supervise::SuperviseError::Disabled { .. } => {
5782 ("module_disabled", err.to_string())
5783 }
5784 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5785 ("swap_in_progress", err.to_string())
5786 }
5787 _ => (
5788 "target_unavailable",
5789 format!("failed to restart module_id '{module_id}': {err}"),
5790 ),
5791 };
5792 return Ok(vec![control_error_frame(&frame, code, message)?]);
5793 }
5794
5795 let response = ClientControlResponse::SupervisorAck {
5796 module_id,
5797 applied: true,
5798 };
5799 Ok(vec![control_response_body_frame(
5800 &frame,
5801 &response,
5802 "ClientControlResponse::SupervisorAck",
5803 )?])
5804 }
5805
5806 async fn handle_supervisor_swap(
5810 &self,
5811 frame: Frame,
5812 module_id: String,
5813 ready_timeout_ms: Option<u64>,
5814 requester: Principal,
5815 ) -> Result<Vec<Frame>, RouterError> {
5816 let module = {
5823 let operation_lock = self.supervisor.operation_lock();
5824 let _operation_guard = operation_lock.lock().await;
5825 self.supervisor.get(&module_id)
5826 };
5827 let Some(module) = module else {
5828 return Ok(vec![control_error_frame(
5829 &frame,
5830 "unknown_module",
5831 format!("module_id '{module_id}' is not supervised"),
5832 )?]);
5833 };
5834
5835 self.route_outages.mark_operator_action(&module_id);
5836 if let Err(err) = module
5837 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5838 .await
5839 {
5840 self.route_outages
5841 .operator_action_ended_unrefused(&module_id);
5842 use crate::supervise::SuperviseError;
5843 let message = err.to_string();
5844 let error = match err {
5845 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5846 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5847 code: "swap_refused".to_string(),
5848 message,
5849 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5850 },
5851 SuperviseError::SwapFailed {
5852 arm,
5853 candidate_exit,
5854 ..
5855 } => ErrorBody {
5856 code: "swap_failed".to_string(),
5857 message,
5858 detail: Some(serde_json::json!({
5859 "arm": arm.as_str(),
5860 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5861 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5862 })),
5863 },
5864 _ => ErrorBody::new(
5865 "target_unavailable",
5866 format!("failed to swap module_id '{module_id}': {message}"),
5867 ),
5868 };
5869 return Ok(vec![control_error_body_frame(&frame, error)?]);
5870 }
5871 self.route_outages
5875 .operator_action_ended_unrefused(&module_id);
5876
5877 let response = ClientControlResponse::SupervisorAck {
5878 module_id,
5879 applied: true,
5880 };
5881 Ok(vec![control_response_body_frame(
5882 &frame,
5883 &response,
5884 "ClientControlResponse::SupervisorAck",
5885 )?])
5886 }
5887
5888 async fn handle_supervisor_reload(
5889 &self,
5890 frame: Frame,
5891 module_id: String,
5892 requester: Principal,
5893 ) -> Result<Vec<Frame>, RouterError> {
5894 let operation_lock = self.supervisor.operation_lock();
5895 let _operation_guard = operation_lock.lock().await;
5896 let Some(module) = self.supervisor.get(&module_id) else {
5897 return Ok(vec![control_error_frame(
5898 &frame,
5899 "unknown_module",
5900 format!("module_id '{module_id}' is not supervised"),
5901 )?]);
5902 };
5903
5904 self.route_outages.mark_operator_action(&module_id);
5905 if let Err(err) = module.reload_requested(Some(requester)).await {
5906 self.route_outages
5907 .operator_action_ended_unrefused(&module_id);
5908 let (code, message) = match err {
5909 crate::supervise::SuperviseError::Disabled { .. } => {
5910 ("module_disabled", err.to_string())
5911 }
5912 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5913 ("swap_in_progress", err.to_string())
5914 }
5915 _ => (
5916 "reload_failed",
5917 format!("failed to reload module_id '{module_id}': {err}"),
5918 ),
5919 };
5920 return Ok(vec![control_error_frame(&frame, code, message)?]);
5921 }
5922
5923 let response = ClientControlResponse::SupervisorAck {
5924 module_id,
5925 applied: true,
5926 };
5927 Ok(vec![control_response_body_frame(
5928 &frame,
5929 &response,
5930 "ClientControlResponse::SupervisorAck",
5931 )?])
5932 }
5933
5934 async fn handle_supervisor_rescan(
5938 &self,
5939 frame: Frame,
5940 mode: RescanMode,
5941 requester: Principal,
5942 ) -> Result<Vec<Frame>, RouterError> {
5943 let preview = mode.previews();
5944 let Some(context) = self.rescan.clone() else {
5945 return Ok(vec![control_error_frame(
5946 &frame,
5947 "rescan_unavailable",
5948 "the daemon was not started with a reloadable config path".to_string(),
5949 )?]);
5950 };
5951
5952 let operation_lock = self.supervisor.operation_lock();
5953 let _operation_guard = operation_lock.lock().await;
5954 let loaded = match self.load_rescan_config(&mode, &context) {
5958 Ok(config) => config,
5959 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5960 };
5961 let Some((config, activity_config)) = loaded else {
5969 return Ok(vec![control_error_frame(
5970 &frame,
5971 "invalid_daemon_config",
5972 format!(
5973 "daemon config not found at {}; refusing to rescan (an absent config would \
5974 retire every supervised module)",
5975 context.config_path.display()
5976 ),
5977 )?]);
5978 };
5979 let (
5980 configured_port,
5981 storage_config,
5982 admission_facts_carrier_module_id,
5983 admission_facts_targets,
5984 scope_authority_owners,
5985 privileged_operator_prompt,
5986 modules,
5987 reserved_capabilities,
5988 ) = (
5989 config.port,
5990 config.storage,
5991 config.admission_facts_carrier_module_id,
5992 config.admission_facts_targets,
5993 config.scope_authority_owners,
5994 config.privileged_operator_prompt,
5995 config.modules,
5996 config.reserved_capabilities,
5997 );
5998
5999 let mut restart_required = Vec::new();
6008 for section in RestartRequiredSection::ALL {
6009 let changed = match section {
6010 RestartRequiredSection::Port => configured_port != context.configured_port,
6011 RestartRequiredSection::Storage => storage_config != context.storage_config,
6012 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6013 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6014 }
6015 RestartRequiredSection::AdmissionFactsTargets => {
6016 admission_facts_targets != context.admission_facts_targets
6017 }
6018 RestartRequiredSection::ScopeAuthorityOwners => {
6019 scope_authority_owners != context.scope_authority_owners
6020 }
6021 RestartRequiredSection::PrivilegedOperatorPrompt => {
6022 privileged_operator_prompt != context.privileged_operator_prompt
6023 }
6024 };
6025 if changed {
6026 restart_required.push(section.label().to_string());
6027 }
6028 }
6029 if !restart_required.is_empty() {
6030 warn!(
6031 config_path = %context.config_path.display(),
6032 sections = %restart_required.join(", "),
6033 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6034 );
6035 }
6036
6037 for configured in &modules {
6038 if let Err(err) = validate_spec(&configured.module_spec()) {
6039 return Ok(vec![control_error_frame(
6040 &frame,
6041 "invalid_daemon_config",
6042 format!("supervisor rescan rejected daemon config: {err}"),
6043 )?]);
6044 }
6045 }
6046
6047 let configured_capabilities = modules
6048 .iter()
6049 .map(|module| (module.module_id.clone(), module.enabled))
6050 .collect::<Vec<_>>();
6051 let preview_capability_warnings = if preview {
6052 let (_, registrations) = self.runtime_capability_snapshot()?;
6053 let current_modules = self
6054 .supervisor
6055 .list()
6056 .into_iter()
6057 .map(|module| module.module_id().to_string())
6058 .collect::<BTreeSet<_>>();
6059 let resulting_modules = configured_capabilities.clone();
6060 let removed = current_modules
6061 .into_iter()
6062 .filter(|module_id| {
6063 !resulting_modules
6064 .iter()
6065 .any(|(configured_id, _)| configured_id == module_id)
6066 })
6067 .collect::<Vec<_>>();
6068 let disabled = match &mode {
6071 RescanMode::Candidate(_) => resulting_modules
6072 .iter()
6073 .filter(|(_, enabled)| !enabled)
6074 .map(|(module_id, _)| module_id.clone())
6075 .collect::<Vec<_>>(),
6076 _ => Vec::new(),
6077 };
6078 self.capability_evaluator.preview_warnings(
6079 resulting_modules,
6080 &removed,
6081 &disabled,
6082 ®istrations,
6083 )
6084 } else {
6085 Vec::new()
6086 };
6087 let reconciled = match &mode {
6088 RescanMode::Candidate(candidate) => self
6089 .preview_candidate(&context, modules, candidate)
6090 .map(|result| (result, Vec::new())),
6091 _ => {
6092 self.reconcile_supervised_modules(
6093 &context.supervisor,
6094 modules,
6095 preview,
6096 requester.clone(),
6097 )
6098 .await
6099 }
6100 };
6101 let (result, changed) = match reconciled {
6102 Ok(result) => result,
6103 Err(message) => {
6104 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6105 }
6106 };
6107 if !preview {
6108 if let Some(ledger) = &self.activity {
6109 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6110 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6111 }
6112 if let Err(code) = self.configure_activity(activity_config).await {
6113 warn!(
6114 code,
6115 "activity retention could not be applied during rescan"
6116 );
6117 }
6118 self.capability_evaluator
6119 .configure(configured_capabilities, reserved_capabilities);
6120 self.capability_evaluator.wake_deadline_loop();
6121 self.refresh_capability_requirements();
6122 }
6123 let mut result = result;
6124 result.restart_required = restart_required;
6125 result.capability_warnings = preview_capability_warnings;
6126 let response = ClientControlResponse::SupervisorRescan { result };
6127 Ok(vec![control_response_body_frame(
6128 &frame,
6129 &response,
6130 "ClientControlResponse::SupervisorRescan",
6131 )?])
6132 }
6133
6134 async fn handle_supervisor_release_reserved(
6135 &self,
6136 frame: Frame,
6137 module_id: String,
6138 ) -> Result<Vec<Frame>, RouterError> {
6139 let Some(context) = self.rescan.clone() else {
6140 return Ok(vec![control_error_frame(
6141 &frame,
6142 "release_unavailable",
6143 "reserved-id release requires a daemon started with a reloadable config path",
6144 )?]);
6145 };
6146 let operation_lock = self.supervisor.operation_lock();
6147 let _operation_guard = operation_lock.lock().await;
6148 let loaded = match crate::daemon_config::load(&context.config_path) {
6149 Ok(Some(config)) => config,
6150 Ok(None) => {
6151 return Ok(vec![control_error_frame(
6152 &frame,
6153 "invalid_daemon_config",
6154 format!(
6155 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6156 context.config_path.display()
6157 ),
6158 )?])
6159 }
6160 Err(err) => {
6161 return Ok(vec![control_error_frame(
6162 &frame,
6163 "invalid_daemon_config",
6164 format!("unable to verify reserved-id release against daemon config: {err}"),
6165 )?])
6166 }
6167 };
6168 if loaded
6169 .modules
6170 .iter()
6171 .any(|configured| configured.module_id == module_id)
6172 {
6173 return Ok(vec![control_error_frame(
6174 &frame,
6175 "reserved_module_configured",
6176 format!(
6177 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6178 ),
6179 )?]);
6180 }
6181 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6182 return Ok(vec![control_error_frame(
6183 &frame,
6184 "reserved_gate_not_retained",
6185 format!(
6186 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6187 ),
6188 )?]);
6189 }
6190
6191 let response = ClientControlResponse::SupervisorAck {
6192 module_id,
6193 applied: true,
6194 };
6195 Ok(vec![control_response_body_frame(
6196 &frame,
6197 &response,
6198 "ClientControlResponse::SupervisorAck",
6199 )?])
6200 }
6201
6202 fn plan_rescan(
6206 &self,
6207 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6208 ) -> Result<RescanPlan, String> {
6209 let mut current = BTreeMap::new();
6210 for module in self.supervisor.list() {
6211 let (spec, health) = module.configuration().map_err(|err| {
6212 format!(
6213 "failed to read configuration for module_id '{}': {err}",
6214 module.module_id()
6215 )
6216 })?;
6217 let enabled = module
6218 .status()
6219 .map_err(|err| {
6220 format!(
6221 "failed to read status for module_id '{}': {err}",
6222 module.module_id()
6223 )
6224 })?
6225 .enabled;
6226 current.insert(
6227 module.module_id().to_string(),
6228 (module, spec, health, enabled),
6229 );
6230 }
6231 let configured = configured_modules
6232 .into_iter()
6233 .map(|module| (module.module_id.clone(), module))
6234 .collect::<BTreeMap<_, _>>();
6235
6236 let added = configured
6237 .keys()
6238 .filter(|module_id| !current.contains_key(*module_id))
6239 .cloned()
6240 .collect::<Vec<_>>();
6241 let removed = current
6242 .keys()
6243 .filter(|module_id| !configured.contains_key(*module_id))
6244 .cloned()
6245 .collect::<Vec<_>>();
6246 let mut changed_pending_reload = Vec::new();
6247 let mut configuration_changes = BTreeSet::new();
6248 let mut enabled_changes = BTreeSet::new();
6249 let mut unchanged = 0_u32;
6250
6251 for (module_id, configured_module) in &configured {
6252 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6253 else {
6254 continue;
6255 };
6256 let launch_changed = *current_spec != configured_module.module_spec();
6260 let configuration_changed =
6261 launch_changed || *current_health != configured_module.health;
6262 let enabled_changed = *current_enabled != configured_module.enabled;
6263 if configuration_changed {
6264 configuration_changes.insert(module_id.clone());
6265 }
6266 if launch_changed {
6267 changed_pending_reload.push(module_id.clone());
6268 }
6269 if enabled_changed {
6270 enabled_changes.insert(module_id.clone());
6271 }
6272 if !configuration_changed && !enabled_changed {
6273 unchanged = unchanged.saturating_add(1);
6274 }
6275 }
6276 Ok(RescanPlan {
6277 current,
6278 configured,
6279 added,
6280 removed,
6281 changed_pending_reload,
6282 configuration_changes,
6283 enabled_changes,
6284 unchanged,
6285 })
6286 }
6287
6288 async fn reconcile_supervised_modules(
6297 &self,
6298 supervisor: &Supervisor,
6299 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6300 preview: bool,
6301 requester: Principal,
6302 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6303 let plan = self.plan_rescan(configured_modules)?;
6304
6305 if preview {
6311 return Ok((plan.preview_result(), Vec::new()));
6312 }
6313 let changed = plan
6314 .configuration_changes
6315 .union(&plan.enabled_changes)
6316 .cloned()
6317 .collect::<Vec<_>>();
6318 let RescanPlan {
6319 current,
6320 configured,
6321 added,
6322 removed,
6323 changed_pending_reload,
6324 configuration_changes,
6325 enabled_changes,
6326 unchanged,
6327 } = plan;
6328 if !(added.is_empty()
6333 && removed.is_empty()
6334 && configuration_changes.is_empty()
6335 && enabled_changes.is_empty())
6336 {
6337 if let Some(context) = &self.rescan {
6338 context
6339 .generation
6340 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6341 }
6342 }
6343
6344 for module_id in &removed {
6345 let module = ¤t
6346 .get(module_id)
6347 .expect("removed module came from current supervisor state")
6348 .0;
6349 module
6350 .retire_requested(Some(requester.clone()))
6351 .await
6352 .map_err(|err| {
6353 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6354 })?;
6355 self.supervisor.record_rescan_removal(module_id);
6389 self.supervisor.retire(module_id);
6390 self.route_outages.forget(module_id);
6391 }
6392
6393 for module_id in configured.keys() {
6394 let Some((module, _, _, _)) = current.get(module_id) else {
6395 continue;
6396 };
6397 let configured_module = configured
6398 .get(module_id)
6399 .expect("configured module id came from configured map");
6400 if configuration_changes.contains(module_id) {
6401 module
6402 .update_configuration(
6403 configured_module.module_spec(),
6404 configured_module.health.clone(),
6405 configured_module.drain_timeout_ms,
6406 )
6407 .await
6408 .map_err(|err| {
6409 format!(
6410 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6411 )
6412 })?;
6413 }
6414 if enabled_changes.contains(module_id) {
6415 self.route_outages.mark_operator_action(module_id);
6418 module
6419 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6420 .await
6421 .map_err(|err| {
6422 self.route_outages.operator_action_ended_unrefused(module_id);
6423 format!(
6424 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6425 configured_module.enabled
6426 )
6427 })?;
6428 }
6429 }
6430
6431 for module_id in &added {
6432 let configured_module = configured
6433 .get(module_id)
6434 .expect("added module id came from configured map");
6435 supervisor
6436 .supervise_configured_requested(
6437 configured_module.module_spec(),
6438 configured_module.enabled,
6439 configured_module.health.clone(),
6440 configured_module.drain_timeout_ms,
6441 configured_module.restart,
6442 configured_module.register_timeout,
6443 Some(requester.clone()),
6444 )
6445 .map_err(|err| {
6446 format!("failed to add module_id '{module_id}' during rescan: {err}")
6447 })?;
6448 }
6449
6450 Ok((
6451 SupervisorRescanResult {
6452 added,
6453 removed,
6454 changed_pending_reload,
6455 enabled_changes: enabled_changes.iter().cloned().collect(),
6456 unchanged,
6457 preview: false,
6458 restart_required: Vec::new(),
6461 capability_warnings: Vec::new(),
6462 module_actions: Vec::new(),
6463 earlier_pending: Vec::new(),
6464 expect: None,
6465 },
6466 changed,
6467 ))
6468 }
6469
6470 async fn handle_supervisor_set_enabled(
6471 &self,
6472 frame: Frame,
6473 module_id: String,
6474 enabled: bool,
6475 requester: Principal,
6476 ) -> Result<Vec<Frame>, RouterError> {
6477 let operation_lock = self.supervisor.operation_lock();
6478 let _operation_guard = operation_lock.lock().await;
6479 let Some(module) = self.supervisor.get(&module_id) else {
6480 return Ok(vec![control_error_frame(
6481 &frame,
6482 "unknown_module",
6483 format!("module_id '{module_id}' is not supervised"),
6484 )?]);
6485 };
6486
6487 self.route_outages.mark_operator_action(&module_id);
6490 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6491 Ok(applied) => applied,
6492 Err(err) => {
6493 self.route_outages
6494 .operator_action_ended_unrefused(&module_id);
6495 return Ok(vec![control_error_frame(
6496 &frame,
6497 "target_unavailable",
6498 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6499 )?]);
6500 }
6501 };
6502 if !applied {
6503 self.route_outages
6506 .operator_action_ended_unrefused(&module_id);
6507 }
6508
6509 self.capability_evaluator.wake_deadline_loop();
6510 self.refresh_capability_requirements();
6511 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6512 Ok(vec![control_response_body_frame(
6513 &frame,
6514 &response,
6515 "ClientControlResponse::SupervisorAck",
6516 )?])
6517 }
6518
6519 async fn handle_supervisor_health_probe(
6520 &self,
6521 frame: Frame,
6522 module_id: String,
6523 ) -> Result<Vec<Frame>, RouterError> {
6524 self.refresh_capability_requirements();
6525 let Some(registration) = self
6526 .registry
6527 .get_module(&module_id)
6528 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6529 else {
6530 return Ok(vec![control_error_frame(
6531 &frame,
6532 "unknown_module",
6533 format!("module_id '{module_id}' is not registered"),
6534 )?]);
6535 };
6536
6537 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6550 {
6551 return Ok(vec![control_error_frame(
6552 &frame,
6553 "health_not_advertised",
6554 format!("module_id '{module_id}' did not advertise health.check"),
6555 )?]);
6556 }
6557
6558 let deadline = Instant::now() + self.health_probe_timeout;
6559 let pending = match self.forwarding.begin_module_control_rpc_for(
6560 &module_id,
6561 MODULE_CONTROL_OP_HEALTH_CHECK,
6562 deadline,
6563 ) {
6564 Ok(pending) => pending,
6565 Err(err) => {
6566 return Ok(vec![control_error_frame(
6567 &frame,
6568 forwarding_error_code(&err),
6569 err.to_string(),
6570 )?])
6571 }
6572 };
6573
6574 let PendingModuleControlRpc {
6575 endpoint,
6576 module_sink,
6577 negotiated_ver,
6578 corr: probe_corr,
6579 receiver,
6580 } = pending;
6581 let mut guard =
6582 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6583 let probe_body =
6584 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6585 RouterError::backend(
6586 0,
6587 frame.header.corr,
6588 format!("failed to encode health.check request: {err}"),
6589 )
6590 })?;
6591 let probe_frame = Frame::build_with_version(
6592 negotiated_ver,
6593 FrameType::Request,
6594 control_flags(),
6595 0,
6596 0,
6597 probe_corr,
6598 probe_body,
6599 )
6600 .map_err(RouterError::FrameBuild)?;
6601
6602 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6605 Ok(Ok(())) => {}
6606 Ok(Err(err)) => {
6607 return Ok(vec![control_error_frame(
6608 &frame,
6609 "target_unavailable",
6610 err.to_string(),
6611 )?]);
6612 }
6613 Err(_) => {
6614 return Ok(vec![control_error_frame(
6615 &frame,
6616 "module_timeout",
6617 format!(
6618 "module_id '{module_id}' did not answer health.check within {:?}",
6619 self.health_probe_timeout
6620 ),
6621 )?]);
6622 }
6623 }
6624
6625 match timeout_at(deadline, receiver).await {
6626 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6627 guard.disarm();
6628 let Some(report) = response.health_report() else {
6629 return Ok(vec![control_error_frame(
6630 &frame,
6631 "invalid_control_body",
6632 "health.check RPC returned a non-health response",
6633 )?]);
6634 };
6635 let HealthReport {
6640 status,
6641 detail,
6642 metrics,
6643 } = report;
6644 let capability_detail = self
6645 .capability_evaluator
6646 .required_problem_detail(&module_id);
6647 let response = ClientControlResponse::SupervisorHealthProbe {
6648 module_id,
6649 status,
6650 detail: append_capability_problem_detail(detail, capability_detail),
6651 metrics,
6652 };
6653 Ok(vec![control_response_body_frame(
6654 &frame,
6655 &response,
6656 "ClientControlResponse::SupervisorHealthProbe",
6657 )?])
6658 }
6659 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6660 guard.disarm();
6661 Ok(vec![control_error_body_frame(&frame, body)?])
6662 }
6663 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6664 guard.disarm();
6665 Ok(vec![control_error_frame(
6666 &frame,
6667 "target_unavailable",
6668 message,
6669 )?])
6670 }
6671 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6672 guard.disarm();
6673 Ok(vec![control_error_frame(
6674 &frame,
6675 "invalid_control_body",
6676 message,
6677 )?])
6678 }
6679 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6680 guard.disarm();
6681 Ok(vec![control_error_frame(
6682 &frame,
6683 "invalid_control_body",
6684 format!("expected module-control op '{expected}', got '{actual}'"),
6685 )?])
6686 }
6687 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6688 guard.disarm();
6689 Ok(vec![control_error_frame(
6690 &frame,
6691 "module_timeout",
6692 format!(
6693 "module_id '{module_id}' answered health.check after {:?}",
6694 self.health_probe_timeout
6695 ),
6696 )?])
6697 }
6698 Ok(Err(_)) => Ok(vec![control_error_frame(
6699 &frame,
6700 "target_unavailable",
6701 "health.check waiter was canceled before the module responded",
6702 )?]),
6703 Err(_) => Ok(vec![control_error_frame(
6704 &frame,
6705 "module_timeout",
6706 format!(
6707 "module_id '{module_id}' did not answer health.check within {:?}",
6708 self.health_probe_timeout
6709 ),
6710 )?]),
6711 }
6712 }
6713
6714 fn supervisor_status(
6715 &self,
6716 module_id: &str,
6717 corr: u64,
6718 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6719 self.supervisor
6720 .get(module_id)
6721 .map(|module| {
6722 let warming = module.is_warming_for_control("status").map_err(|err| {
6723 RouterError::backend(
6724 0,
6725 corr,
6726 format!(
6727 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6728 ),
6729 )
6730 })?;
6731 module.status_for_control("status").map_err(|err| {
6732 RouterError::backend(
6733 0,
6734 corr,
6735 format!(
6736 "failed to read supervisor status for module_id '{module_id}': {err}"
6737 ),
6738 )
6739 }).map(|status| (status, warming))
6740 })
6741 .transpose()
6742 }
6743
6744 fn guard_module_control_op(
6745 &self,
6746 frame: &Frame,
6747 module_id: &str,
6748 op: &str,
6749 ) -> Result<Option<Frame>, RouterError> {
6750 if self.module_grants_op(module_id, op, frame.header.corr)? {
6751 return Ok(None);
6752 }
6753
6754 Ok(Some(control_error_frame(
6755 frame,
6756 "op_not_allowed",
6757 format!("module_id '{module_id}' did not grant control op '{op}'"),
6758 )?))
6759 }
6760
6761 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6762 let Some(registration) = self
6763 .registry
6764 .get_module(module_id)
6765 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6766 else {
6767 return Ok(false);
6768 };
6769 Ok(module_registration_grants_op(®istration.control_ops, op))
6770 }
6771
6772 fn handle_status_update(
6773 &self,
6774 endpoint: ModuleEndpointId,
6775 frame: Frame,
6776 ) -> Result<Vec<Frame>, RouterError> {
6777 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6778 Ok(update) => update,
6779 Err(err) => {
6780 if is_known_module_push_op(&frame.body) {
6785 return Ok(vec![control_error_frame(
6786 &frame,
6787 "invalid_control_body",
6788 format!("malformed module control push body: {err}"),
6789 )?]);
6790 }
6791 return Ok(Vec::new());
6792 }
6793 };
6794
6795 match update {
6796 ModuleControlPush::RouteStatus {
6797 route_channel,
6798 route_epoch,
6799 status,
6800 } => {
6801 self.forwarding
6802 .cache_status(endpoint, route_channel, route_epoch, status)
6803 .map_err(RouterError::Forwarding)?;
6804 }
6805 }
6806 Ok(Vec::new())
6807 }
6808
6809 fn handle_route_poll(
6810 &self,
6811 ctx: &RouteCtx,
6812 frame: Frame,
6813 route_channel: u16,
6814 route_epoch: u32,
6815 kind: PollKind,
6816 ) -> Result<Vec<Frame>, RouterError> {
6817 let snapshot = self
6818 .forwarding
6819 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6820 .map_err(RouterError::Forwarding)?;
6821 let response = match (kind, snapshot) {
6822 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6823 ClientControlResponse::RoutePoll {
6824 route_channel,
6825 route_epoch,
6826 status,
6827 live: None,
6828 }
6829 }
6830 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6831 route_channel,
6832 route_epoch,
6833 status: None,
6834 live: None,
6835 },
6836 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6837 let live = self
6863 .process_liveness
6864 .as_ref()
6865 .and_then(|source| source.process_live(&module_id))
6866 .unwrap_or(true);
6867 ClientControlResponse::RoutePoll {
6868 route_channel,
6869 route_epoch,
6870 status: None,
6871 live: Some(live),
6872 }
6873 }
6874 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6875 route_channel,
6876 route_epoch,
6877 status: None,
6878 live: Some(false),
6879 },
6880 };
6881
6882 Ok(vec![control_response_body_frame(
6883 &frame,
6884 &response,
6885 "ClientControlResponse::RoutePoll",
6886 )?])
6887 }
6888
6889 pub(crate) fn observe_module_control_completion(
6890 &self,
6891 completion: ModuleControlRpcCompletion,
6892 ) -> bool {
6893 match completion {
6894 ModuleControlRpcCompletion::Unknown => false,
6895 ModuleControlRpcCompletion::Settled => true,
6896 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6897 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6898 info!(
6899 module_id = %module_id,
6900 latency_ms,
6901 "late health.check answer proves the module is alive"
6902 );
6903 match self
6904 .supervisor
6905 .record_late_health_answer(&module_id, latency_ms)
6906 {
6907 Ok(true) => {}
6908 Ok(false) => debug!(
6909 module_id = %module_id,
6910 latency_ms,
6911 "late health.check answer has no active supervisor snapshot"
6912 ),
6913 Err(err) => warn!(
6914 module_id = %module_id,
6915 latency_ms,
6916 error = %err,
6917 "failed to record late health.check answer"
6918 ),
6919 }
6920 true
6921 }
6922 }
6923 }
6924
6925 fn refuse_to_end_module_connection_for_a_client(
6946 &self,
6947 module_connection_id: ConnectionId,
6948 corr: u64,
6949 err: ForwardingError,
6950 ) -> Result<(), RouterError> {
6951 if let ForwardingError::ConnectionClosing { connection_id } = err {
6952 if connection_id != module_connection_id {
6953 warn!(
6954 module_connection_id = module_connection_id.get(),
6955 client_connection_id = connection_id.get(),
6956 corr,
6957 "dropping a route.bind response for a closing client; the module connection keeps serving"
6958 );
6959 return Ok(());
6960 }
6961 }
6962 Err(RouterError::Forwarding(err))
6963 }
6964
6965 fn handle_module_relay_response(
6966 &self,
6967 connection_id: ConnectionId,
6968 frame: Frame,
6969 ) -> Result<Vec<Frame>, RouterError> {
6970 let mut secondary_error = None;
6971 let outcome = match frame.header.ty {
6972 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6973 Ok(probe) if probe.op == "route.bind" => {
6974 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6975 Ok(ModuleControlResponse::RouteBindAck {}) => {
6976 RouteBindRelayOutcome::Accepted
6977 }
6978 Ok(other) => {
6979 let message =
6980 format!("route.bind response carried unexpected body: {other:?}");
6981 secondary_error = Some(control_error_frame(
6982 &frame,
6983 "invalid_control_body",
6984 message.clone(),
6985 )?);
6986 RouteBindRelayOutcome::ModuleGone(message)
6987 }
6988 Err(err) => {
6989 let message = format!("malformed route.bind response body: {err}");
6990 secondary_error = Some(control_error_frame(
6991 &frame,
6992 "invalid_control_body",
6993 message.clone(),
6994 )?);
6995 RouteBindRelayOutcome::ModuleGone(message)
6996 }
6997 }
6998 }
6999 Ok(probe) => {
7000 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7001 {
7002 Ok(response) => ModuleControlRpcOutcome::Response(response),
7003 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7004 "malformed {} response body: {err}",
7005 probe.op
7006 )),
7007 };
7008 let completion = self
7009 .forwarding
7010 .complete_module_control_rpc(
7011 connection_id,
7012 frame.header.corr,
7013 Some(&probe.op),
7014 outcome,
7015 )
7016 .map_err(RouterError::Forwarding)?;
7017 if !self.observe_module_control_completion(completion) {
7018 debug!(
7019 connection_id = connection_id.get(),
7020 corr = frame.header.corr,
7021 op = %probe.op,
7022 "dropping late or unknown module-control RPC response"
7023 );
7024 }
7025 return Ok(Vec::new());
7026 }
7027 Err(err) => {
7028 if let Some(expected_op) = self
7029 .forwarding
7030 .pending_module_control_op(connection_id, frame.header.corr)
7031 .map_err(RouterError::Forwarding)?
7032 {
7033 let completion = self
7034 .forwarding
7035 .complete_module_control_rpc(
7036 connection_id,
7037 frame.header.corr,
7038 None,
7039 ModuleControlRpcOutcome::MalformedResponse(format!(
7040 "malformed {expected_op} response body: {err}"
7041 )),
7042 )
7043 .map_err(RouterError::Forwarding)?;
7044 if !self.observe_module_control_completion(completion) {
7045 debug!(
7046 connection_id = connection_id.get(),
7047 corr = frame.header.corr,
7048 "dropping late malformed module-control RPC response"
7049 );
7050 }
7051 return Ok(Vec::new());
7052 }
7053 let message = format!("malformed route.bind response body: {err}");
7054 secondary_error = Some(control_error_frame(
7055 &frame,
7056 "invalid_control_body",
7057 message.clone(),
7058 )?);
7059 RouteBindRelayOutcome::ModuleGone(message)
7060 }
7061 },
7062 FrameType::Error => {
7063 if self
7064 .forwarding
7065 .pending_module_control_op(connection_id, frame.header.corr)
7066 .map_err(RouterError::Forwarding)?
7067 .is_some()
7068 {
7069 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7070 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7071 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7072 "malformed module-control ERROR body: {err}"
7073 )),
7074 };
7075 let completion = self
7076 .forwarding
7077 .complete_module_control_rpc(
7078 connection_id,
7079 frame.header.corr,
7080 None,
7081 outcome,
7082 )
7083 .map_err(RouterError::Forwarding)?;
7084 if !self.observe_module_control_completion(completion) {
7085 debug!(
7086 connection_id = connection_id.get(),
7087 corr = frame.header.corr,
7088 "dropping late or unknown module-control RPC error"
7089 );
7090 }
7091 return Ok(Vec::new());
7092 }
7093 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7094 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7095 Err(err) => {
7096 let message = format!("malformed route.bind ERROR body: {err}");
7097 secondary_error = Some(control_error_frame(
7098 &frame,
7099 "invalid_control_body",
7100 message.clone(),
7101 )?);
7102 RouteBindRelayOutcome::ModuleGone(message)
7103 }
7104 }
7105 }
7106 ty => {
7107 return Ok(vec![control_error_frame(
7108 &frame,
7109 "unsupported_control_frame",
7110 format!("unsupported module channel-0 frame {ty:?}"),
7111 )?])
7112 }
7113 };
7114
7115 let settled =
7116 self.forwarding
7117 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7118 let completion = match settled {
7119 Ok(completion) => completion,
7120 Err(err) => {
7121 self.refuse_to_end_module_connection_for_a_client(
7122 connection_id,
7123 frame.header.corr,
7124 err,
7125 )?;
7126 return Ok(secondary_error.into_iter().collect());
7127 }
7128 };
7129 if let Some(target) = completion.abandoned.as_ref() {
7130 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7131 }
7132 if !completion.settled {
7133 debug!(
7134 connection_id = connection_id.get(),
7135 corr = frame.header.corr,
7136 frame_type = ?frame.header.ty,
7137 "dropping late or unknown route.bind relay response"
7138 );
7139 }
7140 Ok(secondary_error.into_iter().collect())
7141 }
7142
7143 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7144 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7145 self.cleanup_connection_with_end_reason(
7149 connection_id,
7150 RegistrationEndReason::ExplicitGoodbye,
7151 )
7152 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7153 Ok(Vec::new())
7154 }
7155}
7156
7157impl Default for ControlHandler {
7158 fn default() -> Self {
7159 Self::new(Arc::new(Registry::default()))
7160 }
7161}
7162
7163impl crate::supervise::SwapPromotionObserver for ControlHandler {
7164 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7165 self.apply_registration_capabilities(registration);
7166 }
7167}
7168
7169fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7170 CapabilityRequirementStatus {
7171 consumer: status.consumer,
7172 capability: status.capability,
7173 need: match status.need {
7174 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7175 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7176 },
7177 verdict: status.verdict.as_str().to_string(),
7178 episode_seq: status.episode_seq,
7179 config_satisfiable: status.config_satisfiable,
7180 runtime_available: status.runtime_available,
7181 detail: status.detail,
7182 }
7183}
7184
7185fn append_capability_problem_detail(
7186 detail: Option<String>,
7187 capability_detail: Option<String>,
7188) -> Option<String> {
7189 match (detail, capability_detail) {
7190 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7191 (Some(detail), None) => Some(detail),
7192 (None, Some(capability_detail)) => Some(capability_detail),
7193 (None, None) => None,
7194 }
7195}
7196
7197fn subc_ops() -> Vec<String> {
7198 SUBC_CONTROL_OPS
7199 .iter()
7200 .map(|op| (*op).to_string())
7201 .collect()
7202}
7203
7204fn module_subc_ops() -> Vec<String> {
7205 SUBC_CONTROL_OPS
7206 .iter()
7207 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7208 .map(|op| (*op).to_string())
7209 .collect()
7210}
7211
7212#[cfg(test)]
7213fn module_baseline_control_ops() -> Vec<String> {
7214 MODULE_BASELINE_CONTROL_OPS
7215 .iter()
7216 .map(|op| (*op).to_string())
7217 .collect()
7218}
7219
7220fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7221 let mut seen = HashSet::new();
7222 let mut effective = Vec::new();
7223 for op in MODULE_BASELINE_CONTROL_OPS {
7224 if seen.insert((*op).to_string()) {
7225 effective.push((*op).to_string());
7226 }
7227 }
7228 for op in declared.unwrap_or_default() {
7229 if seen.insert(op.clone()) {
7230 effective.push(op);
7231 }
7232 }
7233 effective
7234}
7235
7236fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7237 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7238}
7239
7240fn target_module_id(target: &RouteTarget) -> &str {
7241 match target {
7242 RouteTarget::ToolProvider { module_id }
7243 | RouteTarget::ManagementSurface { module_id }
7244 | RouteTarget::InternalService { module_id, .. } => module_id,
7245 }
7246}
7247
7248fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7249 roles.iter().any(|role| match (target, role) {
7250 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7251 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7252 (
7253 RouteTarget::InternalService { service_id, .. },
7254 ProviderRole::InternalService {
7255 service_id: provided,
7256 ..
7257 },
7258 ) => service_id == provided,
7259 _ => false,
7260 })
7261}
7262
7263fn is_routable_role(role: &ProviderRole) -> bool {
7264 matches!(
7265 role,
7266 ProviderRole::ToolProvider { .. }
7267 | ProviderRole::ManagementSurface { .. }
7268 | ProviderRole::InternalService { .. }
7269 )
7270}
7271
7272#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7273enum ControlRequestBodyError {
7274 UnknownOp,
7275 InvalidBody,
7276}
7277
7278#[derive(Debug, Deserialize)]
7279struct ControlOpProbe {
7280 op: String,
7281}
7282
7283const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7286
7287fn is_known_module_push_op(body: &[u8]) -> bool {
7288 serde_json::from_slice::<ControlOpProbe>(body)
7289 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7290 .unwrap_or(false)
7291}
7292
7293fn is_known_module_request_op(body: &[u8]) -> bool {
7294 serde_json::from_slice::<ControlOpProbe>(body)
7295 .map(|probe| is_module_to_subc_op(&probe.op))
7296 .unwrap_or(false)
7297}
7298
7299fn is_module_to_subc_op(op: &str) -> bool {
7300 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7301}
7302
7303fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7304 debug!(
7305 op = %op,
7306 connection_id = connection_id.get(),
7307 corr,
7308 "control dispatch"
7309 );
7310}
7311
7312fn log_slow_control_dispatch(
7313 dispatch_started_at: Option<StdInstant>,
7314 op: &'static str,
7315 connection_id: ConnectionId,
7316 corr: u64,
7317 route_open_timings: Option<&RouteOpenTimings>,
7318) {
7319 let Some(dispatch_started_at) = dispatch_started_at else {
7320 return;
7321 };
7322 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7324 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7325 if let Some(timings) = route_open_timings {
7326 warn!(
7327 op = %op,
7328 connection_id = connection_id.get(),
7329 corr,
7330 elapsed_ms = elapsed.as_millis() as u64,
7331 module_id = ?timings.module_id,
7332 principal = %timings.principal,
7333 admission_ms = timings.admission.as_millis() as u64,
7334 project_root_ms = timings.project_root.as_millis() as u64,
7335 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7336 bind_send_ms = timings.bind_send.as_millis() as u64,
7337 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7338 "slow control dispatch"
7339 );
7340 return;
7341 }
7342 warn!(
7343 op = %op,
7344 connection_id = connection_id.get(),
7345 corr,
7346 elapsed_ms = elapsed.as_millis() as u64,
7347 "slow control dispatch"
7348 );
7349 }
7350}
7351
7352#[cfg(test)]
7354type TestCallerLookup =
7355 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7356
7357fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7359 match request {
7360 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7361 Some(PrivilegedOperation::Restart { module_id })
7362 }
7363 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7364 Some(PrivilegedOperation::Swap { module_id })
7365 }
7366 ClientControlRequest::SupervisorReload { module_id } => {
7367 Some(PrivilegedOperation::Reload { module_id })
7368 }
7369 ClientControlRequest::SupervisorRescan { preview } => {
7370 if *preview {
7371 None
7372 } else {
7373 Some(PrivilegedOperation::Rescan)
7374 }
7375 }
7376 ClientControlRequest::SupervisorRescanChecked {
7381 preview,
7382 candidate,
7383 expect,
7384 } => (!*preview && candidate.is_none() && expect.is_some())
7385 .then_some(PrivilegedOperation::Rescan),
7386 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7387 Some(PrivilegedOperation::SetEnabled {
7388 module_id,
7389 enabled: *enabled,
7390 })
7391 }
7392 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7393 Some(PrivilegedOperation::ReleaseReserved { module_id })
7394 }
7395 ClientControlRequest::ServerDescribe {}
7396 | ClientControlRequest::CatalogList { .. }
7397 | ClientControlRequest::RouteOpen { .. }
7398 | ClientControlRequest::RoutePoll { .. }
7399 | ClientControlRequest::SupervisorList {}
7400 | ClientControlRequest::SupervisorHealth {}
7401 | ClientControlRequest::SupervisorHealthProbe { .. }
7402 | ClientControlRequest::SupervisorRoutes { .. }
7403 | ClientControlRequest::SupervisorProvenance { .. }
7404 | ClientControlRequest::SupervisorStderrTail { .. }
7405 | ClientControlRequest::SupervisorTerminals { .. }
7406 | ClientControlRequest::SupervisorSpawnSnapshot {}
7407 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7408 }
7409}
7410
7411fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7412 match request {
7413 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7414 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7415 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7416 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7417 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7418 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7419 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7420 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7421 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7422 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7423 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7424 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7425 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7426 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7427 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7428 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7429 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7430 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7431 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7432 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7433 }
7434}
7435
7436struct RouteOpenCaller {
7444 principal: String,
7448 scope: String,
7451 harness: String,
7454}
7455
7456impl RouteOpenCaller {
7457 fn new(
7458 principal: Option<&Principal>,
7459 scope: Option<&ScopeSelector>,
7460 identity: &BindIdentity,
7461 ) -> Self {
7462 Self {
7463 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7464 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7465 harness: bounded_log_text(&identity.harness),
7466 }
7467 }
7468
7469 fn unknown() -> Self {
7472 Self {
7473 principal: "unknown".to_string(),
7474 scope: "unknown".to_string(),
7475 harness: "unknown".to_string(),
7476 }
7477 }
7478}
7479
7480const MAX_LOGGED_TEXT_CHARS: usize = 64;
7483
7484fn bounded_log_text(value: &str) -> String {
7488 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7489 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7490 bounded.push('…');
7491 }
7492 format!("{bounded:?}")
7493}
7494
7495fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7501 use subc_protocol::scope::ScopeKind;
7502 let kind = match stamp.kind {
7503 ScopeKind::Head => "head",
7504 ScopeKind::Worker => "worker",
7505 ScopeKind::Ephemeral => "ephemeral",
7506 };
7507 format!(
7508 "{kind}/{}/{}@{}",
7509 principal_label(&stamp.owner),
7510 bounded_log_text(&stamp.scope_ref),
7511 stamp.scope_epoch
7512 )
7513}
7514
7515fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7522 let epoch = selector
7523 .scope_epoch
7524 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7525 format!(
7526 "requested/{}/{}@{epoch}",
7527 bounded_log_text(&principal_label(&selector.owner)),
7528 bounded_log_text(&selector.scope_ref),
7529 )
7530}
7531
7532fn principal_label(principal: &Principal) -> String {
7533 match principal {
7534 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7535 Principal::Direct => "direct".to_string(),
7536 other => format!("{other:?}"),
7537 }
7538}
7539
7540fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7541 match request {
7542 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7543 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7544 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7545 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7546 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7547 }
7548}
7549
7550fn parse_client_control_request(
7551 body: &[u8],
7552) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7553 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7554 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7555 Ok(probe)
7556 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7557 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7558 {
7559 ControlRequestBodyError::InvalidBody
7560 }
7561 Ok(_) => ControlRequestBodyError::UnknownOp,
7562 Err(_) => ControlRequestBodyError::InvalidBody,
7563 };
7564 (err, classification)
7565 })
7566}
7567
7568fn parse_module_control_request_from_module(
7569 body: &[u8],
7570) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7571 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7572 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7573 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7574 Ok(_) => ControlRequestBodyError::UnknownOp,
7575 Err(_) => ControlRequestBodyError::InvalidBody,
7576 };
7577 (err, classification)
7578 })
7579}
7580
7581#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7582enum ProviderRoleKind {
7583 ToolProvider,
7584 PipelineStage,
7585 ManagementSurface,
7586 InternalService,
7587}
7588
7589fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7590 match role {
7591 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7592 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7593 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7594 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7595 }
7596}
7597
7598fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7599 roles.iter().map(provider_role_kind).collect()
7600}
7601
7602const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7605
7606#[derive(Debug, Default, PartialEq, Eq)]
7608struct ScopeOutcomeCounts {
7609 created: usize,
7610 replaced: usize,
7611 updated: usize,
7612 unchanged: usize,
7613 refused: usize,
7614}
7615
7616impl ScopeOutcomeCounts {
7617 fn of(results: &[ScopeRecordResult]) -> Self {
7618 let mut counts = Self::default();
7619 for result in results {
7620 let slot = match result.outcome {
7621 ScopeRecordOutcome::Created => &mut counts.created,
7622 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7623 ScopeRecordOutcome::Updated => &mut counts.updated,
7624 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7625 ScopeRecordOutcome::Refused => &mut counts.refused,
7626 };
7627 *slot += 1;
7628 }
7629 counts
7630 }
7631}
7632
7633#[cfg(test)]
7634mod scope_outcome_count_tests {
7635 use super::*;
7636
7637 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7638 ScopeRecordResult {
7639 scope_ref: "r".to_string(),
7640 scope_epoch: 1,
7641 outcome,
7642 code: None,
7643 message: None,
7644 version: None,
7645 parent_state: None,
7646 }
7647 }
7648
7649 #[test]
7652 fn every_outcome_is_counted_in_its_own_field() {
7653 let results = [
7654 result(ScopeRecordOutcome::Created),
7655 result(ScopeRecordOutcome::Created),
7656 result(ScopeRecordOutcome::Replaced),
7657 result(ScopeRecordOutcome::Updated),
7658 result(ScopeRecordOutcome::Unchanged),
7659 result(ScopeRecordOutcome::Refused),
7660 result(ScopeRecordOutcome::Refused),
7661 result(ScopeRecordOutcome::Refused),
7662 ];
7663 assert_eq!(
7664 ScopeOutcomeCounts::of(&results),
7665 ScopeOutcomeCounts {
7666 created: 2,
7667 replaced: 1,
7668 updated: 1,
7669 unchanged: 1,
7670 refused: 3,
7671 }
7672 );
7673 }
7674}
7675
7676fn capability_census_trigger(
7680 old: Option<&CapabilityDeclarations>,
7681 new: Option<&CapabilityDeclarations>,
7682) -> bool {
7683 let old_provides = old
7684 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7685 .unwrap_or_default();
7686 let old_denies = old
7687 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7688 .unwrap_or_default();
7689 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7690 provides: Vec::new(),
7691 requires: Vec::new(),
7692 must_never_reach: Vec::new(),
7693 });
7694
7695 new.provides
7696 .iter()
7697 .any(|capability| !old_provides.contains(capability))
7698 || new
7699 .must_never_reach
7700 .iter()
7701 .any(|capability| !old_denies.contains(capability))
7702}
7703
7704fn denied_capability<'a>(
7707 opening_manifest: &'a ModuleManifest,
7708 target_manifest: &ModuleManifest,
7709) -> Option<&'a str> {
7710 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7711 let target_capabilities = target_manifest.capabilities.as_ref()?;
7712 opening_capabilities
7713 .must_never_reach
7714 .iter()
7715 .find(|denied| {
7716 target_capabilities
7717 .provides
7718 .iter()
7719 .any(|provided| provided == *denied)
7720 })
7721 .map(String::as_str)
7722}
7723
7724fn catalog_update_frozen_field_message(
7725 registered: &ModuleManifest,
7726 provides: &[ProviderRole],
7727) -> Option<String> {
7728 let old_has_provides = !registered.provides.is_empty();
7729 let new_has_provides = !provides.is_empty();
7730 if old_has_provides != new_has_provides {
7731 return Some(format!(
7732 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7733 registered.module_id
7734 ));
7735 }
7736
7737 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7738 return Some(format!(
7739 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7740 registered.module_id
7741 ));
7742 }
7743
7744 let registered_concurrency = manifest_concurrency(registered);
7745 let mut candidate = registered.clone();
7746 candidate.provides = provides.to_vec();
7747 let candidate_concurrency = manifest_concurrency(&candidate);
7748 if candidate_concurrency != registered_concurrency {
7749 return Some(format!(
7750 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7751 registered.module_id, registered_concurrency, candidate_concurrency
7752 ));
7753 }
7754
7755 None
7758}
7759
7760fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7761 manifest.provides.iter().any(is_routable_role)
7762}
7763
7764fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7770 manifest
7771 .provides
7772 .iter()
7773 .find_map(|provider| match provider {
7774 ProviderRole::ToolProvider { concurrency, .. }
7775 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7776 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7777 })
7778 .unwrap_or(Concurrency::ModuleManaged)
7779}
7780
7781fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7788 let has_management_surface = manifest
7789 .provides
7790 .iter()
7791 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7792 if !has_management_surface {
7793 return false;
7794 }
7795 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7796 return false;
7797 };
7798 let Some(provides) = raw
7799 .get("manifest")
7800 .and_then(|manifest| manifest.get("provides"))
7801 .and_then(serde_json::Value::as_array)
7802 else {
7803 return false;
7804 };
7805 provides.iter().any(|role| {
7810 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7811 && role.get("concurrency").is_none()
7812 })
7813}
7814
7815fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7816 if peer_version != PROTOCOL_VERSION {
7817 return Err(format!(
7818 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7819 ));
7820 }
7821 Ok(PROTOCOL_VERSION)
7822}
7823
7824fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7825 Frame::build_with_version(
7826 response_version(frame),
7827 FrameType::Pong,
7828 frame.header.flags,
7829 0,
7830 0,
7831 frame.header.corr,
7832 Vec::new(),
7833 )
7834 .map_err(RouterError::FrameBuild)
7835}
7836
7837pub(crate) fn control_error_frame(
7838 frame: &Frame,
7839 code: &'static str,
7840 message: impl Into<String>,
7841) -> Result<Frame, RouterError> {
7842 control_error_body_frame(
7843 frame,
7844 ErrorBody {
7845 code: code.to_string(),
7846 message: message.into(),
7847 detail: None,
7848 },
7849 )
7850}
7851
7852fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7853 let body = serde_json::to_vec(&error).map_err(|err| {
7854 RouterError::backend(
7855 0,
7856 frame.header.corr,
7857 format!("failed to encode control ERROR: {err}"),
7858 )
7859 })?;
7860
7861 Frame::build_with_version(
7862 response_version(frame),
7863 FrameType::Error,
7864 control_flags(),
7865 0,
7866 0,
7867 frame.header.corr,
7868 body,
7869 )
7870 .map_err(RouterError::FrameBuild)
7871}
7872
7873fn control_response_body_frame<T: Serialize>(
7874 frame: &Frame,
7875 reply: &T,
7876 label: &'static str,
7877) -> Result<Frame, RouterError> {
7878 let body = serde_json::to_vec(reply).map_err(|err| {
7879 RouterError::backend(
7880 0,
7881 frame.header.corr,
7882 format!("failed to encode {label}: {err}"),
7883 )
7884 })?;
7885
7886 Frame::build_with_version(
7887 response_version(frame),
7888 FrameType::Response,
7889 control_flags(),
7890 0,
7891 0,
7892 frame.header.corr,
7893 body,
7894 )
7895 .map_err(RouterError::FrameBuild)
7896}
7897
7898fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7928 match err {
7929 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7930 ForwardingError::NoModuleConnection => "target_unavailable",
7931 ForwardingError::ModuleReloading { .. } => "module_reloading",
7932 ForwardingError::ClientRouteChannelExhausted { .. }
7933 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7934 ForwardingError::StaleModuleEndpoint
7935 | ForwardingError::UnknownReservation { .. }
7936 | ForwardingError::ConnectionClosing { .. }
7937 | ForwardingError::ClientEgressClosed { .. }
7938 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7939 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7942 ForwardingError::RelayCorrelationExhausted
7943 | ForwardingError::RouteOpenBuild(_)
7944 | ForwardingError::Poisoned => "forwarding_error",
7945 }
7946}
7947
7948fn response_version(frame: &Frame) -> u8 {
7949 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7950 frame.header.ver
7951 } else {
7952 PROTOCOL_VERSION
7953 }
7954}
7955
7956fn control_flags() -> Flags {
7957 Flags::new(false, Priority::Passive, false)
7958}
7959
7960fn send_goodbye_target_best_effort(
7965 counters: &DaemonCounters,
7966 target: &GoodbyeTarget,
7967 context: &'static str,
7968) {
7969 let Ok(frame) = Frame::build_with_version(
7970 target.negotiated_ver,
7971 FrameType::Goodbye,
7972 control_flags(),
7973 target.channel,
7974 target.epoch,
7975 0,
7976 Vec::new(),
7977 ) else {
7978 return;
7979 };
7980 crate::forwarding::send_module_route_goodbye(
7981 counters,
7982 &target.sink,
7983 frame,
7984 target.module_id.as_deref(),
7985 context,
7986 );
7987}
7988
7989pub(crate) fn send_route_control_pushes(
7990 forwarding: &ForwardingTable,
7991 routes: Vec<EndpointRoute>,
7992 push: ClientControlPush,
7993) {
7994 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7995 for route in routes {
7996 let target = route.goodbye_target;
7997 if let Some((existing, channels)) = targets
7998 .iter_mut()
7999 .find(|(existing, _)| existing.connection_id == target.connection_id)
8000 {
8001 debug_assert_eq!(
8002 existing.negotiated_ver, target.negotiated_ver,
8003 "one connection cannot negotiate multiple frame versions"
8004 );
8005 if !channels.contains(&target.channel) {
8006 channels.push(target.channel);
8007 }
8008 continue;
8009 }
8010 let channel = target.channel;
8011 targets.push((target, vec![channel]));
8012 }
8013 for (target, mut channels) in targets {
8014 channels.sort_unstable();
8015 let mut push = push.clone();
8016 match &mut push {
8017 ClientControlPush::RouteClosing {
8018 channels: covered, ..
8019 }
8020 | ClientControlPush::RouteClosed {
8021 channels: covered, ..
8022 } => *covered = channels,
8023 }
8024 let body = match serde_json::to_vec(&push) {
8025 Ok(body) => body,
8026 Err(err) => {
8027 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8028 continue;
8029 }
8030 };
8031 let frame = match Frame::build_with_version(
8032 target.negotiated_ver,
8033 FrameType::Push,
8034 control_flags(),
8035 0,
8036 0,
8037 0,
8038 body.clone(),
8039 ) {
8040 Ok(frame) => frame,
8041 Err(err) => {
8042 warn!(
8043 route_channel = target.channel,
8044 error = %err,
8045 "failed to build route lifecycle control PUSH frame"
8046 );
8047 continue;
8048 }
8049 };
8050 if let Err(err) = target.sink.try_send(frame) {
8051 if target.close_on_delivery_failure() {
8052 warn!(
8053 target_connection_id = target.connection_id.get(),
8054 route_channel = target.channel,
8055 error = %err,
8056 "route lifecycle control PUSH was not delivered to client; closing target connection"
8057 );
8058 let _ = forwarding.escalate_client_delivery_failure(
8059 target.connection_id,
8060 target.channel,
8061 target.epoch,
8062 CloseReason::new(
8063 "route_lifecycle_push_delivery_failed",
8064 format!(
8065 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8066 target.channel
8067 ),
8068 ),
8069 crate::forwarding::UndeliveredFrame {
8070 module_id: target.module_id.as_deref(),
8071 sink: &target.sink,
8072 },
8073 );
8074 }
8075 }
8076 }
8077}
8078
8079#[cfg(test)]
8080mod tests {
8081 mod activity_read {
8082 include!("activity_read_tests.rs");
8083 }
8084
8085 mod lifecycle_activity {
8086 include!("control_lifecycle_tests.rs");
8087 }
8088 #[cfg(unix)]
8089 #[tokio::test]
8090 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8091 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8092 let path = dir.join("subc.jsonc");
8093 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8094 "program":"/bin/sleep","args":["60"],"protocol":"none",
8095 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8096 }}}).to_string()).unwrap();
8097 let mut configured = crate::daemon_config::load(&path)
8098 .unwrap()
8099 .unwrap()
8100 .modules
8101 .pop()
8102 .unwrap();
8103 let registry = std::sync::Arc::new(crate::Registry::default());
8104 let handle = crate::SupervisorHandle::new();
8105 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8106 .with_handle(handle.clone());
8107 let module = supervisor
8108 .supervise_configured_with_health(
8109 configured.module_spec(),
8110 true,
8111 configured.health.clone(),
8112 None,
8113 configured.restart,
8114 )
8115 .unwrap();
8116 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8117 let before = module.status().unwrap().pid;
8118 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8119 configured.health.cadence = std::time::Duration::from_secs(3600);
8120 let health_only = handler
8121 .reconcile_supervised_modules(
8122 &supervisor,
8123 vec![configured.clone()],
8124 false,
8125 Principal::Direct,
8126 )
8127 .await
8128 .unwrap();
8129 assert!(
8130 health_only.0.changed_pending_reload.is_empty(),
8131 "health policy is already applied live"
8132 );
8133 assert_eq!(module.status().unwrap().pid, before);
8134 assert_eq!(
8135 module.configuration().unwrap().1.http,
8136 configured.health.http
8137 );
8138 configured.args = vec!["61".into()];
8139 let launch = handler
8140 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8141 .await
8142 .unwrap();
8143 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8144 assert_eq!(
8145 module.status().unwrap().pid,
8146 before,
8147 "a launch edit is stored until reload"
8148 );
8149 module.drain().await.unwrap();
8150 }
8151 use cortexkit_test_support::ScratchDir;
8152 use std::{
8153 collections::BTreeMap,
8154 fmt,
8155 path::PathBuf,
8156 sync::{Arc, Mutex},
8157 time::Duration,
8158 };
8159
8160 use serde_json::{json, Value};
8161 use subc_protocol::{
8162 manifest::{
8163 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8164 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8165 },
8166 session::HealthStatus,
8167 FrameType,
8168 };
8169
8170 use super::*;
8171 use crate::{
8172 forwarding::{DataRoute, DataRouteState},
8173 registry::ChannelState,
8174 router::FrameSink,
8175 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8176 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8177 RouteCtx, Router,
8178 };
8179 use tokio::{
8180 sync::mpsc,
8181 time::{sleep, Instant},
8182 };
8183 use tracing::{
8184 field::{Field, Visit},
8185 Event, Subscriber,
8186 };
8187 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8188
8189 fn fake_aft_stub_path() -> PathBuf {
8208 let mut path = std::env::current_exe().expect("current_exe available in tests");
8209 path.pop(); path.pop(); path.push(if cfg!(windows) {
8212 "fake-aft-stub.exe"
8213 } else {
8214 "fake-aft-stub"
8215 });
8216 assert!(
8217 path.exists(),
8218 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8219 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8220 path.display()
8221 );
8222 path
8223 }
8224
8225 fn client_retries(code: &str) -> bool {
8230 subc_protocol::error_codes::is_retryable_route_open(code)
8231 }
8232
8233 #[test]
8244 fn retryability_of_forwarding_codes_matches_the_failure() {
8245 let transient = [
8248 ForwardingError::NoModuleConnection,
8249 ForwardingError::ModuleReloading {
8250 module_id: "m".into(),
8251 },
8252 ForwardingError::StaleModuleEndpoint,
8253 ForwardingError::UnknownReservation {
8254 client_channel: 1,
8255 module_channel: 1,
8256 },
8257 ForwardingError::ConnectionClosing {
8258 connection_id: ConnectionId::new(1),
8259 },
8260 ForwardingError::ClientEgressClosed {
8261 connection_id: ConnectionId::new(1),
8262 },
8263 ForwardingError::ModuleEgressUnavailable {
8264 connection_id: ConnectionId::new(1),
8265 },
8266 ];
8267 for err in transient {
8268 let code = forwarding_error_code(&err);
8269 assert!(
8270 client_retries(code),
8271 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8272 );
8273 }
8274
8275 let permanent = [
8280 ForwardingError::ConnectionRoleConflict {
8281 connection_id: ConnectionId::new(1),
8282 },
8283 ForwardingError::ClientRouteChannelExhausted {
8284 connection_id: ConnectionId::new(1),
8285 },
8286 ForwardingError::ModuleRouteChannelExhausted {
8287 endpoint: ModuleEndpointId {
8288 connection_id: ConnectionId::new(1),
8289 generation: 1,
8290 },
8291 },
8292 ForwardingError::RelayCorrelationExhausted,
8293 ForwardingError::RouteOpenBuild("x".into()),
8294 ForwardingError::Poisoned,
8295 ];
8296 for err in permanent {
8297 let code = forwarding_error_code(&err);
8298 assert!(
8299 !client_retries(code),
8300 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8301 );
8302 }
8303 }
8304
8305 #[tokio::test]
8333 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8334 let handler = ControlHandler::default();
8335 let frame =
8336 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8337
8338 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8342 assert_eq!(
8343 stamped,
8344 Principal::Direct,
8345 "a caller that proved nothing must not be stamped as a supervised module"
8346 );
8347
8348 let forged = handler
8353 .route_open_principal(
8354 &frame,
8355 Some(ConsumerIdentity {
8356 module_id: "aft".to_string(),
8357 launch_nonce: "not-a-real-nonce".to_string(),
8358 }),
8359 )
8360 .unwrap();
8361 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8362 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8363 }
8364
8365 #[test]
8382 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8383 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8384 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8385 let ClientControlRequest::RouteOpen {
8386 consumer_identity, ..
8387 } = parsed
8388 else {
8389 panic!("route.open body must parse as RouteOpen");
8390 };
8391 assert_eq!(
8392 consumer_identity,
8393 Some(ConsumerIdentity {
8394 module_id: "aft".to_string(),
8395 launch_nonce: "n".to_string(),
8396 }),
8397 "the wire field name must reach the value route_open_principal reads"
8398 );
8399 }
8400
8401 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8402 ModuleManifest::builder(module_id, "0.1.0")
8403 .protocol_ver(protocol_ver)
8404 .provides(vec![ProviderRole::ToolProvider {
8405 tools: vec![Tool {
8406 name: "read".to_string(),
8407 description: None,
8408 execution_mode: ExecutionMode::Pure,
8409 schema: json!({"type": "object"}),
8410 }],
8411 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8412 concurrency: Concurrency::ModuleManaged,
8413 emits_push: true,
8414 sub_supervises: true,
8415 }])
8416 .build()
8417 }
8418
8419 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8420 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8421 }
8422
8423 fn hello_frame_with_control_ops(
8424 module_id: &str,
8425 protocol_ver: u8,
8426 corr: u64,
8427 control_ops: Option<Vec<String>>,
8428 ) -> Frame {
8429 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8430 }
8431
8432 fn hello_frame_with_nonce(
8433 module_id: &str,
8434 protocol_ver: u8,
8435 corr: u64,
8436 launch_nonce: Option<&str>,
8437 ) -> Frame {
8438 hello_frame_full(
8439 module_id,
8440 protocol_ver,
8441 corr,
8442 None,
8443 launch_nonce.map(ToOwned::to_owned),
8444 )
8445 }
8446
8447 fn hello_frame_full(
8448 module_id: &str,
8449 protocol_ver: u8,
8450 corr: u64,
8451 control_ops: Option<Vec<String>>,
8452 launch_nonce: Option<String>,
8453 ) -> Frame {
8454 let body = serde_json::to_vec(&ModuleHelloBody {
8455 manifest: manifest(module_id, protocol_ver),
8456 protocol_ver,
8457 control_ops,
8458 launch_nonce,
8459 })
8460 .unwrap();
8461 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8462 }
8463
8464 fn hello_frame_with_rich_ui(
8465 module_id: &str,
8466 protocol_ver: u8,
8467 corr: u64,
8468 launch_nonce: Option<&str>,
8469 rich_ui: Value,
8470 ) -> Frame {
8471 let mut body = serde_json::to_value(ModuleHelloBody {
8472 manifest: manifest(module_id, protocol_ver),
8473 protocol_ver,
8474 control_ops: None,
8475 launch_nonce: launch_nonce.map(str::to_string),
8476 })
8477 .unwrap();
8478 body["manifest"]["rich_ui"] = rich_ui;
8479 Frame::build(
8480 FrameType::Hello,
8481 control_flags(),
8482 0,
8483 0,
8484 corr,
8485 serde_json::to_vec(&body).unwrap(),
8486 )
8487 .unwrap()
8488 }
8489
8490 fn non_routable_hello_frame_with_control_ops(
8491 module_id: &str,
8492 corr: u64,
8493 control_ops: Option<Vec<String>>,
8494 ) -> Frame {
8495 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8496 manifest.provides.clear();
8497 let body = serde_json::to_vec(&ModuleHelloBody {
8498 manifest,
8499 protocol_ver: PROTOCOL_VERSION,
8500 control_ops,
8501 launch_nonce: None,
8502 })
8503 .unwrap();
8504 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8505 }
8506
8507 fn capability_grammar_hello_frame(
8508 capabilities: Value,
8509 runtime_computed: Option<Value>,
8510 corr: u64,
8511 ) -> Frame {
8512 let mut body = serde_json::to_value(ModuleHelloBody {
8513 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8514 protocol_ver: PROTOCOL_VERSION,
8515 control_ops: None,
8516 launch_nonce: None,
8517 })
8518 .expect("HELLO body serializes");
8519 body["manifest"]["capabilities"] = capabilities;
8520 if let Some(runtime_computed) = runtime_computed {
8521 body["runtime_computed"] = runtime_computed;
8522 }
8523 Frame::build(
8524 FrameType::Hello,
8525 control_flags(),
8526 0,
8527 0,
8528 corr,
8529 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8530 )
8531 .expect("HELLO frame builds")
8532 }
8533
8534 fn channel_request(channel: u16, corr: u64) -> Frame {
8535 Frame::build(
8536 FrameType::Request,
8537 Flags::new(true, Priority::Interactive, false),
8538 channel,
8539 0,
8540 corr,
8541 b"opaque".to_vec(),
8542 )
8543 .unwrap()
8544 }
8545
8546 fn route_ctx(
8547 connection_id: ConnectionId,
8548 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8549 let (tx, rx) = mpsc::channel(8);
8550 (
8551 RouteCtx {
8552 connection_id,
8553 egress: FrameSink::new(tx),
8554 },
8555 rx,
8556 )
8557 }
8558
8559 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8560 serde_json::from_slice(&frame.body).unwrap()
8561 }
8562
8563 async fn hello_via_sink(
8568 handler: &ControlHandler,
8569 ctx: &RouteCtx,
8570 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8571 hello: Frame,
8572 ) -> Frame {
8573 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8574 assert!(
8575 replies.is_empty(),
8576 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8577 );
8578 let ack = rx
8579 .try_recv()
8580 .expect("HELLO_ACK is queued on the module sink")
8581 .frame;
8582 assert_eq!(ack.header.ty, FrameType::HelloAck);
8583 ack
8584 }
8585
8586 fn parse_error(frame: &Frame) -> Value {
8587 serde_json::from_slice(&frame.body).unwrap()
8588 }
8589
8590 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8591 serde_json::from_slice(&frame.body).unwrap()
8592 }
8593
8594 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8595 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8596 route_channel,
8597 route_epoch: 0,
8598 kind,
8599 })
8600 .unwrap();
8601 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8602 }
8603
8604 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8605 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8606 module_id: module_id.to_string(),
8607 })
8608 .unwrap();
8609 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8610 }
8611
8612 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8613 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8614 }
8615
8616 fn route_open_frame_with_consumer_capabilities(
8617 corr: u64,
8618 module_id: &str,
8619 project_root: ScratchDir,
8620 consumer_capabilities: Option<Vec<String>>,
8621 ) -> Frame {
8622 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8623 target: RouteTarget::ToolProvider {
8624 module_id: module_id.to_string(),
8625 },
8626 identity: BindIdentity::new(
8627 project_root.path().to_path_buf(),
8628 "unit".to_string(),
8629 "session".to_string(),
8630 ),
8631 consumer_identity: None,
8632 consumer_capabilities,
8633 role_versions: None,
8634 admission_facts: None,
8635 scope: None,
8636 })
8637 .unwrap();
8638 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8639 }
8640
8641 fn route_open_frame_with_role_versions(
8642 corr: u64,
8643 module_id: &str,
8644 project_root: ScratchDir,
8645 role_versions: Option<BTreeMap<String, String>>,
8646 ) -> Frame {
8647 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8648 target: RouteTarget::ToolProvider {
8649 module_id: module_id.to_string(),
8650 },
8651 identity: BindIdentity::new(
8652 project_root.path().to_path_buf(),
8653 "unit".to_string(),
8654 format!("session-{corr}"),
8655 ),
8656 consumer_identity: None,
8657 consumer_capabilities: None,
8658 role_versions,
8659 admission_facts: None,
8660 scope: None,
8661 })
8662 .unwrap();
8663 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8664 }
8665
8666 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8667 entries
8668 .iter()
8669 .map(|(role, version)| (role.to_string(), version.to_string()))
8670 .collect()
8671 }
8672
8673 fn route_open_frame_with_admission_facts(
8674 corr: u64,
8675 module_id: &str,
8676 project_root: ScratchDir,
8677 consumer_identity: Option<subc_control::ConsumerIdentity>,
8678 facts: Option<Value>,
8679 ) -> Frame {
8680 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8681 target: RouteTarget::ToolProvider {
8682 module_id: module_id.to_string(),
8683 },
8684 identity: BindIdentity::new(
8685 project_root.path().to_path_buf(),
8686 "unit".to_string(),
8687 format!("session-{corr}"),
8688 ),
8689 consumer_identity,
8690 consumer_capabilities: None,
8691 role_versions: None,
8692 admission_facts: facts,
8693 scope: None,
8694 })
8695 .unwrap();
8696 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8697 }
8698
8699 #[derive(Clone, Default)]
8700 struct EventCapture {
8701 events: Arc<Mutex<Vec<CapturedEvent>>>,
8702 }
8703
8704 #[derive(Clone, Debug)]
8705 struct CapturedEvent {
8706 target: String,
8707 level: tracing::Level,
8708 fields: BTreeMap<String, String>,
8709 }
8710
8711 impl EventCapture {
8712 fn events(&self) -> Vec<CapturedEvent> {
8713 self.events.lock().unwrap().clone()
8714 }
8715 }
8716
8717 impl<S> Layer<S> for EventCapture
8718 where
8719 S: Subscriber,
8720 {
8721 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8722 let mut visitor = EventFieldVisitor::default();
8723 event.record(&mut visitor);
8724 self.events.lock().unwrap().push(CapturedEvent {
8725 target: event.metadata().target().to_string(),
8726 level: *event.metadata().level(),
8727 fields: visitor.fields,
8728 });
8729 }
8730 }
8731
8732 #[derive(Default)]
8733 struct EventFieldVisitor {
8734 fields: BTreeMap<String, String>,
8735 }
8736
8737 impl Visit for EventFieldVisitor {
8738 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8739 self.fields
8740 .insert(field.name().to_string(), format!("{value:?}"));
8741 }
8742 }
8743
8744 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8745 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8746 status,
8747 detail: Some("warming".to_string()),
8748 metrics: Some(json!({"queue_depth": 3})),
8749 })
8750 .unwrap();
8751 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8752 }
8753
8754 fn route_bind_ack(corr: u64) -> Frame {
8755 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8756 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8757 }
8758
8759 fn unique_project_root(label: &str) -> ScratchDir {
8760 ScratchDir::new(label)
8761 }
8762
8763 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8764 match parse_route_poll(frame) {
8765 ClientControlResponse::RoutePoll {
8766 status: None,
8767 live: Some(live),
8768 ..
8769 } => assert_eq!(live, expected_live),
8770 other => panic!("unexpected route.poll response: {other:?}"),
8771 }
8772 }
8773
8774 fn bind_liveness_route(
8775 registry: &Registry,
8776 forwarding: &ForwardingTable,
8777 module_id: &str,
8778 ) -> (RouteCtx, u16, u32) {
8779 let module_connection = ConnectionId::new(101);
8780 let client_connection = ConnectionId::new(202);
8781 let registration = registry
8782 .register_with_control_ops(
8783 manifest(module_id, PROTOCOL_VERSION),
8784 PROTOCOL_VERSION,
8785 module_connection,
8786 module_baseline_control_ops(),
8787 )
8788 .unwrap();
8789 let (module_tx, _module_rx) = mpsc::channel(8);
8790 let endpoint = forwarding
8791 .register_module_connection(
8792 module_connection,
8793 module_id.to_string(),
8794 PROTOCOL_VERSION,
8795 manifest_concurrency(®istration.manifest),
8796 FrameSink::new(module_tx),
8797 )
8798 .unwrap();
8799 let (client_ctx, _client_rx) = route_ctx(client_connection);
8800 let pending = forwarding
8801 .begin_route_bind_relay_for_test(
8802 client_connection,
8803 client_ctx.egress.clone(),
8804 1,
8805 module_id,
8806 )
8807 .unwrap();
8808 assert_eq!(pending.endpoint, endpoint);
8809 let route_channel = pending.client_channel;
8810 let route_epoch = pending.client_epoch;
8811 forwarding
8812 .complete_pending_relay(
8813 module_connection,
8814 pending.corr,
8815 RouteBindRelayOutcome::Accepted,
8816 )
8817 .unwrap();
8818 (client_ctx, route_channel, route_epoch)
8819 }
8820
8821 struct FakeProcessLiveness {
8822 live: Option<bool>,
8823 }
8824
8825 impl ModuleProcessLiveness for FakeProcessLiveness {
8826 fn process_live(&self, _module_id: &str) -> Option<bool> {
8827 self.live
8828 }
8829 }
8830
8831 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8832 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8833 {
8834 let registry = Arc::new(Registry::default());
8835 let supervisor_handle = SupervisorHandle::new();
8836 let supervisor = Supervisor::new_for_test(
8837 Arc::clone(®istry),
8838 RestartPolicy::new(1, Duration::from_millis(10)),
8839 )
8840 .with_handle(supervisor_handle.clone());
8841 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8842 let module = supervisor
8843 .spawn(ModuleSpec {
8844 module_id: "stderr-tail-wire".to_string(),
8845 program: fake_aft_stub_path(),
8846 args: Vec::new(),
8847 env: vec![
8848 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8849 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8850 ],
8851 reserved: false,
8852 reserved_prefixes: Vec::new(),
8853 protocol: ModuleProtocol::Subc,
8854 overlap: Default::default(),
8855 })
8856 .unwrap();
8857
8858 let deadline = Instant::now() + Duration::from_secs(5);
8859 loop {
8860 let tail = module.stderr_tail(None, None);
8861 if tail
8862 .entries
8863 .iter()
8864 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8865 && tail.entries.iter().any(|entry| {
8866 matches!(
8867 entry,
8868 TailEntry::Line {
8869 truncated: true,
8870 ..
8871 }
8872 )
8873 })
8874 {
8875 break;
8876 }
8877 assert!(
8878 Instant::now() < deadline,
8879 "module did not produce a truncated line and restart boundary: {tail:?}"
8880 );
8881 sleep(Duration::from_millis(10)).await;
8882 }
8883
8884 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8885 let request = ClientControlRequest::SupervisorStderrTail {
8886 module_id: "stderr-tail-wire".to_string(),
8887 max_lines: None,
8888 max_bytes: None,
8889 };
8890 let frame = Frame::build(
8891 FrameType::Request,
8892 control_flags(),
8893 0,
8894 0,
8895 1,
8896 serde_json::to_vec(&request).unwrap(),
8897 )
8898 .unwrap();
8899 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8900 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8901 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8902 serde_json::from_slice(&responses[0].body).unwrap()
8903 else {
8904 panic!("expected supervisor.stderr_tail response");
8905 };
8906
8907 assert!(
8908 tail.entries
8909 .iter()
8910 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8911 "the control response lost the restart boundary"
8912 );
8913 let Some(StderrTailEntry::Line {
8914 text,
8915 truncated,
8916 at_ms,
8917 }) = tail.entries.iter().find(|entry| {
8918 matches!(
8919 entry,
8920 StderrTailEntry::Line {
8921 truncated: true,
8922 ..
8923 }
8924 )
8925 })
8926 else {
8927 panic!("the control response lost the truncated line");
8928 };
8929 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8930 assert!(*truncated);
8931 assert!(
8932 at_ms.is_some(),
8933 "the control response lost the line's capture time"
8934 );
8935 }
8936
8937 #[tokio::test(flavor = "current_thread")]
8942 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8943 let dir = ScratchDir::new("terminals-off-worker");
8944 let journal_path = dir.join("terminals.jsonl");
8945 let registry = Arc::new(Registry::default());
8946 let supervisor_handle = SupervisorHandle::new();
8947 let supervisor =
8948 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8949 .with_handle(supervisor_handle.clone())
8950 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8951 let module = supervisor
8952 .spawn(ModuleSpec {
8953 module_id: "terminal-off-worker".to_string(),
8954 program: fake_aft_stub_path(),
8955 args: Vec::new(),
8956 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8957 reserved: false,
8958 reserved_prefixes: Vec::new(),
8959 protocol: ModuleProtocol::Subc,
8960 overlap: Default::default(),
8961 })
8962 .unwrap();
8963 let deadline = Instant::now() + Duration::from_secs(5);
8964 while module.terminal_history().entries.len() != 2 {
8965 assert!(Instant::now() < deadline, "module did not record two exits");
8966 sleep(Duration::from_millis(10)).await;
8967 }
8968
8969 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8970 let handler =
8971 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8972 let frame = Frame::build(
8973 FrameType::Request,
8974 control_flags(),
8975 0,
8976 0,
8977 1,
8978 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8979 module_id: "terminal-off-worker".to_string(),
8980 })
8981 .unwrap(),
8982 )
8983 .unwrap();
8984 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8985 let spawned_at = std::time::Instant::now();
8986 let read = tokio::spawn({
8987 let handler = Arc::clone(&handler);
8988 async move { handler.handle_control_frame(&ctx, frame).await }
8989 });
8990 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8993 .await
8994 .unwrap()
8995 .expect("the history read reached its pause");
8996 let elapsed = spawned_at.elapsed();
8997 assert!(
8998 elapsed < Duration::from_secs(2) && !read.is_finished(),
8999 "this task could not run while the history read was paused \
9000 (resumed after {elapsed:?}, read finished: {})",
9001 read.is_finished()
9002 );
9003
9004 drop(release);
9005 let responses = read.await.unwrap().unwrap();
9006 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9007 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9008 panic!("expected supervisor.terminals response");
9009 };
9010 assert_eq!(terminals.entries.len(), 2);
9011 assert_eq!(terminals.journal_skipped_lines, 0);
9012 assert_eq!(terminals.journal_read_errors, 0);
9013 }
9014
9015 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9016 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9017 let registry = Arc::new(Registry::default());
9018 let supervisor_handle = SupervisorHandle::new();
9019 let supervisor =
9020 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9021 .with_handle(supervisor_handle.clone());
9022 let module = supervisor
9023 .spawn(ModuleSpec {
9024 module_id: "terminal-golden".to_string(),
9025 program: fake_aft_stub_path(),
9026 args: Vec::new(),
9027 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9028 reserved: false,
9029 reserved_prefixes: Vec::new(),
9030 protocol: ModuleProtocol::Subc,
9031 overlap: Default::default(),
9032 })
9033 .unwrap();
9034
9035 let deadline = Instant::now() + Duration::from_secs(5);
9036 while module.terminal_history().entries.len() != 2 {
9037 assert!(
9038 Instant::now() < deadline,
9039 "module did not retain two terminal exits: {:?}",
9040 module.terminal_history()
9041 );
9042 sleep(Duration::from_millis(10)).await;
9043 }
9044
9045 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9046 let request = ClientControlRequest::SupervisorTerminals {
9047 module_id: "terminal-golden".to_string(),
9048 };
9049 let frame = Frame::build(
9050 FrameType::Request,
9051 control_flags(),
9052 0,
9053 0,
9054 1,
9055 serde_json::to_vec(&request).unwrap(),
9056 )
9057 .unwrap();
9058 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9059 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9060 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9061 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9062 panic!("expected supervisor.terminals response");
9063 };
9064 assert_eq!(terminals.entries.len(), 2);
9065 assert_eq!(terminals.dropped, 0);
9066
9067 let mut rendered = serde_json::to_value(response).unwrap();
9068 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9071 for (index, entry) in rendered["entries"]
9072 .as_array_mut()
9073 .expect("terminal response entries array")
9074 .iter_mut()
9075 .enumerate()
9076 {
9077 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9078 }
9079
9080 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9081 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9082 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9083 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9084 std::fs::write(&golden_path, &serialized).unwrap();
9085 }
9086 let expected: Value =
9087 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9088 assert_eq!(rendered, expected);
9089 }
9090
9091 #[test]
9092 fn hello_registers_manifest_and_returns_ack() {
9093 let registry = Arc::new(Registry::default());
9094 let handler = ControlHandler::new(Arc::clone(®istry));
9095 let conn = ConnectionId::new(1);
9096
9097 let responses = handler
9098 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9099 .unwrap();
9100
9101 assert_eq!(responses.len(), 1);
9102 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9103 assert_eq!(responses[0].header.channel, 0);
9104 assert_eq!(responses[0].header.corr, 7);
9105 let ack = parse_ack(&responses[0]);
9106 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9107 assert!(ack
9108 .subc_capabilities
9109 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9110 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9111 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9112 assert!(ack
9113 .subc_ops
9114 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9115 assert!(ack
9116 .subc_ops
9117 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9118
9119 let registration = registry.get_module("aft").unwrap().unwrap();
9120 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9121 assert_eq!(registration.state, ChannelState::Active);
9122 assert_eq!(registration.connection_id, conn);
9123 assert_eq!(registration.control_ops, module_baseline_control_ops());
9124 }
9125
9126 #[test]
9127 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9128 let invalid_identifiers = [
9129 ("case_change", "credentials-Provider/v1"),
9130 ("leading_zero", "credentials-provider/v01"),
9131 ("trailing_hyphen", "credentials-provider-/v1"),
9132 ("consecutive_hyphens", "credentials--provider/v1"),
9133 ("uppercase", "Credentials-provider/v1"),
9134 ("missing_v", "credentials-provider/1"),
9135 ("whitespace", "credentials provider/v1"),
9136 ("zero_version", "credentials-provider/v0"),
9137 ("out_of_range_version", "credentials-provider/v4294967296"),
9138 (
9139 "overlength_name",
9140 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9141 ),
9142 ];
9143 let mut cases = invalid_identifiers
9144 .into_iter()
9145 .map(|(name, identifier)| {
9146 (
9147 format!("identifier_{name}"),
9148 "capabilities.provides[0]".to_string(),
9149 identifier.to_string(),
9150 json!({ "provides": [identifier] }),
9151 None,
9152 )
9153 })
9154 .collect::<Vec<_>>();
9155 cases.extend([
9156 (
9157 "unknown_need".to_string(),
9158 "capabilities.requires[0].need".to_string(),
9159 "deferred".to_string(),
9160 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9161 None,
9162 ),
9163 (
9164 "duplicate_provides".to_string(),
9165 "capabilities.provides[1]".to_string(),
9166 "credentials-provider/v1".to_string(),
9167 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9168 None,
9169 ),
9170 (
9171 "duplicate_must_never_reach".to_string(),
9172 "capabilities.must_never_reach[1]".to_string(),
9173 "credentials-provider/v1".to_string(),
9174 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9175 None,
9176 ),
9177 (
9178 "duplicate_requires_same_need".to_string(),
9179 "capabilities.requires[1]".to_string(),
9180 "credentials-provider/v1".to_string(),
9181 json!({ "requires": [
9182 { "capability": "credentials-provider/v1", "need": "required" },
9183 { "capability": "credentials-provider/v1", "need": "required" }
9184 ] }),
9185 None,
9186 ),
9187 (
9188 "duplicate_requires_conflicting_need".to_string(),
9189 "capabilities.requires[1]".to_string(),
9190 "credentials-provider/v1".to_string(),
9191 json!({ "requires": [
9192 { "capability": "credentials-provider/v1", "need": "required" },
9193 { "capability": "credentials-provider/v1", "need": "optional" }
9194 ] }),
9195 None,
9196 ),
9197 (
9198 "capabilities_root_pointer".to_string(),
9199 "runtime_computed[0]".to_string(),
9200 "/capabilities".to_string(),
9201 json!({}),
9202 Some(json!(["/capabilities"])),
9203 ),
9204 (
9205 "capabilities_descendant_pointer".to_string(),
9206 "runtime_computed[0]".to_string(),
9207 "/capabilities/provides".to_string(),
9208 json!({}),
9209 Some(json!(["/capabilities/provides"])),
9210 ),
9211 (
9212 "malformed_pointer_without_leading_slash".to_string(),
9213 "runtime_computed[0]".to_string(),
9214 "capabilities".to_string(),
9215 json!({}),
9216 Some(json!(["capabilities"])),
9217 ),
9218 (
9219 "malformed_pointer_escape".to_string(),
9220 "runtime_computed[0]".to_string(),
9221 "/roles/~2/tools".to_string(),
9222 json!({}),
9223 Some(json!(["/roles/~2/tools"])),
9224 ),
9225 (
9226 "unknown_capabilities_field".to_string(),
9227 "capabilities.future".to_string(),
9228 "<array>".to_string(),
9229 json!({ "future": [] }),
9230 None,
9231 ),
9232 ]);
9233
9234 for (index, (name, field, value, capabilities, runtime_computed)) in
9235 cases.into_iter().enumerate()
9236 {
9237 let registry = Arc::new(Registry::default());
9238 let handler = ControlHandler::new(Arc::clone(®istry));
9239 let response = handler
9240 .handle_control(
9241 ConnectionId::new((index + 1) as u64),
9242 capability_grammar_hello_frame(
9243 capabilities,
9244 runtime_computed,
9245 index as u64 + 1,
9246 ),
9247 )
9248 .expect("invalid HELLO returns a refusal");
9249
9250 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9251 let error = parse_error(&response[0]);
9252 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9253 let message = error["message"]
9254 .as_str()
9255 .expect("error message is a string");
9256 assert!(
9257 message.contains(&field),
9258 "{name}: field missing from {message}"
9259 );
9260 assert!(
9261 message.contains(&value),
9262 "{name}: value missing from {message}"
9263 );
9264 assert_eq!(
9265 registry
9266 .active_registration_count()
9267 .expect("registry reads"),
9268 0,
9269 "{name}: refused HELLO must not create a catalog entry"
9270 );
9271 }
9272 }
9273
9274 #[test]
9275 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9276 let registry = Arc::new(Registry::default());
9277 let handler = ControlHandler::new(Arc::clone(®istry));
9278 let capabilities = json!({
9279 "provides": ["credentials-provider/v1"],
9280 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9281 "must_never_reach": ["federation-transport/v1"]
9282 });
9283 let response = handler
9284 .handle_control(
9285 ConnectionId::new(99),
9286 capability_grammar_hello_frame(
9287 capabilities.clone(),
9288 Some(json!(["/roles/0/tools"])),
9289 99,
9290 ),
9291 )
9292 .expect("valid HELLO registers");
9293 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9294
9295 let request = Frame::build(
9296 FrameType::Request,
9297 control_flags(),
9298 0,
9299 0,
9300 100,
9301 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9302 .expect("catalog request serializes"),
9303 )
9304 .expect("catalog request frame builds");
9305 let response = handler
9306 .handle_catalog_list(request, None)
9307 .expect("catalog list succeeds");
9308 let ClientControlResponse::CatalogList { modules, .. } =
9309 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9310 else {
9311 panic!("catalog request must return catalog.list");
9312 };
9313 assert_eq!(modules.len(), 1);
9314 assert_eq!(
9315 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9316 capabilities
9317 );
9318 }
9319
9320 #[test]
9321 fn catalog_list_mirrors_management_operation_description() {
9322 let registry = Arc::new(Registry::default());
9323 let handler = ControlHandler::new(Arc::clone(®istry));
9324 let description = "List managed records and return their identifiers and metadata.";
9325 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9326 manifest.provides = vec![ProviderRole::ManagementSurface {
9327 operations: vec![ManagementOperation {
9328 name: "records.list".to_string(),
9329 kind: ManagementOperationKind::Query,
9330 description: Some(description.to_string()),
9331 }],
9332 config_schema: json!({"type": "object"}),
9333 observability: vec![ObservabilitySurface {
9334 name: "records.stats".to_string(),
9335 kind: ObservabilityKind::Snapshot,
9336 }],
9337 identity_scope: vec![IdentityScope::Project],
9338 concurrency: Concurrency::ModuleManaged,
9339 }];
9340 registry
9341 .register_with_control_ops(
9342 manifest,
9343 PROTOCOL_VERSION,
9344 ConnectionId::new(99),
9345 Vec::new(),
9346 )
9347 .expect("described management manifest registers");
9348
9349 let request = Frame::build(
9350 FrameType::Request,
9351 control_flags(),
9352 0,
9353 0,
9354 100,
9355 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9356 .expect("catalog request serializes"),
9357 )
9358 .expect("catalog request frame builds");
9359 let response = handler
9360 .handle_catalog_list(request, None)
9361 .expect("catalog list succeeds");
9362 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9363 assert_eq!(
9364 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9365 "catalog.list must preserve the declared operation description verbatim"
9366 );
9367 }
9368
9369 #[test]
9370 fn catalog_list_mirrors_registered_operation_delivery() {
9371 let registry = Arc::new(Registry::default());
9372 let handler = ControlHandler::new(Arc::clone(®istry));
9373 let delivery =
9374 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9375 let mut body = serde_json::to_value(ModuleHelloBody {
9376 manifest: manifest("watch-management", PROTOCOL_VERSION),
9377 protocol_ver: PROTOCOL_VERSION,
9378 control_ops: None,
9379 launch_nonce: None,
9380 })
9381 .unwrap();
9382 body["manifest"]["provides"] = json!([{
9383 "role": "management_surface",
9384 "operations": [
9385 {"name": "status", "kind": "query"},
9386 {"name": "future", "kind": "query"},
9387 {"name": "reset", "kind": "mutate"}
9388 ],
9389 "config_schema": {}, "observability": [], "identity_scope": []
9390 }]);
9391 body["manifest"]["operation_delivery"] = delivery.clone();
9392 let hello = Frame::build(
9393 FrameType::Hello,
9394 control_flags(),
9395 0,
9396 0,
9397 99,
9398 serde_json::to_vec(&body).unwrap(),
9399 )
9400 .unwrap();
9401 let response = handler
9402 .handle_control(ConnectionId::new(99), hello)
9403 .unwrap();
9404 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9405
9406 let (_, registrations) = registry.list_modules().unwrap();
9407 let registered = ®istrations[0].manifest;
9408 assert!(registered.is_watch_operation("status"));
9409 assert!(!registered.is_watch_operation("future"));
9410 assert!(!registered.is_watch_operation("reset"));
9411 assert!(!registered.is_watch_operation("missing"));
9412
9413 let request = Frame::build(
9414 FrameType::Request,
9415 control_flags(),
9416 0,
9417 0,
9418 100,
9419 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9420 )
9421 .unwrap();
9422 let response = handler.handle_catalog_list(request, None).unwrap();
9423 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9424 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9425 let ClientControlResponse::CatalogList { modules, .. } =
9426 serde_json::from_slice(&response[0].body).unwrap()
9427 else {
9428 panic!("expected catalog.list");
9429 };
9430 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9431 }
9432
9433 #[test]
9434 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9435 let registry = Arc::new(Registry::default());
9436 let supervisor = SupervisorHandle::new();
9437 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9438 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9439 let descriptor = json!({
9440 "schema": "ckios.richui.manifest/1",
9441 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9442 "bytes": 4096,
9443 "chunks": 4
9444 });
9445 let connection = ConnectionId::new(201);
9446 let hello = handler
9447 .handle_control(
9448 connection,
9449 hello_frame_with_rich_ui(
9450 "vault",
9451 PROTOCOL_VERSION,
9452 201,
9453 Some("the-real-nonce"),
9454 descriptor.clone(),
9455 ),
9456 )
9457 .unwrap();
9458 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9459
9460 let request =
9461 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9462 let provides = registry
9463 .get_module_by_connection(connection)
9464 .unwrap()
9465 .unwrap()
9466 .manifest
9467 .provides;
9468 handler
9469 .handle_catalog_update(connection, request, provides, None, Some(false))
9470 .unwrap();
9471
9472 let request = Frame::build(
9473 FrameType::Request,
9474 control_flags(),
9475 0,
9476 0,
9477 203,
9478 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9479 )
9480 .unwrap();
9481 let response = handler.handle_catalog_list(request, None).unwrap();
9482 let ClientControlResponse::CatalogList { modules, .. } =
9483 serde_json::from_slice(&response[0].body).unwrap()
9484 else {
9485 panic!("expected catalog.list");
9486 };
9487 assert_eq!(
9488 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9489 descriptor
9490 );
9491 assert!(modules[0].nonce_verified);
9492 }
9493
9494 fn hello_with_role_declarations(declarations: Value) -> Frame {
9495 let mut body = serde_json::to_value(ModuleHelloBody {
9496 manifest: manifest("role-test", PROTOCOL_VERSION),
9497 protocol_ver: PROTOCOL_VERSION,
9498 control_ops: None,
9499 launch_nonce: None,
9500 })
9501 .unwrap();
9502 body["manifest"]["role_declarations"] = declarations;
9503 Frame::build(
9504 FrameType::Hello,
9505 control_flags(),
9506 0,
9507 0,
9508 301,
9509 serde_json::to_vec(&body).unwrap(),
9510 )
9511 .unwrap()
9512 }
9513
9514 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9515 let request = Frame::build(
9516 FrameType::Request,
9517 control_flags(),
9518 0,
9519 0,
9520 302,
9521 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9522 )
9523 .unwrap();
9524 let response = handler.handle_catalog_list(request, None).unwrap();
9525 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9526 let ClientControlResponse::CatalogList { mut modules, .. } =
9527 serde_json::from_slice(&response[0].body).unwrap()
9528 else {
9529 panic!("expected catalog.list");
9530 };
9531 (raw["modules"][0].clone(), modules.remove(0))
9532 }
9533
9534 #[test]
9535 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9536 let registry = Arc::new(Registry::default());
9537 let handler = ControlHandler::new(Arc::clone(®istry));
9538 let declarations = json!({
9539 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9540 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9541 });
9542 let connection = ConnectionId::new(301);
9543 let response = handler
9544 .handle_control(
9545 connection,
9546 hello_with_role_declarations(declarations.clone()),
9547 )
9548 .unwrap();
9549 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9550 let registered = registry
9551 .get_module_by_connection(connection)
9552 .unwrap()
9553 .unwrap()
9554 .manifest;
9555 assert_eq!(
9559 serde_json::to_value(®istered.role_declarations).unwrap(),
9560 declarations
9561 );
9562 let (raw, entry) = role_declarations_catalog(&handler);
9563 assert_eq!(raw["role_declarations"], declarations);
9564 assert_eq!(entry.role_declarations, registered.role_declarations);
9565
9566 let request =
9567 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9568 handler
9569 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9570 .unwrap();
9571 let (raw, entry) = role_declarations_catalog(&handler);
9572 assert_eq!(raw["role_declarations"], declarations);
9573 assert_eq!(entry.role_declarations, registered.role_declarations);
9574 }
9575
9576 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9577 let registry = Arc::new(Registry::default());
9578 let handler = ControlHandler::new(Arc::clone(®istry));
9579 let capture = EventCapture::default();
9580 let _subscriber =
9581 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9582 let response = handler
9583 .handle_control(
9584 ConnectionId::new(301),
9585 hello_with_role_declarations(declarations),
9586 )
9587 .unwrap();
9588 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9589 assert!(registry
9590 .get_module_by_connection(ConnectionId::new(301))
9591 .unwrap()
9592 .unwrap()
9593 .manifest
9594 .role_declarations
9595 .is_empty());
9596 let (raw, entry) = role_declarations_catalog(&handler);
9597 assert!(raw.get("role_declarations").is_none());
9598 assert!(entry.role_declarations.is_empty());
9599 let warnings: Vec<_> = capture
9600 .events()
9601 .into_iter()
9602 .filter(|event| {
9603 event.fields.get("message").map(String::as_str)
9604 == Some("dropping whole role_declarations map from HELLO")
9605 })
9606 .collect();
9607 assert_eq!(warnings.len(), 1);
9608 assert_eq!(warnings[0].level, tracing::Level::WARN);
9609 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9610 assert_eq!(warnings[0].fields["reason"], reason);
9611 }
9612
9613 #[test]
9614 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9615 for value in [
9616 json!(null),
9617 json!(false),
9618 json!(0),
9619 json!("opaque"),
9620 json!([]),
9621 ] {
9622 assert_hello_drops_role_declarations_with_warning(
9623 json!({"routing/v1": {}, "broken/v1": value}),
9624 "role_declarations values must be objects",
9625 );
9626 }
9627 assert_hello_drops_role_declarations_with_warning(
9628 json!([]),
9629 "role_declarations must be an object",
9630 );
9631 }
9632
9633 #[test]
9634 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9635 assert_hello_drops_role_declarations_with_warning(
9636 json!({"routing/v1": {}, "bad key": {}}),
9637 "role_declarations key must use capability-name grammar",
9638 );
9639 }
9640
9641 #[test]
9642 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9643 assert_hello_drops_role_declarations_with_warning(
9644 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9645 "role_declarations exceeds 16384 compact JSON bytes",
9646 );
9647 }
9648
9649 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
9650 let mut body = serde_json::to_value(ModuleHelloBody {
9651 manifest: manifest("flow-ops", PROTOCOL_VERSION),
9652 protocol_ver: PROTOCOL_VERSION,
9653 control_ops: None,
9654 launch_nonce: None,
9655 })
9656 .unwrap();
9657 body["manifest"]["provides"] = json!([{
9658 "role": "management_surface",
9659 "operations": [
9660 {"name": "diagnostics.findings", "kind": "query"},
9661 {"name": "reset", "kind": "mutate"}
9662 ],
9663 "config_schema": {}, "observability": [], "identity_scope": []
9664 }]);
9665 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
9666 if let Some(flow_operations) = flow_operations {
9667 body["manifest"]["flow_operations"] = flow_operations;
9668 }
9669 Frame::build(
9670 FrameType::Hello,
9671 control_flags(),
9672 0,
9673 0,
9674 401,
9675 serde_json::to_vec(&body).unwrap(),
9676 )
9677 .unwrap()
9678 }
9679
9680 #[test]
9681 fn catalog_list_mirrors_flow_operations_unchanged() {
9682 for (capability, flow_operations) in [
9685 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
9686 ("call-modes/v1", json!(["diagnostics.findings"])),
9687 ("flow-scopes/v1", json!([])),
9688 ] {
9689 let registry = Arc::new(Registry::default());
9690 let handler = ControlHandler::new(Arc::clone(®istry));
9691 let connection = ConnectionId::new(401);
9692 let response = handler
9693 .handle_control(
9694 connection,
9695 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
9696 )
9697 .unwrap();
9698 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9699 let registered = registry
9700 .get_module_by_connection(connection)
9701 .unwrap()
9702 .unwrap()
9703 .manifest;
9704 assert_eq!(
9705 serde_json::to_value(®istered.flow_operations).unwrap(),
9706 flow_operations
9707 );
9708 let (raw, entry) = role_declarations_catalog(&handler);
9709 assert_eq!(raw["flow_operations"], flow_operations);
9710 assert_eq!(entry.flow_operations, registered.flow_operations);
9711 }
9712
9713 let registry = Arc::new(Registry::default());
9714 let handler = ControlHandler::new(Arc::clone(®istry));
9715 let response = handler
9716 .handle_control(
9717 ConnectionId::new(401),
9718 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
9719 )
9720 .unwrap();
9721 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9722 let (raw, entry) = role_declarations_catalog(&handler);
9723 assert!(raw.get("flow_operations").is_none());
9724 assert_eq!(entry.flow_operations, None);
9725 }
9726
9727 fn assert_hello_drops_flow_operations_with_warning(
9728 capabilities: Value,
9729 flow_operations: Value,
9730 reason: &str,
9731 ) {
9732 let registry = Arc::new(Registry::default());
9733 let handler = ControlHandler::new(Arc::clone(®istry));
9734 let capture = EventCapture::default();
9735 let _subscriber =
9736 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9737 let response = handler
9738 .handle_control(
9739 ConnectionId::new(401),
9740 hello_with_flow_operations(capabilities, Some(flow_operations)),
9741 )
9742 .unwrap();
9743 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9744 let registered = registry
9745 .get_module_by_connection(ConnectionId::new(401))
9746 .unwrap()
9747 .expect("the module still registers")
9748 .manifest;
9749 assert_eq!(registered.module_id, "flow-ops");
9750 assert_eq!(registered.flow_operations, None);
9751 let (raw, entry) = role_declarations_catalog(&handler);
9752 assert_eq!(raw["module_id"], "flow-ops");
9753 assert!(raw.get("flow_operations").is_none());
9754 assert_eq!(entry.flow_operations, None);
9755 let warnings: Vec<_> = capture
9756 .events()
9757 .into_iter()
9758 .filter(|event| {
9759 event.fields.get("message").map(String::as_str)
9760 == Some("dropping flow_operations from HELLO")
9761 })
9762 .collect();
9763 assert_eq!(warnings.len(), 1);
9764 assert_eq!(warnings[0].level, tracing::Level::WARN);
9765 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
9766 assert_eq!(warnings[0].fields["reason"], reason);
9767 }
9768
9769 #[test]
9770 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
9771 assert_hello_drops_flow_operations_with_warning(
9772 json!(["flow-scopes/v1"]),
9773 json!(["diagnostics.findings", "missing"]),
9774 "flow_operations names 'missing', which is not a declared management operation",
9775 );
9776 }
9777
9778 #[test]
9779 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
9780 assert_hello_drops_flow_operations_with_warning(
9781 json!([]),
9782 json!(["diagnostics.findings"]),
9783 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
9784 );
9785 assert_hello_drops_flow_operations_with_warning(
9787 json!(["agent-run-scopes/v1"]),
9788 json!([]),
9789 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
9790 );
9791 }
9792
9793 #[test]
9794 fn hello_malformed_flow_operations_is_dropped_with_warning() {
9795 assert_hello_drops_flow_operations_with_warning(
9796 json!(["flow-scopes/v1"]),
9797 json!(["reset", "reset"]),
9798 "flow_operations names 'reset' more than once",
9799 );
9800 assert_hello_drops_flow_operations_with_warning(
9801 json!(["flow-scopes/v1"]),
9802 json!(vec!["reset"; 257]),
9803 "flow_operations lists 257 names, more than 256",
9804 );
9805 assert_hello_drops_flow_operations_with_warning(
9806 json!(["flow-scopes/v1"]),
9807 json!("reset"),
9808 "flow_operations must be an array of operation names",
9809 );
9810 }
9811
9812 #[test]
9813 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9814 let registry = Arc::new(Registry::default());
9815 let handler = ControlHandler::new(registry.clone());
9816 let descriptor = json!({
9817 "schema": "ckios.richui.manifest/1",
9818 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9819 "bytes": 128,
9820 "chunks": 2
9821 });
9822 let malformed = [
9823 ("schema", json!("")),
9824 ("schema", json!("a".repeat(65))),
9825 ("schema", json!("Bad schema")),
9826 (
9827 "manifest_digest",
9828 json!(format!("sha256:{}", "A".repeat(64))),
9829 ),
9830 ("manifest_digest", json!("sha256:abc")),
9831 ("bytes", json!(0)),
9832 ("bytes", json!(16_777_217)),
9833 ("bytes", json!("128")),
9834 ("chunks", json!(0)),
9835 ("chunks", json!(1025)),
9836 ("chunks", json!("2")),
9837 ];
9838 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9839 let mut invalid = descriptor.clone();
9840 invalid[field] = invalid_value;
9841 let response = handler
9842 .handle_control(
9843 ConnectionId::new(210 + index as u64),
9844 hello_frame_with_rich_ui(
9845 &format!("bad-rich-ui-{index}"),
9846 PROTOCOL_VERSION,
9847 210 + index as u64,
9848 None,
9849 invalid,
9850 ),
9851 )
9852 .unwrap();
9853 assert_eq!(response[0].header.ty, FrameType::Error);
9854 let error = parse_error(&response[0]);
9855 assert_eq!(error["code"], "invalid_manifest");
9856 assert!(error["message"]
9857 .as_str()
9858 .unwrap()
9859 .contains(&format!("rich_ui.{field}")));
9860 assert!(registry
9861 .get_module(&format!("bad-rich-ui-{index}"))
9862 .unwrap()
9863 .is_none());
9864 }
9865 }
9866
9867 #[test]
9868 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9869 let registry = Arc::new(Registry::default());
9870 let handler = ControlHandler::new(Arc::clone(®istry));
9871 let connection = ConnectionId::new(219);
9872 registry
9873 .register_with_control_ops(
9874 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9875 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9876 "bad schema",
9877 format!("sha256:{}", "a".repeat(64)),
9878 128,
9879 2,
9880 )))
9881 .build(),
9882 PROTOCOL_VERSION,
9883 connection,
9884 Vec::new(),
9885 )
9886 .unwrap();
9887 let request =
9888 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9889 let response = handler
9890 .handle_catalog_update(connection, request, Vec::new(), None, None)
9891 .unwrap();
9892 assert_eq!(response[0].header.ty, FrameType::Error);
9893 let error = parse_error(&response[0]);
9894 assert_eq!(error["code"], "invalid_manifest");
9895 assert!(error["message"]
9896 .as_str()
9897 .unwrap()
9898 .contains("rich_ui.schema"));
9899 }
9900
9901 #[test]
9902 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9903 let registry = Arc::new(Registry::default());
9904 let supervisor = SupervisorHandle::new();
9905 supervisor.open_swap("provider", "candidate-nonce".to_string());
9906 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9907 let connection = ConnectionId::new(218);
9908 let hello = handler
9909 .handle_control(
9910 connection,
9911 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9912 )
9913 .unwrap();
9914 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9915 assert!(registry.list_modules().unwrap().1.is_empty());
9916
9917 registry.promote_candidate("provider").unwrap().unwrap();
9918 let request = Frame::build(
9919 FrameType::Request,
9920 control_flags(),
9921 0,
9922 0,
9923 219,
9924 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9925 )
9926 .unwrap();
9927 let response = handler.handle_catalog_list(request, None).unwrap();
9928 let ClientControlResponse::CatalogList { modules, .. } =
9929 serde_json::from_slice(&response[0].body).unwrap()
9930 else {
9931 panic!("expected catalog.list");
9932 };
9933 assert_eq!(modules.len(), 1);
9934 assert!(modules[0].nonce_verified);
9935 }
9936
9937 #[test]
9938 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9939 let registry = Arc::new(Registry::default());
9940 let handler = ControlHandler::new(Arc::clone(®istry));
9941 let hello = Frame::build(
9942 FrameType::Hello,
9943 control_flags(),
9944 0,
9945 0,
9946 220,
9947 serde_json::to_vec(&ModuleHelloBody {
9948 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9949 protocol_ver: PROTOCOL_VERSION,
9950 control_ops: None,
9951 launch_nonce: Some("unverified".to_string()),
9952 })
9953 .unwrap(),
9954 )
9955 .unwrap();
9956 assert_eq!(
9957 handler
9958 .handle_control(ConnectionId::new(220), hello)
9959 .unwrap()[0]
9960 .header
9961 .ty,
9962 FrameType::HelloAck
9963 );
9964 let request = Frame::build(
9965 FrameType::Request,
9966 control_flags(),
9967 0,
9968 0,
9969 221,
9970 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9971 )
9972 .unwrap();
9973 let response = handler.handle_catalog_list(request, None).unwrap();
9974 let ClientControlResponse::CatalogList { modules, .. } =
9975 serde_json::from_slice(&response[0].body).unwrap()
9976 else {
9977 panic!("expected catalog.list");
9978 };
9979 assert_eq!(
9980 serde_json::to_vec(&modules[0]).unwrap(),
9981 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9982 );
9983 assert!(!modules[0].nonce_verified);
9984 }
9985
9986 #[test]
9987 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9988 let registry = Arc::new(Registry::default());
9989 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9990 [("vault".to_string(), true), ("squatter".to_string(), true)],
9991 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9992 );
9993 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9994 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9995 provides: vec!["credentials-provider/v1".to_string()],
9996 requires: Vec::new(),
9997 must_never_reach: Vec::new(),
9998 });
9999 let frame = Frame::build(
10000 FrameType::Hello,
10001 control_flags(),
10002 0,
10003 0,
10004 77,
10005 serde_json::to_vec(&ModuleHelloBody {
10006 manifest: squatter,
10007 protocol_ver: PROTOCOL_VERSION,
10008 control_ops: None,
10009 launch_nonce: None,
10010 })
10011 .expect("HELLO serializes"),
10012 )
10013 .expect("HELLO frame builds");
10014 let response = handler
10015 .handle_control(ConnectionId::new(77), frame)
10016 .expect("reserved claim receives a typed refusal");
10017 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10018 assert_eq!(
10019 registry
10020 .active_registration_count()
10021 .expect("registry reads"),
10022 0,
10023 "a reserved capability refusal must not leave a catalog entry"
10024 );
10025 }
10026
10027 #[test]
10028 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10029 for settlement in ["timeout", "inconclusive", "drop"] {
10030 let breakers = RouteBindBreakers::default();
10031 let RouteBindAdmission::Admitted {
10032 guard: mut old,
10033 probe: false,
10034 } = breakers.admit("prov")
10035 else {
10036 panic!("ordinary relay admitted")
10037 };
10038 let RouteBindAdmission::Admitted {
10039 guard: mut opener, ..
10040 } = breakers.admit("prov")
10041 else {
10042 panic!("second relay admitted")
10043 };
10044 assert!(
10045 !opener
10046 .record_timeout(1, Duration::ZERO)
10047 .unwrap()
10048 .reopened_after_probe
10049 );
10050 let RouteBindAdmission::Admitted {
10051 guard: mut probe,
10052 probe: true,
10053 } = breakers.admit("prov")
10054 else {
10055 panic!("one cooldown probe admitted")
10056 };
10057 match settlement {
10058 "timeout" => assert!(
10059 !old.record_timeout(1, Duration::ZERO)
10060 .unwrap()
10061 .reopened_after_probe
10062 ),
10063 "inconclusive" => old.record_inconclusive(),
10064 "drop" => drop(old),
10065 _ => unreachable!(),
10066 }
10067 assert!(
10068 matches!(
10069 breakers.admit("prov"),
10070 RouteBindAdmission::Refused {
10071 probe_in_flight: true,
10072 ..
10073 }
10074 ),
10075 "{settlement} of a pre-open relay cannot release the real probe"
10076 );
10077 assert!(
10078 probe
10079 .record_timeout(1, Duration::ZERO)
10080 .unwrap()
10081 .reopened_after_probe
10082 );
10083 assert!(matches!(
10084 breakers.admit("prov"),
10085 RouteBindAdmission::Admitted { probe: true, .. }
10086 ));
10087 }
10088 let breakers = RouteBindBreakers::default();
10089 let admit = || match breakers.admit("prov") {
10090 RouteBindAdmission::Admitted { guard, .. } => guard,
10091 _ => panic!("relay admitted"),
10092 };
10093 admit().record_timeout(1, Duration::ZERO);
10094 let mut old_probe = admit();
10095 breakers.reset_for_new_module_connection("prov");
10096 admit().record_timeout(1, Duration::ZERO);
10097 let _new_probe = admit();
10098 old_probe.record_inconclusive();
10099 assert!(matches!(
10100 breakers.admit("prov"),
10101 RouteBindAdmission::Refused {
10102 probe_in_flight: true,
10103 ..
10104 }
10105 ));
10106 }
10107
10108 #[tokio::test]
10109 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10110 for candidate in [false, true] {
10111 let registry = Arc::new(Registry::default());
10112 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10113 [("vault".to_string(), true), ("squatter".to_string(), true)],
10114 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10115 );
10116 let conn = ConnectionId::new(77);
10117 let (ctx, mut rx) = route_ctx(conn);
10118 let initial = capability_manifest("squatter", &[], &[]);
10119 if candidate {
10120 registry
10121 .register_candidate_with_control_ops(
10122 initial.clone(),
10123 PROTOCOL_VERSION,
10124 conn,
10125 module_baseline_control_ops(),
10126 )
10127 .unwrap();
10128 handler
10129 .forwarding
10130 .register_candidate_module_connection(
10131 conn,
10132 "squatter".to_string(),
10133 PROTOCOL_VERSION,
10134 manifest_concurrency(&initial),
10135 ctx.egress.clone(),
10136 )
10137 .unwrap();
10138 } else {
10139 hello_via_sink(
10140 &handler,
10141 &ctx,
10142 &mut rx,
10143 hello_frame_with_manifest(initial.clone(), 1),
10144 )
10145 .await;
10146 }
10147 let response = handler
10148 .handle_control_frame(
10149 &ctx,
10150 catalog_update_with_capabilities_frame(
10151 2,
10152 capability_manifest("squatter", &["credentials-provider/v1"], &[])
10153 .capabilities
10154 .unwrap(),
10155 ),
10156 )
10157 .await
10158 .unwrap();
10159 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10160 assert_eq!(
10161 registry
10162 .get_module_by_connection(conn)
10163 .unwrap()
10164 .unwrap()
10165 .manifest,
10166 initial
10167 );
10168 }
10169 }
10170
10171 #[test]
10172 fn server_describe_surfaces_required_capability_verdict_fields() {
10173 let registry = Arc::new(Registry::default());
10174 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10175 [
10176 ("consumer".to_string(), true),
10177 ("provider".to_string(), false),
10178 ],
10179 BTreeMap::new(),
10180 );
10181 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10182 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10183 provides: Vec::new(),
10184 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10185 capability: "credentials-provider/v1".to_string(),
10186 need: subc_protocol::manifest::CapabilityNeed::Required,
10187 }],
10188 must_never_reach: Vec::new(),
10189 });
10190 let hello = Frame::build(
10191 FrameType::Hello,
10192 control_flags(),
10193 0,
10194 0,
10195 78,
10196 serde_json::to_vec(&ModuleHelloBody {
10197 manifest: consumer,
10198 protocol_ver: PROTOCOL_VERSION,
10199 control_ops: None,
10200 launch_nonce: None,
10201 })
10202 .expect("HELLO serializes"),
10203 )
10204 .expect("HELLO frame builds");
10205 handler
10206 .handle_control(ConnectionId::new(78), hello)
10207 .expect("consumer registers");
10208 let describe = Frame::build(
10209 FrameType::Request,
10210 control_flags(),
10211 0,
10212 0,
10213 79,
10214 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10215 .expect("request serializes"),
10216 )
10217 .expect("describe frame builds");
10218 let response = handler
10219 .handle_server_describe(describe)
10220 .expect("server.describe succeeds");
10221 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10222 let requirement = &rendered["capability_requirements"][0];
10223 assert_eq!(requirement["consumer"], "consumer");
10224 assert_eq!(requirement["verdict"], "never_provided");
10225 assert_eq!(requirement["episode_seq"], 1);
10226 assert_eq!(requirement["config_satisfiable"], false);
10227 assert_eq!(requirement["runtime_available"], false);
10228 assert!(requirement["detail"]
10229 .as_str()
10230 .expect("detail string")
10231 .contains("credentials-provider/v1"));
10232 }
10233
10234 fn deregister_required_provider(
10240 daemon_shutdown_begun: bool,
10241 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10242 const CAPABILITY: &str = "project-identity/v1";
10243 let registry = Arc::new(Registry::default());
10244 let daemon_shutdown = DaemonShutdownFlag::default();
10245 let handler = ControlHandler::new(Arc::clone(®istry))
10246 .with_capability_config(
10247 [
10248 ("consumer".to_string(), true),
10249 ("provider".to_string(), true),
10250 ],
10251 BTreeMap::new(),
10252 )
10253 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10254 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10255 consumer.capabilities = Some(CapabilityDeclarations {
10256 provides: Vec::new(),
10257 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10258 capability: CAPABILITY.to_string(),
10259 need: subc_protocol::manifest::CapabilityNeed::Required,
10260 }],
10261 must_never_reach: Vec::new(),
10262 });
10263 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10264 let provider_conn = ConnectionId::new(182);
10265 for (conn, manifest, corr) in [
10266 (ConnectionId::new(181), consumer, 181),
10267 (provider_conn, provider, 182),
10268 ] {
10269 let hello = handler
10270 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10271 .expect("module registers");
10272 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10273 }
10274 assert_eq!(
10275 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10276 Some(CapabilityVerdict::Provided),
10277 "the registered provider must satisfy the requirement first"
10278 );
10279 if daemon_shutdown_begun {
10280 daemon_shutdown.set_for_test();
10281 }
10282
10283 let capture = EventCapture::default();
10284 let _subscriber =
10285 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10286 handler
10287 .cleanup_connection(provider_conn)
10288 .expect("provider deregisters");
10289 let events = capture
10290 .events()
10291 .into_iter()
10292 .filter(|event| {
10293 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10294 })
10295 .collect();
10296 (
10297 events,
10298 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10299 )
10300 }
10301
10302 #[test]
10303 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10304 let (events, verdict) = deregister_required_provider(true);
10305 assert_eq!(
10306 verdict,
10307 Some(CapabilityVerdict::NeverProvided),
10308 "shutdown must not change the verdict route.open refusals read"
10309 );
10310 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10311 let event = &events[0];
10312 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10313 assert_eq!(
10314 event.fields.get("cause").map(String::as_str),
10315 Some("\"daemon_shutdown\""),
10316 "{event:?}"
10317 );
10318 assert_eq!(
10319 event.fields.get("verdict").map(String::as_str),
10320 Some("\"never_provided\""),
10321 "{event:?}"
10322 );
10323 }
10324
10325 #[test]
10326 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10327 let (events, verdict) = deregister_required_provider(false);
10328 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10329 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10330 let event = &events[0];
10331 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10332 assert_eq!(event.fields.get("cause"), None, "{event:?}");
10333 assert_eq!(
10334 event.fields.get("verdict").map(String::as_str),
10335 Some("\"never_provided\""),
10336 "{event:?}"
10337 );
10338 }
10339
10340 #[test]
10341 fn catalog_list_omits_capabilities_for_legacy_manifest() {
10342 let registry = Arc::new(Registry::default());
10343 let handler = ControlHandler::new(Arc::clone(®istry));
10344 let hello = handler
10345 .handle_control(
10346 ConnectionId::new(101),
10347 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10348 )
10349 .expect("legacy HELLO registers");
10350 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10351
10352 let request = Frame::build(
10353 FrameType::Request,
10354 control_flags(),
10355 0,
10356 0,
10357 102,
10358 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10359 .expect("catalog request serializes"),
10360 )
10361 .expect("catalog request frame builds");
10362 let response = handler
10363 .handle_catalog_list(request, None)
10364 .expect("catalog list succeeds");
10365 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10366 assert!(
10367 body["modules"][0].get("capabilities").is_none(),
10368 "legacy manifest must retain an absent capabilities field on catalog.list"
10369 );
10370 }
10371
10372 #[test]
10373 fn hello_ack_omits_storage_when_no_storage_config() {
10374 let registry = Arc::new(Registry::default());
10375 let handler = ControlHandler::new(Arc::clone(®istry));
10376 let responses = handler
10377 .handle_control(
10378 ConnectionId::new(1),
10379 hello_frame("aft", PROTOCOL_VERSION, 7),
10380 )
10381 .unwrap();
10382 let ack = parse_ack(&responses[0]);
10383 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10384 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10385 }
10386
10387 #[tokio::test]
10388 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10389 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10390 let registry = Arc::new(Registry::default());
10391 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
10392 let responses = handler
10393 .handle_control(
10394 ConnectionId::new(1),
10395 hello_frame("aft", PROTOCOL_VERSION, 7),
10396 )
10397 .unwrap();
10398 let ack = parse_ack(&responses[0]);
10399 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10400
10401 let described = handler
10402 .handle_control_frame(
10403 &route_ctx(ConnectionId::new(2)).0,
10404 Frame::build(
10405 FrameType::Request,
10406 control_flags(),
10407 0,
10408 0,
10409 9,
10410 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10411 )
10412 .unwrap(),
10413 )
10414 .await
10415 .unwrap();
10416 let ClientControlResponse::ServerDescribe { machine_id, .. } =
10417 serde_json::from_slice(&described[0].body).unwrap()
10418 else {
10419 panic!("server.describe answered with another shape");
10420 };
10421 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10422 }
10423
10424 #[test]
10425 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10426 let registry = Arc::new(Registry::default());
10429 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
10430 crate::daemon_config::StorageConfig::Sqlite {
10431 data_home: std::path::PathBuf::from("/data"),
10432 },
10433 ));
10434
10435 let responses = handler
10436 .handle_control(
10437 ConnectionId::new(1),
10438 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10439 )
10440 .unwrap();
10441 let ack = parse_ack(&responses[0]);
10442 assert_eq!(
10443 ack.storage,
10444 Some(serde_json::json!({
10445 "module_id": "alfonso-routing",
10446 "storage_namespace": "default",
10447 "isolation": { "kind": "module" },
10448 "backend": {
10449 "backend": "sqlite",
10450 "path": "/data/cortexkit/alfonso-routing/store.db"
10451 }
10452 })),
10453 "the delivered descriptor is the module's own sqlite store path"
10454 );
10455 }
10456
10457 #[test]
10458 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10459 let registry = Arc::new(Registry::default());
10460 let handler = ControlHandler::new(Arc::clone(®istry));
10461 let conn = ConnectionId::new(1);
10462 let responses = handler
10463 .handle_control(
10464 conn,
10465 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10466 )
10467 .unwrap();
10468 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10469 let registration = registry.get_module("aft").unwrap().unwrap();
10470 assert_eq!(registration.control_ops, module_baseline_control_ops());
10471
10472 let frame =
10473 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10474 assert!(handler
10475 .guard_module_control_op(&frame, "aft", "route.bind")
10476 .unwrap()
10477 .is_none());
10478 let error = handler
10479 .guard_module_control_op(&frame, "aft", "test.synthetic")
10480 .unwrap()
10481 .expect("synthetic ungranted op should be rejected");
10482 assert_eq!(error.header.ty, FrameType::Error);
10483 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10484 }
10485
10486 #[test]
10487 fn hello_control_ops_some_adds_optional_grants() {
10488 let registry = Arc::new(Registry::default());
10489 let handler = ControlHandler::new(Arc::clone(®istry));
10490 handler
10491 .handle_control(
10492 ConnectionId::new(1),
10493 hello_frame_with_control_ops(
10494 "aft",
10495 PROTOCOL_VERSION,
10496 7,
10497 Some(vec![
10498 "future.synthetic".to_string(),
10499 "route.bind".to_string(),
10500 ]),
10501 ),
10502 )
10503 .unwrap();
10504 let registration = registry.get_module("aft").unwrap().unwrap();
10505 assert_eq!(
10506 registration.control_ops,
10507 vec![
10508 "route.bind".to_string(),
10509 "route.status".to_string(),
10510 "future.synthetic".to_string(),
10511 ]
10512 );
10513 let frame =
10514 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10515 assert!(handler
10516 .guard_module_control_op(&frame, "aft", "future.synthetic")
10517 .unwrap()
10518 .is_none());
10519 }
10520
10521 #[tokio::test]
10522 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10523 let registry = Arc::new(Registry::default());
10524 let forwarding = Arc::new(ForwardingTable::default());
10525 let handler =
10526 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10527 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10528 hello_via_sink(
10529 &handler,
10530 &module_ctx,
10531 &mut module_rx,
10532 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10533 )
10534 .await;
10535
10536 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10537 let responses = handler
10538 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10539 .await
10540 .unwrap();
10541 assert_eq!(responses.len(), 1);
10542 assert_eq!(responses[0].header.ty, FrameType::Error);
10543 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10544 assert!(module_rx.try_recv().is_err());
10545 }
10546
10547 #[tokio::test]
10548 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10549 let registry = Arc::new(Registry::default());
10550 let forwarding = Arc::new(ForwardingTable::default());
10551 let handler =
10552 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10553 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10554 hello_via_sink(
10555 &handler,
10556 &module_ctx,
10557 &mut module_rx,
10558 hello_frame_with_control_ops(
10559 "aft",
10560 PROTOCOL_VERSION,
10561 7,
10562 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10563 ),
10564 )
10565 .await;
10566
10567 let project_root = unique_project_root("demux");
10568 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10569 let route_handler = handler.clone();
10570 let route_task = tokio::spawn(async move {
10571 route_handler
10572 .handle_control_frame(
10573 &route_client_ctx,
10574 route_open_frame(100, "aft", project_root),
10575 )
10576 .await
10577 .unwrap()
10578 });
10579 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10580 .await
10581 .unwrap()
10582 .unwrap();
10583 assert!(matches!(
10584 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10585 ModuleControlRequest::RouteBind { .. }
10586 ));
10587
10588 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10589 let health_handler = handler.clone();
10590 let health_task = tokio::spawn(async move {
10591 health_handler
10592 .handle_control_frame(
10593 &health_client_ctx,
10594 supervisor_health_probe_frame(101, "aft"),
10595 )
10596 .await
10597 .unwrap()
10598 });
10599 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10600 .await
10601 .unwrap()
10602 .unwrap();
10603 assert_eq!(
10604 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10605 ModuleControlRequest::HealthCheck {}
10606 );
10607
10608 handler
10609 .handle_control_frame(
10610 &module_ctx,
10611 health_response(health_frame.header.corr, HealthStatus::Degraded),
10612 )
10613 .await
10614 .unwrap();
10615 let health_response = health_task.await.unwrap();
10616 assert_eq!(health_response.len(), 1);
10617 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10618 ClientControlResponse::SupervisorHealthProbe {
10619 module_id,
10620 status,
10621 detail,
10622 metrics,
10623 } => {
10624 assert_eq!(module_id, "aft");
10625 assert_eq!(status, HealthStatus::Degraded);
10626 assert_eq!(detail.as_deref(), Some("warming"));
10627 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10628 }
10629 other => panic!("unexpected health response: {other:?}"),
10630 }
10631
10632 handler
10633 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10634 .await
10635 .unwrap();
10636 let route_response = route_task.await.unwrap();
10637 assert!(route_response.is_empty());
10638 let published = route_client_rx.recv().await.unwrap();
10639 assert!(matches!(
10640 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10641 ClientControlResponse::RouteOpen { .. }
10642 ));
10643 }
10644
10645 async fn relay_route_open(
10650 handler: &ControlHandler,
10651 client_connection: ConnectionId,
10652 client_egress: &FrameSink,
10653 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10654 corr: u64,
10655 module_id: &str,
10656 project_root_label: &str,
10657 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10658 let ctx = RouteCtx {
10659 connection_id: client_connection,
10660 egress: client_egress.clone(),
10661 };
10662 let handler = handler.clone();
10663 let project_root = unique_project_root(project_root_label);
10664 let module_id = module_id.to_string();
10665 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10666 let task = tokio::spawn(async move {
10667 let _guard = tracing::dispatcher::set_default(&dispatch);
10668 handler
10669 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10670 .await
10671 .unwrap()
10672 });
10673 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10674 .await
10675 .expect("module receives the relayed route.bind")
10676 .expect("module egress is open");
10677 (task, bind.frame)
10678 }
10679
10680 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10681 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10682 ModuleControlRequest::RouteBind {
10683 route_channel,
10684 epoch,
10685 ..
10686 } => (route_channel, epoch),
10687 other => panic!("expected a route.bind request, got {other:?}"),
10688 }
10689 }
10690
10691 fn published_route(frame: &Frame) -> (u16, u32) {
10692 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10693 ClientControlResponse::RouteOpen {
10694 route_channel,
10695 route_epoch,
10696 } => (route_channel, route_epoch),
10697 other => panic!("expected a route.open response, got {other:?}"),
10698 }
10699 }
10700
10701 #[tokio::test]
10721 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10722 let registry = Arc::new(Registry::default());
10723 let forwarding = Arc::new(ForwardingTable::default());
10724 let handler =
10725 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10726
10727 let module_connection = ConnectionId::new(30);
10728 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10729 hello_via_sink(
10730 &handler,
10731 &module_ctx,
10732 &mut module_rx,
10733 hello_frame("aft", PROTOCOL_VERSION, 7),
10734 )
10735 .await;
10736
10737 let dying_client = ConnectionId::new(31);
10738 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10739
10740 let (first_task, first_bind) = relay_route_open(
10743 &handler,
10744 dying_client,
10745 &dying_ctx.egress,
10746 &mut module_rx,
10747 100,
10748 "aft",
10749 "closing-first",
10750 )
10751 .await;
10752 handler
10753 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10754 .await
10755 .unwrap();
10756 assert!(first_task.await.unwrap().is_empty());
10757 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10758
10759 let (second_task, second_bind) = relay_route_open(
10761 &handler,
10762 dying_client,
10763 &dying_ctx.egress,
10764 &mut module_rx,
10765 101,
10766 "aft",
10767 "closing-second",
10768 )
10769 .await;
10770 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10771
10772 assert!(forwarding
10775 .escalate_client_delivery_failure(
10776 dying_client,
10777 first_channel,
10778 first_epoch,
10779 CloseReason::new(
10780 "module_to_client_delivery_failed",
10781 "client egress refused a module frame",
10782 ),
10783 crate::forwarding::UndeliveredFrame {
10784 module_id: None,
10785 sink: &dying_ctx.egress,
10786 },
10787 )
10788 .unwrap());
10789 assert!(!dying_ctx.egress.is_closed());
10790
10791 let ack = handler
10793 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10794 .await;
10795 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10796 if module_loop_error.is_some() {
10797 handler.cleanup_connection(module_connection).unwrap();
10801 }
10802 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10806 .await
10807 .ok()
10808 .flatten();
10809
10810 assert!(
10812 registry
10813 .get_module_by_connection(module_connection)
10814 .unwrap()
10815 .is_some(),
10816 "one client's closing connection ended the shared module connection: \
10817 {module_loop_error:?}"
10818 );
10819 let cotenant = ConnectionId::new(32);
10821 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10822 let (cotenant_task, cotenant_bind) = relay_route_open(
10823 &handler,
10824 cotenant,
10825 &cotenant_ctx.egress,
10826 &mut module_rx,
10827 102,
10828 "aft",
10829 "closing-cotenant",
10830 )
10831 .await;
10832 handler
10833 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10834 .await
10835 .unwrap();
10836 assert!(cotenant_task.await.unwrap().is_empty());
10837 let (cotenant_channel, cotenant_epoch) =
10838 published_route(&cotenant_rx.recv().await.unwrap());
10839 assert!(matches!(
10840 forwarding
10841 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10842 .unwrap(),
10843 DataRoute::Client(DataRouteState::Bound(_))
10844 ));
10845
10846 let goodbye = post_ack_module_frame
10849 .expect("module receives a GOODBYE for the abandoned route channel");
10850 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10851 assert_eq!(goodbye.header.channel, abandoned_channel);
10852 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10853
10854 assert!(dying_rx.try_recv().is_err());
10858 let second_response = second_task.await.unwrap();
10859 assert_eq!(second_response.len(), 1);
10860 assert_eq!(
10861 parse_error(&second_response[0])["code"],
10862 "target_unavailable"
10863 );
10864 }
10865
10866 async fn catalog_snapshot(
10869 handler: &ControlHandler,
10870 client_ctx: &RouteCtx,
10871 corr: u64,
10872 ) -> (u64, Vec<(String, bool)>) {
10873 let body =
10874 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10875 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10876 let reply = handler
10877 .handle_control_frame(client_ctx, request)
10878 .await
10879 .unwrap()
10880 .pop()
10881 .unwrap();
10882 let ClientControlResponse::CatalogList {
10883 generation,
10884 modules,
10885 ..
10886 } = serde_json::from_slice(&reply.body).unwrap()
10887 else {
10888 panic!("not a catalog.list reply");
10889 };
10890 (
10891 generation,
10892 modules
10893 .into_iter()
10894 .map(|entry| (entry.module_id, entry.ready))
10895 .collect(),
10896 )
10897 }
10898
10899 fn hello_held_before_forwarding() -> (
10905 ControlHandler,
10906 Arc<std::sync::Barrier>,
10907 std::thread::JoinHandle<Vec<Frame>>,
10908 RouteCtx,
10909 mpsc::Receiver<crate::router::OutboundFrame>,
10910 ) {
10911 let hold = Arc::new(std::sync::Barrier::new(2));
10912 let handler = ControlHandler::with_forwarding(
10913 Arc::new(Registry::default()),
10914 Arc::new(ForwardingTable::default()),
10915 )
10916 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10917 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10918 let hello = {
10919 let handler = handler.clone();
10920 let module_ctx = module_ctx.clone();
10921 std::thread::spawn(move || {
10922 handler
10923 .handle_hello(
10924 module_ctx.connection_id,
10925 Some(module_ctx.egress.clone()),
10926 hello_frame("aft", PROTOCOL_VERSION, 7),
10927 )
10928 .unwrap()
10929 })
10930 };
10931 (handler, hold, hello, module_ctx, module_rx)
10932 }
10933
10934 #[tokio::test]
10943 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10944 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10945 let client = ConnectionId::new(41);
10946 let (client_ctx, mut client_rx) = route_ctx(client);
10947
10948 hold.wait();
10949 assert!(handler.registry.get_module("aft").unwrap().is_some());
10951 assert!(!handler
10952 .forwarding
10953 .has_live_module_connection("aft")
10954 .unwrap());
10955 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10956 assert!(
10957 listed.is_empty(),
10958 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10959 );
10960 let refused = handler
10961 .handle_control_frame(
10962 &client_ctx,
10963 route_open_frame(301, "aft", unique_project_root("listed-window")),
10964 )
10965 .await
10966 .unwrap();
10967 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10968 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10969 let duplicate = handler
10970 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10971 .await
10972 .unwrap();
10973 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10974
10975 hold.wait();
10976 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10977 let ack = module_rx.try_recv().unwrap().frame;
10978 assert_eq!(ack.header.ty, FrameType::HelloAck);
10979 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10980 assert_eq!(listed, vec![("aft".to_string(), true)]);
10981
10982 let (open_task, bind) = relay_route_open(
10983 &handler,
10984 client,
10985 &client_ctx.egress,
10986 &mut module_rx,
10987 303,
10988 "aft",
10989 "listed-after",
10990 )
10991 .await;
10992 assert!(handler
10993 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10994 .await
10995 .unwrap()
10996 .is_empty());
10997 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10998 published_route(&client_rx.try_recv().unwrap().frame);
10999 }
11000
11001 #[tokio::test]
11008 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11009 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11010 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11011 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11012
11013 hold.wait();
11014 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11015 assert!(listed.is_empty());
11016 assert_eq!(
11017 during, before,
11018 "the generation moved while the module was not listed"
11019 );
11020
11021 hold.wait();
11022 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11023 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11024 assert_eq!(listed, vec![("aft".to_string(), true)]);
11025 assert_eq!(
11026 after,
11027 before + 1,
11028 "the module appeared without exactly one generation bump"
11029 );
11030 }
11031
11032 #[tokio::test]
11037 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11038 let registry = Arc::new(Registry::default());
11039 let handler = ControlHandler::with_forwarding(
11040 Arc::clone(®istry),
11041 Arc::new(ForwardingTable::default()),
11042 );
11043 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11044 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11045 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11046 drop(module_rx);
11047
11048 let replies = handler
11049 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11050 .await
11051 .unwrap();
11052 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11053 assert!(registry.get_module("aft").unwrap().is_none());
11054 assert!(registry
11055 .get_module_by_connection(module_ctx.connection_id)
11056 .unwrap()
11057 .is_none());
11058 assert_eq!(
11059 catalog_snapshot(&handler, &client_ctx, 301).await,
11060 (before, Vec::new())
11061 );
11062 }
11063
11064 async fn route_open_held_after_bind_send() -> (
11068 ControlHandler,
11069 Arc<tokio::sync::Barrier>,
11070 RouteCtx,
11071 mpsc::Receiver<crate::router::OutboundFrame>,
11072 ) {
11073 let hold = Arc::new(tokio::sync::Barrier::new(2));
11074 let handler = ControlHandler::with_forwarding(
11075 Arc::new(Registry::default()),
11076 Arc::new(ForwardingTable::default()),
11077 )
11078 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11079 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11080 hello_via_sink(
11081 &handler,
11082 &module_ctx,
11083 &mut module_rx,
11084 hello_frame("aft", PROTOCOL_VERSION, 7),
11085 )
11086 .await;
11087 (handler, hold, module_ctx, module_rx)
11088 }
11089
11090 #[tokio::test]
11097 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11098 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11099 let client = ConnectionId::new(41);
11100 let (client_ctx, mut client_rx) = route_ctx(client);
11101
11102 let (open_task, bind) = relay_route_open(
11103 &handler,
11104 client,
11105 &client_ctx.egress,
11106 &mut module_rx,
11107 200,
11108 "aft",
11109 "acked-before-mark",
11110 )
11111 .await;
11112 let (module_channel, module_epoch) = route_bind_channel(&bind);
11113 hold.wait().await;
11115
11116 assert!(handler
11118 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11119 .await
11120 .unwrap()
11121 .is_empty());
11122 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11123
11124 hold.wait().await;
11125 assert!(
11126 open_task.await.unwrap().is_empty(),
11127 "route.open answers accepted"
11128 );
11129
11130 match module_rx.try_recv() {
11133 Err(mpsc::error::TryRecvError::Empty) => {}
11134 Ok(outbound) => panic!(
11135 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11136 outbound.frame.header.ty,
11137 outbound.frame.header.channel,
11138 outbound.frame.header.epoch
11139 ),
11140 Err(err) => panic!("module egress closed: {err}"),
11141 }
11142
11143 let router = Router::with_control_handler(Arc::new(handler.clone()));
11146 let request = Frame::build(
11147 FrameType::Request,
11148 Flags::new(true, Priority::Interactive, false),
11149 client_channel,
11150 client_epoch,
11151 201,
11152 b"first request".to_vec(),
11153 )
11154 .unwrap();
11155 router
11156 .route_for_connection(&client_ctx, request)
11157 .await
11158 .unwrap();
11159 let delivered = module_rx.try_recv().unwrap().frame;
11160 assert_eq!(delivered.header.ty, FrameType::Request);
11161 assert_eq!(
11162 (delivered.header.channel, delivered.header.epoch),
11163 (module_channel, module_epoch)
11164 );
11165 assert_eq!(delivered.body, b"first request");
11166 }
11167
11168 #[tokio::test]
11174 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11175 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11176 let client = ConnectionId::new(42);
11177 let (client_ctx, _client_rx) = route_ctx(client);
11178
11179 let (open_task, bind) = relay_route_open(
11180 &handler,
11181 client,
11182 &client_ctx.egress,
11183 &mut module_rx,
11184 300,
11185 "aft",
11186 "released-before-mark",
11187 )
11188 .await;
11189 let (module_channel, module_epoch) = route_bind_channel(&bind);
11190 hold.wait().await;
11191
11192 handler.cleanup_connection(client).unwrap();
11193 assert!(
11194 module_rx.try_recv().is_err(),
11195 "client teardown must not GOODBYE a bind route.open is still queueing"
11196 );
11197
11198 hold.wait().await;
11199 let response = open_task.await.unwrap();
11200 assert_eq!(response.len(), 1);
11201 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11202
11203 let goodbye = module_rx
11204 .try_recv()
11205 .expect("module is told to drop the binding the daemon abandoned")
11206 .frame;
11207 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11208 assert_eq!(
11209 (goodbye.header.channel, goodbye.header.epoch),
11210 (module_channel, module_epoch)
11211 );
11212 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11213 }
11214
11215 #[tokio::test]
11220 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11221 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11222 let client = ConnectionId::new(43);
11223 let (client_ctx, client_rx) = route_ctx(client);
11224
11225 let (open_task, bind) = relay_route_open(
11226 &handler,
11227 client,
11228 &client_ctx.egress,
11229 &mut module_rx,
11230 400,
11231 "aft",
11232 "gone-client-before-mark",
11233 )
11234 .await;
11235 let (module_channel, module_epoch) = route_bind_channel(&bind);
11236 hold.wait().await;
11237
11238 drop(client_rx);
11239 assert!(client_ctx.egress.is_closed());
11240 handler
11241 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11242 .await
11243 .unwrap();
11244
11245 hold.wait().await;
11246 let response = open_task.await.unwrap();
11247 assert_eq!(response.len(), 1);
11248 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11249
11250 let goodbye = module_rx
11251 .try_recv()
11252 .expect("module is told to drop the binding it acked")
11253 .frame;
11254 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11255 assert_eq!(
11256 (goodbye.header.channel, goodbye.header.epoch),
11257 (module_channel, module_epoch)
11258 );
11259 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11260 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11261 }
11262
11263 #[test]
11270 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11271 let handler = ControlHandler::default();
11272 let module_connection = ConnectionId::new(30);
11273 let client_connection = ConnectionId::new(31);
11274
11275 handler
11276 .refuse_to_end_module_connection_for_a_client(
11277 module_connection,
11278 77,
11279 ForwardingError::ConnectionClosing {
11280 connection_id: client_connection,
11281 },
11282 )
11283 .expect("a closing client must never end the module connection");
11284
11285 assert!(matches!(
11286 handler.refuse_to_end_module_connection_for_a_client(
11287 module_connection,
11288 78,
11289 ForwardingError::ConnectionClosing {
11290 connection_id: module_connection,
11291 },
11292 ),
11293 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11294 connection_id
11295 })) if connection_id == module_connection
11296 ));
11297 assert!(matches!(
11298 handler.refuse_to_end_module_connection_for_a_client(
11299 module_connection,
11300 79,
11301 ForwardingError::Poisoned,
11302 ),
11303 Err(RouterError::Forwarding(ForwardingError::Poisoned))
11304 ));
11305 assert!(matches!(
11306 handler.refuse_to_end_module_connection_for_a_client(
11307 module_connection,
11308 80,
11309 ForwardingError::StaleModuleEndpoint,
11310 ),
11311 Err(RouterError::Forwarding(
11312 ForwardingError::StaleModuleEndpoint
11313 ))
11314 ));
11315 }
11316
11317 #[tokio::test]
11330 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11331 let registry = Arc::new(Registry::default());
11332 let forwarding = Arc::new(ForwardingTable::default());
11333 let supervisor = SupervisorHandle::new();
11334 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11335 let handler =
11336 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11337 .with_supervisor(supervisor);
11338
11339 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11340 hello_via_sink(
11341 &handler,
11342 &target_ctx,
11343 &mut target_rx,
11344 hello_frame("target", PROTOCOL_VERSION, 1),
11345 )
11346 .await;
11347
11348 let wrong_nonce = handler
11352 .handle_control_frame(
11353 &route_ctx(ConnectionId::new(91)).0,
11354 route_open_frame_with_admission_facts(
11355 20,
11356 "target",
11357 unique_project_root("admission-facts"),
11358 Some(subc_control::ConsumerIdentity {
11359 module_id: "fed".to_string(),
11360 launch_nonce: "not-the-real-nonce".to_string(),
11361 }),
11362 None,
11363 ),
11364 )
11365 .await
11366 .unwrap();
11367 assert_eq!(
11368 parse_error(&wrong_nonce[0])["code"],
11369 "bad_consumer_identity",
11370 "a mismatched launch nonce must be refused, not stamped Reserved"
11371 );
11372
11373 let never_spawned = handler
11377 .handle_control_frame(
11378 &route_ctx(ConnectionId::new(92)).0,
11379 route_open_frame_with_admission_facts(
11380 21,
11381 "target",
11382 unique_project_root("admission-facts"),
11383 Some(subc_control::ConsumerIdentity {
11384 module_id: "never-spawned".to_string(),
11385 launch_nonce: "any-nonce".to_string(),
11386 }),
11387 None,
11388 ),
11389 )
11390 .await
11391 .unwrap();
11392 assert_eq!(
11393 parse_error(&never_spawned[0])["code"],
11394 "bad_consumer_identity",
11395 "an unspawned module_id must be refused rather than accepted for lack of a record"
11396 );
11397 }
11398
11399 #[tokio::test]
11422 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11423 let registry = Arc::new(Registry::default());
11424 let forwarding = Arc::new(ForwardingTable::default());
11425 let supervisor = SupervisorHandle::new();
11426 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11427 let handler =
11428 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11429 .with_supervisor(supervisor);
11430
11431 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11432 hello_via_sink(
11433 &handler,
11434 &target_ctx,
11435 &mut target_rx,
11436 hello_frame("target", PROTOCOL_VERSION, 1),
11437 )
11438 .await;
11439
11440 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11441 let route_handler = handler.clone();
11442 let route_task = tokio::spawn(async move {
11443 route_handler
11444 .handle_control_frame(
11445 &client_ctx,
11446 route_open_frame_with_admission_facts(
11447 30,
11448 "target",
11449 unique_project_root("admission-facts"),
11450 Some(subc_control::ConsumerIdentity {
11451 module_id: "fed".to_string(),
11452 launch_nonce: "fed-nonce".to_string(),
11453 }),
11454 None,
11455 ),
11456 )
11457 .await
11458 .unwrap()
11459 });
11460
11461 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11468 .await
11469 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11470 .expect("module control channel closed before route.bind");
11471 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11472 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11473 panic!("expected route.bind")
11474 };
11475 assert_eq!(
11476 principal,
11477 Some(Principal::Reserved {
11478 module_id: "fed".to_string()
11479 }),
11480 "a correctly attested consumer must be stamped Reserved for its own id"
11481 );
11482
11483 handler
11484 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11485 .await
11486 .unwrap();
11487 assert!(route_task.await.unwrap().is_empty());
11488 assert!(
11489 matches!(
11490 serde_json::from_slice::<ClientControlResponse>(
11491 &client_rx.recv().await.unwrap().body
11492 )
11493 .unwrap(),
11494 ClientControlResponse::RouteOpen { .. }
11495 ),
11496 "the route must actually open, not merely avoid an error"
11497 );
11498 }
11499
11500 #[tokio::test(start_paused = true)]
11501 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11502 let registry = Arc::new(Registry::default());
11503 let forwarding = Arc::new(ForwardingTable::default());
11504 let supervisor = SupervisorHandle::new();
11505 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11506 let handler =
11507 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11508 .with_supervisor(supervisor);
11509
11510 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11511 hello_via_sink(
11512 &handler,
11513 &target_ctx,
11514 &mut target_rx,
11515 hello_frame("target", PROTOCOL_VERSION, 1),
11516 )
11517 .await;
11518
11519 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11520 let direct_handler = handler.clone();
11521 let direct_open = tokio::spawn(async move {
11522 direct_handler
11523 .handle_control_frame(
11524 &direct_ctx,
11525 route_open_frame(2, "target", unique_project_root("route-census-direct")),
11526 )
11527 .await
11528 .unwrap()
11529 });
11530 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11531 .await
11532 .expect("no direct route.bind within 5s")
11533 .expect("target control channel closed before direct route.bind");
11534 handler
11535 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11536 .await
11537 .unwrap();
11538 assert!(direct_open.await.unwrap().is_empty());
11539 let _ = direct_rx.recv().await.unwrap();
11540
11541 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11542 let reserved_handler = handler.clone();
11543 let reserved_open = tokio::spawn(async move {
11544 reserved_handler
11545 .handle_control_frame(
11546 &reserved_ctx,
11547 route_open_frame_with_admission_facts(
11548 3,
11549 "target",
11550 unique_project_root("admission-facts"),
11551 Some(ConsumerIdentity {
11552 module_id: "fed".to_string(),
11553 launch_nonce: "fed-nonce".to_string(),
11554 }),
11555 None,
11556 ),
11557 )
11558 .await
11559 .unwrap()
11560 });
11561 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11562 .await
11563 .expect("no reserved route.bind within 5s")
11564 .expect("target control channel closed before reserved route.bind");
11565 handler
11566 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11567 .await
11568 .unwrap();
11569 assert!(reserved_open.await.unwrap().is_empty());
11570 let _ = reserved_rx.recv().await.unwrap();
11571
11572 forwarding
11573 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11574 .unwrap();
11575 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11576 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11577 module_id: Some("target".to_string()),
11578 })
11579 .unwrap();
11580 let census_frame =
11581 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11582 let response = handler
11583 .handle_control_frame(&census_ctx, census_frame)
11584 .await
11585 .unwrap()
11586 .pop()
11587 .unwrap();
11588 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11589 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11590 assert!(matches!(
11591 decoded,
11592 ClientControlResponse::SupervisorRoutes { .. }
11593 ));
11594 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11595 assert_eq!(routes.len(), 2);
11596 assert!(routes.iter().all(|route| route["draining"] == true));
11597 assert!(
11600 routes.iter().all(|route| route["drain_reason"] == "reload"),
11601 "draining routes must name the drain's reason: {routes:?}"
11602 );
11603 assert!(routes.iter().any(|route| {
11604 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11605 }));
11606 assert!(routes.iter().any(|route| {
11607 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11608 }));
11609
11610 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11611 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11612 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11613 std::fs::write(
11614 &golden_path,
11615 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11616 )
11617 .unwrap();
11618 }
11619 let expected: Value =
11620 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11621 assert_eq!(actual, expected);
11622 }
11623
11624 async fn query_live_roots(
11625 handler: &ControlHandler,
11626 module_ctx: &RouteCtx,
11627 ) -> ModuleControlResponseToModule {
11628 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11629 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11630 let response = handler
11631 .handle_control_frame(module_ctx, frame)
11632 .await
11633 .unwrap()
11634 .pop()
11635 .unwrap();
11636 serde_json::from_slice(&response.body).unwrap()
11637 }
11638
11639 #[tokio::test(start_paused = true)]
11640 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11641 let registry = Arc::new(Registry::default());
11642 let forwarding = Arc::new(ForwardingTable::default());
11643 let handler = ControlHandler::with_forwarding(registry, forwarding);
11644 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11645 hello_via_sink(
11646 &handler,
11647 &target_ctx,
11648 &mut target_rx,
11649 hello_frame("target", PROTOCOL_VERSION, 1),
11650 )
11651 .await;
11652 let root = unique_project_root("live-roots-known");
11653 let path = ProjectRootId::from_path_allowing_missing(root.path())
11654 .unwrap()
11655 .as_path()
11656 .to_path_buf();
11657 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11658 let open_handler = handler.clone();
11659 let opened = tokio::spawn(async move {
11660 open_handler
11661 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11662 .await
11663 .unwrap()
11664 });
11665 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11666 .await
11667 .unwrap()
11668 .unwrap();
11669 handler
11670 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11671 .await
11672 .unwrap();
11673 assert!(opened.await.unwrap().is_empty());
11674 let _ = client_rx.recv().await.unwrap();
11675
11676 let root = unique_project_root("live-roots-pending");
11677 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11678 .unwrap()
11679 .as_path()
11680 .to_path_buf();
11681 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11682 let open_handler = handler.clone();
11683 let pending = tokio::spawn(async move {
11684 open_handler
11685 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11686 .await
11687 .unwrap()
11688 });
11689 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11690 .await
11691 .unwrap()
11692 .unwrap();
11693 let actual = query_live_roots(&handler, &target_ctx).await;
11694 let ModuleControlResponseToModule::LiveRoots {
11695 roots,
11696 unknown_root_bindings,
11697 total_bindings,
11698 } = actual
11699 else {
11700 panic!("expected live roots")
11701 };
11702 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11703 assert_eq!(unknown_root_bindings, 0);
11704 assert_eq!(
11705 roots.len(),
11706 2,
11707 "root-known arm must retain each canonical root"
11708 );
11709 assert_eq!(
11710 total_bindings,
11711 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11712 );
11713 let counts = roots
11714 .iter()
11715 .map(|root| (root.project_root.clone(), root.bound, root.pending))
11716 .collect::<Vec<_>>();
11717 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11718 expected.sort_by(|a, b| a.0.cmp(&b.0));
11719 assert_eq!(
11720 counts, expected,
11721 "roots must sort by path and count pending separately"
11722 );
11723 handler
11724 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11725 .await
11726 .unwrap();
11727 assert!(pending.await.unwrap().is_empty());
11728 }
11729
11730 #[tokio::test(start_paused = true)]
11731 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11732 let forwarding = Arc::new(ForwardingTable::default());
11733 let handler =
11734 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11735 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11736 hello_via_sink(
11737 &handler,
11738 &target_ctx,
11739 &mut target_rx,
11740 hello_frame("target", PROTOCOL_VERSION, 1),
11741 )
11742 .await;
11743 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11744 let pending = forwarding
11745 .begin_route_bind_relay_for_test(
11746 client_ctx.connection_id,
11747 client_ctx.egress.clone(),
11748 2,
11749 "target",
11750 )
11751 .unwrap();
11752 forwarding
11753 .complete_pending_relay(
11754 target_ctx.connection_id,
11755 pending.corr,
11756 RouteBindRelayOutcome::Accepted,
11757 )
11758 .unwrap();
11759 let actual = query_live_roots(&handler, &target_ctx).await;
11760 let ModuleControlResponseToModule::LiveRoots {
11761 roots,
11762 unknown_root_bindings,
11763 total_bindings,
11764 } = actual
11765 else {
11766 panic!("expected live roots")
11767 };
11768 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11769 assert_eq!(
11770 unknown_root_bindings, 1,
11771 "unknown-root arm must not read as no bindings"
11772 );
11773 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11774 assert_eq!(
11775 total_bindings,
11776 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11777 );
11778 }
11779
11780 #[tokio::test(start_paused = true)]
11788 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11789 let forwarding = Arc::new(ForwardingTable::default());
11790 let handler =
11791 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11792 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11793 let replies = handler
11794 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11795 .await
11796 .unwrap();
11797 let queued_by_hello = module_rx.len();
11798
11799 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11800 let open_handler = handler.clone();
11801 let open = tokio::spawn(async move {
11802 open_handler
11803 .handle_control_frame(
11804 &client_ctx,
11805 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11806 )
11807 .await
11808 .unwrap()
11809 });
11810 let mut spins = 0;
11812 while module_rx.len() == queued_by_hello {
11813 spins += 1;
11814 assert!(spins < 10_000, "route.open never queued a route.bind");
11815 tokio::task::yield_now().await;
11816 }
11817
11818 for reply in replies {
11820 module_ctx.egress.send(reply).await.unwrap();
11821 }
11822
11823 let first = module_rx.recv().await.unwrap().frame;
11824 assert_eq!(
11825 first.header.ty,
11826 FrameType::HelloAck,
11827 "the first frame a registering module reads must be its HELLO_ACK"
11828 );
11829 assert_eq!(first.header.corr, 7);
11830 let second = module_rx.recv().await.unwrap().frame;
11831 assert_eq!(second.header.ty, FrameType::Request);
11832 assert!(
11833 matches!(
11834 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11835 ModuleControlRequest::RouteBind { .. }
11836 ),
11837 "the route.bind follows the ack"
11838 );
11839 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11840
11841 handler
11842 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11843 .await
11844 .unwrap();
11845 assert!(open.await.unwrap().is_empty());
11846 let _ = client_rx.recv().await.unwrap();
11847 }
11848
11849 #[tokio::test(start_paused = true)]
11850 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11851 let handler = ControlHandler::with_forwarding(
11852 Arc::new(Registry::default()),
11853 Arc::new(ForwardingTable::default()),
11854 );
11855 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11856 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11857 hello_via_sink(
11858 &handler,
11859 &first_ctx,
11860 &mut first_rx,
11861 hello_frame("first", PROTOCOL_VERSION, 1),
11862 )
11863 .await;
11864 hello_via_sink(
11865 &handler,
11866 &second_ctx,
11867 &mut second_rx,
11868 hello_frame("second", PROTOCOL_VERSION, 2),
11869 )
11870 .await;
11871 let root = unique_project_root("second-only");
11872 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11873 let cloned = handler.clone();
11874 let open = tokio::spawn(async move {
11875 cloned
11876 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11877 .await
11878 .unwrap()
11879 });
11880 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11881 .await
11882 .unwrap()
11883 .unwrap();
11884 let first = query_live_roots(&handler, &first_ctx).await;
11885 let second = query_live_roots(&handler, &second_ctx).await;
11886 assert!(
11887 matches!(
11888 first,
11889 ModuleControlResponseToModule::LiveRoots {
11890 total_bindings: 0,
11891 ..
11892 }
11893 ),
11894 "cross-module scope must not expose another module's roots"
11895 );
11896 assert!(
11897 matches!(
11898 second,
11899 ModuleControlResponseToModule::LiveRoots {
11900 total_bindings: 1,
11901 ..
11902 }
11903 ),
11904 "second module must see its pending route"
11905 );
11906 handler
11907 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11908 .await
11909 .unwrap();
11910 assert!(open.await.unwrap().is_empty());
11911 }
11912
11913 #[tokio::test(start_paused = true)]
11914 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11915 let handler = ControlHandler::with_forwarding(
11916 Arc::new(Registry::default()),
11917 Arc::new(ForwardingTable::default()),
11918 );
11919 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11920 hello_via_sink(
11921 &handler,
11922 &target_ctx,
11923 &mut target_rx,
11924 hello_frame("target", PROTOCOL_VERSION, 1),
11925 )
11926 .await;
11927 let actual = query_live_roots(&handler, &target_ctx).await;
11928 let ModuleControlResponseToModule::LiveRoots {
11929 roots,
11930 unknown_root_bindings,
11931 total_bindings,
11932 } = actual
11933 else {
11934 panic!("expected live roots")
11935 };
11936 assert!(roots.is_empty());
11937 assert_eq!(unknown_root_bindings, 0);
11938 assert_eq!(total_bindings, 0);
11939 assert_eq!(
11940 total_bindings,
11941 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11942 );
11943 }
11944
11945 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11952 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11953 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11954 let text = std::fs::read_to_string(&path)
11955 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11956 let vectors: Vec<(String, Value)> = text
11957 .lines()
11958 .filter(|line| !line.trim().is_empty())
11959 .map(|line| {
11960 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11961 let id = entry["corpus_id"]
11962 .as_str()
11963 .expect("every vector carries a corpus_id")
11964 .to_string();
11965 (id, entry["package"].clone())
11966 })
11967 .collect();
11968 assert_eq!(
11972 vectors.len(),
11973 3,
11974 "vendored fed corpus changed size; re-sync from subc-federation"
11975 );
11976
11977 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11992 let richest = vectors
11993 .iter()
11994 .filter_map(|(_, package)| package.as_object())
11995 .map(|object| {
11996 object
11997 .keys()
11998 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11999 .count()
12000 })
12001 .max()
12002 .unwrap_or(0);
12003 assert!(
12004 richest >= 2,
12005 "vendored corpus no longer carries a package with unmodelled fields, \
12006 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12007 );
12008
12009 vectors
12010 }
12011
12012 #[tokio::test]
12020 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12021 for (corpus_id, package) in fed_admission_facts_vectors() {
12022 let registry = Arc::new(Registry::default());
12023 let forwarding = Arc::new(ForwardingTable::default());
12024 let supervisor = SupervisorHandle::new();
12025 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12026 let handler =
12027 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12028 .with_supervisor(supervisor)
12029 .with_admission_facts_config(
12030 Some("fed".to_string()),
12031 Some(vec!["target".to_string()]),
12032 );
12033
12034 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12035 hello_via_sink(
12036 &handler,
12037 &target_ctx,
12038 &mut target_rx,
12039 hello_frame("target", PROTOCOL_VERSION, 1),
12040 )
12041 .await;
12042
12043 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12044 let route_handler = handler.clone();
12045 let expected = package.clone();
12046 let route_task = tokio::spawn(async move {
12047 route_handler
12048 .handle_control_frame(
12049 &client_ctx,
12050 route_open_frame_with_admission_facts(
12051 20,
12052 "target",
12053 unique_project_root("admission-facts"),
12054 Some(subc_control::ConsumerIdentity {
12055 module_id: "fed".to_string(),
12056 launch_nonce: "fed-nonce".to_string(),
12057 }),
12058 Some(package),
12059 ),
12060 )
12061 .await
12062 .unwrap()
12063 });
12064
12065 let bind_frame = target_rx.recv().await.unwrap();
12066 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12067 let ModuleControlRequest::RouteBind {
12068 admission_facts, ..
12069 } = bind
12070 else {
12071 panic!("{corpus_id}: expected route.bind")
12072 };
12073 assert_eq!(
12074 admission_facts,
12075 Some(expected),
12076 "{corpus_id}: relay must not add, drop or reshape any field"
12077 );
12078
12079 handler
12080 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12081 .await
12082 .unwrap();
12083 route_task.await.unwrap();
12084 }
12085 }
12086
12087 #[tokio::test]
12088 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12089 let registry = Arc::new(Registry::default());
12090 let forwarding = Arc::new(ForwardingTable::default());
12091 let supervisor = SupervisorHandle::new();
12092 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12093 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12094 let handler =
12095 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12096 .with_supervisor(supervisor)
12097 .with_admission_facts_config(
12098 Some("fed".to_string()),
12099 Some(vec!["target".to_string()]),
12100 );
12101
12102 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12103 hello_via_sink(
12104 &handler,
12105 &target_ctx,
12106 &mut target_rx,
12107 hello_frame("target", PROTOCOL_VERSION, 1),
12108 )
12109 .await;
12110 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12111 hello_via_sink(
12112 &handler,
12113 &other_ctx,
12114 &mut other_rx,
12115 hello_frame("other", PROTOCOL_VERSION, 2),
12116 )
12117 .await;
12118
12119 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12120 let expected_facts = facts.clone();
12121 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12122 let route_handler = handler.clone();
12123 let route_task = tokio::spawn(async move {
12124 route_handler
12125 .handle_control_frame(
12126 &client_ctx,
12127 route_open_frame_with_admission_facts(
12128 10,
12129 "target",
12130 unique_project_root("admission-facts"),
12131 Some(subc_control::ConsumerIdentity {
12132 module_id: "fed".to_string(),
12133 launch_nonce: "fed-nonce".to_string(),
12134 }),
12135 Some(facts.clone()),
12136 ),
12137 )
12138 .await
12139 .unwrap()
12140 });
12141 let bind_frame = target_rx.recv().await.unwrap();
12142 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12143 let ModuleControlRequest::RouteBind {
12144 admission_facts, ..
12145 } = bind
12146 else {
12147 panic!("expected route.bind")
12148 };
12149 assert_eq!(admission_facts, Some(expected_facts));
12150 handler
12151 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12152 .await
12153 .unwrap();
12154 assert!(route_task.await.unwrap().is_empty());
12155 assert!(matches!(
12156 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12157 .unwrap(),
12158 ClientControlResponse::RouteOpen { .. }
12159 ));
12160
12161 let direct = handler
12162 .handle_control_frame(
12163 &route_ctx(ConnectionId::new(73)).0,
12164 route_open_frame_with_admission_facts(
12165 11,
12166 "target",
12167 unique_project_root("admission-facts"),
12168 None,
12169 Some(json!({"x": 1})),
12170 ),
12171 )
12172 .await
12173 .unwrap();
12174 assert_eq!(
12175 parse_error(&direct[0])["code"],
12176 "admission_facts_not_permitted"
12177 );
12178
12179 let different_reserved = handler
12180 .handle_control_frame(
12181 &route_ctx(ConnectionId::new(77)).0,
12182 route_open_frame_with_admission_facts(
12183 15,
12184 "target",
12185 unique_project_root("admission-facts"),
12186 Some(subc_control::ConsumerIdentity {
12187 module_id: "other".to_string(),
12188 launch_nonce: "other-nonce".to_string(),
12189 }),
12190 Some(json!({"x": 1})),
12191 ),
12192 )
12193 .await
12194 .unwrap();
12195 assert_eq!(
12196 parse_error(&different_reserved[0])["code"],
12197 "admission_facts_not_permitted"
12198 );
12199
12200 let other_target = handler
12201 .handle_control_frame(
12202 &route_ctx(ConnectionId::new(74)).0,
12203 route_open_frame_with_admission_facts(
12204 12,
12205 "other",
12206 unique_project_root("admission-facts"),
12207 Some(subc_control::ConsumerIdentity {
12208 module_id: "fed".to_string(),
12209 launch_nonce: "fed-nonce".to_string(),
12210 }),
12211 Some(json!({"x": 1})),
12212 ),
12213 )
12214 .await
12215 .unwrap();
12216 assert_eq!(
12217 parse_error(&other_target[0])["code"],
12218 "admission_facts_target_not_allowed"
12219 );
12220
12221 let nonexistent = handler
12222 .handle_control_frame(
12223 &route_ctx(ConnectionId::new(75)).0,
12224 route_open_frame_with_admission_facts(
12225 13,
12226 "missing",
12227 unique_project_root("admission-facts"),
12228 None,
12229 Some(json!({"x": 1})),
12230 ),
12231 )
12232 .await
12233 .unwrap();
12234 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12235
12236 let described = handler
12237 .handle_control_frame(
12238 &route_ctx(ConnectionId::new(76)).0,
12239 Frame::build(
12240 FrameType::Request,
12241 control_flags(),
12242 0,
12243 0,
12244 14,
12245 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12246 )
12247 .unwrap(),
12248 )
12249 .await
12250 .unwrap();
12251 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12252 serde_json::from_slice(&described[0].body).unwrap()
12253 else {
12254 panic!("expected server.describe response")
12255 };
12256 assert!(capabilities
12257 .iter()
12258 .any(|cap| cap == "admission_facts_relay_v1"));
12259 }
12260
12261 #[tokio::test]
12262 async fn admission_facts_without_configured_carrier_are_rejected() {
12263 let registry = Arc::new(Registry::default());
12264 let forwarding = Arc::new(ForwardingTable::default());
12265 let handler = ControlHandler::with_forwarding(registry, forwarding);
12266 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12267 hello_via_sink(
12268 &handler,
12269 &target_ctx,
12270 &mut target_rx,
12271 hello_frame("target", PROTOCOL_VERSION, 1),
12272 )
12273 .await;
12274
12275 let responses = handler
12276 .handle_control_frame(
12277 &route_ctx(ConnectionId::new(79)).0,
12278 route_open_frame_with_admission_facts(
12279 16,
12280 "target",
12281 unique_project_root("admission-facts"),
12282 None,
12283 Some(json!({"x": 1})),
12284 ),
12285 )
12286 .await
12287 .unwrap();
12288 assert_eq!(
12289 parse_error(&responses[0])["code"],
12290 "admission_facts_not_permitted"
12291 );
12292 }
12293
12294 #[tokio::test]
12295 async fn route_open_relays_consumer_capabilities_verbatim() {
12296 let registry = Arc::new(Registry::default());
12297 let forwarding = Arc::new(ForwardingTable::default());
12298 let handler =
12299 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12300 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12301 hello_via_sink(
12302 &handler,
12303 &module_ctx,
12304 &mut module_rx,
12305 hello_frame("aft", PROTOCOL_VERSION, 7),
12306 )
12307 .await;
12308
12309 let expected = vec!["elicitation".to_string(), "roots".to_string()];
12310 let expected_for_request = expected.clone();
12311 let project_root = unique_project_root("consumer-capabilities-present");
12312 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12313 let route_handler = handler.clone();
12314 let route_task = tokio::spawn(async move {
12315 route_handler
12316 .handle_control_frame(
12317 &client_ctx,
12318 route_open_frame_with_consumer_capabilities(
12319 401,
12320 "aft",
12321 project_root,
12322 Some(expected_for_request),
12323 ),
12324 )
12325 .await
12326 .unwrap()
12327 });
12328 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12329 .await
12330 .unwrap()
12331 .unwrap();
12332 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12333 let ModuleControlRequest::RouteBind {
12334 consumer_capabilities,
12335 ..
12336 } = bind
12337 else {
12338 panic!("expected route.bind request, got {bind:?}");
12339 };
12340 assert_eq!(consumer_capabilities, Some(expected.clone()));
12341
12342 handler
12343 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12344 .await
12345 .unwrap();
12346 let route_response = route_task.await.unwrap();
12347 assert!(route_response.is_empty());
12348 let published = client_rx.recv().await.unwrap();
12349 assert!(matches!(
12350 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12351 ClientControlResponse::RouteOpen { .. }
12352 ));
12353 }
12354
12355 #[tokio::test]
12356 async fn route_open_without_consumer_capabilities_relays_none() {
12357 let registry = Arc::new(Registry::default());
12358 let forwarding = Arc::new(ForwardingTable::default());
12359 let handler =
12360 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12361 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12362 hello_via_sink(
12363 &handler,
12364 &module_ctx,
12365 &mut module_rx,
12366 hello_frame("aft", PROTOCOL_VERSION, 7),
12367 )
12368 .await;
12369
12370 let project_root = unique_project_root("consumer-capabilities-absent");
12371 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12372 let route_handler = handler.clone();
12373 let route_task = tokio::spawn(async move {
12374 route_handler
12375 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12376 .await
12377 .unwrap()
12378 });
12379 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12380 .await
12381 .unwrap()
12382 .unwrap();
12383 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12384 let ModuleControlRequest::RouteBind {
12385 consumer_capabilities,
12386 ..
12387 } = bind
12388 else {
12389 panic!("expected route.bind request, got {bind:?}");
12390 };
12391 assert_eq!(consumer_capabilities, None);
12392
12393 handler
12394 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12395 .await
12396 .unwrap();
12397 let route_response = route_task.await.unwrap();
12398 assert!(route_response.is_empty());
12399 let published = client_rx.recv().await.unwrap();
12400 assert!(matches!(
12401 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12402 ClientControlResponse::RouteOpen { .. }
12403 ));
12404 }
12405
12406 async fn bind_role_versions_for(
12410 sent: Option<BTreeMap<String, String>>,
12411 connection: u64,
12412 ) -> Option<BTreeMap<String, String>> {
12413 let registry = Arc::new(Registry::default());
12414 let forwarding = Arc::new(ForwardingTable::default());
12415 let handler =
12416 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12417 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12418 hello_via_sink(
12419 &handler,
12420 &module_ctx,
12421 &mut module_rx,
12422 hello_frame("aft", PROTOCOL_VERSION, 7),
12423 )
12424 .await;
12425 let project_root = unique_project_root("role-versions");
12426 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12427 let route_handler = handler.clone();
12428 let route_task = tokio::spawn(async move {
12429 route_handler
12430 .handle_control_frame(
12431 &client_ctx,
12432 route_open_frame_with_role_versions(403, "aft", project_root, sent),
12433 )
12434 .await
12435 .unwrap()
12436 });
12437 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12438 .await
12439 .expect("a well-formed route.open reaches the module as a bind")
12440 .unwrap();
12441 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12442 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12443 panic!("expected route.bind request, got {bind:?}");
12444 };
12445 handler
12446 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12447 .await
12448 .unwrap();
12449 assert!(route_task.await.unwrap().is_empty());
12450 let published = client_rx.recv().await.unwrap();
12451 assert!(matches!(
12452 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12453 ClientControlResponse::RouteOpen { .. }
12454 ));
12455 role_versions
12456 }
12457
12458 #[tokio::test]
12459 async fn route_open_relays_role_versions_verbatim() {
12460 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12461 assert_eq!(
12462 bind_role_versions_for(Some(sent.clone()), 141).await,
12463 Some(sent)
12464 );
12465 }
12466
12467 #[tokio::test]
12470 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12471 assert_eq!(bind_role_versions_for(None, 143).await, None);
12472 assert_eq!(
12473 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12474 None
12475 );
12476 }
12477
12478 #[tokio::test]
12479 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12480 let registry = Arc::new(Registry::default());
12481 let forwarding = Arc::new(ForwardingTable::default());
12482 let handler =
12483 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12484 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12485 hello_via_sink(
12486 &handler,
12487 &module_ctx,
12488 &mut module_rx,
12489 hello_frame("aft", PROTOCOL_VERSION, 7),
12490 )
12491 .await;
12492
12493 let nine: BTreeMap<String, String> = (0..9)
12494 .map(|index| (format!("role-{index}"), "v1".to_string()))
12495 .collect();
12496 for (label, malformed) in [
12497 (
12498 "invalid role name",
12499 role_versions(&[("Tool_Provider", "v1")]),
12500 ),
12501 ("invalid version", role_versions(&[("tool-provider", "v0")])),
12502 ("nine entries", nine),
12503 ] {
12504 let responses = tokio::time::timeout(
12507 Duration::from_secs(1),
12508 handler.handle_control_frame(
12509 &route_ctx(ConnectionId::new(148)).0,
12510 route_open_frame_with_role_versions(
12511 404,
12512 "aft",
12513 unique_project_root("role-versions-malformed"),
12514 Some(malformed),
12515 ),
12516 ),
12517 )
12518 .await
12519 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12520 .unwrap();
12521 assert_eq!(responses.len(), 1, "{label}");
12522 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12523 let error = parse_error(&responses[0]);
12524 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12525 assert_eq!(
12526 error["detail"]["field"], "role_versions",
12527 "{label}: {error}"
12528 );
12529 assert!(
12530 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12531 "{label}: a malformed declaration is terminal"
12532 );
12533 assert!(
12534 module_rx.try_recv().is_err(),
12535 "{label}: the module must never see a bind"
12536 );
12537 }
12538 }
12539
12540 #[tokio::test]
12544 async fn route_role_versions_capability_is_advertised() {
12545 let handler = ControlHandler::new(Arc::new(Registry::default()));
12546 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12547 let ack = hello_via_sink(
12548 &handler,
12549 &ctx,
12550 &mut rx,
12551 hello_frame("m", PROTOCOL_VERSION, 1),
12552 )
12553 .await;
12554 let ack = parse_ack(&ack);
12555 assert!(
12556 ack.subc_capabilities
12557 .iter()
12558 .any(|c| c == "route-role-versions/v1"),
12559 "{:?}",
12560 ack.subc_capabilities
12561 );
12562
12563 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12564 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12565 let reply = handler
12566 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12567 .await
12568 .unwrap()
12569 .pop()
12570 .unwrap();
12571 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12572 serde_json::from_slice(&reply.body).unwrap()
12573 else {
12574 panic!("not a server.describe reply");
12575 };
12576 assert!(
12577 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12578 "{capabilities:?}"
12579 );
12580 }
12581
12582 #[tokio::test]
12586 async fn catalog_events_capability_is_advertised() {
12587 let handler = ControlHandler::new(Arc::new(Registry::default()));
12588 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12589 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12590 let reply = handler
12591 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12592 .await
12593 .unwrap()
12594 .pop()
12595 .unwrap();
12596 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12597 serde_json::from_slice(&reply.body).unwrap()
12598 else {
12599 panic!("not a server.describe reply");
12600 };
12601 assert!(
12602 capabilities
12603 .iter()
12604 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12605 "{capabilities:?}"
12606 );
12607 }
12608
12609 #[tokio::test]
12610 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12611 let handler = ControlHandler::new(Arc::new(Registry::default()));
12612 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12613 let ack = hello_via_sink(
12614 &handler,
12615 &module,
12616 &mut module_rx,
12617 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12618 )
12619 .await;
12620 let ack = parse_ack(&ack);
12621 assert!(
12622 ack.subc_capabilities
12623 .iter()
12624 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12625 "{:?}",
12626 ack.subc_capabilities
12627 );
12628 assert!(ack
12629 .subc_capabilities
12630 .iter()
12631 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12632
12633 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12634 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12635 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12636 let reply = handler
12637 .handle_control_frame(&client, frame)
12638 .await
12639 .unwrap()
12640 .pop()
12641 .unwrap();
12642 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12643 serde_json::from_slice(&reply.body).unwrap()
12644 else {
12645 panic!("not a server.describe reply");
12646 };
12647 assert!(
12648 capabilities
12649 .iter()
12650 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12651 "{capabilities:?}"
12652 );
12653 assert!(capabilities
12654 .iter()
12655 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12656 }
12657
12658 #[tokio::test]
12659 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12660 let handler = ControlHandler::new(Arc::new(Registry::default()));
12661 let (module, mut rx) = route_ctx(ConnectionId::new(156));
12662 let ack = hello_via_sink(
12663 &handler,
12664 &module,
12665 &mut rx,
12666 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12667 )
12668 .await;
12669 assert!(parse_ack(&ack)
12670 .subc_capabilities
12671 .iter()
12672 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12673 assert!(parse_ack(&ack)
12674 .subc_capabilities
12675 .iter()
12676 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12677 let (client, _rx) = route_ctx(ConnectionId::new(157));
12678 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12679 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12680 let reply = handler
12681 .handle_control_frame(&client, frame)
12682 .await
12683 .unwrap()
12684 .pop()
12685 .unwrap();
12686 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12687 serde_json::from_slice(&reply.body).unwrap()
12688 else {
12689 panic!("server.describe reply")
12690 };
12691 assert!(capabilities
12692 .iter()
12693 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12694 assert!(capabilities
12695 .iter()
12696 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12697 }
12698
12699 #[tokio::test]
12700 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12701 let handler = ControlHandler::new(Arc::new(Registry::default()));
12702 let (module, mut rx) = route_ctx(ConnectionId::new(158));
12703 let ack = hello_via_sink(
12704 &handler,
12705 &module,
12706 &mut rx,
12707 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12708 )
12709 .await;
12710 assert!(parse_ack(&ack)
12711 .subc_capabilities
12712 .iter()
12713 .any(|cap| cap == "scope-workdir/v1"));
12714 let (client, _rx) = route_ctx(ConnectionId::new(159));
12715 let frame = Frame::build(
12716 FrameType::Request,
12717 control_flags(),
12718 0,
12719 0,
12720 5,
12721 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12722 )
12723 .unwrap();
12724 let reply = handler
12725 .handle_control_frame(&client, frame)
12726 .await
12727 .unwrap()
12728 .remove(0);
12729 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12730 serde_json::from_slice(&reply.body).unwrap()
12731 else {
12732 panic!("server.describe reply")
12733 };
12734 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12735 }
12736
12737 #[tokio::test]
12738 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
12739 let handler = ControlHandler::new(Arc::new(Registry::default()));
12740 let (module, mut rx) = route_ctx(ConnectionId::new(160));
12741 let ack = hello_via_sink(
12742 &handler,
12743 &module,
12744 &mut rx,
12745 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
12746 )
12747 .await;
12748 assert!(parse_ack(&ack)
12749 .subc_capabilities
12750 .iter()
12751 .any(|cap| cap == "worker-runs/v1"));
12752 let (client, _rx) = route_ctx(ConnectionId::new(161));
12753 let frame = Frame::build(
12754 FrameType::Request,
12755 control_flags(),
12756 0,
12757 0,
12758 5,
12759 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12760 )
12761 .unwrap();
12762 let reply = handler
12763 .handle_control_frame(&client, frame)
12764 .await
12765 .unwrap()
12766 .remove(0);
12767 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12768 serde_json::from_slice(&reply.body).unwrap()
12769 else {
12770 panic!("server.describe reply")
12771 };
12772 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
12773 }
12774
12775 #[tokio::test]
12776 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12777 let handler = ControlHandler::new(Arc::new(Registry::default()));
12778 let (module, mut rx) = route_ctx(ConnectionId::new(154));
12779 let ack = hello_via_sink(
12780 &handler,
12781 &module,
12782 &mut rx,
12783 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12784 )
12785 .await;
12786 assert!(parse_ack(&ack)
12787 .subc_capabilities
12788 .iter()
12789 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12790 let (client, _rx) = route_ctx(ConnectionId::new(155));
12791 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12792 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12793 let reply = handler
12794 .handle_control_frame(&client, frame)
12795 .await
12796 .unwrap()
12797 .pop()
12798 .unwrap();
12799 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12800 serde_json::from_slice(&reply.body).unwrap()
12801 else {
12802 panic!("not a server.describe reply")
12803 };
12804 assert!(capabilities
12805 .iter()
12806 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12807 }
12808
12809 #[tokio::test]
12810 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12811 let registry = Arc::new(Registry::default());
12812 let forwarding = Arc::new(ForwardingTable::default());
12813 let handler =
12814 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12815 .with_health_probe_timeout(Duration::from_secs(5));
12816 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12817 hello_via_sink(
12818 &handler,
12819 &module_ctx,
12820 &mut module_rx,
12821 non_routable_hello_frame_with_control_ops(
12822 "mcp",
12823 300,
12824 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12825 ),
12826 )
12827 .await;
12828 assert!(registry
12829 .get_module("mcp")
12830 .unwrap()
12831 .unwrap()
12832 .manifest
12833 .provides
12834 .is_empty());
12835
12836 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12837 let route_response = handler
12838 .handle_control_frame(
12839 &route_client_ctx,
12840 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12841 )
12842 .await
12843 .unwrap();
12844 assert_eq!(route_response[0].header.ty, FrameType::Error);
12845 assert_eq!(
12846 parse_error(&route_response[0])["code"],
12847 "target_unavailable"
12848 );
12849 assert!(parse_error(&route_response[0])["message"]
12850 .as_str()
12851 .unwrap()
12852 .contains("does not provide the requested target"));
12853 assert!(module_rx.try_recv().is_err());
12854
12855 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12856 let health_handler = handler.clone();
12857 let health_task = tokio::spawn(async move {
12858 health_handler
12859 .handle_control_frame(
12860 &health_client_ctx,
12861 supervisor_health_probe_frame(302, "mcp"),
12862 )
12863 .await
12864 .unwrap()
12865 });
12866 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12867 .await
12868 .unwrap()
12869 .unwrap();
12870 assert_eq!(
12871 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12872 ModuleControlRequest::HealthCheck {}
12873 );
12874 handler
12875 .handle_control_frame(
12876 &module_ctx,
12877 health_response(health_frame.header.corr, HealthStatus::Ok),
12878 )
12879 .await
12880 .unwrap();
12881 let health_response = health_task.await.unwrap();
12882 assert_eq!(health_response[0].header.ty, FrameType::Response);
12883 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12884 ClientControlResponse::SupervisorHealthProbe {
12885 module_id, status, ..
12886 } => {
12887 assert_eq!(module_id, "mcp");
12888 assert_eq!(status, HealthStatus::Ok);
12889 }
12890 other => panic!("unexpected health response: {other:?}"),
12891 }
12892
12893 forwarding
12898 .cleanup_connection(module_ctx.connection_id)
12899 .unwrap();
12900 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12901 let cleanup_response = tokio::time::timeout(
12902 Duration::from_millis(200),
12903 handler.handle_control_frame(
12904 &cleanup_probe_ctx,
12905 supervisor_health_probe_frame(303, "mcp"),
12906 ),
12907 )
12908 .await
12909 .expect("probe should fail immediately when the control lane is gone")
12910 .unwrap();
12911 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12912 assert_eq!(
12913 parse_error(&cleanup_response[0])["code"],
12914 "target_unavailable"
12915 );
12916 assert!(parse_error(&cleanup_response[0])["message"]
12917 .as_str()
12918 .unwrap()
12919 .contains("no module connection"));
12920
12921 handler
12922 .cleanup_connection(module_ctx.connection_id)
12923 .unwrap();
12924 }
12925
12926 #[tokio::test]
12927 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12928 let registry = Arc::new(Registry::default());
12929 let supervisor_handle = SupervisorHandle::new();
12930 let supervisor =
12931 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12932 .with_handle(supervisor_handle.clone())
12933 .with_connection_file_path(
12934 std::env::temp_dir()
12935 .join(format!("subc-route-open-warming-{}", std::process::id())),
12936 );
12937 let module = supervisor
12938 .supervise_configured(
12939 ModuleSpec {
12940 module_id: "warming".to_string(),
12941 program: fake_aft_stub_path(),
12942 args: Vec::new(),
12943 env: Vec::new(),
12944 reserved: false,
12945 reserved_prefixes: Vec::new(),
12946 protocol: ModuleProtocol::Subc,
12947 overlap: Default::default(),
12948 },
12949 true,
12950 )
12951 .unwrap();
12952 assert_eq!(module.state().unwrap(), ModuleState::Running);
12953
12954 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12955 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12956 let response = handler
12957 .handle_control_frame(
12958 &ctx,
12959 route_open_frame(304, "warming", unique_project_root("warming")),
12960 )
12961 .await
12962 .unwrap();
12963 module.stop().await.unwrap();
12964
12965 assert_eq!(response[0].header.ty, FrameType::Error);
12966 let error = parse_error(&response[0]);
12967 assert_eq!(error["code"], "module_warming");
12968 assert!(error["message"]
12969 .as_str()
12970 .unwrap()
12971 .contains("state=running, enabled=true, live=false"));
12972 }
12973
12974 #[test]
12975 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12976 let handler = ControlHandler::new(Arc::new(Registry::default()));
12977 let capture = EventCapture::default();
12978 let _subscriber =
12979 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12980 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12981 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12982 let pending = (0..limit).collect::<Vec<_>>();
12983 let response = handler
12984 .route_open_capacity_refusal(
12985 &ctx,
12986 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12987 "busy",
12988 pending.len(),
12989 limit,
12990 )
12991 .unwrap();
12992 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12993 let event = capture
12994 .events()
12995 .into_iter()
12996 .find(|event| {
12997 event.target == "control"
12998 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12999 })
13000 .expect("connection admission refusal event");
13001 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13002 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13003 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13006 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13007 }
13008
13009 #[test]
13010 fn route_open_target_cap_logs_admission_reason_and_capacity() {
13011 let handler = ControlHandler::new(Arc::new(Registry::default()));
13012 let capture = EventCapture::default();
13013 let _subscriber =
13014 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13015 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13016 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13017 let guards = (0..limit)
13018 .map(|_| {
13019 handler
13020 .route_bind_concurrency
13021 .try_admit("busy", limit)
13022 .unwrap()
13023 })
13024 .collect::<Vec<_>>();
13025 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13026 Err(in_flight) => in_flight,
13027 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13028 };
13029 let response = handler
13030 .route_open_target_capacity_refusal(
13031 &ctx,
13032 &RouteOpenCaller::unknown(),
13033 &route_open_frame(397, "busy", unique_project_root("target-cap")),
13034 "busy",
13035 in_flight,
13036 )
13037 .unwrap();
13038 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13039 let event = capture
13040 .events()
13041 .into_iter()
13042 .find(|event| {
13043 event.target == "control"
13044 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13045 })
13046 .expect("target admission refusal event");
13047 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13048 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13049 drop(guards);
13050 }
13051
13052 #[cfg(unix)]
13053 #[tokio::test(start_paused = true)]
13054 async fn supervisor_list_shows_a_failed_modules_next_retry() {
13055 let registry = Arc::new(Registry::default());
13056 let handle = SupervisorHandle::new();
13057 let supervisor =
13058 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13059 .with_handle(handle.clone());
13060 let module = supervisor
13061 .spawn(crate::ModuleSpec {
13062 module_id: "retry-list".into(),
13063 program: PathBuf::from("/bin/sh"),
13064 args: vec!["-c".into(), "exit 17".into()],
13065 env: vec![],
13066 reserved: false,
13067 reserved_prefixes: vec![],
13068 protocol: ModuleProtocol::None,
13069 overlap: Default::default(),
13070 })
13071 .unwrap();
13072 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13073 while module.state().unwrap() != crate::ModuleState::Failed {
13074 assert!(
13075 std::time::Instant::now() < deadline,
13076 "waiting for budget failure"
13077 );
13078 tokio::task::yield_now().await;
13079 }
13080 let handler = ControlHandler::new(registry).with_supervisor(handle);
13081 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13082 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13083 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13084 assert_eq!(body["modules"][0]["state"], "failed");
13085 let retry = body["modules"][0]["next_retry_at_ms"]
13086 .as_u64()
13087 .expect("scheduled retry on the wire");
13088 let now = std::time::SystemTime::now()
13089 .duration_since(std::time::UNIX_EPOCH)
13090 .unwrap()
13091 .as_millis() as u64;
13092 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13093 module.set_enabled(false).await.unwrap();
13094 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13095 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13096 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13097 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13098 }
13099
13100 #[tokio::test]
13101 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
13102 let registry = Arc::new(Registry::default());
13103 let handle = SupervisorHandle::new();
13104 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13105 .with_handle(handle.clone());
13106 supervisor
13107 .supervise_configured(
13108 crate::ModuleSpec {
13109 module_id: "drift".into(),
13110 program: PathBuf::from("disabled-module"),
13111 args: Vec::new(),
13112 env: Vec::new(),
13113 reserved: false,
13114 reserved_prefixes: Vec::new(),
13115 protocol: ModuleProtocol::Subc,
13116 overlap: Default::default(),
13117 },
13118 false,
13119 )
13120 .unwrap();
13121 let handler = ControlHandler::new(registry).with_supervisor(handle);
13122 for configured in [None, Some(false), Some(true)] {
13123 handler.capability_evaluator.configure(
13124 configured.map(|enabled| ("drift".to_string(), enabled)),
13125 BTreeMap::new(),
13126 );
13127 let frame =
13128 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13129 let replies = handler
13130 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
13131 .await
13132 .unwrap();
13133 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13134 assert!(body.get("configured_enabled").is_none());
13135 assert_eq!(body["op"], "supervisor.ack");
13136 let frame =
13137 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13138 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13139 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13140 assert_eq!(
13141 body["modules"][0].get("configured_enabled"),
13142 configured.map(Value::Bool).as_ref()
13143 );
13144 }
13145 }
13146
13147 #[tokio::test(flavor = "current_thread")]
13148 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
13149 let handler = ControlHandler::new(Arc::new(Registry::default()));
13150 let capture = EventCapture::default();
13151 let _subscriber =
13152 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13153 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
13154 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13155
13156 let _ = handler
13157 .handle_client_control_request(
13158 &ctx,
13159 frame,
13160 ClientControlRequest::SupervisorSetEnabled {
13161 module_id: "broca".to_string(),
13162 enabled: false,
13163 },
13164 &mut None,
13165 )
13166 .await
13167 .unwrap();
13168
13169 let events = capture
13170 .events()
13171 .into_iter()
13172 .filter(|event| {
13173 event.target == "control"
13174 && event.fields.get("message").map(String::as_str)
13175 == Some("supervisor request received")
13176 })
13177 .collect::<Vec<_>>();
13178 assert_eq!(events.len(), 1, "one supervisor request log line");
13179 let event = &events[0];
13180 assert_eq!(event.level, tracing::Level::INFO);
13181 assert_eq!(
13182 event.fields.get("op"),
13183 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
13184 );
13185 assert_eq!(
13186 event.fields.get("module_id"),
13187 Some(&"\"broca\"".to_string())
13188 );
13189 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
13190 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
13191 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
13192 }
13193
13194 #[tokio::test(flavor = "current_thread")]
13195 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
13196 let registry = Arc::new(Registry::default());
13197 let module_connection = ConnectionId::new(303);
13198 registry
13199 .register_with_control_ops(
13200 manifest("aft", PROTOCOL_VERSION),
13201 PROTOCOL_VERSION,
13202 module_connection,
13203 module_baseline_control_ops(),
13204 )
13205 .unwrap();
13206 let handler = ControlHandler::new(registry);
13207 let capture = EventCapture::default();
13208 let _subscriber =
13209 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13210 let (ctx, _rx) = route_ctx(module_connection);
13211 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13212
13213 let _ = handler
13214 .handle_client_control_request(
13215 &ctx,
13216 frame,
13217 ClientControlRequest::SupervisorSetEnabled {
13218 module_id: "broca".to_string(),
13219 enabled: false,
13220 },
13221 &mut None,
13222 )
13223 .await
13224 .unwrap();
13225
13226 let callers = capture
13227 .events()
13228 .into_iter()
13229 .filter(|event| {
13230 event.fields.get("message").map(String::as_str)
13231 == Some("supervisor request received")
13232 })
13233 .map(|event| event.fields.get("caller").cloned())
13234 .collect::<Vec<_>>();
13235 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13236 }
13237
13238 #[tokio::test(flavor = "current_thread")]
13239 async fn supervisor_list_does_not_log_request_received() {
13240 let handler = ControlHandler::new(Arc::new(Registry::default()));
13241 let capture = EventCapture::default();
13242 let _subscriber =
13243 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13244 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13245 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13246
13247 handler
13248 .handle_client_control_request(
13249 &ctx,
13250 frame,
13251 ClientControlRequest::SupervisorList {},
13252 &mut None,
13253 )
13254 .await
13255 .unwrap();
13256
13257 assert!(capture.events().into_iter().all(|event| {
13258 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13259 }));
13260 }
13261
13262 #[tokio::test(flavor = "current_thread")]
13263 async fn supervisor_rescan_preview_does_not_log_request_received() {
13264 let handler = ControlHandler::new(Arc::new(Registry::default()));
13265 let capture = EventCapture::default();
13266 let _subscriber =
13267 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13268 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13269 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13270
13271 handler
13272 .handle_client_control_request(
13273 &ctx,
13274 frame,
13275 ClientControlRequest::SupervisorRescan { preview: true },
13276 &mut None,
13277 )
13278 .await
13279 .unwrap();
13280
13281 assert!(capture.events().into_iter().all(|event| {
13282 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13283 }));
13284 }
13285
13286 struct ApproveOperator;
13287 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13288 fn prompt(
13289 &self,
13290 _: &str,
13291 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13292 ) -> crate::operator_confirm::ProviderResult {
13293 crate::operator_confirm::ProviderResult::Approved
13294 }
13295 }
13296
13297 #[tokio::test]
13298 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13299 assert_eq!(
13300 ControlHandler::default().privileged_operator_prompt,
13301 PrivilegedOperatorPrompt::Log
13302 );
13303 for (initial, changed, running) in [
13304 ("log", "enforce", PrivilegedOperatorPrompt::Log),
13305 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13306 ] {
13307 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13308 let path = dir.join("subc.jsonc");
13309 std::fs::write(
13310 &path,
13311 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13312 )
13313 .unwrap();
13314 let config = crate::daemon_config::load(&path).unwrap().unwrap();
13315 let registry = Arc::new(Registry::default());
13316 let handle = SupervisorHandle::new();
13317 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13318 .with_handle(handle.clone());
13319 let handler = ControlHandler::new(registry)
13320 .with_supervisor(handle)
13321 .with_privileged_operator_prompt(config.privileged_operator_prompt)
13322 .with_supervisor_rescan(supervisor, &path, config.port);
13323 handler.forwarding.operator_confirms().configure(
13324 Arc::new(ApproveOperator),
13325 crate::operator_confirm::OperatorLimits::default(),
13326 );
13327 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13328
13329 for (disk_mode, needs_restart) in [
13330 (Some(initial), false),
13331 (Some(changed), true),
13332 (Some(changed), true),
13333 (None, running == PrivilegedOperatorPrompt::Enforce),
13334 (Some(initial), false),
13335 ] {
13336 let mut doc = json!({"version":1});
13337 if let Some(mode) = disk_mode {
13338 doc["privileged_operator_prompt"] = json!(mode);
13339 }
13340 std::fs::write(&path, doc.to_string()).unwrap();
13341 for preview in [true, false] {
13342 let request = ClientControlRequest::SupervisorRescan { preview };
13343 let frame = Frame::build(
13344 FrameType::Request,
13345 control_flags(),
13346 0,
13347 0,
13348 1,
13349 serde_json::to_vec(&request).unwrap(),
13350 )
13351 .unwrap();
13352 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13353 assert_eq!(frames.len(), 1);
13354 let ClientControlResponse::SupervisorRescan { result } =
13355 serde_json::from_slice(&frames[0].body).unwrap()
13356 else {
13357 panic!("expected a rescan response");
13358 };
13359 let expected = if needs_restart {
13360 vec!["privileged_operator_prompt"]
13361 } else {
13362 Vec::new()
13363 };
13364 assert_eq!(result.restart_required, expected);
13365 assert_eq!(handler.privileged_operator_prompt, running);
13366 assert_eq!(
13367 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13368 running
13369 );
13370 }
13371 }
13372 }
13373 }
13374
13375 #[tokio::test]
13376 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13377 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13378 let path = dir.join("subc.jsonc");
13379 let registry = Arc::new(Registry::default());
13380 let handle = SupervisorHandle::new();
13381 let supervisor =
13382 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13383 let handler = ControlHandler::new(registry)
13384 .with_supervisor(handle)
13385 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13386 .with_supervisor_rescan(supervisor, &path, None);
13387 handler.forwarding.operator_confirms().configure(
13388 Arc::new(ApproveOperator),
13389 crate::operator_confirm::OperatorLimits::default(),
13390 );
13391 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13392 for preview in [true, false] {
13393 std::fs::write(
13394 &path,
13395 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13396 )
13397 .unwrap();
13398 let request = ClientControlRequest::SupervisorRescan { preview };
13399 let frame = Frame::build(
13400 FrameType::Request,
13401 control_flags(),
13402 0,
13403 0,
13404 1,
13405 serde_json::to_vec(&request).unwrap(),
13406 )
13407 .unwrap();
13408 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13409 assert_eq!(frames.len(), 1);
13410 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13411 assert_eq!(
13412 handler.privileged_operator_prompt,
13413 PrivilegedOperatorPrompt::Enforce
13414 );
13415 }
13416 }
13417
13418 #[tokio::test]
13423 async fn route_open_refusal_names_the_check_that_refused() {
13424 let handler = ControlHandler::new(Arc::new(Registry::default()));
13425 let capture = EventCapture::default();
13426 let _subscriber =
13427 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13428 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13429 let response = handler
13430 .handle_control_frame(
13431 &ctx,
13432 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13433 )
13434 .await
13435 .unwrap();
13436
13437 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13438 let event = capture
13439 .events()
13440 .into_iter()
13441 .find(|event| {
13442 event.target == "control"
13443 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13444 })
13445 .expect("route.open refusal event");
13446 assert_eq!(
13447 event.fields.get("reason"),
13448 Some(&"\"not_registered\"".to_string())
13449 );
13450 }
13451
13452 #[tokio::test]
13453 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13454 let registry = Arc::new(Registry::default());
13455 let supervisor_handle = SupervisorHandle::new();
13456 let supervisor =
13457 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13458 .with_handle(supervisor_handle.clone())
13459 .with_connection_file_path(std::env::temp_dir().join(format!(
13460 "subc-route-open-refusal-info-{}",
13461 std::process::id()
13462 )));
13463 let module = supervisor
13464 .supervise_configured(
13465 ModuleSpec {
13466 module_id: "warming".to_string(),
13467 program: fake_aft_stub_path(),
13468 args: Vec::new(),
13469 env: Vec::new(),
13470 reserved: false,
13471 reserved_prefixes: Vec::new(),
13472 protocol: ModuleProtocol::Subc,
13473 overlap: Default::default(),
13474 },
13475 true,
13476 )
13477 .unwrap();
13478 assert_eq!(module.state().unwrap(), ModuleState::Running);
13479
13480 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13481 assert!(handler
13482 .counters()
13483 .snapshot()
13484 .get("route_open_refused_by_code")
13485 .is_none());
13486 let capture = EventCapture::default();
13487 let _subscriber =
13488 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13489 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13490 let response = handler
13491 .handle_control_frame(
13492 &ctx,
13493 route_open_frame(394, "warming", unique_project_root("refusal-info")),
13494 )
13495 .await
13496 .unwrap();
13497 module.stop().await.unwrap();
13498
13499 assert_eq!(parse_error(&response[0])["code"], "module_warming");
13500 let event = capture
13501 .events()
13502 .into_iter()
13503 .find(|event| {
13504 event.target == "control"
13505 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13506 })
13507 .expect("route.open refusal event");
13508 assert_eq!(
13509 event.fields.get("module_id"),
13510 Some(&"\"warming\"".to_string())
13511 );
13512 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13513 assert_eq!(
13514 event.fields.get("reason"),
13515 Some(&"\"supervised_not_registered\"".to_string())
13516 );
13517 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13518 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13519 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13520 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13521 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13522 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13523 assert_eq!(
13524 handler.counters().snapshot()["route_open_refused_by_code"],
13525 json!({ "module_warming": 1 })
13526 );
13527 }
13528
13529 const OUTAGE_START: &str = "route.open refusing module: not serving";
13530 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13531
13532 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13533 capture
13534 .events()
13535 .into_iter()
13536 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13537 .collect()
13538 }
13539
13540 fn supervise_stub(
13541 registry: &Arc<Registry>,
13542 module_id: &str,
13543 enabled: bool,
13544 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13545 let supervisor_handle = SupervisorHandle::new();
13546 let supervisor =
13547 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13548 .with_handle(supervisor_handle.clone())
13549 .with_connection_file_path(std::env::temp_dir().join(format!(
13550 "subc-route-outage-{module_id}-{}",
13551 std::process::id()
13552 )));
13553 let module = supervisor
13554 .supervise_configured(
13555 ModuleSpec {
13556 module_id: module_id.to_string(),
13557 program: fake_aft_stub_path(),
13558 args: Vec::new(),
13559 env: Vec::new(),
13560 reserved: false,
13561 reserved_prefixes: Vec::new(),
13562 protocol: ModuleProtocol::Subc,
13563 overlap: Default::default(),
13564 },
13565 enabled,
13566 )
13567 .unwrap();
13568 (supervisor_handle, module)
13569 }
13570
13571 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13572 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13573 module_id: module_id.to_string(),
13574 drain_timeout_ms: Some(50),
13575 })
13576 .unwrap();
13577 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13578 }
13579
13580 #[test]
13584 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13585 let registry = Arc::new(Registry::default());
13586 let forwarding = Arc::new(ForwardingTable::default());
13587 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13588 let second = ControlHandler::with_forwarding(registry, forwarding);
13589 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13590 }
13591
13592 #[tokio::test(flavor = "current_thread")]
13596 async fn route_open_unknown_module_refusals_add_no_outage_state() {
13597 let handler = ControlHandler::new(Arc::new(Registry::default()));
13598 let capture = EventCapture::default();
13599 let _subscriber =
13600 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13601 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13602 for corr in 0..8 {
13603 let response = handler
13604 .handle_control_frame(
13605 &ctx,
13606 route_open_frame(
13607 380 + corr,
13608 &format!("nobody-{corr}"),
13609 unique_project_root("outage-unknown"),
13610 ),
13611 )
13612 .await
13613 .unwrap();
13614 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13615 }
13616
13617 assert_eq!(handler.route_outages.tracked_module_count(), 0);
13618 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13619 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13620 }
13621
13622 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13627 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13628 let registry = Arc::new(Registry::default());
13629 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
13630 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13631 let capture = EventCapture::default();
13632 let _subscriber =
13633 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13634 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13635 handler.route_outages.record_accepted("outage-restart");
13638
13639 let response = handler
13640 .handle_control_frame(
13641 &ctx,
13642 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13643 )
13644 .await
13645 .unwrap();
13646 assert_eq!(response[0].header.ty, FrameType::Error);
13647 let starts = outage_lines(&capture, OUTAGE_START);
13648 assert_eq!(starts.len(), 1, "{starts:?}");
13649 assert_eq!(starts[0].level, tracing::Level::WARN);
13650 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13651 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13652 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13653 handler.route_outages.record_accepted("outage-restart");
13654 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13655
13656 let restart = handler
13657 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13658 .await
13659 .unwrap();
13660 assert_eq!(
13661 restart[0].header.ty,
13662 FrameType::Response,
13663 "{:?}",
13664 parse_error(&restart[0])
13665 );
13666 handler
13667 .handle_control_frame(
13668 &ctx,
13669 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13670 )
13671 .await
13672 .unwrap();
13673 module.stop().await.unwrap();
13674
13675 let starts = outage_lines(&capture, OUTAGE_START);
13676 assert_eq!(starts.len(), 2, "{starts:?}");
13677 assert_eq!(starts[1].level, tracing::Level::INFO);
13678 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13679 }
13680
13681 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13685 async fn failed_operator_restart_leaves_no_operator_mark() {
13686 let registry = Arc::new(Registry::default());
13687 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
13688 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13689 let capture = EventCapture::default();
13690 let _subscriber =
13691 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13692 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13693 handler.route_outages.record_accepted("outage-disabled");
13694
13695 let restart = handler
13696 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13697 .await
13698 .unwrap();
13699 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13700 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13701
13702 handler
13703 .handle_control_frame(
13704 &ctx,
13705 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13706 )
13707 .await
13708 .unwrap();
13709 let starts = outage_lines(&capture, OUTAGE_START);
13710 assert_eq!(starts.len(), 1, "{starts:?}");
13711 assert_eq!(starts[0].level, tracing::Level::WARN);
13712 }
13713
13714 #[tokio::test(flavor = "current_thread")]
13715 async fn route_open_unknown_module_escapes_target_module_id() {
13716 let handler = ControlHandler::new(Arc::new(Registry::default()));
13717 let capture = EventCapture::default();
13718 let _subscriber =
13719 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13720 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13721 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13722 let response = handler
13723 .handle_control_frame(
13724 &ctx,
13725 route_open_frame(
13726 395,
13727 hostile_module_id,
13728 unique_project_root("hostile-target-module-id"),
13729 ),
13730 )
13731 .await
13732 .unwrap();
13733
13734 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13735 let event = capture
13736 .events()
13737 .into_iter()
13738 .find(|event| {
13739 event.target == "control"
13740 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13741 })
13742 .expect("route.open unknown-module refusal event");
13743 let logged = event.fields.get("module_id").expect("module_id field");
13744 assert!(!logged.bytes().any(|byte| byte < 0x20));
13745 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13746 }
13747
13748 #[tokio::test(flavor = "current_thread")]
13749 async fn route_open_module_rejection_uses_daemon_counter_key() {
13750 let registry = Arc::new(Registry::default());
13751 let forwarding = Arc::new(ForwardingTable::default());
13752 let handler =
13753 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13754 let module_connection = ConnectionId::new(95);
13755 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13756 hello_via_sink(
13757 &handler,
13758 &module_ctx,
13759 &mut module_rx,
13760 hello_frame("aft", PROTOCOL_VERSION, 395),
13761 )
13762 .await;
13763
13764 let client_connection = ConnectionId::new(96);
13765 let (client_ctx, _client_rx) = route_ctx(client_connection);
13766 let capture = EventCapture::default();
13767 let _subscriber =
13768 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13769 let (route_task, bind) = relay_route_open(
13770 &handler,
13771 client_connection,
13772 &client_ctx.egress,
13773 &mut module_rx,
13774 396,
13775 "aft",
13776 "hostile-module-code",
13777 )
13778 .await;
13779 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13780 let rejection = Frame::build(
13781 FrameType::Error,
13782 control_flags(),
13783 0,
13784 0,
13785 bind.header.corr,
13786 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13787 )
13788 .unwrap();
13789 handler
13790 .handle_control_frame(&module_ctx, rejection)
13791 .await
13792 .unwrap();
13793
13794 let response = route_task.await.unwrap();
13795 assert_eq!(parse_error(&response[0])["code"], hostile_code);
13796 let counters = handler.counters().snapshot();
13797 assert_eq!(
13798 counters["route_open_refused_by_code"],
13799 json!({ "module_rejected": 1 })
13800 );
13801 assert!(counters["route_open_refused_by_code"]
13802 .get(hostile_code)
13803 .is_none());
13804
13805 let event = capture
13806 .events()
13807 .into_iter()
13808 .find(|event| {
13809 event.target == "control"
13810 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13811 })
13812 .expect("route.open module-rejection refusal event");
13813 let logged = event.fields.get("module_code").expect("module_code field");
13814 assert!(!logged.bytes().any(|byte| byte < 0x20));
13815 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13816 }
13817
13818 #[tokio::test]
13819 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13820 let registry = Arc::new(Registry::default());
13821 let supervisor_handle = SupervisorHandle::new();
13822 let missing_program = std::env::temp_dir().join(format!(
13823 "subc-route-open-missing-program-{}",
13824 std::process::id()
13825 ));
13826 let supervisor =
13827 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13828 .with_handle(supervisor_handle.clone());
13829 let module = supervisor
13830 .supervise_configured(
13831 ModuleSpec {
13832 module_id: "failed".to_string(),
13833 program: missing_program,
13834 args: Vec::new(),
13835 env: Vec::new(),
13836 reserved: false,
13837 reserved_prefixes: Vec::new(),
13838 protocol: ModuleProtocol::Subc,
13839 overlap: Default::default(),
13840 },
13841 true,
13842 )
13843 .unwrap();
13844 assert_eq!(module.state().unwrap(), ModuleState::Failed);
13845
13846 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13847 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13848 let response = handler
13849 .handle_control_frame(
13850 &ctx,
13851 route_open_frame(305, "failed", unique_project_root("failed")),
13852 )
13853 .await
13854 .unwrap();
13855
13856 assert_eq!(response[0].header.ty, FrameType::Error);
13857 let error = parse_error(&response[0]);
13858 assert_eq!(error["code"], "target_unavailable");
13859 assert!(error["message"]
13860 .as_str()
13861 .unwrap()
13862 .contains("state=failed, enabled=true, live=false"));
13863 }
13864
13865 #[tokio::test]
13866 async fn route_open_role_mismatch_remains_target_unavailable() {
13867 let registry = Arc::new(Registry::default());
13868 let handler = ControlHandler::new(Arc::clone(®istry));
13869 handler
13870 .handle_control(
13871 ConnectionId::new(41),
13872 non_routable_hello_frame_with_control_ops("health-only", 306, None),
13873 )
13874 .unwrap();
13875
13876 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13877 let response = handler
13878 .handle_control_frame(
13879 &ctx,
13880 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13881 )
13882 .await
13883 .unwrap();
13884
13885 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13886 assert!(parse_error(&response[0])["message"]
13887 .as_str()
13888 .unwrap()
13889 .contains("does not provide the requested target"));
13890 }
13891
13892 #[tokio::test]
13893 async fn route_open_inactive_registration_remains_target_unavailable() {
13894 let registry = Arc::new(Registry::default());
13895 let handler = ControlHandler::new(Arc::clone(®istry));
13896 handler
13897 .handle_control(
13898 ConnectionId::new(43),
13899 hello_frame("inactive", PROTOCOL_VERSION, 308),
13900 )
13901 .unwrap();
13902 assert!(registry
13903 .set_module_state_for_test("inactive", ChannelState::Closed)
13904 .unwrap());
13905
13906 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13907 let response = handler
13908 .handle_control_frame(
13909 &ctx,
13910 route_open_frame(309, "inactive", unique_project_root("inactive")),
13911 )
13912 .await
13913 .unwrap();
13914
13915 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13916 assert!(parse_error(&response[0])["message"]
13917 .as_str()
13918 .unwrap()
13919 .contains("is not active"));
13920 }
13921
13922 #[tokio::test]
13923 async fn late_health_reply_is_recorded_through_the_module_response_path() {
13924 let registry = Arc::new(Registry::default());
13925 let forwarding = Arc::new(ForwardingTable::default());
13926 let supervisor_handle = SupervisorHandle::new();
13927 let supervisor =
13928 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
13929 .with_forwarding(Arc::clone(&forwarding))
13930 .with_handle(supervisor_handle.clone());
13931 let module = supervisor
13932 .supervise_configured(
13933 crate::ModuleSpec {
13934 module_id: "late-health-response".to_string(),
13935 program: PathBuf::from("disabled-module"),
13936 args: Vec::new(),
13937 env: Vec::new(),
13938 reserved: false,
13939 reserved_prefixes: Vec::new(),
13940 protocol: ModuleProtocol::Subc,
13941 overlap: Default::default(),
13942 },
13943 false,
13944 )
13945 .unwrap();
13946 let handler =
13947 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13948 .with_supervisor(supervisor_handle);
13949 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13950 handler
13951 .handle_control_frame(
13952 &module_ctx,
13953 hello_frame_with_control_ops(
13954 "late-health-response",
13955 PROTOCOL_VERSION,
13956 7,
13957 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13958 ),
13959 )
13960 .await
13961 .unwrap();
13962 let probe_started_at = Instant::now() - Duration::from_millis(80);
13963 let pending = forwarding
13964 .begin_health_probe_rpc_for(
13965 "late-health-response",
13966 MODULE_CONTROL_OP_HEALTH_CHECK,
13967 probe_started_at,
13968 Instant::now() - Duration::from_millis(1),
13969 )
13970 .unwrap();
13971 assert!(forwarding
13972 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13973 .unwrap());
13974
13975 let responses = handler
13976 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13977 .await
13978 .unwrap();
13979
13980 assert!(responses.is_empty());
13981 let health = module.status().unwrap().health;
13982 assert_eq!(health.late_answer_count, 1);
13983 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13984 }
13985
13986 #[tokio::test]
13987 async fn health_probe_timeout_and_module_death_are_typed() {
13988 let registry = Arc::new(Registry::default());
13989 let forwarding = Arc::new(ForwardingTable::default());
13990 let handler =
13991 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13992 .with_health_probe_timeout(Duration::from_millis(50));
13993 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13994 hello_via_sink(
13995 &handler,
13996 &module_ctx,
13997 &mut module_rx,
13998 hello_frame_with_control_ops(
13999 "aft",
14000 PROTOCOL_VERSION,
14001 7,
14002 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14003 ),
14004 )
14005 .await;
14006
14007 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14008 let responses = handler
14009 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
14010 .await
14011 .unwrap();
14012 assert_eq!(responses[0].header.ty, FrameType::Error);
14013 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
14014 let _ = module_rx.try_recv();
14015
14016 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
14017 let health_handler = handler.clone();
14018 let death_task = tokio::spawn(async move {
14019 health_handler
14020 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
14021 .await
14022 .unwrap()
14023 });
14024 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14025 .await
14026 .unwrap()
14027 .unwrap();
14028 handler
14029 .cleanup_connection(module_ctx.connection_id)
14030 .unwrap();
14031 let responses = death_task.await.unwrap();
14032 assert_eq!(responses[0].header.ty, FrameType::Error);
14033 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
14034 }
14035
14036 #[test]
14037 fn hello_requires_exact_protocol_version() {
14038 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
14039 let registry = Arc::new(Registry::default());
14040 let handler = ControlHandler::new(Arc::clone(®istry));
14041 let responses = handler
14042 .handle_control(
14043 ConnectionId::new(connection),
14044 hello_frame("aft", offered, 9),
14045 )
14046 .unwrap();
14047
14048 assert_eq!(responses.len(), 1);
14049 assert_eq!(responses[0].header.ty, FrameType::Error);
14050 let error = parse_error(&responses[0]);
14051 assert_eq!(error["code"], "version_unsupported");
14052 assert!(registry.get_module("aft").unwrap().is_none());
14053 assert_eq!(registry.active_registration_count().unwrap(), 0);
14054 }
14055 }
14056
14057 #[test]
14058 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
14059 let registry = Arc::new(Registry::default());
14060 let forwarding = Arc::new(ForwardingTable::default());
14061 let handler =
14062 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14063 let module_connection = ConnectionId::new(301);
14064 let registration = registry
14065 .register_with_control_ops(
14066 manifest("aft-push", PROTOCOL_VERSION),
14067 PROTOCOL_VERSION,
14068 module_connection,
14069 module_baseline_control_ops(),
14070 )
14071 .unwrap();
14072 let (module_tx, _module_rx) = mpsc::channel(8);
14073 let endpoint = forwarding
14074 .register_module_connection(
14075 module_connection,
14076 "aft-push".to_string(),
14077 PROTOCOL_VERSION,
14078 manifest_concurrency(®istration.manifest),
14079 FrameSink::new(module_tx),
14080 )
14081 .unwrap();
14082
14083 let unknown = Frame::build(
14085 FrameType::Push,
14086 control_flags(),
14087 0,
14088 0,
14089 5,
14090 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
14091 )
14092 .unwrap();
14093 let out = handler.handle_status_update(endpoint, unknown).unwrap();
14094 assert!(
14095 out.is_empty(),
14096 "unknown push op must be ignored, got {out:?}"
14097 );
14098
14099 let malformed = Frame::build(
14101 FrameType::Push,
14102 control_flags(),
14103 0,
14104 0,
14105 6,
14106 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
14107 )
14108 .unwrap();
14109 let out = handler.handle_status_update(endpoint, malformed).unwrap();
14110 assert_eq!(out.len(), 1);
14111 assert_eq!(out[0].header.ty, FrameType::Error);
14112 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
14113 }
14114
14115 #[test]
14116 fn hello_rejected_when_connection_already_owns_client_routes() {
14117 let registry = Arc::new(Registry::default());
14118 let forwarding = Arc::new(ForwardingTable::default());
14119 let handler =
14120 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14121 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
14123 let client_connection = ConnectionId::new(202);
14124
14125 let responses = handler
14128 .handle_control(
14129 client_connection,
14130 hello_frame("aft-second", PROTOCOL_VERSION, 9),
14131 )
14132 .unwrap();
14133 assert_eq!(responses[0].header.ty, FrameType::Error);
14134 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
14135 assert!(registry.get_module("aft-second").unwrap().is_none());
14136 }
14137
14138 #[tokio::test]
14139 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
14140 let registry = Arc::new(Registry::default());
14141 let forwarding = Arc::new(ForwardingTable::default());
14142 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
14143 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
14144 hello_via_sink(
14145 &handler,
14146 &module_ctx,
14147 &mut module_rx,
14148 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
14149 )
14150 .await;
14151 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
14152 let pending = forwarding
14153 .begin_route_bind_relay_for_test(
14154 client_ctx.connection_id,
14155 client_ctx.egress.clone(),
14156 2,
14157 "alpha",
14158 )
14159 .unwrap();
14160 forwarding
14161 .complete_pending_relay(
14162 module_ctx.connection_id,
14163 pending.corr,
14164 RouteBindRelayOutcome::Accepted,
14165 )
14166 .unwrap();
14167 client_rx.try_recv().unwrap();
14168 for module_id in ["beta", "alpha"] {
14169 let replies = handler
14170 .handle_control_frame(
14171 &module_ctx,
14172 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
14173 )
14174 .await
14175 .unwrap();
14176 assert_eq!(replies.len(), 1, "second HELLO must be refused");
14177 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
14178 }
14179 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
14180 assert!(registry.get_module("beta").unwrap().is_none());
14181 assert!(matches!(
14182 forwarding
14183 .lookup_data_route(
14184 client_ctx.connection_id,
14185 pending.client_channel,
14186 pending.client_epoch,
14187 )
14188 .unwrap(),
14189 DataRoute::Client(DataRouteState::Bound(_))
14190 ));
14191 assert!(handler
14192 .hello_launch_nonces
14193 .lock()
14194 .unwrap()
14195 .presented(module_ctx.connection_id, Some("alpha-nonce")));
14196 assert!(module_rx.try_recv().is_err());
14197 }
14198
14199 #[test]
14200 fn reserved_module_hello_requires_matching_launch_nonce() {
14201 let registry = Arc::new(Registry::default());
14202 let supervisor = SupervisorHandle::new();
14203 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
14206 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14207
14208 let no_nonce = handler
14210 .handle_control(
14211 ConnectionId::new(1),
14212 hello_frame("vault", PROTOCOL_VERSION, 1),
14213 )
14214 .unwrap();
14215 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
14216 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
14217 assert!(registry.get_module("vault").unwrap().is_none());
14218
14219 let wrong = handler
14221 .handle_control(
14222 ConnectionId::new(2),
14223 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
14224 )
14225 .unwrap();
14226 assert_eq!(wrong[0].header.ty, FrameType::Error);
14227 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14228 assert!(registry.get_module("vault").unwrap().is_none());
14229
14230 let ok = handler
14232 .handle_control(
14233 ConnectionId::new(3),
14234 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14235 )
14236 .unwrap();
14237 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14238 assert!(registry.get_module("vault").unwrap().is_some());
14239 }
14240
14241 #[test]
14242 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14243 let registry = Arc::new(Registry::default());
14244 let supervisor = SupervisorHandle::new();
14245 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14246 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14247 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14248
14249 let squat = handler
14250 .handle_control(
14251 ConnectionId::new(1),
14252 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14253 )
14254 .unwrap();
14255 assert_eq!(squat[0].header.ty, FrameType::Error);
14256 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14257 assert!(parse_error(&squat[0])["message"]
14258 .as_str()
14259 .unwrap()
14260 .contains("fed:"));
14261
14262 let accepted_peer = handler
14263 .handle_control(
14264 ConnectionId::new(2),
14265 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14266 )
14267 .unwrap();
14268 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14269
14270 let accepted_short = handler
14271 .handle_control(
14272 ConnectionId::new(3),
14273 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14274 )
14275 .unwrap();
14276 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14277
14278 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14279 let response = handler
14280 .handle_control(
14281 ConnectionId::new(conn),
14282 hello_frame(module_id, PROTOCOL_VERSION, conn),
14283 )
14284 .unwrap();
14285 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14286 }
14287 }
14288
14289 #[test]
14290 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14291 let registry = Arc::new(Registry::default());
14292 let supervisor = SupervisorHandle::new();
14293 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14294 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14295 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14296 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14297
14298 let owner_nonce = handler
14299 .handle_control(
14300 ConnectionId::new(1),
14301 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14302 )
14303 .unwrap();
14304 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14305 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14306 assert!(registry.get_module("fed:special").unwrap().is_none());
14307
14308 let exact_nonce = handler
14309 .handle_control(
14310 ConnectionId::new(2),
14311 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14312 )
14313 .unwrap();
14314 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14315 assert!(registry.get_module("fed:special").unwrap().is_some());
14316 }
14317
14318 #[test]
14319 fn non_reserved_module_ignores_launch_nonce() {
14320 let registry = Arc::new(Registry::default());
14321 let handler = ControlHandler::new(Arc::clone(®istry));
14324 let no_nonce = handler
14325 .handle_control(
14326 ConnectionId::new(1),
14327 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14328 )
14329 .unwrap();
14330 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14331 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14332
14333 let echoed_nonce = handler
14334 .handle_control(
14335 ConnectionId::new(2),
14336 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14337 )
14338 .unwrap();
14339 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14340 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14341 }
14342
14343 #[test]
14344 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14345 let handler = ControlHandler::default();
14346 let conn = ConnectionId::new(1);
14347 let malformed = Frame::build(
14348 FrameType::Hello,
14349 control_flags(),
14350 0,
14351 0,
14352 3,
14353 b"{not json".to_vec(),
14354 )
14355 .unwrap();
14356
14357 let error = handler.handle_control(conn, malformed).unwrap();
14358 assert_eq!(error[0].header.ty, FrameType::Error);
14359 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14360
14361 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14362 let pong = handler.handle_control(conn, ping).unwrap();
14363 assert_eq!(pong[0].header.ty, FrameType::Pong);
14364 assert_eq!(pong[0].header.corr, 4);
14365 }
14366
14367 #[test]
14368 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14369 let registry = Arc::new(Registry::default());
14370 let handler = ControlHandler::new(Arc::clone(®istry));
14371
14372 handler
14373 .handle_control(
14374 ConnectionId::new(1),
14375 hello_frame("aft", PROTOCOL_VERSION, 1),
14376 )
14377 .unwrap();
14378 let duplicate = handler
14379 .handle_control(
14380 ConnectionId::new(2),
14381 hello_frame("aft", PROTOCOL_VERSION, 2),
14382 )
14383 .unwrap();
14384
14385 assert_eq!(duplicate[0].header.ty, FrameType::Error);
14386 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14387 let registration = registry.get_module("aft").unwrap().unwrap();
14388 assert_eq!(registration.connection_id, ConnectionId::new(1));
14389 }
14390
14391 #[test]
14392 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14393 let registry = Arc::new(Registry::default());
14394 let forwarding = Arc::new(ForwardingTable::default());
14395 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14396 let handler =
14397 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14398 .with_process_liveness(process_liveness);
14399 let (ctx, route_channel, route_epoch) =
14400 bind_liveness_route(®istry, &forwarding, "aft-dead");
14401 let responses = handler
14402 .handle_route_poll(
14403 &ctx,
14404 route_poll_frame(41, PollKind::Liveness, route_channel),
14405 route_channel,
14406 route_epoch,
14407 PollKind::Liveness,
14408 )
14409 .unwrap();
14410
14411 assert_eq!(responses.len(), 1);
14412 assert_eq!(responses[0].header.ty, FrameType::Response);
14413 assert_route_poll_liveness(&responses[0], false);
14414 }
14415
14416 #[test]
14417 fn liveness_poll_without_process_source_uses_bound_route() {
14418 let registry = Arc::new(Registry::default());
14419 let forwarding = Arc::new(ForwardingTable::default());
14420 let handler =
14421 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14422 let (ctx, route_channel, route_epoch) =
14423 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
14424 let responses = handler
14425 .handle_route_poll(
14426 &ctx,
14427 route_poll_frame(42, PollKind::Liveness, route_channel),
14428 route_channel,
14429 route_epoch,
14430 PollKind::Liveness,
14431 )
14432 .unwrap();
14433
14434 assert_route_poll_liveness(&responses[0], true);
14435 }
14436
14437 #[test]
14438 fn liveness_poll_untracked_process_source_uses_bound_route() {
14439 let registry = Arc::new(Registry::default());
14440 let forwarding = Arc::new(ForwardingTable::default());
14441 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14442 let handler =
14443 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14444 .with_process_liveness(process_liveness);
14445 let (ctx, route_channel, route_epoch) =
14446 bind_liveness_route(®istry, &forwarding, "aft-untracked");
14447 let responses = handler
14448 .handle_route_poll(
14449 &ctx,
14450 route_poll_frame(43, PollKind::Liveness, route_channel),
14451 route_channel,
14452 route_epoch,
14453 PollKind::Liveness,
14454 )
14455 .unwrap();
14456
14457 assert_route_poll_liveness(&responses[0], true);
14458 }
14459
14460 #[tokio::test]
14461 async fn unknown_op_returns_unknown_control_op() {
14462 let handler = ControlHandler::default();
14463 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14464 let request = Frame::build(
14465 FrameType::Request,
14466 control_flags(),
14467 0,
14468 0,
14469 55,
14470 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14471 )
14472 .unwrap();
14473
14474 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14475
14476 assert_eq!(response.len(), 1);
14477 assert_eq!(response[0].header.ty, FrameType::Error);
14478 assert_eq!(response[0].header.corr, 55);
14479 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14480 }
14481
14482 #[tokio::test]
14483 async fn supervisor_provenance_rejects_unknown_exact_module() {
14484 let handler = ControlHandler::default();
14485 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14486 let request = Frame::build(
14487 FrameType::Request,
14488 control_flags(),
14489 0,
14490 0,
14491 57,
14492 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14493 )
14494 .unwrap();
14495
14496 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14497
14498 assert_eq!(response.len(), 1);
14499 assert_eq!(response[0].header.ty, FrameType::Error);
14500 assert_eq!(response[0].header.corr, 57);
14501 let error = parse_error(&response[0]);
14502 assert_eq!(error["code"], "unknown_module");
14503 assert_eq!(error["message"], "module_id 'missing' is not supervised");
14504 }
14505
14506 #[test]
14507 fn provenance_probe_override_keeps_handler_tests_deterministic() {
14508 let expected = subc_control::RunningImageAgreement::Unavailable {
14509 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14510 };
14511 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14512 assert_eq!(handler.provenance_probe_override, Some(expected));
14513 }
14514
14515 #[test]
14516 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14517 let verdict = reload_verdict(
14518 std::path::Path::new("/bin/new"),
14519 Some(std::path::Path::new("/bin/old")),
14520 subc_control::RunningImageAgreement::Unavailable {
14521 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14522 },
14523 );
14524 assert!(matches!(
14525 verdict.path,
14526 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14527 if configured == std::path::Path::new("/bin/new")
14528 && spawned_from == std::path::Path::new("/bin/old")
14529 ));
14530 }
14531
14532 #[test]
14533 fn launch_spec_without_running_digest_is_unavailable_not_match() {
14534 use subc_control::{
14535 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14536 };
14537 let configured = crate::supervise::ModuleSpec {
14538 module_id: "legacy".into(),
14539 program: "/module".into(),
14540 args: Vec::new(),
14541 env: Vec::new(),
14542 reserved: false,
14543 reserved_prefixes: Vec::new(),
14544 protocol: subc_control::ModuleProtocol::Subc,
14545 overlap: Default::default(),
14546 };
14547 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14548 assert_eq!(
14549 launch_spec_agreement(&configured, false, Some(&digest)),
14550 Agreement::Unavailable {
14551 reason: Reason::NotRunning
14552 }
14553 );
14554 assert_eq!(
14555 launch_spec_agreement(&configured, true, None),
14556 Agreement::Unavailable {
14557 reason: Reason::SpawnedSpecUnavailable
14558 }
14559 );
14560 assert_eq!(
14561 launch_spec_agreement(&configured, true, Some(&digest)),
14562 Agreement::Match
14563 );
14564 }
14565
14566 #[cfg(windows)]
14567 #[test]
14568 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14569 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14570 let configured = dir.join("module.exe");
14571 std::fs::write(&configured, b"executable path fixture").unwrap();
14572 let spawned = std::fs::canonicalize(&configured).unwrap();
14573 assert_ne!(
14574 configured, spawned,
14575 "fixture must exercise different path spellings"
14576 );
14577 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14578 let lower_case = configured.to_string_lossy().to_lowercase();
14579 for spelling in [
14580 &configured,
14581 Path::new(&forward_slashes),
14582 Path::new(&lower_case),
14583 ] {
14584 let verdict = reload_verdict(
14585 spelling,
14586 Some(&spawned),
14587 subc_control::RunningImageAgreement::Unavailable {
14588 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14589 },
14590 );
14591 assert_eq!(
14592 verdict.path,
14593 ReloadPathAgreement::Match,
14594 "configured {spelling:?}, spawned {spawned:?}"
14595 );
14596 }
14597 }
14598
14599 #[cfg(windows)]
14600 #[test]
14601 fn windows_reload_path_keeps_different_existing_program_pending() {
14602 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14603 let old = dir.join("old.exe");
14604 let new = dir.join("new.exe");
14605 std::fs::write(&old, b"same executable content").unwrap();
14606 std::fs::copy(&old, &new).unwrap();
14607 let spawned = std::fs::canonicalize(&old).unwrap();
14608 let image = subc_control::RunningImageAgreement::Unavailable {
14609 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14610 };
14611 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14612 assert!(
14613 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14614 if configured == new && spawned_from == spawned)
14615 );
14616 let missing = dir.join("missing.exe");
14617 assert!(matches!(
14618 reload_verdict(&missing, Some(&spawned), image).path,
14619 ReloadPathAgreement::Mismatch { .. }
14620 ));
14621 }
14622
14623 #[test]
14624 fn reload_verdict_detects_replaced_image_at_same_path() {
14625 let image = subc_control::RunningImageAgreement::Mismatch {
14626 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14627 digest: "old".into(),
14628 },
14629 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14630 digest: "new".into(),
14631 },
14632 };
14633 let verdict = reload_verdict(
14634 std::path::Path::new("/bin/same"),
14635 Some(std::path::Path::new("/bin/same")),
14636 image.clone(),
14637 );
14638 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14639 assert_eq!(verdict.image, image);
14640 }
14641
14642 #[test]
14643 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14644 let image = subc_control::RunningImageAgreement::Unavailable {
14645 reason: subc_control::RunningImageUnavailableReason::NotRunning,
14646 };
14647 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14648 assert_eq!(
14649 verdict.path,
14650 subc_control::ReloadPathAgreement::Unavailable {
14651 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14652 }
14653 );
14654 assert_eq!(verdict.image, image);
14655
14656 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14657 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14658 };
14659 let verdict = reload_verdict(
14660 std::path::Path::new("/bin/same"),
14661 Some(std::path::Path::new("/bin/same")),
14662 unconfirmed.clone(),
14663 );
14664 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14665 assert_eq!(verdict.image, unconfirmed);
14666 }
14667
14668 #[test]
14669 fn reload_verdict_preserves_each_image_unavailability_reason() {
14670 use subc_control::RunningImageUnavailableReason as Reason;
14671
14672 for reason in [
14673 Reason::NotRunning,
14674 Reason::UnsupportedPlatform,
14675 Reason::RunningExecutableUnreadable,
14676 Reason::SpawnedPathUnreadable,
14677 Reason::HashFailed,
14678 Reason::ProcessIdentityUnconfirmed,
14679 Reason::Unknown("future_probe_reason".to_string()),
14680 ] {
14681 let image = subc_control::RunningImageAgreement::Unavailable {
14682 reason: reason.clone(),
14683 };
14684 let verdict = reload_verdict(
14685 std::path::Path::new("/bin/same"),
14686 Some(std::path::Path::new("/bin/same")),
14687 image.clone(),
14688 );
14689 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14690 assert_eq!(verdict.image, image, "{reason:?}");
14691 }
14692 }
14693
14694 #[tokio::test]
14695 async fn malformed_control_bodies_return_invalid_control_body() {
14696 let handler = ControlHandler::default();
14697 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14698
14699 for (corr, body) in [
14700 (56, br#"{"route_channel":1}"#.as_slice()),
14701 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14702 (
14703 58,
14704 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14705 ),
14706 ] {
14707 let request = Frame::build(
14708 FrameType::Request,
14709 control_flags(),
14710 0,
14711 0,
14712 corr,
14713 body.to_vec(),
14714 )
14715 .unwrap();
14716 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14717
14718 assert_eq!(response.len(), 1);
14719 assert_eq!(response[0].header.ty, FrameType::Error);
14720 assert_eq!(response[0].header.corr, corr);
14721 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14722 }
14723 }
14724
14725 #[tokio::test]
14726 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14727 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14728 let registry = Arc::new(Registry::default());
14729 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14730 let router = Router::with_control_handler(Arc::clone(&control));
14731 let connection = router.begin_connection();
14732 let (ctx, mut rx) = route_ctx(connection.id());
14733
14734 router
14735 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14736 .await
14737 .unwrap();
14738 let response = rx.recv().await.unwrap();
14739 let ack = parse_ack(&response);
14740 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14741 let channel = 1;
14742
14743 let goodbye =
14744 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14745 router.route_for_connection(&ctx, goodbye).await.unwrap();
14746 assert!(rx.try_recv().is_err());
14747 assert!(registry.get_module("aft").unwrap().is_none());
14748
14749 router
14750 .route_for_connection(&ctx, channel_request(channel, 13))
14751 .await
14752 .unwrap();
14753 let error_frame = rx.recv().await.unwrap();
14754 assert_eq!(error_frame.header.ty, FrameType::Error);
14755 assert_eq!(error_frame.header.channel, channel);
14756 let captured = crate::router::test_log::captured_logs(&logs);
14757 assert_eq!(
14758 captured
14759 .lines()
14760 .filter(|line| {
14761 line.contains("module_id=aft")
14762 && line.contains("reason=explicit_goodbye")
14763 && line.contains("module registration ended")
14764 })
14765 .count(),
14766 1,
14767 "unexpected GOODBYE registry log: {captured}"
14768 );
14769 }
14770
14771 #[tokio::test]
14772 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14773 let registry = Arc::new(Registry::default());
14774 let handler = ControlHandler::new(registry).with_capability_config(
14775 [("prov".to_string(), true), ("cons".to_string(), true)],
14776 BTreeMap::new(),
14777 );
14778 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14779 register_capability_manifest(
14780 &handler,
14781 &provider_ctx,
14782 &mut provider_rx,
14783 capability_manifest("prov", &["thing/v1"], &[]),
14784 1,
14785 )
14786 .await;
14787 let mut consumer = capability_manifest("cons", &[], &[]);
14788 consumer.capabilities.as_mut().unwrap().requires.push(
14789 subc_protocol::manifest::CapabilityRequirement {
14790 capability: "thing/v1".to_string(),
14791 need: subc_protocol::manifest::CapabilityNeed::Required,
14792 },
14793 );
14794 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14795 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14796 assert_eq!(
14797 handler.capability_evaluator.verdict("cons", "thing/v1"),
14798 Some(CapabilityVerdict::Provided)
14799 );
14800 let (mut client_rx, _) = open_route_for_capability_test(
14801 &handler,
14802 &provider_ctx,
14803 &mut provider_rx,
14804 703,
14805 3,
14806 "prov",
14807 None,
14808 )
14809 .await;
14810 let goodbye =
14811 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14812 handler
14813 .handle_control_frame(&provider_ctx, goodbye)
14814 .await
14815 .unwrap();
14816 assert_eq!(
14817 handler.capability_evaluator.verdict("cons", "thing/v1"),
14818 Some(CapabilityVerdict::NeverProvided)
14819 );
14820 let closed = client_rx
14821 .try_recv()
14822 .expect("GOODBYE pushes route.closed before route GOODBYE");
14823 assert!(
14824 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14825 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14826 );
14827 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14828 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14829 }
14830
14831 struct RouteClosedFixture {
14832 handler: ControlHandler,
14833 #[cfg(unix)]
14834 supervisor: Supervisor,
14835 module_ctx: RouteCtx,
14836 #[cfg(unix)]
14837 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14838 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14839 route: Arc<crate::forwarding::RouteBinding>,
14840 }
14841
14842 async fn route_closed_fixture() -> RouteClosedFixture {
14843 let registry = Arc::new(Registry::default());
14844 let handler = ControlHandler::new(Arc::clone(®istry));
14845 #[cfg(unix)]
14846 let supervisor = Supervisor::new(registry, RestartPolicy::default())
14847 .with_forwarding(Arc::clone(&handler.forwarding));
14848 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14849 register_capability_manifest(
14850 &handler,
14851 &module_ctx,
14852 &mut module_rx,
14853 capability_manifest("prov", &[], &[]),
14854 1,
14855 )
14856 .await;
14857 let (client_rx, bind) = open_route_for_capability_test(
14858 &handler,
14859 &module_ctx,
14860 &mut module_rx,
14861 702,
14862 2,
14863 "prov",
14864 None,
14865 )
14866 .await;
14867 let ModuleControlRequest::RouteBind {
14868 route_channel: channel,
14869 epoch: route_epoch,
14870 ..
14871 } = bind
14872 else {
14873 panic!("expected route.bind");
14874 };
14875 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14876 handler
14877 .forwarding
14878 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14879 .unwrap()
14880 else {
14881 panic!("the acknowledged route must be bound");
14882 };
14883 RouteClosedFixture {
14884 handler,
14885 #[cfg(unix)]
14886 supervisor,
14887 module_ctx,
14888 #[cfg(unix)]
14889 module_rx,
14890 client_rx,
14891 route,
14892 }
14893 }
14894
14895 impl RouteClosedFixture {
14896 #[cfg(unix)]
14897 async fn cut(&mut self) {
14898 let notices = async {
14899 let mut draining = self.module_rx.recv().await.unwrap();
14900 assert!(matches!(
14901 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14902 &draining.body
14903 )
14904 .unwrap(),
14905 subc_protocol::session::ModuleControlCommand::Draining {
14906 reason: RouteCloseReason::Restart,
14907 ..
14908 }
14909 ));
14910 draining.flushed.take().unwrap().send(()).unwrap();
14911 let mut closing = self.client_rx.recv().await.unwrap();
14912 assert!(matches!(
14913 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14914 ClientControlPush::RouteClosing {
14915 reason: RouteCloseReason::Restart,
14916 ..
14917 }
14918 ));
14919 closing.flushed.take().unwrap().send(()).unwrap();
14920 };
14921 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14922 result.unwrap();
14923 }
14924
14925 fn disconnect(&mut self) -> ClientControlPush {
14926 self.handler
14927 .cleanup_connection(self.module_ctx.connection_id)
14928 .unwrap();
14929 let closed = self.client_rx.try_recv().unwrap();
14930 assert_eq!(closed.header.ty, FrameType::Push);
14931 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14932 assert!(matches!(
14933 &push,
14934 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14935 if channels == &[self.route.client_channel]
14936 ));
14937 assert_eq!(
14938 self.client_rx.try_recv().unwrap().header.ty,
14939 FrameType::Goodbye
14940 );
14941 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14942 push
14943 }
14944 }
14945
14946 #[cfg(unix)]
14947 #[tokio::test(start_paused = true)]
14948 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14949 let mut fixture = route_closed_fixture().await;
14950 fixture.cut().await;
14951 assert!(matches!(
14952 fixture.disconnect(),
14953 ClientControlPush::RouteClosed {
14954 reason: RouteCloseReason::Restart,
14955 drained: true,
14956 ..
14957 }
14958 ));
14959 }
14960
14961 #[cfg(unix)]
14962 #[tokio::test(start_paused = true)]
14963 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14964 let mut fixture = route_closed_fixture().await;
14965 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14966 let started = tokio::time::Instant::now();
14967 fixture.cut().await;
14968 assert!(started.elapsed() >= Duration::from_secs(2));
14969 assert_eq!(fixture.route.flow.in_flight(), 1);
14970 assert!(matches!(
14971 fixture.disconnect(),
14972 ClientControlPush::RouteClosed {
14973 reason: RouteCloseReason::Restart,
14974 drained: false,
14975 ..
14976 }
14977 ));
14978 }
14979
14980 #[tokio::test]
14981 async fn module_connection_crash_reports_crash_and_not_drained() {
14982 let mut fixture = route_closed_fixture().await;
14983 assert!(matches!(
14987 fixture.disconnect(),
14988 ClientControlPush::RouteClosed {
14989 reason: RouteCloseReason::Crash,
14990 drained: false,
14991 ..
14992 }
14993 ));
14994 }
14995
14996 #[tokio::test]
14997 async fn dropping_router_connection_releases_registration() {
14998 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14999 let registry = Arc::new(Registry::default());
15000 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15001 let router = Router::with_control_handler(Arc::clone(&control));
15002 let connection = router.begin_connection();
15003 let connection_id = connection.id();
15004 let (ctx, mut rx) = route_ctx(connection_id);
15005
15006 router
15007 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
15008 .await
15009 .unwrap();
15010 let response = rx.recv().await.unwrap();
15011 let ack = parse_ack(&response);
15012 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15013 assert!(registry.get_module("aft").unwrap().is_some());
15014
15015 drop(connection);
15016
15017 assert!(registry.get_module("aft").unwrap().is_none());
15018 assert_eq!(registry.active_registration_count().unwrap(), 0);
15019
15020 control
15021 .cleanup_connection(ConnectionId::new(u64::MAX))
15022 .unwrap();
15023 let captured = crate::router::test_log::captured_logs(&logs);
15024 println!("captured registry lifecycle logs:\n{captured}");
15025 let events: Vec<_> = captured
15026 .lines()
15027 .filter(|line| {
15028 line.contains("module registered module_id=aft ")
15029 || line.contains("module registration ended")
15030 })
15031 .collect();
15032 assert_eq!(
15033 events.len(),
15034 2,
15035 "unexpected registry lifecycle logs: {captured}"
15036 );
15037 assert!(events[0].contains("module registered module_id=aft "));
15038 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
15039 assert!(events[1].contains(&format!(
15040 "module_id=aft connection_id={}",
15041 connection_id.get()
15042 )));
15043 assert!(events[1].contains("reason=connection_closed"));
15044 assert!(events[1].contains("module registration ended"));
15045 assert_eq!(
15046 captured
15047 .lines()
15048 .filter(|line| line.contains("module registered module_id=aft "))
15049 .count(),
15050 1,
15051 "legacy registration admission line must appear once: {captured}"
15052 );
15053 }
15054
15055 #[tokio::test]
15056 async fn hello_registration_keeps_legacy_module_registered_line_once() {
15057 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15058 let registry = Arc::new(Registry::default());
15059 let control = ControlHandler::new(Arc::clone(®istry));
15060 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
15061 hello_via_sink(
15062 &control,
15063 &ctx,
15064 &mut rx,
15065 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
15066 )
15067 .await;
15068
15069 let captured = crate::router::test_log::captured_logs(&logs);
15070 let admissions: Vec<_> = captured
15071 .lines()
15072 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
15073 .collect();
15074 assert_eq!(
15075 admissions.len(),
15076 1,
15077 "legacy admission line must remain exactly once: {captured}"
15078 );
15079 assert!(admissions[0].contains("routable_provider=true"));
15080 assert!(admissions[0].contains("connection_id=777"));
15081 }
15082
15083 fn capability_manifest(
15084 module_id: &str,
15085 provides: &[&str],
15086 must_never_reach: &[&str],
15087 ) -> ModuleManifest {
15088 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
15089 manifest.capabilities = Some(CapabilityDeclarations {
15090 provides: provides
15091 .iter()
15092 .map(|capability| (*capability).to_string())
15093 .collect(),
15094 requires: Vec::new(),
15095 must_never_reach: must_never_reach
15096 .iter()
15097 .map(|capability| (*capability).to_string())
15098 .collect(),
15099 });
15100 manifest
15101 }
15102
15103 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
15104 Frame::build(
15105 FrameType::Hello,
15106 control_flags(),
15107 0,
15108 0,
15109 corr,
15110 serde_json::to_vec(&ModuleHelloBody {
15111 protocol_ver: manifest.protocol_ver,
15112 manifest,
15113 control_ops: None,
15114 launch_nonce: None,
15115 })
15116 .expect("capability test HELLO serializes"),
15117 )
15118 .expect("capability test HELLO frame builds")
15119 }
15120
15121 fn catalog_update_with_capabilities_frame(
15122 corr: u64,
15123 capabilities: CapabilityDeclarations,
15124 ) -> Frame {
15125 Frame::build(
15126 FrameType::Request,
15127 control_flags(),
15128 0,
15129 0,
15130 corr,
15131 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15132 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
15133 capabilities: Some(capabilities),
15134 ready: None,
15135 })
15136 .expect("capability catalog.update serializes"),
15137 )
15138 .expect("capability catalog.update frame builds")
15139 }
15140
15141 async fn register_capability_manifest(
15142 handler: &ControlHandler,
15143 ctx: &RouteCtx,
15144 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15145 manifest: ModuleManifest,
15146 corr: u64,
15147 ) {
15148 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
15149 }
15150
15151 async fn open_route_for_capability_test(
15152 handler: &ControlHandler,
15153 target_ctx: &RouteCtx,
15154 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15155 client_connection_id: u64,
15156 corr: u64,
15157 target_module_id: &str,
15158 consumer_identity: Option<ConsumerIdentity>,
15159 ) -> (
15160 mpsc::Receiver<crate::router::OutboundFrame>,
15161 ModuleControlRequest,
15162 ) {
15163 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
15164 let route_handler = handler.clone();
15165 let target_module_id = target_module_id.to_string();
15166 let route_task = tokio::spawn(async move {
15167 route_handler
15168 .handle_control_frame(
15169 &client_ctx,
15170 route_open_frame_with_admission_facts(
15171 corr,
15172 &target_module_id,
15173 unique_project_root("admission-facts"),
15174 consumer_identity,
15175 None,
15176 ),
15177 )
15178 .await
15179 .expect("capability test route.open succeeds")
15180 });
15181 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
15182 .await
15183 .expect("capability test route.open must reach route.bind")
15184 .expect("target control receiver stays open");
15185 let bind_request: ModuleControlRequest =
15186 serde_json::from_slice(&bind.body).expect("route.bind decodes");
15187 handler
15188 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
15189 .await
15190 .expect("capability test route.bind ACK succeeds");
15191 assert!(route_task.await.expect("route.open task joins").is_empty());
15192 let opened = client_rx
15193 .recv()
15194 .await
15195 .expect("successful route.open publishes a response");
15196 assert!(matches!(
15197 serde_json::from_slice::<ClientControlResponse>(&opened.body),
15198 Ok(ClientControlResponse::RouteOpen { .. })
15199 ));
15200 (client_rx, bind_request)
15201 }
15202
15203 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
15204 assert_eq!(frame.header.ty, FrameType::Push);
15205 assert_eq!(frame.header.channel, 0);
15206 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
15207 .expect("route.closed control push decodes");
15208 let ClientControlPush::RouteClosed { channels, .. } = &push else {
15209 panic!("expected route.closed");
15210 };
15211 assert_eq!(channels.len(), 1, "exactly one violating route closed");
15212 let closed_channel = channels[0];
15213 let channels = channels.clone();
15214 assert_eq!(
15215 push,
15216 ClientControlPush::RouteClosed {
15217 module_id: target_module_id.to_string(),
15218 channels,
15219 reason: RouteCloseReason::CapabilityDenied,
15220 drained: false,
15221 abandoned: 0,
15222 excluded_subscriptions: 0,
15223 terminal: Some(false),
15224 }
15225 );
15226 closed_channel
15227 }
15228
15229 #[tokio::test]
15230 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15231 let registry = Arc::new(Registry::default());
15232 let forwarding = Arc::new(ForwardingTable::default());
15233 let supervisor = SupervisorHandle::new();
15234 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15235 let handler =
15236 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15237 .with_supervisor(supervisor);
15238 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15239 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15240 register_capability_manifest(
15241 &handler,
15242 &target_ctx,
15243 &mut target_rx,
15244 capability_manifest("target", &["credentials-provider/v1"], &[]),
15245 1,
15246 )
15247 .await;
15248 register_capability_manifest(
15249 &handler,
15250 &opener_ctx,
15251 &mut opener_rx,
15252 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15253 2,
15254 )
15255 .await;
15256
15257 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15258 let replies = handler
15259 .handle_control_frame(
15260 &client_ctx,
15261 route_open_frame_with_admission_facts(
15262 3,
15263 "target",
15264 unique_project_root("admission-facts"),
15265 Some(ConsumerIdentity {
15266 module_id: "opener".to_string(),
15267 launch_nonce: "opener-nonce".to_string(),
15268 }),
15269 None,
15270 ),
15271 )
15272 .await
15273 .expect("denied route.open returns a typed frame");
15274 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15275 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15276 assert!(
15277 target_rx.try_recv().is_err(),
15278 "forbidden route.open must not relay route.bind"
15279 );
15280 }
15281
15282 #[tokio::test]
15283 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15284 let registry = Arc::new(Registry::default());
15285 let forwarding = Arc::new(ForwardingTable::default());
15286 let supervisor = SupervisorHandle::new();
15287 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15288 let handler =
15289 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15290 .with_supervisor(supervisor);
15291 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15292 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15293 register_capability_manifest(
15294 &handler,
15295 &target_ctx,
15296 &mut target_rx,
15297 capability_manifest("target", &["credentials-provider/v1"], &[]),
15298 1,
15299 )
15300 .await;
15301 register_capability_manifest(
15302 &handler,
15303 &old_opener_ctx,
15304 &mut old_opener_rx,
15305 capability_manifest("opener", &[], &[]),
15306 2,
15307 )
15308 .await;
15309 let (mut client_rx, _) = open_route_for_capability_test(
15310 &handler,
15311 &target_ctx,
15312 &mut target_rx,
15313 712,
15314 3,
15315 "target",
15316 Some(ConsumerIdentity {
15317 module_id: "opener".to_string(),
15318 launch_nonce: "opener-nonce".to_string(),
15319 }),
15320 )
15321 .await;
15322 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15323
15324 handler
15325 .cleanup_connection(old_opener_ctx.connection_id)
15326 .expect("old opener registration cleans up");
15327 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15328 register_capability_manifest(
15329 &handler,
15330 &new_opener_ctx,
15331 &mut new_opener_rx,
15332 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15333 4,
15334 )
15335 .await;
15336
15337 let closed_channel = assert_capability_denied_push(
15338 client_rx
15339 .try_recv()
15340 .expect("HELLO deny addition must emit route.closed")
15341 .frame,
15342 "target",
15343 );
15344 let goodbye = client_rx
15345 .try_recv()
15346 .expect("HELLO deny addition must send the client a route GOODBYE")
15347 .frame;
15348 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15349 assert_eq!(goodbye.header.channel, closed_channel);
15350 assert_ne!(goodbye.header.channel, 0);
15351 assert_ne!(goodbye.header.epoch, 0);
15352 assert!(client_rx.try_recv().is_err());
15353 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15354 assert!(matches!(
15355 target_rx.try_recv(),
15356 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15357 ));
15358 }
15359
15360 #[tokio::test]
15361 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15362 let registry = Arc::new(Registry::default());
15363 let forwarding = Arc::new(ForwardingTable::default());
15364 let supervisor = SupervisorHandle::new();
15365 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15366 let handler =
15367 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15368 .with_supervisor(supervisor);
15369 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15370 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15371 register_capability_manifest(
15372 &handler,
15373 &target_ctx,
15374 &mut target_rx,
15375 capability_manifest("target", &[], &[]),
15376 1,
15377 )
15378 .await;
15379 register_capability_manifest(
15380 &handler,
15381 &opener_ctx,
15382 &mut opener_rx,
15383 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15384 2,
15385 )
15386 .await;
15387 let (mut client_rx, _) = open_route_for_capability_test(
15388 &handler,
15389 &target_ctx,
15390 &mut target_rx,
15391 722,
15392 3,
15393 "target",
15394 Some(ConsumerIdentity {
15395 module_id: "opener".to_string(),
15396 launch_nonce: "opener-nonce".to_string(),
15397 }),
15398 )
15399 .await;
15400 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15401
15402 let replies = handler
15403 .handle_control_frame(
15404 &target_ctx,
15405 catalog_update_with_capabilities_frame(
15406 4,
15407 CapabilityDeclarations {
15408 provides: vec!["credentials-provider/v1".to_string()],
15409 requires: Vec::new(),
15410 must_never_reach: Vec::new(),
15411 },
15412 ),
15413 )
15414 .await
15415 .expect("claim catalog.update succeeds");
15416 assert!(matches!(
15417 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15418 Ok(ModuleControlResponseToModule::CatalogUpdate {})
15419 ));
15420 let closed_channel = assert_capability_denied_push(
15421 client_rx
15422 .try_recv()
15423 .expect("claim addition must emit route.closed")
15424 .frame,
15425 "target",
15426 );
15427 let goodbye = client_rx
15428 .try_recv()
15429 .expect("claim addition must send the client a route GOODBYE")
15430 .frame;
15431 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15432 assert_eq!(goodbye.header.channel, closed_channel);
15433 assert_ne!(goodbye.header.channel, 0);
15434 assert_ne!(goodbye.header.epoch, 0);
15435 assert!(client_rx.try_recv().is_err());
15436 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15437 assert!(matches!(
15438 target_rx.try_recv(),
15439 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15440 ));
15441 }
15442
15443 #[tokio::test]
15444 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15445 let registry = Arc::new(Registry::default());
15446 let forwarding = Arc::new(ForwardingTable::default());
15447 let supervisor = SupervisorHandle::new();
15448 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15449 let handler =
15450 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15451 .with_supervisor(supervisor);
15452 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15453 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15454 register_capability_manifest(
15455 &handler,
15456 &target_ctx,
15457 &mut target_rx,
15458 capability_manifest("target", &["credentials-provider/v1"], &[]),
15459 1,
15460 )
15461 .await;
15462 register_capability_manifest(
15463 &handler,
15464 &opener_ctx,
15465 &mut opener_rx,
15466 capability_manifest("opener", &[], &[]),
15467 2,
15468 )
15469 .await;
15470 let (mut client_rx, _) = open_route_for_capability_test(
15471 &handler,
15472 &target_ctx,
15473 &mut target_rx,
15474 732,
15475 3,
15476 "target",
15477 Some(ConsumerIdentity {
15478 module_id: "opener".to_string(),
15479 launch_nonce: "opener-nonce".to_string(),
15480 }),
15481 )
15482 .await;
15483 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15484
15485 handler
15486 .handle_control_frame(
15487 &target_ctx,
15488 catalog_update_with_capabilities_frame(
15489 4,
15490 CapabilityDeclarations {
15491 provides: Vec::new(),
15492 requires: Vec::new(),
15493 must_never_reach: Vec::new(),
15494 },
15495 ),
15496 )
15497 .await
15498 .expect("claim removal catalog.update succeeds");
15499 assert_eq!(
15500 forwarding.active_binding_count().unwrap(),
15501 1,
15502 "removing an attested target claim must leave the route census unchanged"
15503 );
15504 assert!(
15505 client_rx.try_recv().is_err(),
15506 "claim removal must not emit route.closed or a client route GOODBYE"
15507 );
15508 assert!(
15509 target_rx.try_recv().is_err(),
15510 "claim removal must not send the target a route GOODBYE"
15511 );
15512 }
15513
15514 #[tokio::test]
15517 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15518 let registry = Arc::new(Registry::default());
15519 let forwarding = Arc::new(ForwardingTable::default());
15520 let supervisor = SupervisorHandle::new();
15521 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15522 let handler =
15523 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15524 .with_supervisor(supervisor);
15525 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15526 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15527 register_capability_manifest(
15528 &handler,
15529 &target_ctx,
15530 &mut target_rx,
15531 capability_manifest("target", &["credentials-provider/v1"], &[]),
15532 1,
15533 )
15534 .await;
15535 register_capability_manifest(
15536 &handler,
15537 &opener_ctx,
15538 &mut opener_rx,
15539 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15540 2,
15541 )
15542 .await;
15543
15544 let (_client_rx, bind) = open_route_for_capability_test(
15545 &handler,
15546 &target_ctx,
15547 &mut target_rx,
15548 742,
15549 3,
15550 "target",
15551 None,
15552 )
15553 .await;
15554 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15555 panic!("direct scope-honesty route must bind");
15556 };
15557 assert_eq!(principal, Some(Principal::Direct));
15558 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15559 }
15560
15561 #[tokio::test]
15564 async fn must_never_reach_self_route_is_capability_forbidden() {
15565 let registry = Arc::new(Registry::default());
15566 let forwarding = Arc::new(ForwardingTable::default());
15567 let supervisor = SupervisorHandle::new();
15568 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15569 let handler =
15570 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15571 .with_supervisor(supervisor);
15572 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15573 register_capability_manifest(
15574 &handler,
15575 &self_ctx,
15576 &mut self_rx,
15577 capability_manifest(
15578 "self-provider",
15579 &["credentials-provider/v1"],
15580 &["credentials-provider/v1"],
15581 ),
15582 1,
15583 )
15584 .await;
15585
15586 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15587 let replies = handler
15588 .handle_control_frame(
15589 &client_ctx,
15590 route_open_frame_with_admission_facts(
15591 2,
15592 "self-provider",
15593 unique_project_root("admission-facts"),
15594 Some(ConsumerIdentity {
15595 module_id: "self-provider".to_string(),
15596 launch_nonce: "self-nonce".to_string(),
15597 }),
15598 None,
15599 ),
15600 )
15601 .await
15602 .expect("self-route refusal returns a typed frame");
15603 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15604 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15605 assert!(
15606 self_rx.try_recv().is_err(),
15607 "self denial must not relay route.bind"
15608 );
15609 }
15610
15611 #[test]
15612 fn unsupported_channel_zero_frame_returns_error() {
15613 let handler = ControlHandler::default();
15614 let request = Frame::build(
15615 FrameType::Request,
15616 control_flags(),
15617 0,
15618 0,
15619 21,
15620 b"opaque".to_vec(),
15621 )
15622 .unwrap();
15623
15624 let response = handler
15625 .handle_control(ConnectionId::new(1), request)
15626 .unwrap();
15627
15628 assert_eq!(response[0].header.ty, FrameType::Error);
15629 assert_eq!(
15630 parse_error(&response[0])["code"],
15631 "unsupported_control_frame"
15632 );
15633 }
15634
15635 mod swap {
15640 use super::*;
15641
15642 const INCUMBENT: ConnectionId = ConnectionId::new(30);
15643 const CANDIDATE: ConnectionId = ConnectionId::new(40);
15644
15645 struct Swap {
15646 registry: Arc<Registry>,
15647 forwarding: Arc<ForwardingTable>,
15648 handler: ControlHandler,
15649 incumbent_ctx: RouteCtx,
15650 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15651 candidate_ctx: RouteCtx,
15652 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15653 }
15654
15655 async fn swap_with_incumbent() -> Swap {
15656 let registry = Arc::new(Registry::default());
15657 let forwarding = Arc::new(ForwardingTable::default());
15658 let handler =
15659 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15660 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15661 hello_via_sink(
15662 &handler,
15663 &incumbent_ctx,
15664 &mut incumbent_rx,
15665 hello_frame("aft", PROTOCOL_VERSION, 7),
15666 )
15667 .await;
15668 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15669 Swap {
15670 registry,
15671 forwarding,
15672 handler,
15673 incumbent_ctx,
15674 incumbent_rx,
15675 candidate_ctx,
15676 candidate_rx,
15677 }
15678 }
15679
15680 fn register_candidate(swap: &Swap, ready: Option<bool>) {
15681 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15682 candidate_manifest.ready = ready;
15683 let registration = swap
15684 .registry
15685 .register_candidate_with_control_ops(
15686 candidate_manifest,
15687 PROTOCOL_VERSION,
15688 CANDIDATE,
15689 module_baseline_control_ops(),
15690 )
15691 .unwrap();
15692 swap.forwarding
15693 .register_candidate_module_connection(
15694 CANDIDATE,
15695 "aft".to_string(),
15696 PROTOCOL_VERSION,
15697 manifest_concurrency(®istration.manifest),
15698 swap.candidate_ctx.egress.clone(),
15699 )
15700 .unwrap();
15701 }
15702
15703 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15704 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15705 swap.registry.promote_candidate("aft").unwrap().unwrap();
15706 cutover.incumbent.unwrap()
15707 }
15708
15709 #[tokio::test]
15718 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15719 let mut swap = swap_with_incumbent().await;
15720 let hold = Arc::new(std::sync::Barrier::new(2));
15721 let handler = swap
15722 .handler
15723 .clone()
15724 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15725 handler
15726 .supervisor
15727 .open_swap("aft", "candidate-nonce".to_string());
15728 let hello = {
15729 let handler = handler.clone();
15730 let candidate_ctx = swap.candidate_ctx.clone();
15731 std::thread::spawn(move || {
15732 handler
15733 .handle_hello(
15734 candidate_ctx.connection_id,
15735 Some(candidate_ctx.egress.clone()),
15736 hello_frame_with_nonce(
15737 "aft",
15738 PROTOCOL_VERSION,
15739 8,
15740 Some("candidate-nonce"),
15741 ),
15742 )
15743 .unwrap()
15744 })
15745 };
15746
15747 hold.wait();
15748 assert!(swap
15751 .registry
15752 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15753 .unwrap()
15754 .is_some());
15755 assert!(swap
15756 .registry
15757 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15758 .unwrap()
15759 .is_none());
15760 assert!(swap
15761 .forwarding
15762 .module_endpoint_for_connection(CANDIDATE)
15763 .unwrap()
15764 .is_none());
15765
15766 hold.wait();
15767 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15768 assert_eq!(
15769 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15770 FrameType::HelloAck
15771 );
15772 let published = swap
15773 .registry
15774 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15775 .unwrap()
15776 .expect("candidate is published once its HELLO completes");
15777 assert_eq!(published.connection_id, CANDIDATE);
15778 assert!(swap
15779 .forwarding
15780 .module_endpoint_for_connection(CANDIDATE)
15781 .unwrap()
15782 .is_some());
15783 }
15784
15785 fn keyed_total(counters: &Value, key: &str) -> u64 {
15786 counters[key]
15787 .as_object()
15788 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15789 .unwrap_or(0)
15790 }
15791
15792 #[tokio::test]
15793 async fn replacement_logs_old_end_and_new_admission_once() {
15794 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15795 let mut swap = swap_with_incumbent().await;
15796 swap.handler
15797 .supervisor
15798 .open_swap("aft", "candidate-nonce".to_string());
15799 hello_via_sink(
15800 &swap.handler,
15801 &swap.candidate_ctx,
15802 &mut swap.candidate_rx,
15803 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15804 )
15805 .await;
15806 cutover(&swap);
15807 swap.handler.cleanup_connection(INCUMBENT).unwrap();
15808
15809 let captured = crate::router::test_log::captured_logs(&logs);
15810 println!("captured replacement registry logs:\n{captured}");
15811 let new_admissions: Vec<_> = captured
15812 .lines()
15813 .filter(|line| {
15814 line.contains("connection_id=40")
15815 && line.contains("swap candidate registered; not routable until cutover")
15816 })
15817 .collect();
15818 assert_eq!(
15819 new_admissions.len(),
15820 1,
15821 "unexpected admission logs: {captured}"
15822 );
15823 assert!(new_admissions[0].contains("module_id=aft"));
15824 assert!(new_admissions[0].contains("connection_id=40"));
15825 assert!(
15826 new_admissions[0].contains("swap candidate registered; not routable until cutover")
15827 );
15828 assert!(new_admissions[0].contains("ready=true"));
15829
15830 let old_admissions: Vec<_> = captured
15831 .lines()
15832 .filter(|line| {
15833 line.contains("module registered module_id=aft ")
15834 && line.contains("connection_id=30")
15835 })
15836 .collect();
15837 assert_eq!(
15838 old_admissions.len(),
15839 1,
15840 "unexpected incumbent admission: {captured}"
15841 );
15842
15843 let old_ends: Vec<_> = captured
15844 .lines()
15845 .filter(|line| {
15846 line.contains("connection_id=30") && line.contains("module registration ended")
15847 })
15848 .collect();
15849 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15850 assert!(old_ends[0].contains("module_id=aft"));
15851 assert!(old_ends[0].contains("connection_id=30"));
15852 assert!(old_ends[0].contains("reason=replaced"));
15853 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15854 assert_eq!(
15855 captured
15856 .lines()
15857 .filter(|line| line.contains("module registration promoted"))
15858 .count(),
15859 1,
15860 "unexpected promotion logs: {captured}"
15861 );
15862 }
15863
15864 #[tokio::test]
15870 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15871 let mut swap = swap_with_incumbent().await;
15872 let handler = swap.handler.clone();
15873
15874 let cotenant = ConnectionId::new(31);
15876 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15877 let (cotenant_task, cotenant_bind) = relay_route_open(
15878 &handler,
15879 cotenant,
15880 &cotenant_ctx.egress,
15881 &mut swap.incumbent_rx,
15882 100,
15883 "aft",
15884 "swap-cotenant",
15885 )
15886 .await;
15887 handler
15888 .handle_control_frame(
15889 &swap.incumbent_ctx,
15890 route_bind_ack(cotenant_bind.header.corr),
15891 )
15892 .await
15893 .unwrap();
15894 assert!(cotenant_task.await.unwrap().is_empty());
15895 let (cotenant_channel, cotenant_epoch) =
15896 published_route(&cotenant_rx.recv().await.unwrap());
15897
15898 let caller = ConnectionId::new(32);
15900 let (caller_ctx, mut caller_rx) = route_ctx(caller);
15901 let (caller_task, caller_bind) = relay_route_open(
15902 &handler,
15903 caller,
15904 &caller_ctx.egress,
15905 &mut swap.incumbent_rx,
15906 101,
15907 "aft",
15908 "swap-caller",
15909 )
15910 .await;
15911 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15912
15913 register_candidate(&swap, None);
15914 cutover(&swap);
15915
15916 let ack = handler
15918 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15919 .await;
15920 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15921 if module_loop_error.is_some() {
15922 handler.cleanup_connection(INCUMBENT).unwrap();
15925 }
15926
15927 assert!(
15929 cotenant_rx.try_recv().is_err(),
15930 "the co-tenant route on the incumbent was torn down by one late ack: \
15931 {module_loop_error:?}"
15932 );
15933 assert!(matches!(
15934 swap.forwarding
15935 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15936 .unwrap(),
15937 DataRoute::Client(DataRouteState::Bound(_))
15938 ));
15939 assert_eq!(module_loop_error, None);
15940 assert!(swap
15941 .registry
15942 .get_module_by_connection(INCUMBENT)
15943 .unwrap()
15944 .is_some());
15945
15946 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15948 .await
15949 .expect("the incumbent is told to drop the abandoned binding")
15950 .unwrap()
15951 .frame;
15952 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15953 assert_eq!(goodbye.header.channel, abandoned_channel);
15954 assert_eq!(goodbye.header.epoch, abandoned_epoch);
15955 assert!(swap.incumbent_rx.try_recv().is_err());
15956
15957 let response = caller_task.await.unwrap();
15959 assert_eq!(response.len(), 1);
15960 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15961 assert!(caller_rx.try_recv().is_err());
15962
15963 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15967 let counters = handler.counters().snapshot();
15968 assert_eq!(
15969 keyed_total(&counters, "route_open_accepted_by_principal"),
15970 1
15971 );
15972 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15973 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15974 }
15975
15976 #[tokio::test]
15980 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15981 let mut swap = swap_with_incumbent().await;
15982 register_candidate(&swap, None);
15983 let incumbent = cutover(&swap);
15984 swap.forwarding
15985 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15986 .unwrap()
15987 .expect("the incumbent is still registered");
15988
15989 let client = ConnectionId::new(33);
15990 let (client_ctx, mut client_rx) = route_ctx(client);
15991 let route_handler = swap.handler.clone();
15992 let open_ctx = RouteCtx {
15993 connection_id: client,
15994 egress: client_ctx.egress.clone(),
15995 };
15996 let mut route_task = tokio::spawn(async move {
15997 route_handler
15998 .handle_control_frame(
15999 &open_ctx,
16000 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
16001 )
16002 .await
16003 .unwrap()
16004 });
16005 let bind = tokio::select! {
16006 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
16007 response = &mut route_task => {
16008 let response = response.unwrap();
16009 panic!(
16010 "post-cutover route.open was refused instead of relayed to the candidate: {}",
16011 parse_error(&response[0])["code"]
16012 );
16013 }
16014 };
16015 swap.handler
16016 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
16017 .await
16018 .unwrap();
16019 assert!(route_task.await.unwrap().is_empty());
16020 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
16021 match swap
16022 .forwarding
16023 .lookup_data_route(client, channel, epoch)
16024 .unwrap()
16025 {
16026 DataRoute::Client(DataRouteState::Bound(route)) => {
16027 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
16028 }
16029 other => panic!("expected a bound route on the candidate, got {other:?}"),
16030 }
16031 assert!(swap.incumbent_rx.try_recv().is_err());
16032 }
16033
16034 #[tokio::test]
16039 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
16040 let swap = swap_with_incumbent().await;
16041 register_candidate(&swap, Some(false));
16042 let update = Frame::build(
16043 FrameType::Request,
16044 control_flags(),
16045 0,
16046 0,
16047 55,
16048 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16049 provides: manifest("aft", PROTOCOL_VERSION).provides,
16050 capabilities: None,
16051 ready: Some(true),
16052 })
16053 .unwrap(),
16054 )
16055 .unwrap();
16056
16057 let replies = swap
16058 .handler
16059 .handle_control_frame(&swap.candidate_ctx, update)
16060 .await
16061 .unwrap();
16062
16063 assert_eq!(replies.len(), 1);
16064 assert_eq!(
16065 replies[0].header.ty,
16066 FrameType::Response,
16067 "candidate catalog.update was refused: {:?}",
16068 serde_json::from_slice::<Value>(&replies[0].body).ok()
16069 );
16070 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
16071 assert_eq!(
16072 swap.registry
16073 .get_module("aft")
16074 .unwrap()
16075 .unwrap()
16076 .connection_id,
16077 INCUMBENT
16078 );
16079 }
16080 }
16081
16082 mod swap_admission {
16086 use super::*;
16087
16088 const INCUMBENT_NONCE: &str = "incumbent-nonce";
16089 const CANDIDATE_NONCE: &str = "candidate-nonce";
16090
16091 fn handler_with_incumbent(
16092 module_id: &str,
16093 reserved: bool,
16094 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
16095 let registry = Arc::new(Registry::default());
16096 let supervisor = SupervisorHandle::new();
16097 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
16098 if reserved {
16099 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
16100 }
16101 let handler =
16102 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
16103 let incumbent = handler
16104 .handle_control(
16105 ConnectionId::new(1),
16106 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
16107 )
16108 .unwrap();
16109 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
16110 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
16111 (registry, supervisor, handler)
16112 }
16113
16114 #[test]
16120 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
16121 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
16122
16123 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
16124 let replies = handler
16125 .handle_control(
16126 ConnectionId::new(connection),
16127 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
16128 )
16129 .unwrap();
16130 assert_eq!(replies[0].header.ty, FrameType::Error);
16131 assert_eq!(
16132 parse_error(&replies[0])["code"],
16133 "swap_token_invalid",
16134 "nonce {nonce:?}"
16135 );
16136 }
16137 assert!(registry.get_candidate("aft").unwrap().is_none());
16138 assert_eq!(
16139 registry.get_module("aft").unwrap().unwrap().connection_id,
16140 ConnectionId::new(1)
16141 );
16142
16143 let admitted = handler
16146 .handle_control(
16147 ConnectionId::new(4),
16148 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
16149 )
16150 .unwrap();
16151 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
16152 assert_eq!(
16153 registry
16154 .get_candidate("aft")
16155 .unwrap()
16156 .unwrap()
16157 .connection_id,
16158 ConnectionId::new(4)
16159 );
16160 assert_eq!(
16161 registry.get_module("aft").unwrap().unwrap().connection_id,
16162 ConnectionId::new(1),
16163 "the candidate must not take the active slot"
16164 );
16165 let replayed = handler
16166 .handle_control(
16167 ConnectionId::new(5),
16168 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
16169 )
16170 .unwrap();
16171 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
16172
16173 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
16177 let squatter = handler
16178 .handle_control(
16179 ConnectionId::new(6),
16180 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
16181 )
16182 .unwrap();
16183 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
16184 assert!(
16185 registry.get_module("aft").unwrap().is_none(),
16186 "a squatter took the active slot of an id being swapped"
16187 );
16188 }
16189
16190 #[test]
16195 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
16196 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
16197
16198 let replies = handler
16199 .handle_control(
16200 ConnectionId::new(2),
16201 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16202 )
16203 .unwrap();
16204
16205 assert_eq!(
16206 replies[0].header.ty,
16207 FrameType::HelloAck,
16208 "reserved candidate refused: {:?}",
16209 serde_json::from_slice::<Value>(&replies[0].body).ok()
16210 );
16211 assert_eq!(
16212 registry
16213 .get_candidate("vault")
16214 .unwrap()
16215 .unwrap()
16216 .connection_id,
16217 ConnectionId::new(2)
16218 );
16219 }
16220
16221 #[test]
16224 fn without_an_open_swap_the_ordinary_gates_decide() {
16225 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16226 supervisor.close_swap("vault");
16227
16228 let candidate = handler
16229 .handle_control(
16230 ConnectionId::new(2),
16231 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16232 )
16233 .unwrap();
16234 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16235 let duplicate = handler
16236 .handle_control(
16237 ConnectionId::new(3),
16238 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16239 )
16240 .unwrap();
16241 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16242 assert!(registry.get_candidate("vault").unwrap().is_none());
16243 }
16244 }
16245
16246 mod scopes {
16250 mod idle_tests {
16251 use super::*;
16252 include!("scope_idle_tests.rs");
16253 }
16254 mod route_open_timings {
16255 use super::*;
16256 include!("route_open_timing_tests.rs");
16257 }
16258 mod route_open_refusal_log {
16259 use super::*;
16260 include!("route_open_refusal_log_tests.rs");
16261 }
16262 mod activity {
16263 include!("activity_scope_tests.rs");
16264 }
16265 mod delta {
16266 include!("scope_handler_delta_tests.rs");
16267 }
16268 mod call_modes {
16269 include!("scope_call_mode_tests.rs");
16270 }
16271 #[tokio::test]
16272 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16273 let rig = rig().await;
16274 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16275 "s".into(),
16276 1,
16277 "tool".into(),
16278 ))
16279 .unwrap();
16280 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16281 let (direct, _rx) = wide_ctx(999);
16282 for (ctx, expected) in [
16283 (&rig.owner, FrameType::Response),
16284 (&direct, FrameType::Error),
16285 ] {
16286 let frame =
16287 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16288 .unwrap();
16289 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16290 assert_eq!(replies[0].header.ty, expected);
16291 if expected == FrameType::Response {
16292 let reply: subc_control::ScopeServedByReply =
16293 serde_json::from_slice(&replies[0].body).unwrap();
16294 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16295 } else {
16296 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16297 }
16298 }
16299 let reply = call(
16300 &rig.handler,
16301 &rig.owner,
16302 &ModuleControlRequestFromModule::ScopeDescribe {
16303 owner: Principal::Reserved {
16304 module_id: OWNER.into(),
16305 },
16306 scope_ref: "s".into(),
16307 },
16308 )
16309 .await;
16310 assert!(matches!(
16311 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16312 ModuleControlResponseToModule::ScopeDescribe { .. }
16313 ));
16314 }
16315
16316 use subc_protocol::scope::{
16317 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16318 ScopeStatus,
16319 };
16320
16321 use super::*;
16322
16323 const OWNER: &str = "prefrontal-core";
16324
16325 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16326 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16327 }
16328
16329 async fn call(
16330 handler: &ControlHandler,
16331 ctx: &RouteCtx,
16332 request: &ModuleControlRequestFromModule,
16333 ) -> Frame {
16334 let body = serde_json::to_vec(request).unwrap();
16335 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16336 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16337 assert_eq!(replies.len(), 1, "{replies:?}");
16338 replies.pop().unwrap()
16339 }
16340
16341 async fn sync(
16342 handler: &ControlHandler,
16343 ctx: &RouteCtx,
16344 generation: u64,
16345 scopes: Vec<ScopeRecord>,
16346 ) -> Result<ModuleControlResponseToModule, String> {
16347 let reply = call(
16348 handler,
16349 ctx,
16350 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16351 )
16352 .await;
16353 match reply.header.ty {
16354 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16355 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16356 }
16357 }
16358
16359 async fn describe(
16360 handler: &ControlHandler,
16361 ctx: &RouteCtx,
16362 owner: &str,
16363 scope_ref: &str,
16364 ) -> ModuleControlResponseToModule {
16365 let reply = call(
16366 handler,
16367 ctx,
16368 &ModuleControlRequestFromModule::ScopeDescribe {
16369 owner: Principal::Reserved {
16370 module_id: owner.to_string(),
16371 },
16372 scope_ref: scope_ref.to_string(),
16373 },
16374 )
16375 .await;
16376 assert_eq!(
16377 reply.header.ty,
16378 FrameType::Response,
16379 "{:?}",
16380 parse_error(&reply)
16381 );
16382 serde_json::from_slice(&reply.body).unwrap()
16383 }
16384
16385 #[tokio::test]
16386 async fn scope_describe_never_echoes_bind_proof_or_hash() {
16387 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16388 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16389 let mut rig = rig().await;
16390 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16391 .await;
16392 let _bound = rig
16393 .bound(
16394 Some(AFT),
16395 PLEXUS,
16396 Some(
16397 rig.selector("s", Some(1))
16398 .with_bind_proof(Some(PROOF.into())),
16399 ),
16400 )
16401 .await;
16402 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16403 let output = serde_json::to_string(&reply).unwrap();
16404 assert!(output.contains("\"status\":\"live\""));
16405 assert!(!output.contains(PROOF));
16406 assert!(!output.contains(HASH));
16407 }
16408
16409 async fn module(
16411 handler: &ControlHandler,
16412 connection: u64,
16413 module_id: &str,
16414 nonce: Option<&str>,
16415 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16416 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16417 hello_via_sink(
16418 handler,
16419 &ctx,
16420 &mut rx,
16421 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16422 )
16423 .await;
16424 (ctx, rx)
16425 }
16426
16427 #[tokio::test]
16430 async fn a_client_connection_cannot_sync_or_describe() {
16431 let handler = ControlHandler::new(Arc::new(Registry::default()));
16432 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16433 for request in [
16434 ModuleControlRequestFromModule::ScopeSync {
16435 generation: 1,
16436 scopes: vec![head("s", 1)],
16437 },
16438 ModuleControlRequestFromModule::ScopeDescribe {
16439 owner: Principal::Direct,
16440 scope_ref: "s".to_string(),
16441 },
16442 ] {
16443 let reply = call(&handler, &ctx, &request).await;
16444 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16445 }
16446 assert!(
16447 !handler
16448 .scopes
16449 .read()
16450 .unwrap()
16451 .describe(
16452 &Principal::Reserved {
16453 module_id: OWNER.to_string()
16454 },
16455 "s"
16456 )
16457 .owner_synced
16458 );
16459 }
16460
16461 #[tokio::test]
16464 async fn a_module_without_a_supervised_launch_cannot_sync() {
16465 let handler = ControlHandler::new(Arc::new(Registry::default()));
16466 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16467 assert_eq!(
16468 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16469 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16470 );
16471 }
16472
16473 #[tokio::test]
16474 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16475 let supervisor = SupervisorHandle::new();
16476 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16477 let handler = ControlHandler::new(Arc::new(Registry::default()))
16478 .with_supervisor(supervisor.clone());
16479 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16480 sync(&handler, &incumbent, 1, vec![head("s", 1)])
16481 .await
16482 .expect("the current launch syncs");
16483
16484 supervisor.open_swap(OWNER, "n2".to_string());
16487 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16488 assert_eq!(
16489 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16490 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16491 );
16492 sync(&handler, &incumbent, 2, vec![head("s", 1)])
16493 .await
16494 .expect("the serving owner syncs during the swap");
16495
16496 supervisor.close_swap(OWNER);
16499 assert_eq!(
16500 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16501 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16502 );
16503 sync(&handler, &incumbent, 3, vec![head("s", 1)])
16504 .await
16505 .expect("the serving owner syncs after the rollback");
16506 handler.cleanup_connection(candidate.connection_id).unwrap();
16507
16508 supervisor.open_swap(OWNER, "n3".to_string());
16513 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16514 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16515 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16516 .await
16517 .expect("the promoted launch takes authority");
16518 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16519 panic!("unexpected reply {reply:?}");
16520 };
16521 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16522 assert_eq!(
16523 sync(&handler, &incumbent, 4, Vec::new()).await,
16524 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16525 );
16526 }
16527
16528 #[tokio::test]
16531 async fn closing_the_authority_connection_frees_sync_authority() {
16532 let supervisor = SupervisorHandle::new();
16533 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16534 let handler = ControlHandler::new(Arc::new(Registry::default()))
16535 .with_supervisor(supervisor.clone());
16536 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16537 sync(&handler, &first, 10, vec![head("s", 1)])
16538 .await
16539 .unwrap();
16540 handler.cleanup_connection(first.connection_id).unwrap();
16541
16542 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16543 sync(&handler, &second, 1, vec![head("s", 1)])
16544 .await
16545 .expect("the next connection takes the released authority");
16546 }
16547
16548 #[tokio::test]
16549 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16550 let supervisor = SupervisorHandle::new();
16551 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16552 let handler =
16553 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16554 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16555 sync(&handler, &first, 10, vec![head("s", 1)])
16556 .await
16557 .unwrap();
16558 handler
16559 .handle_control_frame(
16560 &first,
16561 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16562 )
16563 .await
16564 .unwrap();
16565 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16566 sync(&handler, &second, 1, vec![head("s", 1)])
16567 .await
16568 .expect("GOODBYE releases authority even if the old socket remains open");
16569 }
16570
16571 #[tokio::test]
16572 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16573 let registry = Arc::new(Registry::default());
16574 let supervisor_handle = SupervisorHandle::new();
16575 let supervisor =
16576 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16577 .with_handle(supervisor_handle.clone())
16578 .with_daemon_incarnation("incarnation-7".to_string());
16579 supervisor
16582 .supervise_configured(
16583 ModuleSpec {
16584 module_id: OWNER.to_string(),
16585 program: PathBuf::from("/nonexistent/prefrontal-core"),
16586 args: Vec::new(),
16587 env: Vec::new(),
16588 reserved: false,
16589 reserved_prefixes: Vec::new(),
16590 protocol: ModuleProtocol::Subc,
16591 overlap: Default::default(),
16592 },
16593 false,
16594 )
16595 .unwrap();
16596 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16597 let handler =
16598 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16599 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16600
16601 let ModuleControlResponseToModule::ScopeDescribe {
16603 status,
16604 daemon_incarnation,
16605 owner_synced,
16606 owner_configured,
16607 scope,
16608 ..
16609 } = describe(&handler, &reader, OWNER, "s").await
16610 else {
16611 panic!("not a describe reply");
16612 };
16613 assert_eq!(status, ScopeStatus::NotLive);
16614 assert_eq!(daemon_incarnation, "incarnation-7");
16615 assert!(!owner_synced);
16616 assert!(owner_configured);
16617 assert!(scope.is_none());
16618
16619 let ModuleControlResponseToModule::ScopeDescribe {
16622 status,
16623 owner_configured,
16624 ..
16625 } = describe(&handler, &reader, "ghost", "s").await
16626 else {
16627 panic!("not a describe reply");
16628 };
16629 assert_eq!(status, ScopeStatus::NotLive);
16630 assert!(!owner_configured);
16631
16632 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16634 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16635 let ModuleControlResponseToModule::ScopeDescribe {
16636 status,
16637 scope_epoch,
16638 owner_synced,
16639 scope,
16640 ..
16641 } = describe(&handler, &reader, OWNER, "s").await
16642 else {
16643 panic!("not a describe reply");
16644 };
16645 assert_eq!(status, ScopeStatus::Live);
16646 assert_eq!(scope_epoch, Some(4));
16647 assert!(owner_synced);
16648 let stamp = scope.expect("a live scope carries its stamp");
16649 assert!(
16650 stamp.owner_authorized,
16651 "prefrontal-core is the default authority"
16652 );
16653 assert_eq!(stamp.kind, ScopeKind::Head);
16654 }
16655
16656 #[tokio::test]
16657 async fn scope_authority_owners_decides_owner_authorized() {
16658 let supervisor = SupervisorHandle::new();
16659 supervisor.set_spawn_nonce("broca", "b1".to_string());
16660 let handler = ControlHandler::new(Arc::new(Registry::default()))
16661 .with_supervisor(supervisor)
16662 .with_scope_authority_owners(vec!["broca".to_string()]);
16663 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16664 let mut gated = head("s", 1);
16665 gated.attributes.agent_id = Some("agent".to_string());
16666 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16667 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16668 describe(&handler, &broca, "broca", "s").await
16669 else {
16670 panic!("not a describe reply");
16671 };
16672 assert!(scope.unwrap().owner_authorized);
16673 }
16674
16675 #[tokio::test]
16679 async fn scope_ops_and_the_scopes_capability_are_advertised() {
16680 let handler = ControlHandler::new(Arc::new(Registry::default()));
16681 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16682 let ack = hello_via_sink(
16683 &handler,
16684 &ctx,
16685 &mut rx,
16686 hello_frame("m", PROTOCOL_VERSION, 1),
16687 )
16688 .await;
16689 let ack = parse_ack(&ack);
16690 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16691 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16692 }
16693 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16694
16695 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16696 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16697 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16698 let reply = handler
16699 .handle_control_frame(&client, frame)
16700 .await
16701 .unwrap()
16702 .pop()
16703 .unwrap();
16704 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16705 serde_json::from_slice(&reply.body).unwrap()
16706 else {
16707 panic!("not a server.describe reply");
16708 };
16709 assert!(
16710 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16711 "{capabilities:?}"
16712 );
16713 }
16714
16715 const PLEXUS: &str = "plexus";
16718 const OTHER: &str = "other";
16719 const AFT: &str = "aft";
16720 const BROCA: &str = "broca";
16721 const MAGIC: &str = "magic-context";
16722
16723 fn nonce(module_id: &str) -> String {
16724 format!("nonce-{module_id}")
16725 }
16726
16727 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16728 let (tx, rx) = mpsc::channel(64);
16729 (
16730 RouteCtx {
16731 connection_id: ConnectionId::new(connection),
16732 egress: FrameSink::new(tx),
16733 },
16734 rx,
16735 )
16736 }
16737
16738 struct Rig {
16742 handler: ControlHandler,
16743 forwarding: Arc<ForwardingTable>,
16744 owner: RouteCtx,
16745 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16746 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16747 generation: u64,
16748 next_connection: u64,
16749 _supervisor: Supervisor,
16750 }
16751
16752 async fn rig() -> Rig {
16753 rig_with_flow_support(true).await
16754 }
16755
16756 async fn rig_with_flow_support(flow_support: bool) -> Rig {
16757 rig_with_scope_capabilities(flow_support, false).await
16758 }
16759
16760 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16761 let mut provides = Vec::new();
16762 if flow_support {
16763 provides.push("flow-scopes/v1");
16764 }
16765 if run_support {
16766 provides.push("agent-run-scopes/v1");
16767 }
16768 rig_with_target_capabilities(&provides).await
16769 }
16770
16771 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16772 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16773 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16774 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16777 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16778 Frame::build(
16779 FrameType::Hello,
16780 control_flags(),
16781 0,
16782 0,
16783 connection,
16784 serde_json::to_vec(&body).unwrap(),
16785 )
16786 .unwrap()
16787 }
16788
16789 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16790 let registry = Arc::new(Registry::default());
16791 let forwarding = Arc::new(ForwardingTable::default());
16792 let supervisor_handle = SupervisorHandle::new();
16793 let supervisor =
16794 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16795 .with_handle(supervisor_handle.clone());
16796 supervisor
16797 .supervise_configured(
16798 ModuleSpec {
16799 module_id: OWNER.to_string(),
16800 program: PathBuf::from("/nonexistent/prefrontal-core"),
16801 args: Vec::new(),
16802 env: Vec::new(),
16803 reserved: false,
16804 reserved_prefixes: Vec::new(),
16805 protocol: ModuleProtocol::Subc,
16806 overlap: Default::default(),
16807 },
16808 false,
16809 )
16810 .unwrap();
16811 for module_id in [OWNER, AFT, BROCA, MAGIC] {
16812 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16813 }
16814 let handler =
16815 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16816 .with_supervisor(supervisor_handle);
16817 let (owner, mut owner_rx) = wide_ctx(1);
16818 hello_via_sink(
16819 &handler,
16820 &owner,
16821 &mut owner_rx,
16822 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16823 )
16824 .await;
16825 let mut modules = BTreeMap::new();
16826 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16827 let (ctx, mut rx) = wide_ctx(connection);
16828 let hello = hello_with_capabilities(module_id, connection, provides);
16829 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16830 modules.insert(module_id.to_string(), (ctx, rx));
16831 }
16832 Rig {
16833 handler,
16834 forwarding,
16835 owner,
16836 _owner_rx: owner_rx,
16837 modules,
16838 generation: 0,
16839 next_connection: 100,
16840 _supervisor: supervisor,
16841 }
16842 }
16843
16844 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16845 ScopeCarrier::new(Principal::Reserved {
16846 module_id: module_id.to_string(),
16847 })
16848 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16849 }
16850
16851 fn session(scope_epoch: u64) -> ScopeRecord {
16854 let mut record = head("s", scope_epoch);
16855 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16856 record.attributes.agent_id = Some("agent-1".to_string());
16857 record.attributes.delegates = true;
16858 record
16859 }
16860
16861 impl Rig {
16862 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16863 self.generation += 1;
16864 sync(&self.handler, &self.owner, self.generation, scopes)
16865 .await
16866 .expect("the owner's sync is accepted");
16867 }
16868
16869 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16870 ScopeSelector::new(
16871 Principal::Reserved {
16872 module_id: OWNER.to_string(),
16873 },
16874 scope_ref.to_string(),
16875 scope_epoch,
16876 )
16877 }
16878
16879 fn open_frame(
16880 &mut self,
16881 opener: Option<&str>,
16882 target: &str,
16883 scope: Option<ScopeSelector>,
16884 ) -> (
16885 RouteCtx,
16886 mpsc::Receiver<crate::router::OutboundFrame>,
16887 Frame,
16888 ) {
16889 self.next_connection += 1;
16890 let (ctx, rx) = wide_ctx(self.next_connection);
16891 let root = unique_project_root("scoped-open");
16892 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16893 target: RouteTarget::ToolProvider {
16894 module_id: target.to_string(),
16895 },
16896 identity: BindIdentity::new(
16897 root.path().to_path_buf(),
16898 "unit".to_string(),
16899 "session".to_string(),
16900 ),
16901 consumer_identity: opener.map(|module_id| ConsumerIdentity {
16902 module_id: module_id.to_string(),
16903 launch_nonce: nonce(module_id),
16904 }),
16905 consumer_capabilities: None,
16906 role_versions: None,
16907 admission_facts: None,
16908 scope,
16909 })
16910 .unwrap();
16911 let frame = Frame::build(
16912 FrameType::Request,
16913 control_flags(),
16914 0,
16915 0,
16916 self.next_connection,
16917 body,
16918 )
16919 .unwrap();
16920 (ctx, rx, frame)
16921 }
16922
16923 async fn refused(
16925 &mut self,
16926 opener: Option<&str>,
16927 target: &str,
16928 scope: Option<ScopeSelector>,
16929 ) -> String {
16930 self.refusal_body(opener, target, scope).await["code"]
16931 .as_str()
16932 .unwrap()
16933 .to_string()
16934 }
16935
16936 async fn refusal_body(
16937 &mut self,
16938 opener: Option<&str>,
16939 target: &str,
16940 scope: Option<ScopeSelector>,
16941 ) -> Value {
16942 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16943 let replies = tokio::time::timeout(
16944 Duration::from_secs(2),
16945 self.handler.handle_control_frame(&ctx, frame),
16946 )
16947 .await
16948 .expect("the open must be refused before waiting for a bind ack")
16949 .unwrap();
16950 assert_eq!(replies.len(), 1, "{replies:?}");
16951 assert_eq!(replies[0].header.ty, FrameType::Error);
16952 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16953 assert!(
16954 module_rx.try_recv().is_err(),
16955 "a refused open relays nothing"
16956 );
16957 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16958 parse_error(&replies[0])
16959 }
16960
16961 async fn relayed(
16963 &mut self,
16964 opener: Option<&str>,
16965 target: &str,
16966 scope: Option<ScopeSelector>,
16967 ) -> Relayed {
16968 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16969 let handler = self.handler.clone();
16970 let task_ctx = ctx.clone();
16971 let task = tokio::spawn(async move {
16972 handler
16973 .handle_control_frame(&task_ctx, frame)
16974 .await
16975 .unwrap()
16976 });
16977 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16978 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16979 .await
16980 .expect("the target receives the relayed route.bind")
16981 .unwrap()
16982 .frame;
16983 Relayed {
16984 target: target.to_string(),
16985 client: ctx,
16986 client_rx: rx,
16987 task,
16988 bind,
16989 }
16990 }
16991
16992 async fn ack(&self, relayed: &Relayed) {
16993 let (module, _) = &self.modules[&relayed.target];
16994 self.handler
16995 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16996 .await
16997 .unwrap();
16998 }
16999
17000 async fn bound(
17002 &mut self,
17003 opener: Option<&str>,
17004 target: &str,
17005 scope: Option<ScopeSelector>,
17006 ) -> Bound {
17007 let relayed = self.relayed(opener, target, scope).await;
17008 self.ack(&relayed).await;
17009 let Relayed {
17010 target,
17011 client,
17012 mut client_rx,
17013 task,
17014 bind,
17015 } = relayed;
17016 assert!(
17017 task.await.unwrap().is_empty(),
17018 "the open is answered by commit"
17019 );
17020 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
17021 Bound {
17022 target,
17023 client,
17024 client_rx,
17025 channel,
17026 epoch,
17027 bind,
17028 }
17029 }
17030
17031 fn live(&self, route: &Bound) -> bool {
17032 matches!(
17033 self.forwarding
17034 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17035 .unwrap(),
17036 DataRoute::Client(DataRouteState::Bound(_))
17037 )
17038 }
17039 }
17040
17041 struct Relayed {
17042 target: String,
17043 client: RouteCtx,
17044 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17045 task: tokio::task::JoinHandle<Vec<Frame>>,
17046 bind: Frame,
17047 }
17048
17049 struct Bound {
17050 target: String,
17051 client: RouteCtx,
17052 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17053 channel: u16,
17054 epoch: u32,
17055 bind: Frame,
17056 }
17057
17058 impl Bound {
17059 fn closed_reason(&mut self) -> RouteCloseReason {
17062 let mut reason = None;
17063 let mut goodbye = false;
17064 while let Ok(outbound) = self.client_rx.try_recv() {
17065 let frame = outbound.frame;
17066 match frame.header.ty {
17067 FrameType::Goodbye => {
17068 assert_eq!(
17069 (frame.header.channel, frame.header.epoch),
17070 (self.channel, self.epoch)
17071 );
17072 goodbye = true;
17073 }
17074 FrameType::Push => {
17075 let ClientControlPush::RouteClosed {
17076 reason: r,
17077 module_id,
17078 ..
17079 } = serde_json::from_slice(&frame.body).unwrap()
17080 else {
17081 panic!("unexpected push");
17082 };
17083 assert_eq!(module_id, self.target);
17084 reason = Some(r);
17085 }
17086 other => panic!("unexpected frame {other:?}"),
17087 }
17088 }
17089 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
17090 reason.expect("the client is told why the route closed")
17091 }
17092
17093 fn untouched(&mut self) -> bool {
17094 self.client_rx.try_recv().is_err()
17095 }
17096
17097 fn stamp(&self) -> Option<ScopeStamp> {
17098 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
17099 ModuleControlRequest::RouteBind { scope, .. } => scope,
17100 other => panic!("expected a route.bind, got {other:?}"),
17101 }
17102 }
17103 }
17104
17105 #[tokio::test]
17106 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
17107 ) {
17108 let mut rig = rig().await;
17109 let mut record = session(1);
17110 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
17111 record.child_owners = vec![Principal::Reserved {
17112 module_id: MAGIC.to_string(),
17113 }];
17114 rig.sync(vec![record]).await;
17115 let scope = || Some(rig_selector("s", Some(1)));
17116
17117 rig.bound(Some(OWNER), PLEXUS, scope()).await;
17120 rig.bound(Some(AFT), OTHER, scope()).await;
17121 rig.bound(Some(BROCA), PLEXUS, scope()).await;
17122
17123 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
17127 assert_eq!(
17128 rig.refused(opener, target, scope()).await,
17129 error_codes::SCOPE_NOT_CARRIER,
17130 "{opener:?} -> {target}"
17131 );
17132 }
17133 }
17134
17135 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17136 ScopeSelector::new(
17137 Principal::Reserved {
17138 module_id: OWNER.to_string(),
17139 },
17140 scope_ref.to_string(),
17141 scope_epoch,
17142 )
17143 }
17144
17145 #[tokio::test]
17146 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
17147 let mut rig = rig().await;
17148 rig.sync(vec![session(1)]).await;
17149 for opener in [OWNER, AFT] {
17150 assert_eq!(
17151 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
17152 .await,
17153 error_codes::SCOPE_EPOCH_REQUIRED,
17154 "{opener}"
17155 );
17156 }
17157 }
17158
17159 #[tokio::test]
17160 async fn admission_separates_not_synced_not_live_and_ended() {
17161 let mut rig = rig().await;
17162 let code = rig
17164 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17165 .await;
17166 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
17167 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
17168
17169 let ghost = ScopeSelector::new(
17171 Principal::Reserved {
17172 module_id: "ghost".to_string(),
17173 },
17174 "s".to_string(),
17175 Some(1),
17176 );
17177 assert_eq!(
17178 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
17179 error_codes::SCOPE_NOT_LIVE
17180 );
17181
17182 rig.sync(vec![session(2)]).await;
17183 assert_eq!(
17184 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
17185 .await,
17186 error_codes::SCOPE_NOT_LIVE
17187 );
17188 for epoch in [1, 3] {
17189 assert_eq!(
17190 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
17191 .await,
17192 error_codes::SCOPE_ENDED,
17193 "epoch {epoch}"
17194 );
17195 }
17196 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
17198 .await;
17199 }
17200
17201 #[tokio::test]
17202 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
17203 let mut rig = rig().await;
17204 rig.sync(vec![session(1)]).await;
17205 let route = rig
17206 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17207 .await;
17208 let stamp = route.stamp().expect("a scoped bind carries the stamp");
17209 assert_eq!(stamp.scope_ref, "s");
17210 assert_eq!(stamp.scope_epoch, 1);
17211 assert_eq!(stamp.kind, ScopeKind::Head);
17212 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
17213 assert!(stamp.attributes.delegates);
17214 assert!(stamp.owner_authorized);
17215 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17216 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
17217
17218 let (broca, mut broca_rx) = wide_ctx(50);
17221 hello_via_sink(
17222 &rig.handler,
17223 &broca,
17224 &mut broca_rx,
17225 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17226 )
17227 .await;
17228 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17229 .await
17230 .unwrap();
17231 let own = ScopeSelector::new(
17232 Principal::Reserved {
17233 module_id: BROCA.to_string(),
17234 },
17235 "b".to_string(),
17236 Some(1),
17237 );
17238 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17239 assert!(!route.stamp().unwrap().owner_authorized);
17240 }
17241
17242 #[tokio::test]
17243 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17244 let mut rig = rig().await;
17245 let mut record = head("s", 1);
17246 record.carriers = vec![carrier(AFT, None)];
17247 let flow_id = "Flow:run-7/step_2!~";
17248 record.attributes.flow_id = Some(flow_id.to_string());
17249 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17250 .await
17251 .unwrap();
17252 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17253 panic!("not a sync reply");
17254 };
17255 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17256 let route = rig
17257 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17258 .await;
17259 assert!(rig.live(&route), "the stamped bind committed");
17260 let stamp = route.stamp().expect("a flow scope carries a stamp");
17261 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17262 assert_eq!(stamp.attributes.agent_id, None);
17263 assert!(!stamp.attributes.delegates);
17264 assert!(stamp.owner_authorized);
17265 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17266 assert_eq!(unscoped.stamp(), None);
17267 }
17268
17269 #[tokio::test]
17270 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17271 let mut rig = rig_with_flow_support(false).await;
17272 let mut record = session(1);
17273 record.attributes.flow_id = Some("flow:7".to_string());
17274 rig.sync(vec![record]).await;
17275 for opener in [OWNER, AFT] {
17276 let body = rig
17277 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17278 .await;
17279 assert_eq!(body["code"], "target_flow_unsupported");
17280 let message = body["message"].as_str().unwrap();
17281 for required in [PLEXUS, "flow-scopes/v1"] {
17282 assert!(message.contains(required), "{message}");
17283 }
17284 }
17285 }
17286
17287 #[tokio::test]
17288 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17289 let mut rig = rig_with_flow_support(true).await;
17290 let mut record = session(1);
17291 record.attributes.flow_id = Some("flow:7".to_string());
17292 rig.sync(vec![record]).await;
17293 for opener in [OWNER, AFT] {
17294 let route = rig
17295 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17296 .await;
17297 assert!(rig.live(&route));
17298 assert_eq!(
17299 route.stamp().unwrap().attributes.flow_id.as_deref(),
17300 Some("flow:7")
17301 );
17302 }
17303 }
17304
17305 #[tokio::test]
17306 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17307 use std::future::Future;
17308
17309 let mut rig = rig_with_flow_support(true).await;
17310 let mut record = session(1);
17311 record.attributes.flow_id = Some("flow:7".to_string());
17312 rig.sync(vec![record]).await;
17313 let (client, mut client_rx, frame) =
17314 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17315 for _ in 0..64 {
17318 client.egress.try_send(route_bind_ack(1)).unwrap();
17319 }
17320 let handler = rig.handler.clone();
17321 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17322 std::future::poll_fn(|cx| {
17323 assert!(open.as_mut().poll(cx).is_pending());
17324 std::task::Poll::Ready(())
17325 })
17326 .await;
17327 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17328
17329 let old_connection = rig.modules[PLEXUS].0.connection_id;
17330 rig.handler.cleanup_connection(old_connection).unwrap();
17331 let (replacement, mut replacement_rx) = wide_ctx(200);
17332 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17333 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17334 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17335 let hello = Frame::build(
17336 FrameType::Hello,
17337 control_flags(),
17338 0,
17339 0,
17340 200,
17341 serde_json::to_vec(&body).unwrap(),
17342 )
17343 .unwrap();
17344 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17345
17346 client_rx.try_recv().unwrap();
17347 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17348 .await
17349 .expect("the replacement is refused without waiting for a bind ack")
17350 .unwrap();
17351 assert_eq!(replies.len(), 1);
17352 let body = parse_error(&replies[0]);
17353 assert_eq!(body["code"], "target_flow_unsupported");
17354 for required in [PLEXUS, "flow-scopes/v1"] {
17355 assert!(body["message"].as_str().unwrap().contains(required));
17356 }
17357 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17358 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17359 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17360 assert!(rig
17361 .handler
17362 .registry
17363 .get_module_by_connection(replacement.connection_id)
17364 .unwrap()
17365 .is_some());
17366 }
17367
17368 #[tokio::test]
17369 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17370 use std::future::Future;
17371 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17372 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17373 const NEXT_HASH: &str =
17374 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17375 let mut rig = rig().await;
17376 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17377 record.carriers.push(ScopeCarrier::new(Principal::Direct));
17378 rig.sync(vec![record.clone()]).await;
17379 let (client, mut rx, frame) = rig.open_frame(
17380 None,
17381 PLEXUS,
17382 Some(
17383 rig.selector("s", Some(1))
17384 .with_bind_proof(Some(PROOF.into())),
17385 ),
17386 );
17387 for _ in 0..64 {
17390 client.egress.try_send(route_bind_ack(1)).unwrap();
17391 }
17392 let handler = rig.handler.clone();
17393 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17394 std::future::poll_fn(|cx| {
17395 assert!(open.as_mut().poll(cx).is_pending());
17396 std::task::Poll::Ready(())
17397 })
17398 .await;
17399 record.bind_proof_sha256 = Some(NEXT_HASH.into());
17400 rig.sync(vec![record]).await;
17401 rx.try_recv().unwrap();
17402 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17403 .await
17404 .unwrap()
17405 .unwrap();
17406 assert_eq!(
17407 parse_error(&replies[0])["code"],
17408 error_codes::SCOPE_BIND_PROOF_MISMATCH
17409 );
17410 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17411 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17412 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17413 }
17414
17415 #[tokio::test]
17416 async fn scope_bind_proof_reserved_openers_ignore_proof() {
17417 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17418 let mut rig = rig().await;
17419 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17420 .await;
17421 for opener in [OWNER, AFT] {
17422 for proof in [None, Some("malformed+proof=")] {
17423 let route = rig
17424 .bound(
17425 Some(opener),
17426 PLEXUS,
17427 Some(
17428 rig.selector("s", Some(1))
17429 .with_bind_proof(proof.map(str::to_string)),
17430 ),
17431 )
17432 .await;
17433 assert!(rig.live(&route));
17434 }
17435 }
17436 }
17437
17438 #[tokio::test]
17439 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17440 use std::future::Future;
17441 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17442 const NEXT_HASH: &str =
17443 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17444 let mut rig = rig().await;
17445 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17446 .await;
17447 let (client, mut rx, frame) =
17448 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17449 for _ in 0..64 {
17450 client.egress.try_send(route_bind_ack(1)).unwrap();
17451 }
17452 let handler = rig.handler.clone();
17453 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17454 std::future::poll_fn(|cx| {
17455 assert!(open.as_mut().poll(cx).is_pending());
17456 std::task::Poll::Ready(())
17457 })
17458 .await;
17459 rig.sync(vec![
17460 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17461 ])
17462 .await;
17463 rx.try_recv().unwrap();
17464 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17465 let bind = std::future::poll_fn(|cx| {
17466 assert!(open.as_mut().poll(cx).is_pending());
17467 module_rx.poll_recv(cx)
17468 })
17469 .await
17470 .unwrap()
17471 .frame;
17472 handler
17473 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17474 .await
17475 .unwrap();
17476 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17477 .await
17478 .unwrap()
17479 .unwrap();
17480 assert_eq!(replies.len(), 1);
17481 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17484 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17485 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17486 let route = rig
17487 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17488 .await;
17489 assert!(rig.live(&route));
17490 }
17491
17492 #[tokio::test]
17493 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17494 {
17495 let mut rig = rig_with_flow_support(false).await;
17496 rig.sync(vec![session(1)]).await;
17497 let route = rig
17498 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17499 .await;
17500 assert!(rig.live(&route));
17501 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17502 let mut record = session(1);
17503 record.attributes.flow_id = Some("flow:7".to_string());
17504 rig.sync(vec![record]).await;
17505 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17506 assert!(rig.live(&unscoped));
17507 assert_eq!(unscoped.stamp(), None);
17508 }
17509
17510 #[tokio::test]
17511 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17512 let mut rig = rig().await;
17513 let mut record = session(1);
17514 record.attributes.flow_id = Some("flow:7".to_string());
17515 rig.sync(vec![record.clone()]).await;
17516 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17517 let mut owner_route = rig
17518 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17519 .await;
17520 let mut carrier_route = rig
17521 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17522 .await;
17523 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17524 rig.sync(vec![record.clone()]).await;
17525 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17526 assert!(rig.live(&owner_route) && owner_route.untouched());
17527 assert!(rig.live(&carrier_route) && carrier_route.untouched());
17528
17529 record.attributes.flow_id = Some("flow:8".to_string());
17530 rig.sync(vec![record]).await;
17531 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17532 let before = before.unwrap();
17533 assert_eq!(after.scope_epoch, before.scope_epoch);
17534 assert!(after.version > before.version);
17535 for route in [&mut owner_route, &mut carrier_route] {
17536 assert!(!rig.live(route));
17537 assert_eq!(
17538 route.closed_reason(),
17539 RouteCloseReason::ScopeDelegationChanged
17540 );
17541 }
17542 assert!(rig.live(&unscoped) && unscoped.untouched());
17543 for target in [PLEXUS, OTHER] {
17546 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17547 let goodbye = module_rx
17548 .try_recv()
17549 .expect("the provider sees the drain")
17550 .frame;
17551 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17552 assert!(module_rx.try_recv().is_err());
17553 }
17554 let rebound = rig
17555 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17556 .await;
17557 assert_eq!(
17558 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17559 Some("flow:8")
17560 );
17561 }
17562
17563 #[tokio::test]
17568 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17569 let mut rig = rig().await;
17570 rig.sync(vec![session(1)]).await;
17571 let mut cotenant = rig
17572 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17573 .await;
17574
17575 let mut changed = session(1);
17576 changed.child_owners.push(Principal::Reserved {
17577 module_id: MAGIC.to_string(),
17578 });
17579 let mut ended = None;
17580 for (code, next) in [
17581 (error_codes::SCOPE_CHANGED, vec![changed]),
17582 (error_codes::SCOPE_ENDED, Vec::new()),
17583 ] {
17584 let relayed = rig
17585 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17586 .await;
17587 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17588 ended = Some(next.is_empty());
17589 rig.sync(next).await;
17590 rig.ack(&relayed).await;
17591 let replies = relayed.task.await.unwrap();
17592 assert_eq!(replies.len(), 1, "{replies:?}");
17593 assert_eq!(parse_error(&replies[0])["code"], code);
17594 assert_eq!(
17595 subc_protocol::error_codes::is_retryable_route_open(code),
17596 code == error_codes::SCOPE_CHANGED
17597 );
17598 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17599 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17601 let mut goodbyes = Vec::new();
17604 while let Ok(outbound) = plexus_rx.try_recv() {
17605 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17606 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17607 }
17608 assert!(
17609 goodbyes.contains(&(bind_channel, bind_epoch)),
17610 "{goodbyes:?}"
17611 );
17612 assert!(rig
17613 .handler
17614 .registry
17615 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17616 .unwrap()
17617 .is_some());
17618 }
17619 assert_eq!(ended, Some(true));
17620 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17623 }
17624
17625 #[tokio::test]
17628 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17629 struct Case {
17630 name: &'static str,
17631 change: fn(&mut ScopeRecord),
17632 closed: [Option<RouteCloseReason>; 4],
17635 }
17636 use RouteCloseReason::*;
17637 let cases = [
17638 Case {
17639 name: "a carrier entry removed",
17640 change: |r| {
17641 r.carriers.retain(|c| {
17642 c.principal
17643 != Principal::Reserved {
17644 module_id: AFT.to_string(),
17645 }
17646 })
17647 },
17648 closed: [None, Some(ScopeCarrierRemoved), None, None],
17649 },
17650 Case {
17651 name: "a target removed from a carrier",
17652 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17653 closed: [None, None, None, Some(ScopeCarrierRemoved)],
17654 },
17655 Case {
17656 name: "a bare carrier narrowed to targets",
17657 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17658 closed: [None, Some(ScopeCarrierRemoved), None, None],
17659 },
17660 Case {
17661 name: "delegates turned off",
17662 change: |r| r.attributes.delegates = false,
17663 closed: [Some(ScopeDelegationChanged); 4],
17664 },
17665 Case {
17666 name: "agent_id changed",
17667 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17668 closed: [Some(ScopeDelegationChanged); 4],
17669 },
17670 Case {
17671 name: "a carrier added, child owners changed, the record re-sent",
17672 change: |r| {
17673 r.carriers.push(carrier(MAGIC, None));
17674 r.child_owners.push(Principal::Reserved {
17675 module_id: MAGIC.to_string(),
17676 });
17677 },
17678 closed: [None; 4],
17679 },
17680 Case {
17681 name: "a target added",
17682 change: |r| {
17683 r.carriers[1]
17684 .targets
17685 .as_mut()
17686 .unwrap()
17687 .push("third".to_string())
17688 },
17689 closed: [None; 4],
17690 },
17691 Case {
17692 name: "delegates turned on",
17693 change: |r| r.attributes.delegates = true,
17694 closed: [None; 4],
17695 },
17696 ];
17697 for case in cases {
17698 let mut rig = rig().await;
17699 rig.sync(vec![session(1)]).await;
17700 let scope = || Some(rig_selector("s", Some(1)));
17701 let mut routes = [
17702 rig.bound(Some(OWNER), PLEXUS, scope()).await,
17703 rig.bound(Some(AFT), PLEXUS, scope()).await,
17704 rig.bound(Some(BROCA), PLEXUS, scope()).await,
17705 rig.bound(Some(BROCA), OTHER, scope()).await,
17706 ];
17707 let mut record = session(1);
17708 (case.change)(&mut record);
17709 rig.sync(vec![record]).await;
17710 for (index, expected) in case.closed.iter().enumerate() {
17711 let route = &mut routes[index];
17712 match expected {
17713 Some(reason) => {
17714 assert!(!rig.live(route), "{}: route {index} still live", case.name);
17715 assert_eq!(
17716 route.closed_reason(),
17717 *reason,
17718 "{}: route {index}",
17719 case.name
17720 );
17721 }
17722 None => {
17723 assert!(rig.live(route), "{}: route {index} closed", case.name);
17724 assert!(
17725 route.untouched(),
17726 "{}: route {index} was told something",
17727 case.name
17728 );
17729 }
17730 }
17731 }
17732 }
17733 }
17734
17735 #[tokio::test]
17736 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17737 for next in [Vec::new(), vec![session(2)]] {
17739 let mut rig = rig().await;
17740 rig.sync(vec![session(1)]).await;
17741 let mut route = rig
17742 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17743 .await;
17744 rig.sync(next).await;
17745 assert!(!rig.live(&route));
17746 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17747 }
17748
17749 let mut rig = rig().await;
17752 let mut child = session(1);
17753 child.scope_ref = "child".to_string();
17754 child.kind = ScopeKind::Worker;
17755 child.parent = Some(ScopeParent::new(
17756 Principal::Reserved {
17757 module_id: OWNER.to_string(),
17758 },
17759 "s".to_string(),
17760 1,
17761 ));
17762 rig.sync(vec![session(1), child.clone()]).await;
17763 let mut child_route = rig
17764 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17765 .await;
17766 assert_eq!(
17767 child_route.stamp().unwrap().parent_state,
17768 Some(ParentState::Linked)
17769 );
17770 rig.sync(vec![child]).await;
17771 assert!(!rig.live(&child_route));
17772 assert_eq!(
17773 child_route.closed_reason(),
17774 RouteCloseReason::ScopeParentEnded
17775 );
17776 }
17777
17778 #[tokio::test]
17779 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17780 ) {
17781 let mut rig = rig().await;
17782 rig.sync(vec![session(1)]).await;
17783 let mut route = rig
17784 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17785 .await;
17786 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17787 rig.sync(vec![session(1)]).await;
17788 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17789 assert!(rig.live(&route) && route.untouched());
17790
17791 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17795 .forwarding
17796 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17797 .unwrap()
17798 else {
17799 panic!("the route is bound");
17800 };
17801 binding.flow.acquire_tagged(9, false).await.unwrap();
17802 let mut widened = session(1);
17803 widened.carriers.push(carrier(MAGIC, None));
17804 rig.sync(vec![widened]).await;
17805 assert!(rig.live(&route) && route.untouched());
17806 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17807 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17808 assert_eq!(binding.flow.in_flight(), 1);
17811 binding
17812 .flow
17813 .acquire_tagged(10, false)
17814 .await
17815 .expect("the flow is still open");
17816 }
17817
17818 #[tokio::test]
17821 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17822 let mut rig = rig().await;
17823 rig.sync(vec![session(1)]).await;
17824 let mut on_incumbent = rig
17825 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17826 .await;
17827
17828 let (candidate, _candidate_rx) = wide_ctx(9);
17831 let registration = rig
17832 .handler
17833 .registry
17834 .register_candidate_with_control_ops(
17835 manifest(PLEXUS, PROTOCOL_VERSION),
17836 PROTOCOL_VERSION,
17837 candidate.connection_id,
17838 module_baseline_control_ops(),
17839 )
17840 .unwrap();
17841 rig.forwarding
17842 .register_candidate_module_connection(
17843 candidate.connection_id,
17844 PLEXUS.to_string(),
17845 PROTOCOL_VERSION,
17846 manifest_concurrency(®istration.manifest),
17847 candidate.egress.clone(),
17848 )
17849 .unwrap();
17850 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17851 rig.handler
17852 .registry
17853 .promote_candidate(PLEXUS)
17854 .unwrap()
17855 .unwrap();
17856 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17857
17858 rig.sync(Vec::new()).await;
17859 assert!(!rig.live(&on_incumbent));
17860 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17861 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17862 let goodbye = incumbent_rx
17863 .try_recv()
17864 .expect("the superseded endpoint is told")
17865 .frame;
17866 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17867 }
17868 }
17869}
17870
17871#[cfg(test)]
17872mod concurrency_default_exposure_tests {
17873 use super::*;
17874
17875 fn hello_body(role_json: &str) -> Vec<u8> {
17876 format!(
17877 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17878 )
17879 .into_bytes()
17880 }
17881
17882 fn manifest_from(body: &[u8]) -> ModuleManifest {
17883 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17884 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17885 .expect("manifest parses")
17886 }
17887
17888 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17889
17890 #[test]
17891 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17892 let body = hello_body(&format!(
17893 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17894 ));
17895 let manifest = manifest_from(&body);
17896 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17899 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17900 }
17901
17902 #[test]
17903 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17904 let body = hello_body(&format!(
17905 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17906 ));
17907 let manifest = manifest_from(&body);
17908 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17909 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17910 }
17911
17912 #[test]
17913 fn non_management_roles_are_never_reported() {
17914 let body = hello_body(
17915 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17916 );
17917 let manifest = manifest_from(&body);
17918 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17919 }
17920}