1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_rich_ui_descriptor, validate_hello_self_signal_declarations,
26 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
27 ProviderRole,
28 },
29 scope::{
30 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32 SCOPE_SYNC_OP,
33 },
34 session::{
35 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38 ROLE_VERSIONS_FIELD,
39 },
40 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41 Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47 caller_identity::CallerIdentity,
48 capability_requirements::{
49 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51 RuntimeModule,
52 },
53 child_roster::DaemonShutdownFlag,
54 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55 forwarding::{
56 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59 },
60 observability::{
61 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62 },
63 privileged_prompt::{prompt_text, PrivilegedOperation},
64 provenance::{
65 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66 },
67 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68 router::{RouteCtx, RouterError},
69 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71 stderr_tail::{CaptureState, TailEntry},
72 supervise::{
73 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74 SupervisorHandle, SwapHelloAdmission,
75 },
76 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93 ops::ACTIVITY_RECORD,
94 ops::ACTIVITY_QUERY,
95 ops::ACTIVITY_SUBSCRIBE,
96 ops::SERVER_DESCRIBE,
97 ops::CATALOG_LIST,
98 ops::ROUTE_OPEN,
99 ops::ROUTE_POLL,
100 ops::ROUTE_CLOSING,
101 ops::ROUTE_CLOSED,
102 ops::SUPERVISOR_LIST,
103 ops::SUPERVISOR_RESTART,
104 ops::SUPERVISOR_SWAP,
105 ops::SUPERVISOR_RELOAD,
106 ops::SUPERVISOR_RESCAN,
107 ops::SUPERVISOR_RELEASE_RESERVED,
108 ops::SUPERVISOR_SET_ENABLED,
109 ops::SUPERVISOR_HEALTH_PROBE,
110 ops::SUPERVISOR_HEALTH,
111 ops::SUPERVISOR_STDERR_TAIL,
112 ops::SUPERVISOR_TERMINALS,
113 ops::SUPERVISOR_ROUTES,
114 ops::SUPERVISOR_PROVENANCE,
115 ops::SUPERVISOR_SPAWN_SNAPSHOT,
116 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132 "supervisor.live_roots",
133 SCOPE_SYNC_OP,
134 SCOPE_APPLY_OP,
135 SCOPE_DESCRIBE_OP,
136 "operator.confirm",
137];
138
139const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194 configured: &Path,
195 spawned_from: Option<&Path>,
196 image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198 let path = match spawned_from {
199 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200 ReloadPathAgreement::Match
201 }
202 Some(spawned_from) => ReloadPathAgreement::Mismatch {
203 configured: configured.to_path_buf(),
204 spawned_from: spawned_from.to_path_buf(),
205 },
206 None => ReloadPathAgreement::Unavailable {
207 reason: if matches!(
208 image,
209 subc_control::RunningImageAgreement::Unavailable {
210 reason: subc_control::RunningImageUnavailableReason::NotRunning
211 }
212 ) {
213 ReloadPathUnavailableReason::NotRunning
214 } else {
215 ReloadPathUnavailableReason::SpawnedPathUnavailable
216 },
217 },
218 };
219 PendingReloadVerdict {
220 path,
221 image,
222 spec: None,
223 }
224}
225
226fn launch_spec_agreement(
227 configured: &crate::supervise::ModuleSpec,
228 running: bool,
229 spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232 if !running {
233 return Agreement::Unavailable {
234 reason: Reason::NotRunning,
235 };
236 }
237 match spawned {
238 None => Agreement::Unavailable {
239 reason: Reason::SpawnedSpecUnavailable,
240 },
241 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242 Agreement::Match
243 }
244 Some(_) => Agreement::Mismatch,
245 }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249 if configured == spawned_from {
250 return true;
251 }
252 #[cfg(windows)]
253 {
254 match (
260 std::fs::canonicalize(configured),
261 std::fs::canonicalize(spawned_from),
262 ) {
263 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264 _ => false,
265 }
266 }
267 #[cfg(not(windows))]
268 {
269 false
270 }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275 build: DaemonBuildProvenance,
276 pid: Option<u32>,
277 started_at_ms: Option<u64>,
278 start_clock: Option<crate::clock::StartClock>,
279 executable_path: Option<PathBuf>,
280 executable_identity: Option<SpawnedFileIdentity>,
281 process_start_time: Option<u64>,
282 probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286 fn default() -> Self {
287 Self {
288 build: DaemonBuildProvenance {
289 build_git_sha: None,
290 build_lock_digest: None,
291 },
292 pid: None,
293 started_at_ms: None,
294 start_clock: None,
295 executable_path: None,
296 executable_identity: None,
297 process_start_time: None,
298 probe: ExecutableIdentityProbe::default(),
299 }
300 }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305 supervisor: Supervisor,
306 config_path: PathBuf,
307 configured_port: Option<u16>,
308 storage_config: Option<crate::daemon_config::StorageConfig>,
309 admission_facts_carrier_module_id: Option<String>,
310 admission_facts_targets: Option<Vec<String>>,
311 scope_authority_owners: Vec<String>,
312 privileged_operator_prompt: PrivilegedOperatorPrompt,
313 generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342 "reloading",
343 "supervisor_not_live",
344 "registration_not_active",
345 "no_forwarding_connection",
346 "relay_send_failed",
347];
348
349#[derive(Clone)]
351pub struct ControlHandler {
352 registry: Arc<Registry>,
353 activity: Option<crate::activity::ActivityLedger>,
354 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355 forwarding: Arc<ForwardingTable>,
356 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357 supervisor: SupervisorHandle,
358 subc_capabilities: Arc<[String]>,
359 route_bind_relay_timeout: Duration,
363 route_bind_relay_timeouts: BTreeMap<String, Duration>,
367 route_bind_breakers: RouteBindBreakers,
370 route_bind_concurrency: RouteBindConcurrency,
373 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378 route_bind_breaker_threshold: u32,
380 route_bind_breaker_cooldown: Duration,
382 health_probe_timeout: Duration,
383 storage_config: Option<crate::daemon_config::StorageConfig>,
386 machine_id: Option<crate::machine_id::MachineId>,
390 admission_facts_carrier_module_id: Option<String>,
391 admission_facts_targets: Option<Vec<String>>,
392 scopes: Arc<RwLock<ScopeTable>>,
396 scope_deadline_added: Arc<tokio::sync::Notify>,
400 scope_authority_owners: Vec<String>,
403 privileged_operator_prompt: PrivilegedOperatorPrompt,
405 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408 #[cfg(test)]
409 caller_lookup: Option<TestCallerLookup>,
410 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413 rescan: Option<SupervisorRescanContext>,
414 connected_clients: ConnectedClients,
415 counters: DaemonCounters,
416 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417 daemon_shutdown: DaemonShutdownFlag,
422 daemon_provenance: DaemonProvenanceFacts,
423 #[cfg(test)]
424 control_dispatch_delay: Option<Duration>,
425 #[cfg(test)]
426 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427 #[cfg(test)]
431 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432 #[cfg(test)]
438 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443 f.debug_struct("ControlHandler")
444 .field("registry", &self.registry)
445 .field("forwarding", &self.forwarding)
446 .field("process_liveness", &self.process_liveness.is_some())
447 .field("supervisor", &self.supervisor)
448 .field("subc_capabilities", &self.subc_capabilities)
449 .finish()
450 }
451}
452
453struct RouteOpenRequest {
454 target: RouteTarget,
455 identity: BindIdentity,
456 consumer_identity: Option<ConsumerIdentity>,
457 consumer_capabilities: Option<Vec<String>>,
458 role_versions: Option<BTreeMap<String, String>>,
459 admission_facts: Option<serde_json::Value>,
460 scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465 Admission,
466 ProjectRoot,
467 EgressReservation,
468 BindSend,
469 BindAckWait,
470}
471
472struct RouteOpenTimings {
476 module_id: String,
477 principal: String,
478 admission: Duration,
479 project_root: Duration,
480 egress_reservation: Duration,
481 bind_send: Duration,
482 bind_ack_wait: Duration,
483 phase: Option<RouteOpenPhase>,
484 phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488 fn new() -> Self {
489 Self {
490 module_id: String::new(),
491 principal: "unresolved".to_string(),
493 admission: Duration::ZERO,
494 project_root: Duration::ZERO,
495 egress_reservation: Duration::ZERO,
496 bind_send: Duration::ZERO,
497 bind_ack_wait: Duration::ZERO,
498 phase: Some(RouteOpenPhase::Admission),
499 phase_started_at: Instant::now(),
500 }
501 }
502
503 fn finish(&mut self) {
504 let elapsed = self.phase_started_at.elapsed();
505 match self.phase.take() {
506 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511 None => {}
512 }
513 }
514
515 fn enter(&mut self, phase: RouteOpenPhase) {
516 self.finish();
517 self.phase = Some(phase);
518 self.phase_started_at = Instant::now();
519 }
520}
521
522struct RouteBindReservationGuard {
523 forwarding: Arc<ForwardingTable>,
524 endpoint: ModuleEndpointId,
525 relay_corr: u64,
526 armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530 forwarding: Arc<ForwardingTable>,
531 endpoint: ModuleEndpointId,
532 corr: u64,
533 armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538 Self {
539 forwarding,
540 endpoint,
541 corr,
542 armed: true,
543 }
544 }
545
546 fn disarm(&mut self) {
547 self.armed = false;
548 }
549}
550
551impl Drop for ModuleControlRpcGuard {
552 fn drop(&mut self) {
553 if self.armed {
554 let _ = self
555 .forwarding
556 .cancel_module_control_rpc(self.endpoint, self.corr);
557 }
558 }
559}
560
561impl RouteBindReservationGuard {
562 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563 Self {
564 forwarding,
565 endpoint,
566 relay_corr,
567 armed: true,
568 }
569 }
570
571 fn release_and_disarm(&mut self) {
572 if !self.armed {
573 return;
574 }
575 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576 self.endpoint,
577 self.relay_corr,
578 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579 ) {
580 send_goodbye_target_best_effort(
581 &self.forwarding.counters(),
582 &target,
583 "canceled route.bind",
584 );
585 }
586 self.armed = false;
587 }
588
589 fn disarm(&mut self) {
590 self.armed = false;
591 }
592}
593
594impl Drop for RouteBindReservationGuard {
595 fn drop(&mut self) {
596 self.release_and_disarm();
597 }
598}
599
600#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636 modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640 concurrency: RouteBindConcurrency,
641 module_id: String,
642}
643
644impl RouteBindConcurrency {
645 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648 let mut modules = self
649 .modules
650 .lock()
651 .expect("route.bind concurrency mutex poisoned");
652 let in_flight = modules.entry(module_id.to_string()).or_default();
653 if *in_flight >= limit {
654 return Err(*in_flight);
655 }
656 *in_flight += 1;
657 Ok(RouteBindConcurrencyGuard {
658 concurrency: self.clone(),
659 module_id: module_id.to_string(),
660 })
661 }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665 fn drop(&mut self) {
666 let mut modules = self
667 .concurrency
668 .modules
669 .lock()
670 .expect("route.bind concurrency mutex poisoned");
671 let remove = {
672 let in_flight = modules
673 .get_mut(&self.module_id)
674 .expect("admitted route.bind has a concurrency entry");
675 *in_flight -= 1;
676 *in_flight == 0
677 };
678 if remove {
679 modules.remove(&self.module_id);
680 }
681 }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686 consecutive_timeouts: u32,
688 cooldown_until: Option<Instant>,
691 probe_in_flight: Option<Arc<()>>,
696}
697
698enum RouteBindAdmission<'a> {
700 Admitted {
701 guard: RouteBindBreakerGuard<'a>,
702 probe: bool,
705 },
706 Refused {
707 consecutive_timeouts: u32,
708 retry_in: Duration,
712 probe_in_flight: bool,
713 },
714}
715
716struct RouteBindBreakerGuard<'a> {
723 breakers: RouteBindBreakers,
724 module_id: &'a str,
725 probe_token: Option<Arc<()>>,
726 settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730 fn record_accepted(&mut self) -> bool {
734 self.settled = true;
735 self.breakers.record_accepted(self.module_id)
736 }
737
738 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741 self.settled = true;
742 self.breakers.record_timeout(
743 self.module_id,
744 self.probe_token.as_ref(),
745 threshold,
746 cooldown,
747 )
748 }
749
750 fn record_inconclusive(&mut self) {
759 self.settled = true;
760 self.breakers
761 .record_inconclusive(self.module_id, self.probe_token.as_ref());
762 }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766 fn drop(&mut self) {
767 if !self.settled {
768 self.breakers
769 .record_inconclusive(self.module_id, self.probe_token.as_ref());
770 }
771 }
772}
773
774struct BreakerOpened {
778 consecutive_timeouts: u32,
779 reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786 self.modules
787 .lock()
788 .expect("route.bind breaker mutex poisoned")
789 }
790
791 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795 probe: probe_token.is_some(),
796 guard: RouteBindBreakerGuard {
797 breakers: self.clone(),
798 module_id,
799 probe_token,
800 settled: false,
801 },
802 };
803
804 let mut modules = self.lock();
805 let Some(state) = modules.get_mut(module_id) else {
806 return admitted(None);
807 };
808 let Some(cooldown_until) = state.cooldown_until else {
809 return admitted(None);
810 };
811 if state.probe_in_flight.is_some() {
812 return RouteBindAdmission::Refused {
813 consecutive_timeouts: state.consecutive_timeouts,
814 retry_in: Duration::ZERO,
815 probe_in_flight: true,
816 };
817 }
818 let now = Instant::now();
819 if now < cooldown_until {
820 return RouteBindAdmission::Refused {
821 consecutive_timeouts: state.consecutive_timeouts,
822 retry_in: cooldown_until - now,
823 probe_in_flight: false,
824 };
825 }
826 let token = Arc::new(());
827 state.probe_in_flight = Some(Arc::clone(&token));
828 admitted(Some(token))
829 }
830
831 fn record_accepted(&self, module_id: &str) -> bool {
832 self.lock()
833 .remove(module_id)
834 .is_some_and(|state| state.cooldown_until.is_some())
835 }
836
837 fn record_timeout(
838 &self,
839 module_id: &str,
840 probe_token: Option<&Arc<()>>,
841 threshold: u32,
842 cooldown: Duration,
843 ) -> Option<BreakerOpened> {
844 let mut modules = self.lock();
845 let state = modules.entry(module_id.to_string()).or_default();
846 let was_open = state.cooldown_until.is_some();
847 let was_probe = Self::owns_probe(state, probe_token);
848 if was_probe {
849 state.probe_in_flight = None;
850 }
851 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852 if state.consecutive_timeouts < threshold {
853 return None;
854 }
855 state.cooldown_until = Some(Instant::now() + cooldown);
856 Some(BreakerOpened {
857 consecutive_timeouts: state.consecutive_timeouts,
858 reopened_after_probe: was_open && was_probe,
859 })
860 }
861
862 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863 state
867 .probe_in_flight
868 .as_ref()
869 .zip(token)
870 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871 }
872
873 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874 if let Some(state) = self.lock().get_mut(module_id) {
875 if Self::owns_probe(state, probe_token) {
876 state.probe_in_flight = None;
877 }
878 }
879 }
880
881 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896 self.lock()
897 .remove(module_id)
898 .map(|state| state.consecutive_timeouts)
899 .filter(|discarded| *discarded > 0)
900 }
901
902 fn open_snapshot(&self) -> Option<serde_json::Value> {
909 let now = Instant::now();
910 let modules = self.lock();
911 let open = modules
912 .iter()
913 .filter_map(|(module_id, state)| {
914 let cooldown_until = state.cooldown_until?;
915 Some((
916 module_id.clone(),
917 serde_json::json!({
918 "consecutive_timeouts": state.consecutive_timeouts,
919 "cooldown_remaining_ms":
920 cooldown_until.saturating_duration_since(now).as_millis() as u64,
921 "probe_in_flight": state.probe_in_flight.is_some(),
922 }),
923 ))
924 })
925 .collect::<serde_json::Map<String, serde_json::Value>>();
926 (!open.is_empty()).then_some(serde_json::Value::Object(open))
927 }
928}
929
930impl ControlHandler {
931 pub fn new(registry: Arc<Registry>) -> Self {
932 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933 }
934
935 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936 let counters = forwarding.counters();
937 let route_bind_breakers = forwarding.route_bind_breakers();
941 let route_bind_concurrency = forwarding.route_bind_concurrency();
942 let route_outages = forwarding.route_outages();
943 Self {
944 registry,
945 activity: None,
946 activity_streams: Arc::new(Mutex::new(HashMap::new())),
947 forwarding,
948 process_liveness: None,
949 supervisor: SupervisorHandle::new(),
950 subc_capabilities: Arc::from([
951 CAP_MANIFEST_REGISTRATION.to_string(),
952 CAP_CHANNEL_LIFECYCLE.to_string(),
953 CAP_PING_PONG.to_string(),
954 CAP_SESSION_ATTACH.to_string(),
955 CAP_ADMISSION_FACTS_RELAY.to_string(),
956 CAP_SCOPES_V1.to_string(),
957 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
964 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
965 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
966 ]),
967 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
968 route_bind_relay_timeouts: BTreeMap::new(),
969 route_bind_breakers,
970 route_bind_concurrency,
971 route_outages,
972 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
973 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
974 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
975 storage_config: None,
976 machine_id: None,
977 admission_facts_carrier_module_id: None,
978 admission_facts_targets: None,
979 scopes: Arc::new(RwLock::new(ScopeTable::new(
980 crate::daemon_config::default_scope_authority_owners(),
981 ))),
982 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
983 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
984 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
985 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
986 #[cfg(test)]
987 caller_lookup: None,
988 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
989 rescan: None,
990 connected_clients: ConnectedClients::new(),
991 counters,
992 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
993 daemon_shutdown: DaemonShutdownFlag::default(),
994 daemon_provenance: DaemonProvenanceFacts::default(),
995 #[cfg(test)]
996 control_dispatch_delay: None,
997 #[cfg(test)]
998 provenance_probe_override: None,
999 #[cfg(test)]
1000 route_bind_relay_sent_hold: None,
1001 #[cfg(test)]
1002 hello_registry_to_forwarding_hold: None,
1003 }
1004 }
1005
1006 pub fn with_storage_config(
1009 mut self,
1010 storage_config: Option<crate::daemon_config::StorageConfig>,
1011 ) -> Self {
1012 self.storage_config = storage_config;
1013 self
1014 }
1015
1016 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1019 self.activity = Some(crate::activity::ActivityLedger::open(
1020 run_dir.join("activity.jsonl"),
1021 incarnation,
1022 ));
1023 self.supervisor.set_activity(self.activity.clone());
1024 self
1025 }
1026
1027 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1030 self.machine_id = machine_id;
1031 self
1032 }
1033
1034 pub fn with_admission_facts_config(
1038 mut self,
1039 carrier_module_id: Option<String>,
1040 targets: Option<Vec<String>>,
1041 ) -> Self {
1042 self.admission_facts_carrier_module_id = carrier_module_id;
1043 self.admission_facts_targets = targets;
1044 self
1045 }
1046
1047 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1051 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1052 self.scope_authority_owners = owners;
1053 self
1054 }
1055
1056 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1058 self.privileged_operator_prompt = mode;
1059 self
1060 }
1061
1062 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1065 self.route_bind_relay_timeout = timeout;
1066 self
1067 }
1068
1069 pub fn with_route_bind_relay_timeouts(
1075 mut self,
1076 timeouts: impl IntoIterator<Item = (String, Duration)>,
1077 ) -> Self {
1078 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1079 self
1080 }
1081
1082 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1088 self.route_bind_relay_timeouts
1089 .get(module_id)
1090 .copied()
1091 .unwrap_or(self.route_bind_relay_timeout)
1092 }
1093
1094 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1102 self.route_bind_breaker_threshold = threshold.max(1);
1103 self.route_bind_breaker_cooldown = cooldown;
1104 self
1105 }
1106
1107 #[cfg(test)]
1108 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1109 self.health_probe_timeout = timeout;
1110 self
1111 }
1112
1113 #[cfg(test)]
1114 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1115 self.control_dispatch_delay = Some(delay);
1116 self
1117 }
1118
1119 #[cfg(test)]
1123 pub(crate) fn with_route_bind_relay_sent_hold(
1124 mut self,
1125 hold: Arc<tokio::sync::Barrier>,
1126 ) -> Self {
1127 self.route_bind_relay_sent_hold = Some(hold);
1128 self
1129 }
1130
1131 #[cfg(test)]
1136 pub(crate) fn with_hello_registry_to_forwarding_hold(
1137 mut self,
1138 hold: Arc<std::sync::Barrier>,
1139 ) -> Self {
1140 self.hello_registry_to_forwarding_hold = Some(hold);
1141 self
1142 }
1143
1144 #[cfg(test)]
1145 fn hold_between_registry_and_forwarding(&self) {
1146 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1147 hold.wait();
1148 hold.wait();
1149 }
1150 }
1151
1152 pub fn with_process_liveness(
1153 mut self,
1154 process_liveness: Arc<dyn ModuleProcessLiveness>,
1155 ) -> Self {
1156 self.process_liveness = Some(process_liveness);
1157 self
1158 }
1159
1160 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1161 self.supervisor = supervisor;
1162 self.supervisor.set_activity(self.activity.clone());
1163 self
1164 }
1165
1166 pub fn with_daemon_provenance(
1167 mut self,
1168 pid: u32,
1169 started_at_ms: u64,
1170 executable_path: Option<PathBuf>,
1171 build_git_sha: Option<String>,
1172 build_lock_digest: Option<String>,
1173 ) -> Self {
1174 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1175 let process_start_time = process_start_time(pid);
1176 self.daemon_provenance = DaemonProvenanceFacts {
1177 build: DaemonBuildProvenance {
1178 build_git_sha,
1179 build_lock_digest,
1180 },
1181 pid: Some(pid),
1182 started_at_ms: Some(started_at_ms),
1183 start_clock: None,
1184 executable_path,
1185 executable_identity,
1186 process_start_time,
1187 probe: ExecutableIdentityProbe::default(),
1188 };
1189 self
1190 }
1191
1192 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1193 self.daemon_provenance.start_clock = Some(clock);
1194 self
1195 }
1196
1197 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1199 self.scopes
1200 .write()
1201 .unwrap_or_else(|poisoned| poisoned.into_inner())
1202 .set_wall_clock(Arc::new(clock));
1203 self
1204 }
1205
1206 #[cfg(test)]
1207 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1208 self.provenance_probe_override = Some(result);
1209 self
1210 }
1211
1212 pub fn with_capability_config(
1216 self,
1217 modules: impl IntoIterator<Item = (String, bool)>,
1218 reserved_capabilities: BTreeMap<String, String>,
1219 ) -> Self {
1220 self.capability_evaluator
1221 .configure(modules, reserved_capabilities);
1222 self
1223 }
1224
1225 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1229 self.daemon_shutdown = flag;
1230 self
1231 }
1232
1233 pub fn with_supervisor_rescan(
1234 mut self,
1235 supervisor: Supervisor,
1236 config_path: impl Into<PathBuf>,
1237 configured_port: Option<u16>,
1238 ) -> Self {
1239 self.rescan = Some(SupervisorRescanContext {
1240 supervisor,
1241 config_path: config_path.into(),
1242 configured_port,
1243 storage_config: self.storage_config.clone(),
1244 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1245 admission_facts_targets: self.admission_facts_targets.clone(),
1246 scope_authority_owners: self.scope_authority_owners.clone(),
1247 privileged_operator_prompt: self.privileged_operator_prompt,
1248 generation: Arc::default(),
1249 });
1250 self
1251 }
1252
1253 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1254 self.connected_clients = connected_clients;
1255 self
1256 }
1257
1258 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1259 Arc::clone(&self.forwarding)
1260 }
1261
1262 pub(crate) fn counters(&self) -> DaemonCounters {
1263 self.counters.clone()
1264 }
1265
1266 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1269 tokio::spawn(async move {
1270 loop {
1271 self.capability_evaluator
1272 .wait_for_change_or_deadline()
1273 .await;
1274 self.refresh_capability_requirements();
1275 }
1276 });
1277 }
1278
1279 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1285 tokio::spawn(async move {
1286 loop {
1287 let has_deadlines = self
1288 .scopes
1289 .read()
1290 .unwrap_or_else(|poisoned| poisoned.into_inner())
1291 .has_deadlines();
1292 if !has_deadlines {
1293 self.scope_deadline_added.notified().await;
1294 continue;
1295 }
1296 tokio::time::sleep(Duration::from_secs(1)).await;
1297 if let Err(error) = self.sweep_expired_scopes() {
1298 warn!(%error, "scope expiry sweep failed");
1299 }
1300 }
1301 });
1302 }
1303
1304 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1305 let mut table = self
1306 .scopes
1307 .write()
1308 .unwrap_or_else(|poisoned| poisoned.into_inner());
1309 let swept = table.sweep_expired();
1310 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1311 drop(table);
1312 self.close_scope_drained_routes(drained);
1313 Ok(())
1314 }
1315
1316 fn publish_scope_changes(
1317 &self,
1318 changes: &[crate::scopes::ScopeTagChange],
1319 expired: &[crate::scopes::ScopeExpired],
1320 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1321 if changes.is_empty() {
1322 return Ok(Vec::new());
1323 }
1324 let drained = self
1325 .forwarding
1326 .publish_scope_changes(changes)
1327 .map_err(RouterError::Forwarding)?;
1328 let mut counts = HashMap::new();
1329 for route in &drained {
1330 *counts
1331 .entry((
1332 route.scope.owner.as_str(),
1333 route.scope.scope_ref.as_str(),
1334 route.scope.tag.scope_epoch,
1335 ))
1336 .or_insert(0usize) += 1;
1337 }
1338 for scope in expired {
1339 let routes_closed = counts
1340 .get(&(
1341 scope.owner.as_str(),
1342 scope.scope_ref.as_str(),
1343 scope.scope_epoch,
1344 ))
1345 .copied()
1346 .unwrap_or(0);
1347 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1348 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1349 routes_closed, "scope expired");
1350 }
1351 Ok(drained)
1352 }
1353
1354 fn runtime_capability_snapshot(
1355 &self,
1356 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1357 let runtime = self
1358 .supervisor
1359 .list()
1360 .into_iter()
1361 .map(|module| {
1362 let status = module.status().map_err(|err| {
1363 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1364 })?;
1365 Ok(RuntimeModule {
1366 module_id: status.module_id,
1367 state: status.state,
1368 enabled: status.enabled,
1369 })
1370 })
1371 .collect::<Result<Vec<_>, RouterError>>()?;
1372 let registrations = self.registry.list_registered_modules().map_err(|err| {
1378 RouterError::backend(
1379 0,
1380 0,
1381 format!("failed to list capability registrations: {err}"),
1382 )
1383 })?;
1384 let registrations = registrations
1385 .into_iter()
1386 .map(|registration| RegisteredModule {
1387 module_id: registration.manifest.module_id,
1388 module_version: registration.manifest.module_version,
1389 capabilities: registration.manifest.capabilities,
1390 })
1391 .collect();
1392 Ok((runtime, registrations))
1393 }
1394
1395 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1402 let cached_registration = RegisteredModule {
1403 module_id: registration.manifest.module_id.clone(),
1404 module_version: registration.manifest.module_version.clone(),
1405 capabilities: registration.manifest.capabilities.clone(),
1406 };
1407 if self.capability_evaluator.record_hello(&cached_registration) {
1408 warn!(
1409 module_id = %cached_registration.module_id,
1410 "capability claims drifted from the cached manifest"
1411 );
1412 }
1413 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1414 self.enforce_capability_denies();
1415 }
1416 self.refresh_capability_requirements();
1417 }
1418
1419 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1423 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1424 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1425 self.supervisor.set_swap_promotion_observer(observer);
1426 }
1427
1428 pub fn refresh_capability_requirements(&self) {
1429 match self.runtime_capability_snapshot() {
1430 Ok((runtime, registrations)) => {
1431 log_requirement_events(
1432 self.capability_evaluator
1433 .evaluate_now(&runtime, ®istrations),
1434 &self.daemon_shutdown,
1435 );
1436 }
1437 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1438 }
1439 }
1440
1441 fn enforce_capability_denies(&self) {
1445 let registrations = match self.registry.list_registered_modules() {
1446 Ok(snapshot) => snapshot,
1447 Err(err) => {
1448 warn!(error = %err, "failed to read registrations for capability deny census");
1449 return;
1450 }
1451 };
1452 let manifests = registrations
1453 .into_iter()
1454 .map(|registration| {
1455 (
1456 registration.manifest.module_id.clone(),
1457 registration.manifest,
1458 )
1459 })
1460 .collect::<BTreeMap<_, _>>();
1461 let census = match self.forwarding.route_census(None) {
1462 Ok(census) => census,
1463 Err(err) => {
1464 warn!(error = %err, "failed to read route census for capability deny enforcement");
1465 return;
1466 }
1467 };
1468
1469 for (target_module_id, routes) in census {
1470 let Some(target_manifest) = manifests.get(&target_module_id) else {
1471 continue;
1472 };
1473 let mut closed_routes = Vec::new();
1474 let mut module_goodbyes = Vec::new();
1475 for route in routes {
1476 let Principal::Reserved {
1477 module_id: opening_module_id,
1478 } = &route.principal
1479 else {
1480 continue;
1481 };
1482 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1483 continue;
1484 };
1485 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1486 continue;
1487 };
1488
1489 match self.forwarding.release_client_route(
1490 route.goodbye_target.connection_id,
1491 route.goodbye_target.channel,
1492 route.goodbye_target.epoch,
1493 ) {
1494 Ok(RouteRelease::Removed(module_goodbye)) => {
1495 warn!(
1496 opening_module_id,
1497 target_module_id,
1498 capability,
1499 "force-closing route because an attested capability deny edge now matches"
1500 );
1501 closed_routes.push(route);
1502 module_goodbyes.push(module_goodbye);
1503 }
1504 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1505 Err(err) => warn!(
1506 opening_module_id,
1507 target_module_id,
1508 capability,
1509 error = %err,
1510 "failed to force-close capability-denied route"
1511 ),
1512 }
1513 }
1514
1515 if closed_routes.is_empty() {
1516 continue;
1517 }
1518 send_route_control_pushes(
1519 &self.forwarding,
1520 closed_routes,
1521 ClientControlPush::RouteClosed {
1522 module_id: target_module_id,
1523 channels: Vec::new(),
1524 reason: RouteCloseReason::CapabilityDenied,
1525 drained: false,
1526 abandoned: 0,
1527 excluded_subscriptions: 0,
1528 terminal: Some(false),
1529 },
1530 );
1531 self.emit_route_goodbyes(module_goodbyes);
1532 }
1533 }
1534
1535 fn not_ready_reason(
1541 &self,
1542 registration: &crate::registry::ModuleRegistration,
1543 ) -> Option<NotReadyReason> {
1544 if !registration.ready {
1545 return Some(NotReadyReason {
1546 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1547 capability: None,
1548 });
1549 }
1550 self.first_unprovided_required_capability(registration)
1551 .map(|capability| NotReadyReason {
1552 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1553 capability: Some(capability),
1554 })
1555 }
1556
1557 fn first_unprovided_required_capability(
1575 &self,
1576 registration: &crate::registry::ModuleRegistration,
1577 ) -> Option<String> {
1578 let required = registration
1579 .manifest
1580 .capabilities
1581 .iter()
1582 .flat_map(|declarations| declarations.requires.iter())
1583 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1584 .map(|requirement| requirement.capability.as_str())
1585 .collect::<BTreeSet<_>>();
1586 if required.is_empty() {
1587 return None;
1588 }
1589 let module_id = registration.manifest.module_id.as_str();
1590 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1591 if required
1592 .iter()
1593 .any(|capability| verdict(capability).is_none())
1594 {
1595 self.refresh_capability_requirements();
1596 }
1597 required
1598 .into_iter()
1599 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1600 .map(str::to_string)
1601 }
1602
1603 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1604 self.capability_evaluator
1605 .statuses()
1606 .into_iter()
1607 .map(capability_requirement_status)
1608 .collect()
1609 }
1610
1611 fn deregister_connection(
1618 &self,
1619 connection_id: ConnectionId,
1620 reason: RegistrationEndReason,
1621 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1622 self.registry
1623 .deregister_connection_with_reason(connection_id, reason)
1624 }
1625
1626 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1627 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1628 return None;
1629 }
1630 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1631 parse_client_control_request(&frame.body)
1632 else {
1633 return None;
1634 };
1635 Some(target_module_id(&target).to_string())
1636 }
1637
1638 pub(crate) fn route_open_capacity_refusal(
1639 &self,
1640 ctx: &RouteCtx,
1641 frame: &Frame,
1642 target_module_id: &str,
1643 in_flight: usize,
1644 limit: usize,
1645 ) -> Result<Frame, RouterError> {
1646 self.route_open_admission_refusal_frame(
1647 ctx,
1648 frame,
1649 target_module_id,
1650 "open_admission_full",
1651 (in_flight, limit),
1652 format!(
1653 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1654 ),
1655 )
1656 }
1657
1658 fn route_open_target_capacity_refusal(
1659 &self,
1660 ctx: &RouteCtx,
1661 frame: &Frame,
1662 target_module_id: &str,
1663 in_flight: usize,
1664 ) -> Result<Frame, RouterError> {
1665 self.route_open_admission_refusal_frame(
1666 ctx,
1667 frame,
1668 target_module_id,
1669 "target_binds_full",
1670 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1671 format!(
1672 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1673 ),
1674 )
1675 }
1676
1677 fn route_open_admission_refusal_frame(
1685 &self,
1686 ctx: &RouteCtx,
1687 frame: &Frame,
1688 target_module_id: &str,
1689 reason: &'static str,
1690 (in_flight, limit): (usize, usize),
1691 message: impl Into<String>,
1692 ) -> Result<Frame, RouterError> {
1693 let code = error_codes::TARGET_UNAVAILABLE;
1694 self.counters.increment_route_open_refused(code);
1695 info!(
1696 target: "control",
1697 code,
1698 reason,
1699 module_id = ?target_module_id,
1700 connection_id = ctx.connection_id.get(),
1701 in_flight,
1702 limit,
1703 "route.open refused"
1704 );
1705 control_error_frame(frame, code, message.into())
1706 }
1707
1708 #[cfg(test)]
1714 pub fn handle_control(
1715 &self,
1716 connection_id: ConnectionId,
1717 frame: Frame,
1718 ) -> Result<Vec<Frame>, RouterError> {
1719 match frame.header.ty {
1720 FrameType::Ping => Ok(vec![pong(&frame)?]),
1721 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1722 FrameType::Goodbye => self.handle_goodbye(connection_id),
1723 ty => Ok(vec![control_error_frame(
1724 &frame,
1725 "unsupported_control_frame",
1726 format!("unsupported channel-0 frame {ty:?}"),
1727 )?]),
1728 }
1729 }
1730
1731 pub async fn handle_control_frame(
1732 &self,
1733 ctx: &RouteCtx,
1734 frame: Frame,
1735 ) -> Result<Vec<Frame>, RouterError> {
1736 self.handle_control_frame_timed(ctx, frame, None).await
1737 }
1738
1739 pub(crate) fn record_caller_endpoints(
1740 &self,
1741 connection: ConnectionId,
1742 endpoints: Option<(SocketAddr, SocketAddr)>,
1743 ) {
1744 let mut callers = self
1745 .caller_endpoints
1746 .lock()
1747 .unwrap_or_else(|p| p.into_inner());
1748 callers.remove(&connection);
1749 if let Some(endpoints) = endpoints {
1750 callers.insert(connection, endpoints);
1751 }
1752 }
1753
1754 #[cfg(test)]
1755 pub(crate) fn with_caller_lookup_for_test(
1756 mut self,
1757 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1758 ) -> Self {
1759 self.caller_lookup = Some(Arc::new(lookup));
1760 self
1761 }
1762
1763 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1764 let endpoints = self
1765 .caller_endpoints
1766 .lock()
1767 .unwrap_or_else(|p| p.into_inner())
1768 .get(&connection)
1769 .copied();
1770 #[cfg(test)]
1771 let lookup = self.caller_lookup.clone();
1772 tokio::task::spawn_blocking(move || {
1773 #[cfg(test)]
1774 if let Some(lookup) = lookup {
1775 return lookup(endpoints);
1776 }
1777 endpoints
1778 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1779 .unwrap_or_default()
1780 })
1781 .await
1782 .unwrap_or_default()
1783 }
1784
1785 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1786 frame.header.channel == 0
1787 && frame.header.ty == FrameType::Request
1788 && self
1789 .forwarding
1790 .module_endpoint_for_connection(ctx.connection_id)
1791 .is_ok_and(|endpoint| endpoint.is_none())
1792 && parse_client_control_request(&frame.body)
1793 .ok()
1794 .is_some_and(|request| privileged_operation(&request).is_some())
1795 }
1796
1797 pub(crate) async fn handle_control_frame_timed(
1798 &self,
1799 ctx: &RouteCtx,
1800 frame: Frame,
1801 dispatch_started_at: Option<StdInstant>,
1802 ) -> Result<Vec<Frame>, RouterError> {
1803 match frame.header.ty {
1804 FrameType::Ping => Ok(vec![pong(&frame)?]),
1805 FrameType::Hello => {
1806 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1807 }
1808 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1809 FrameType::Cancel => {
1810 if self
1813 .forwarding
1814 .operator_confirms()
1815 .cancel(ctx.connection_id, frame.header.corr)
1816 || self
1817 .supervisor
1818 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1819 || self.cancel_activity_subscription(ctx, &frame)?
1820 {
1821 Ok(Vec::new())
1822 } else {
1823 Ok(vec![control_error_frame(
1824 &frame,
1825 "unknown_subscription",
1826 "no subscription has this correlation id",
1827 )?])
1828 }
1829 }
1830 FrameType::Request => {
1831 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1834 if op
1835 .as_ref()
1836 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1837 {
1838 return self.handle_activity_record(ctx, frame).await;
1839 }
1840 if op.as_ref().is_some_and(|probe| {
1841 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1842 }) {
1843 return self.handle_activity_read(ctx, frame).await;
1844 }
1845 if op
1846 .as_ref()
1847 .is_some_and(|probe| probe.op == "operator.confirm")
1848 {
1849 return self.handle_operator_confirm(ctx, frame);
1850 }
1851 if op
1852 .as_ref()
1853 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1854 {
1855 return self.handle_scope_served_by(ctx.connection_id, frame);
1856 }
1857 if self
1858 .forwarding
1859 .module_endpoint_for_connection(ctx.connection_id)
1860 .map_err(RouterError::Forwarding)?
1861 .is_some()
1862 {
1863 if !is_known_module_request_op(&frame.body) {
1864 return Ok(vec![control_error_frame(
1865 &frame,
1866 "unsupported_control_frame",
1867 "module-originated channel-0 REQUEST is not supported",
1868 )?]);
1869 }
1870 let request = match parse_module_control_request_from_module(&frame.body) {
1871 Ok(request) => request,
1872 Err((err, ControlRequestBodyError::UnknownOp)) => {
1873 return Ok(vec![control_error_frame(
1874 &frame,
1875 "unsupported_control_frame",
1876 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1877 )?])
1878 }
1879 Err((err, ControlRequestBodyError::InvalidBody)) => {
1880 return Ok(vec![control_error_frame(
1881 &frame,
1882 "invalid_control_body",
1883 format!("malformed module control body: {err}"),
1884 )?])
1885 }
1886 };
1887 let op = module_control_request_op(&request);
1888 let corr = frame.header.corr;
1889 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1890 let result =
1891 self.handle_module_control_request(ctx.connection_id, frame, request);
1892 log_slow_control_dispatch(
1893 dispatch_started_at,
1894 op,
1895 ctx.connection_id,
1896 corr,
1897 None,
1898 );
1899 return result;
1900 }
1901
1902 if is_known_module_request_op(&frame.body) {
1903 return Ok(vec![control_error_frame(
1904 &frame,
1905 "not_registered",
1906 "catalog.update requires an active module registration owned by this connection",
1907 )?]);
1908 }
1909
1910 let request = match parse_client_control_request(&frame.body) {
1911 Ok(request) => request,
1912 Err((err, ControlRequestBodyError::UnknownOp)) => {
1913 return Ok(vec![control_error_frame(
1914 &frame,
1915 "unknown_control_op",
1916 format!("unknown client control op: {err}"),
1917 )?])
1918 }
1919 Err((err, ControlRequestBodyError::InvalidBody)) => {
1920 return Ok(vec![control_error_frame(
1921 &frame,
1922 "invalid_control_body",
1923 format!("malformed client control body: {err}"),
1924 )?])
1925 }
1926 };
1927 let op = client_control_request_op(&request);
1928 let corr = frame.header.corr;
1929 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1930 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1931 #[cfg(test)]
1932 if let Some(delay) = self.control_dispatch_delay {
1933 tokio::time::sleep(delay).await;
1934 }
1935 let result = self
1936 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1937 .await;
1938 if let Some(timings) = route_open_timings.as_mut() {
1939 timings.finish();
1940 }
1941 log_slow_control_dispatch(
1942 dispatch_started_at,
1943 op,
1944 ctx.connection_id,
1945 corr,
1946 route_open_timings.as_ref(),
1947 );
1948 result
1949 }
1950 FrameType::Push => {
1951 let Some(endpoint) = self
1952 .forwarding
1953 .module_endpoint_for_connection(ctx.connection_id)
1954 .map_err(RouterError::Forwarding)?
1955 else {
1956 return Ok(vec![control_error_frame(
1957 &frame,
1958 "unsupported_control_frame",
1959 "client-originated channel-0 PUSH is not supported",
1960 )?]);
1961 };
1962 self.handle_status_update(endpoint, frame)
1963 }
1964 FrameType::Response | FrameType::Error
1965 if self
1966 .forwarding
1967 .module_endpoint_for_connection(ctx.connection_id)
1968 .map_err(RouterError::Forwarding)?
1969 .is_some() =>
1970 {
1971 self.handle_module_relay_response(ctx.connection_id, frame)
1972 }
1973 ty => Ok(vec![control_error_frame(
1974 &frame,
1975 "unsupported_control_frame",
1976 format!("unsupported channel-0 frame {ty:?}"),
1977 )?]),
1978 }
1979 }
1980
1981 pub fn cleanup_connection(
1982 &self,
1983 connection_id: ConnectionId,
1984 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1985 self.caller_endpoints
1986 .lock()
1987 .unwrap_or_else(|p| p.into_inner())
1988 .remove(&connection_id);
1989 self.forwarding
1990 .operator_confirms()
1991 .connection_closed(connection_id);
1992 self.cleanup_connection_with_end_reason(
1993 connection_id,
1994 RegistrationEndReason::ConnectionClosed,
1995 )
1996 }
1997
1998 fn cleanup_connection_with_end_reason(
1999 &self,
2000 connection_id: ConnectionId,
2001 requested_reason: RegistrationEndReason,
2002 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2003 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2004 self.registry
2005 .get_module_by_connection(connection_id)?
2006 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2007 .and_then(|module| module.registration_end_reason().ok().flatten())
2008 .unwrap_or(requested_reason)
2009 } else {
2010 requested_reason
2011 };
2012 let crash_closed = self
2013 .registry
2014 .get_module_by_connection(connection_id)?
2015 .and_then(|registration| {
2016 self.forwarding
2017 .module_endpoint_for_connection(connection_id)
2018 .ok()
2019 .flatten()
2020 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2021 .map(|routes| (registration.manifest.module_id, routes))
2022 });
2023 let crash_closed = crash_closed.map(|(module_id, routes)| {
2024 let terminal = match self.supervisor.get(&module_id) {
2025 None => false,
2026 Some(module) => match module.will_recover_after_connection_loss() {
2027 Ok(will_recover) => !will_recover,
2028 Err(err) => {
2029 warn!(
2030 %module_id,
2031 error = %err,
2032 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2033 );
2034 false
2035 }
2036 },
2037 };
2038 let (reason, drained) = match self
2042 .forwarding
2043 .daemon_drain_result_for_connection(connection_id)
2044 {
2045 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2046 Ok(None) => (RouteCloseReason::Crash, false),
2047 Err(err) => {
2048 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2049 (RouteCloseReason::Crash, false)
2050 }
2051 };
2052 (module_id, routes, reason, drained, terminal)
2053 });
2054 let registrations = self.deregister_connection(connection_id, end_reason);
2055 let cleanup = if crash_closed.is_some() {
2056 self.forwarding.cleanup_connection_counted(connection_id)
2057 } else {
2058 self.forwarding
2064 .cleanup_connection(connection_id)
2065 .map(|released| crate::forwarding::ConnectionCleanup {
2066 released,
2067 abandoned_relays: 0,
2068 })
2069 };
2070 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2075 let abandoned = cleanup
2076 .as_ref()
2077 .map_or(0, |cleanup| cleanup.abandoned_relays);
2078 send_route_control_pushes(
2079 &self.forwarding,
2080 routes,
2081 ClientControlPush::RouteClosed {
2082 module_id,
2083 channels: Vec::new(),
2084 reason,
2085 drained,
2086 abandoned,
2087 excluded_subscriptions: 0,
2088 terminal: Some(terminal),
2089 },
2090 );
2091 }
2092 if let Ok(cleanup) = cleanup {
2093 self.emit_route_goodbyes(cleanup.released);
2094 }
2095 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2099 crate::supervise::notify_registration_release();
2100 self.capability_evaluator.wake_deadline_loop();
2101 self.refresh_capability_requirements();
2102 }
2103 self.supervisor.remove_spawn_subscribers(connection_id);
2104 self.activity_streams
2105 .lock()
2106 .unwrap_or_else(|p| p.into_inner())
2107 .retain(|(id, _), task| {
2108 if *id == connection_id {
2109 task.abort();
2110 false
2111 } else {
2112 true
2113 }
2114 });
2115 self.hello_launch_nonces
2118 .lock()
2119 .unwrap_or_else(|poisoned| poisoned.into_inner())
2120 .forget(connection_id);
2121 self.scopes
2122 .write()
2123 .unwrap_or_else(|poisoned| poisoned.into_inner())
2124 .release_connection(connection_id);
2125 registrations
2126 }
2127
2128 pub(crate) fn handle_route_goodbye(
2129 &self,
2130 connection_id: ConnectionId,
2131 route_channel: u16,
2132 route_epoch: u32,
2133 ) -> Result<bool, RouterError> {
2134 debug!(
2135 connection_id = connection_id.get(),
2136 route_channel, route_epoch, "handling route GOODBYE"
2137 );
2138 let RouteRelease::Removed(released_route) = self
2139 .forwarding
2140 .release_client_route(connection_id, route_channel, route_epoch)
2141 .map_err(RouterError::Forwarding)?
2142 else {
2143 return Ok(false);
2144 };
2145 self.emit_route_goodbyes(vec![released_route]);
2146 Ok(true)
2147 }
2148
2149 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2150 for released in released_routes {
2151 let frame = match Frame::build_with_version(
2152 released.negotiated_ver,
2153 FrameType::Goodbye,
2154 control_flags(),
2155 released.channel,
2156 released.epoch,
2157 0,
2158 Vec::new(),
2159 ) {
2160 Ok(frame) => frame,
2161 Err(err) => {
2162 warn!(
2163 route_channel = released.channel,
2164 error = %err,
2165 "failed to build route GOODBYE frame"
2166 );
2167 continue;
2168 }
2169 };
2170 if !released.close_on_delivery_failure() {
2171 crate::forwarding::send_module_route_goodbye(
2172 &self.counters,
2173 &released.sink,
2174 frame,
2175 released.module_id.as_deref(),
2176 "client route released",
2177 );
2178 continue;
2179 }
2180 if let Err(err) = released.sink.try_send(frame) {
2181 warn!(
2182 target_connection_id = released.connection_id.get(),
2183 route_channel = released.channel,
2184 error = %err,
2185 "route GOODBYE was not delivered to client; closing target connection"
2186 );
2187 if self
2188 .forwarding
2189 .escalate_client_delivery_failure(
2190 released.connection_id,
2191 released.channel,
2192 released.epoch,
2193 CloseReason::new(
2194 "route_goodbye_delivery_failed",
2195 format!(
2196 "failed to enqueue route GOODBYE for channel {}: {err}",
2197 released.channel
2198 ),
2199 ),
2200 crate::forwarding::UndeliveredFrame {
2201 module_id: released.module_id.as_deref(),
2202 sink: &released.sink,
2203 },
2204 )
2205 .unwrap_or(false)
2206 {
2207 self.counters.increment_goodbye_relay_client_failed();
2208 }
2209 }
2210 }
2211 }
2212
2213 fn send_abandoned_route_bind_goodbye(
2225 &self,
2226 module_sink: &crate::FrameSink,
2227 negotiated_ver: u8,
2228 module_channel: u16,
2229 module_epoch: u32,
2230 ) {
2231 let frame = match Frame::build_with_version(
2232 negotiated_ver,
2233 FrameType::Goodbye,
2234 control_flags(),
2235 module_channel,
2236 module_epoch,
2237 0,
2238 Vec::new(),
2239 ) {
2240 Ok(frame) => frame,
2241 Err(err) => {
2242 warn!(
2243 route_channel = module_channel,
2244 error = %err,
2245 "failed to build GOODBYE for abandoned route.bind"
2246 );
2247 return;
2248 }
2249 };
2250 crate::forwarding::send_module_route_goodbye(
2251 &self.counters,
2252 module_sink,
2253 frame,
2254 None,
2255 "abandoned route.bind",
2256 );
2257 }
2258
2259 fn handle_hello(
2260 &self,
2261 connection_id: ConnectionId,
2262 sink: Option<crate::FrameSink>,
2263 frame: Frame,
2264 ) -> Result<Vec<Frame>, RouterError> {
2265 debug!(
2266 connection_id = connection_id.get(),
2267 corr = frame.header.corr,
2268 "handling HELLO"
2269 );
2270 if self
2274 .registry
2275 .get_module_by_connection(connection_id)
2276 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2277 .is_some()
2278 {
2279 return Ok(vec![control_error_frame(
2280 &frame,
2281 "invalid_hello",
2282 "connection is already registered as a module",
2283 )?]);
2284 }
2285 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2286 Ok(value) => value,
2287 Err(err) => {
2288 return Ok(vec![control_error_frame(
2289 &frame,
2290 "invalid_hello",
2291 format!("malformed HELLO body: {err}"),
2292 )?])
2293 }
2294 };
2295 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2296 return Ok(vec![control_error_frame(
2297 &frame,
2298 "invalid_capability_grammar",
2299 err.to_string(),
2300 )?]);
2301 }
2302 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2303 return Ok(vec![control_error_frame(
2304 &frame,
2305 "invalid_manifest",
2306 err.to_string(),
2307 )?]);
2308 }
2309 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2310 return Ok(vec![control_error_frame(
2311 &frame,
2312 "invalid_event_declaration",
2313 err.to_string(),
2314 )?]);
2315 }
2316 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2317 return Ok(vec![control_error_frame(
2318 &frame,
2319 "invalid_manifest",
2320 err.to_string(),
2321 )?]);
2322 }
2323 if let Some(provenance) = hello_value
2324 .get("manifest")
2325 .and_then(|manifest| manifest.get("provenance"))
2326 {
2327 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2328 return Ok(vec![control_error_frame(
2329 &frame,
2330 "invalid_manifest",
2331 format!("malformed manifest provenance: {err}"),
2332 )?]);
2333 }
2334 }
2335 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2336 Ok(hello) => hello,
2337 Err(err) => {
2338 return Ok(vec![control_error_frame(
2339 &frame,
2340 "invalid_hello",
2341 format!("malformed HELLO body: {err}"),
2342 )?])
2343 }
2344 };
2345
2346 if hello.protocol_ver != hello.manifest.protocol_ver {
2347 return Ok(vec![control_error_frame(
2348 &frame,
2349 "invalid_manifest",
2350 format!(
2351 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2352 hello.protocol_ver, hello.manifest.protocol_ver
2353 ),
2354 )?]);
2355 }
2356
2357 if hello.manifest.module_id.trim().is_empty() {
2358 return Ok(vec![control_error_frame(
2359 &frame,
2360 "invalid_manifest",
2361 "manifest module_id must not be empty",
2362 )?]);
2363 }
2364
2365 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2366 Ok(negotiated_ver) => negotiated_ver,
2367 Err(message) => {
2368 return Ok(vec![control_error_frame(
2369 &frame,
2370 "version_unsupported",
2371 message,
2372 )?])
2373 }
2374 };
2375
2376 let swap_admission = self
2384 .supervisor
2385 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2386 if swap_admission == SwapHelloAdmission::Refused {
2387 warn!(
2388 module_id = %hello.manifest.module_id,
2389 connection_id = connection_id.get(),
2390 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2391 );
2392 return Ok(vec![control_error_frame(
2393 &frame,
2394 "swap_token_invalid",
2395 format!(
2396 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2397 hello.manifest.module_id
2398 ),
2399 )?]);
2400 }
2401 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2402
2403 let nonce_verified = if swap_candidate {
2408 true
2409 } else {
2410 match self.supervisor.reserved_hello_verification(
2411 &hello.manifest.module_id,
2412 hello.launch_nonce.as_deref(),
2413 ) {
2414 Ok(verified) => verified,
2415 Err(rejection) => {
2416 let message = match rejection {
2417 ReservedHelloRejection::Exact { module_id } => format!(
2418 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2419 ),
2420 ReservedHelloRejection::Prefix {
2421 prefix,
2422 owner_module_id,
2423 } => format!(
2424 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2425 hello.manifest.module_id
2426 ),
2427 };
2428 return Ok(vec![control_error_frame(
2429 &frame,
2430 "reserved_module",
2431 message,
2432 )?]);
2433 }
2434 }
2435 };
2436
2437 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2438 &hello.manifest.module_id,
2439 hello.manifest.capabilities.as_ref(),
2440 );
2441 if let Some(refusal) = reserved_capability_refusals.first() {
2442 let capability = refusal.capability.clone();
2443 let bound_module = refusal.claimants[0].clone();
2444 log_duplicate_claim_events(reserved_capability_refusals);
2445 return Ok(vec![control_error_frame(
2446 &frame,
2447 "reserved_capability",
2448 format!(
2449 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2450 capability, bound_module, hello.manifest.module_id
2451 ),
2452 )?]);
2453 }
2454
2455 if self
2458 .forwarding
2459 .connection_has_client_routes(connection_id)
2460 .map_err(RouterError::Forwarding)?
2461 {
2462 return Ok(vec![control_error_frame(
2463 &frame,
2464 "invalid_hello",
2465 "connection has open client routes and cannot also register as a module",
2466 )?]);
2467 }
2468
2469 self.hello_launch_nonces
2474 .lock()
2475 .unwrap_or_else(|poisoned| poisoned.into_inner())
2476 .record(connection_id, hello.launch_nonce.as_deref());
2477 let control_ops = effective_module_control_ops(hello.control_ops);
2478 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2481 if swap_candidate {
2482 return self.register_swap_candidate(
2483 connection_id,
2484 sink,
2485 &frame,
2486 hello.manifest,
2487 negotiated_ver,
2488 control_ops,
2489 hello_ack,
2490 );
2491 }
2492 let registration = match self
2499 .registry
2500 .register_unpublished_verified_with_control_ops(
2501 hello.manifest,
2502 negotiated_ver,
2503 connection_id,
2504 control_ops,
2505 nonce_verified,
2506 ) {
2507 Ok(registration) => registration,
2508 Err(RegistryError::DuplicateModuleId { module_id }) => {
2509 return Ok(vec![control_error_frame(
2510 &frame,
2511 "duplicate_module_id",
2512 format!(
2513 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2514 ),
2515 )?])
2516 }
2517 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2518 return Ok(vec![control_error_frame(
2519 &frame,
2520 "invalid_module_id",
2521 err.to_string(),
2522 )?])
2523 }
2524 Err(err) => {
2525 return Ok(vec![control_error_frame(
2526 &frame,
2527 "registry_error",
2528 err.to_string(),
2529 )?])
2530 }
2531 };
2532 #[cfg(test)]
2533 self.hold_between_registry_and_forwarding();
2534
2535 let reply = if let Some(sink) = sink {
2536 let concurrency = manifest_concurrency(®istration.manifest);
2552 if let Err(err) = self.forwarding.register_module_connection_acked(
2553 connection_id,
2554 registration.manifest.module_id.clone(),
2555 negotiated_ver,
2556 concurrency,
2557 sink,
2558 hello_ack,
2559 ) {
2560 if matches!(
2565 self.deregister_connection(
2566 connection_id,
2567 RegistrationEndReason::RegistrationFailed,
2568 ),
2569 Ok(r) if !r.is_empty()
2570 ) {
2571 crate::supervise::notify_registration_release();
2572 }
2573 return Ok(vec![control_error_frame(
2574 &frame,
2575 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2576 "invalid_hello"
2577 } else {
2578 forwarding_error_code(&err)
2579 },
2580 err.to_string(),
2581 )?]);
2582 }
2583 Vec::new()
2584 } else {
2585 vec![hello_ack]
2588 };
2589
2590 if !self
2595 .registry
2596 .publish_registration(connection_id)
2597 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2598 {
2599 debug!(
2600 module_id = %registration.manifest.module_id,
2601 connection_id = connection_id.get(),
2602 "registration was removed before it could be published"
2603 );
2604 }
2605
2606 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2615 info!(
2616 module_id = %registration.manifest.module_id,
2617 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2618 );
2619 }
2620
2621 self.apply_registration_capabilities(®istration);
2622
2623 self.supervisor
2624 .record_activity_registration(®istration.manifest.module_id);
2625
2626 info!(
2627 module_id = %registration.manifest.module_id,
2628 module_version = %registration.manifest.module_version,
2629 negotiated_ver,
2630 routable_provider = manifest_provides_routable_role(®istration.manifest),
2631 connection_id = connection_id.get(),
2632 "module registered"
2633 );
2634
2635 Ok(reply)
2636 }
2637
2638 #[allow(clippy::too_many_arguments)]
2655 fn register_swap_candidate(
2656 &self,
2657 connection_id: ConnectionId,
2658 sink: Option<crate::FrameSink>,
2659 frame: &Frame,
2660 manifest: ModuleManifest,
2661 negotiated_ver: u8,
2662 control_ops: Vec<String>,
2663 hello_ack: Frame,
2664 ) -> Result<Vec<Frame>, RouterError> {
2665 let module_id = manifest.module_id.clone();
2666 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2667 manifest,
2668 negotiated_ver,
2669 connection_id,
2670 control_ops,
2671 ) {
2672 Ok(registration) => registration,
2673 Err(RegistryError::DuplicateModuleId { module_id }) => {
2674 return Ok(vec![control_error_frame(
2675 frame,
2676 "duplicate_module_id",
2677 format!(
2678 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2679 ),
2680 )?])
2681 }
2682 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2683 return Ok(vec![control_error_frame(
2684 frame,
2685 "invalid_module_id",
2686 err.to_string(),
2687 )?])
2688 }
2689 Err(err) => {
2690 return Ok(vec![control_error_frame(
2691 frame,
2692 "registry_error",
2693 err.to_string(),
2694 )?])
2695 }
2696 };
2697 #[cfg(test)]
2698 self.hold_between_registry_and_forwarding();
2699 let reply = if let Some(sink) = sink {
2700 let concurrency = manifest_concurrency(®istration.manifest);
2704 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2705 connection_id,
2706 module_id.clone(),
2707 negotiated_ver,
2708 concurrency,
2709 sink,
2710 hello_ack,
2711 ) {
2712 if matches!(
2713 self.deregister_connection(
2714 connection_id,
2715 RegistrationEndReason::RegistrationFailed,
2716 ),
2717 Ok(r) if !r.is_empty()
2718 ) {
2719 crate::supervise::notify_registration_release();
2720 }
2721 return Ok(vec![control_error_frame(
2722 frame,
2723 forwarding_error_code(&err),
2724 err.to_string(),
2725 )?]);
2726 }
2727 Vec::new()
2728 } else {
2729 vec![hello_ack]
2730 };
2731 if !self
2732 .registry
2733 .publish_registration(connection_id)
2734 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2735 {
2736 debug!(
2737 module_id = %module_id,
2738 connection_id = connection_id.get(),
2739 "swap candidate registration was removed before it could be published"
2740 );
2741 }
2742 self.supervisor.mark_swap_candidate_admitted(&module_id);
2743 info!(
2744 module_id = %module_id,
2745 module_version = %registration.manifest.module_version,
2746 negotiated_ver,
2747 ready = registration.ready,
2748 connection_id = connection_id.get(),
2749 "swap candidate registered; not routable until cutover"
2750 );
2751 Ok(reply)
2752 }
2753
2754 fn build_hello_ack(
2755 &self,
2756 frame: &Frame,
2757 negotiated_ver: u8,
2758 module_id: &str,
2759 ) -> Result<Frame, RouterError> {
2760 let ack = ModuleHelloAckBody {
2761 negotiated_ver,
2762 subc_ops: module_subc_ops(),
2763 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2764 storage: self
2765 .storage_config
2766 .as_ref()
2767 .map(|cfg| cfg.descriptor_for(module_id)),
2768 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2769 };
2770 let body = serde_json::to_vec(&ack).map_err(|err| {
2771 RouterError::backend(
2772 0,
2773 frame.header.corr,
2774 format!("failed to encode HELLO_ACK: {err}"),
2775 )
2776 })?;
2777
2778 Frame::build_with_version(
2779 negotiated_ver,
2780 FrameType::HelloAck,
2781 control_flags(),
2782 0,
2783 0,
2784 frame.header.corr,
2785 body,
2786 )
2787 .map_err(RouterError::FrameBuild)
2788 }
2789
2790 async fn handle_client_control_request(
2791 &self,
2792 ctx: &RouteCtx,
2793 frame: Frame,
2794 request: ClientControlRequest,
2795 route_open_timings: &mut Option<RouteOpenTimings>,
2796 ) -> Result<Vec<Frame>, RouterError> {
2797 if let Some(operation) = privileged_operation(&request) {
2798 let caller = self.resolve_caller(ctx.connection_id).await;
2799 let op = client_control_request_op(&request);
2800 let text = prompt_text(operation, &caller);
2801 match self.privileged_operator_prompt {
2802 PrivilegedOperatorPrompt::Log => {
2803 info!(target: "subc_daemon::privileged_prompt", operation = op,
2804 target = ?operation, caller = %caller.description(), summary = %text,
2805 "privileged request would prompt the operator");
2806 }
2807 PrivilegedOperatorPrompt::Enforce => {
2808 let key =
2809 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2810 let outcome = self
2811 .forwarding
2812 .operator_confirms()
2813 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2814 .await;
2815 if outcome.code != "confirmed" {
2816 return Ok(vec![outcome.frame(&frame)]);
2817 }
2818 }
2819 }
2820 }
2821 match request {
2822 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2823 ClientControlRequest::CatalogList { module_id } => {
2824 self.handle_catalog_list(frame, module_id)
2825 }
2826 ClientControlRequest::RouteOpen {
2827 target,
2828 identity,
2829 consumer_identity,
2830 consumer_capabilities,
2831 role_versions,
2832 admission_facts,
2833 scope,
2834 } => {
2835 self.handle_route_open(
2836 ctx,
2837 frame,
2838 RouteOpenRequest {
2839 target,
2840 identity,
2841 consumer_identity,
2842 consumer_capabilities,
2843 role_versions,
2844 admission_facts,
2845 scope,
2846 },
2847 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2848 )
2849 .await
2850 }
2851 ClientControlRequest::RoutePoll {
2852 route_channel,
2853 route_epoch,
2854 kind,
2855 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2856 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2857 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2858 self.handle_supervisor_spawn_snapshot(frame)
2859 }
2860 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2861 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2862 }
2863 ClientControlRequest::SupervisorRestart {
2864 module_id,
2865 drain_timeout_ms,
2866 } => {
2867 self.log_supervisor_request_received(
2868 ctx,
2869 frame.header.corr,
2870 ops::SUPERVISOR_RESTART,
2871 Some(&module_id),
2872 None,
2873 )?;
2874 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2875 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2876 .await
2877 }
2878 ClientControlRequest::SupervisorSwap {
2879 module_id,
2880 ready_timeout_ms,
2881 } => {
2882 self.log_supervisor_request_received(
2883 ctx,
2884 frame.header.corr,
2885 ops::SUPERVISOR_SWAP,
2886 Some(&module_id),
2887 None,
2888 )?;
2889 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2890 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2891 .await
2892 }
2893 ClientControlRequest::SupervisorReload { module_id } => {
2894 self.log_supervisor_request_received(
2895 ctx,
2896 frame.header.corr,
2897 ops::SUPERVISOR_RELOAD,
2898 Some(&module_id),
2899 None,
2900 )?;
2901 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2902 self.handle_supervisor_reload(frame, module_id, requester)
2903 .await
2904 }
2905 ClientControlRequest::SupervisorRescan { preview } => {
2906 if !preview {
2907 self.log_supervisor_request_received(
2908 ctx,
2909 frame.header.corr,
2910 ops::SUPERVISOR_RESCAN,
2911 None,
2912 None,
2913 )?;
2914 }
2915 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2916 self.handle_supervisor_rescan(
2917 frame,
2918 if preview {
2919 RescanMode::Preview
2920 } else {
2921 RescanMode::Execute
2922 },
2923 requester,
2924 )
2925 .await
2926 }
2927 ClientControlRequest::SupervisorRescanChecked {
2928 preview,
2929 candidate,
2930 expect,
2931 } => {
2932 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2933 .await
2934 }
2935 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2936 self.log_supervisor_request_received(
2937 ctx,
2938 frame.header.corr,
2939 ops::SUPERVISOR_RELEASE_RESERVED,
2940 Some(&module_id),
2941 None,
2942 )?;
2943 self.handle_supervisor_release_reserved(frame, module_id)
2944 .await
2945 }
2946 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2947 self.log_supervisor_request_received(
2948 ctx,
2949 frame.header.corr,
2950 ops::SUPERVISOR_SET_ENABLED,
2951 Some(&module_id),
2952 Some(enabled),
2953 )?;
2954 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2955 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2956 .await
2957 }
2958 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2959 self.handle_supervisor_health_probe(frame, module_id).await
2960 }
2961 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2962 ClientControlRequest::SupervisorRoutes { module_id } => {
2963 self.handle_supervisor_routes(frame, module_id)
2964 }
2965 ClientControlRequest::SupervisorProvenance { module_id } => {
2966 self.handle_supervisor_provenance(frame, module_id).await
2967 }
2968 ClientControlRequest::SupervisorStderrTail {
2969 module_id,
2970 max_lines,
2971 max_bytes,
2972 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2973 ClientControlRequest::SupervisorTerminals { module_id } => {
2974 self.handle_supervisor_terminals(frame, module_id).await
2975 }
2976 }
2977 }
2978
2979 fn log_supervisor_request_received(
2980 &self,
2981 ctx: &RouteCtx,
2982 corr: u64,
2983 op: &'static str,
2984 module_id: Option<&str>,
2985 enabled: Option<bool>,
2986 ) -> Result<(), RouterError> {
2987 let caller = self.supervisor_requester(ctx, corr)?;
2988 let caller = principal_label(&caller);
2989 info!(
2992 target: "control",
2993 op,
2994 module_id,
2995 enabled,
2996 connection_id = ctx.connection_id.get(),
2997 caller = %caller,
2998 "supervisor request received"
2999 );
3000 Ok(())
3001 }
3002
3003 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3004 Ok(self
3005 .registry
3006 .get_module_by_connection(ctx.connection_id)
3007 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3008 .map(|registration| Principal::Reserved {
3009 module_id: registration.manifest.module_id,
3010 })
3011 .unwrap_or(Principal::Direct))
3012 }
3013
3014 fn handle_module_control_request(
3015 &self,
3016 connection_id: ConnectionId,
3017 frame: Frame,
3018 request: ModuleControlRequestFromModule,
3019 ) -> Result<Vec<Frame>, RouterError> {
3020 match request {
3021 ModuleControlRequestFromModule::CatalogUpdate {
3022 provides,
3023 capabilities,
3024 ready,
3025 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3026 ModuleControlRequestFromModule::LiveRoots {} => {
3027 let registered = self
3028 .registry
3029 .get_module_by_connection(connection_id)
3030 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3031 let Some(registration) = registered else {
3032 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3033 };
3034 let response = self
3035 .forwarding
3036 .live_roots(®istration.manifest.module_id)
3037 .map_err(RouterError::Forwarding)?;
3038 Ok(vec![control_response_body_frame(
3039 &frame,
3040 &response,
3041 "ModuleControlResponseToModule::LiveRoots",
3042 )?])
3043 }
3044 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3045 self.handle_scope_sync(connection_id, frame, generation, scopes)
3046 }
3047 ModuleControlRequestFromModule::ScopeApply {
3048 generation,
3049 upsert,
3050 end,
3051 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3052 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3053 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3054 }
3055 }
3056 }
3057
3058 fn handle_scope_served_by(
3059 &self,
3060 connection_id: ConnectionId,
3061 frame: Frame,
3062 ) -> Result<Vec<Frame>, RouterError> {
3063 let registered = self
3064 .registry
3065 .get_module_by_connection(connection_id)
3066 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3067 let Some(registered) = registered else {
3068 return Ok(vec![control_error_frame(
3069 &frame,
3070 "not_registered",
3071 "scope.served_by requires an active module registration owned by this connection",
3072 )?]);
3073 };
3074 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3075 {
3076 Ok(request) => request,
3077 Err(err) => {
3078 return Ok(vec![control_error_frame(
3079 &frame,
3080 "invalid_control_body",
3081 format!("malformed scope.served_by body: {err}"),
3082 )?])
3083 }
3084 };
3085 let answer = self
3086 .scopes
3087 .read()
3088 .unwrap_or_else(|poisoned| poisoned.into_inner())
3089 .served_by(
3090 ®istered.manifest.module_id,
3091 &request.scope_ref,
3092 request.scope_epoch,
3093 &request.module_id,
3094 );
3095 let reply = subc_control::ScopeServedByReply::new(
3096 answer,
3097 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3098 );
3099 Ok(vec![control_response_body_frame(
3100 &frame,
3101 &reply,
3102 "ScopeServedByReply",
3103 )?])
3104 }
3105
3106 fn handle_operator_confirm(
3107 &self,
3108 ctx: &RouteCtx,
3109 frame: Frame,
3110 ) -> Result<Vec<Frame>, RouterError> {
3111 use crate::operator_confirm::{audit, Outcome};
3112 let registration = self
3113 .registry
3114 .get_module_by_connection(ctx.connection_id)
3115 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3116 let Some(registration) = registration else {
3117 let outcome = Outcome::refusal("not_registered");
3118 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3119 return Ok(vec![outcome.frame(&frame)]);
3120 };
3121 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3122 Ok(request) => request,
3123 Err(_) => {
3124 let outcome = Outcome::refusal("invalid_control_body");
3125 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3126 return Ok(vec![outcome.frame(&frame)]);
3127 }
3128 };
3129 let module_id = registration.manifest.module_id;
3130 let nonce = self
3134 .hello_launch_nonces
3135 .lock()
3136 .unwrap_or_else(|p| p.into_inner())
3137 .nonce(ctx.connection_id)
3138 .map(str::to_owned);
3139 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3140 self.supervisor
3141 .spawned_consumer_authorized(&module_id, nonce)
3142 });
3143 let confirms = self.forwarding.operator_confirms();
3144 self.forwarding
3145 .with_operator_route(
3146 ctx.connection_id,
3147 request.route_channel,
3148 request.route_epoch,
3149 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3150 )
3151 .map_err(RouterError::Forwarding)
3152 }
3153
3154 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3157 let mut scopes = ScopeRecords::from(scopes);
3158 for (index, record) in scopes.records.iter_mut().enumerate() {
3159 if let Some(refusal) = Self::canonicalize_workdir(record) {
3160 scopes.refusals.insert(index, refusal);
3161 }
3162 }
3163 scopes
3164 }
3165
3166 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3167 let workdir = record.workdir.as_mut()?;
3168 let path = std::path::Path::new(workdir);
3169 if !path.is_absolute() {
3170 return Some((
3171 error_codes::INVALID_WORKDIR,
3172 "workdir must be absolute".to_string(),
3173 ));
3174 }
3175 match ProjectRootId::from_path_allowing_missing(path) {
3176 Ok(root) => match root.as_path().to_str() {
3177 Some(canonical) => {
3178 *workdir = canonical.to_string();
3179 None
3180 }
3181 None => Some((
3182 error_codes::INVALID_WORKDIR,
3183 "canonical workdir is not UTF-8".to_string(),
3184 )),
3185 },
3186 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3187 }
3188 }
3189
3190 fn handle_scope_sync(
3194 &self,
3195 connection_id: ConnectionId,
3196 frame: Frame,
3197 generation: u64,
3198 scopes: Vec<ScopeRecord>,
3199 ) -> Result<Vec<Frame>, RouterError> {
3200 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3201 }
3202
3203 fn handle_scope_change(
3204 &self,
3205 connection_id: ConnectionId,
3206 frame: Frame,
3207 generation: u64,
3208 scopes: Vec<ScopeRecord>,
3209 end: Option<Vec<ScopeEnd>>,
3210 ) -> Result<Vec<Frame>, RouterError> {
3211 let is_apply = end.is_some();
3212 let Some(registration) = self
3213 .registry
3214 .get_module_by_connection(connection_id)
3215 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3216 else {
3217 return Ok(vec![control_error_frame(
3218 &frame,
3219 "not_registered",
3220 "scope.sync and scope.apply require an active module registration owned by this connection",
3221 )?]);
3222 };
3223 let owner = registration.manifest.module_id;
3224 let scopes = Self::prepare_scope_records(scopes);
3225 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3226 let is_current_launch = |connection: ConnectionId| {
3227 self.hello_launch_nonces
3228 .lock()
3229 .unwrap_or_else(|poisoned| poisoned.into_inner())
3230 .presented(connection, current_nonce.as_deref())
3231 };
3232 let mut table = self
3237 .scopes
3238 .write()
3239 .unwrap_or_else(|poisoned| poisoned.into_inner());
3240 let outcome = match end {
3241 Some(end) => table.apply(
3242 &owner,
3243 connection_id,
3244 is_current_launch,
3245 generation,
3246 scopes,
3247 end,
3248 ),
3249 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3250 };
3251 let drained = match &outcome {
3252 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3253 Err(_) => Vec::new(),
3254 };
3255 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3256 drop(table);
3257 if has_deadlines {
3258 self.scope_deadline_added.notify_one();
3259 }
3260 match outcome {
3261 Ok(applied) => {
3262 let counts = ScopeOutcomeCounts::of(&applied.results);
3263 info!(
3264 owner = %owner,
3265 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3266 generation,
3267 records = applied.results.len(),
3268 created = counts.created,
3269 replaced = counts.replaced,
3270 updated = counts.updated,
3271 unchanged = counts.unchanged,
3272 refused = counts.refused,
3273 ended = applied.ended.len(),
3274 tag_changes = applied.tag_changes.len(),
3275 routes_closed = drained.len(),
3276 "scope change accepted"
3277 );
3278 for refused in applied
3284 .results
3285 .iter()
3286 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3287 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3288 {
3289 warn!(
3290 owner = %owner,
3291 generation,
3292 scope_ref = %refused.scope_ref,
3293 scope_epoch = refused.scope_epoch,
3294 code = refused.code.as_deref().unwrap_or(""),
3295 "scope record refused"
3296 );
3297 }
3298 self.close_scope_drained_routes(drained);
3299 let response = if is_apply {
3300 ModuleControlResponseToModule::ScopeApply {
3301 generation,
3302 results: applied.results,
3303 end_results: applied.end_results,
3304 ended: applied.ended,
3305 }
3306 } else {
3307 ModuleControlResponseToModule::ScopeSync {
3308 generation,
3309 results: applied.results,
3310 ended: applied.ended,
3311 }
3312 };
3313 Ok(vec![control_response_body_frame(
3314 &frame,
3315 &response,
3316 "scope change response",
3317 )?])
3318 }
3319 Err(refusal) => {
3320 info!(
3321 owner = %owner,
3322 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3323 generation,
3324 code = refusal.code,
3325 "scope change refused"
3326 );
3327 Ok(vec![control_error_frame(
3328 &frame,
3329 refusal.code,
3330 refusal.message,
3331 )?])
3332 }
3333 }
3334 }
3335
3336 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3342 if drained.is_empty() {
3343 return;
3344 }
3345 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3346 BTreeMap::new();
3347 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3348 for route in drained {
3349 warn!(
3350 module_id = %route.module_id,
3351 reason = ?route.reason,
3352 client_connection_id = route.client.connection_id.get(),
3353 route_channel = route.client.channel,
3354 "closing route because its scope changed"
3355 );
3356 pushes
3357 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3358 .or_insert_with(|| (route.reason, Vec::new()))
3359 .1
3360 .push(EndpointRoute {
3361 goodbye_target: route.client.clone(),
3362 principal: Principal::Unverified,
3363 bound_at: Instant::now(),
3364 draining: false,
3365 drain_reason: None,
3366 });
3367 goodbyes.push(route.module);
3368 goodbyes.push(route.client);
3369 }
3370 for ((module_id, _), (reason, routes)) in pushes {
3371 send_route_control_pushes(
3372 &self.forwarding,
3373 routes,
3374 ClientControlPush::RouteClosed {
3375 module_id,
3376 channels: Vec::new(),
3377 reason,
3378 drained: false,
3379 abandoned: 0,
3380 excluded_subscriptions: 0,
3381 terminal: Some(false),
3382 },
3383 );
3384 }
3385 self.emit_route_goodbyes(goodbyes);
3386 }
3387
3388 fn handle_scope_describe(
3391 &self,
3392 connection_id: ConnectionId,
3393 frame: Frame,
3394 owner: Principal,
3395 scope_ref: String,
3396 ) -> Result<Vec<Frame>, RouterError> {
3397 let registered = self
3398 .registry
3399 .get_module_by_connection(connection_id)
3400 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3401 let Some(registered) = registered else {
3402 return Ok(vec![control_error_frame(
3403 &frame,
3404 "not_registered",
3405 "scope.describe requires an active module registration owned by this connection",
3406 )?]);
3407 };
3408 let mut description = self
3409 .scopes
3410 .read()
3411 .unwrap_or_else(|poisoned| poisoned.into_inner())
3412 .describe(&owner, &scope_ref);
3413 if !registered
3416 .manifest
3417 .capabilities
3418 .as_ref()
3419 .is_some_and(|caps| {
3420 caps.provides
3421 .iter()
3422 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3423 })
3424 {
3425 if let Some(stamp) = description.stamp.as_mut() {
3426 stamp.attributes.on_behalf_of_agent = None;
3427 }
3428 }
3429 let owner_configured = match &owner {
3430 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3435 _ => false,
3436 };
3437 let response = ModuleControlResponseToModule::ScopeDescribe {
3438 status: description.status,
3439 scope_epoch: description.scope_epoch,
3440 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3441 owner_synced: description.owner_synced,
3442 owner_configured,
3443 scope: description.stamp,
3444 };
3445 Ok(vec![control_response_body_frame(
3446 &frame,
3447 &response,
3448 "ModuleControlResponseToModule::ScopeDescribe",
3449 )?])
3450 }
3451
3452 fn handle_catalog_update(
3453 &self,
3454 connection_id: ConnectionId,
3455 frame: Frame,
3456 provides: Vec<ProviderRole>,
3457 capabilities: Option<CapabilityDeclarations>,
3458 ready: Option<bool>,
3459 ) -> Result<Vec<Frame>, RouterError> {
3460 self.refresh_capability_requirements();
3461 let Some(registration) = self
3462 .registry
3463 .get_module_by_connection(connection_id)
3464 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3465 else {
3466 return Ok(vec![control_error_frame(
3467 &frame,
3468 "not_registered",
3469 "catalog.update requires an active module registration owned by this connection",
3470 )?]);
3471 };
3472
3473 if let Some(message) =
3474 catalog_update_frozen_field_message(®istration.manifest, &provides)
3475 {
3476 return Ok(vec![control_error_frame(
3477 &frame,
3478 "catalog_update_frozen_field",
3479 message,
3480 )?]);
3481 }
3482
3483 let mut candidate = registration.manifest.clone();
3484 candidate.provides = provides.clone();
3485 candidate.capabilities = capabilities
3486 .clone()
3487 .or_else(|| registration.manifest.capabilities.clone());
3488 if let Some(rich_ui) = &candidate.rich_ui {
3489 if let Err(err) = rich_ui.validate() {
3490 return Ok(vec![control_error_frame(
3491 &frame,
3492 "invalid_manifest",
3493 err.to_string(),
3494 )?]);
3495 }
3496 }
3497 if let Err(err) = candidate.validate_capability_grammar() {
3498 return Ok(vec![control_error_frame(
3499 &frame,
3500 "invalid_capability_grammar",
3501 err.to_string(),
3502 )?]);
3503 }
3504
3505 let mut conflicts = self
3508 .capability_evaluator
3509 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3510 if let Some(conflict) = conflicts.first() {
3511 let message = format!(
3512 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3513 conflict.capability, conflict.claimants[0], candidate.module_id
3514 );
3515 for conflict in &mut conflicts {
3516 conflict.source = DuplicateClaimSource::CatalogUpdate;
3517 }
3518 log_duplicate_claim_events(conflicts);
3519 return Ok(vec![control_error_frame(
3520 &frame,
3521 "reserved_capability",
3522 message,
3523 )?]);
3524 }
3525
3526 let updated = self
3527 .registry
3528 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3529 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3530 if updated.is_none() {
3531 return Ok(vec![control_error_frame(
3532 &frame,
3533 "not_registered",
3534 "catalog.update requires an active module registration owned by this connection",
3535 )?]);
3536 }
3537 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3538 log_duplicate_claim_events(
3539 self.capability_evaluator
3540 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3541 );
3542 }
3543 if capability_census_trigger(
3544 registration.manifest.capabilities.as_ref(),
3545 updated
3546 .as_ref()
3547 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3548 ) {
3549 self.enforce_capability_denies();
3550 }
3551 self.refresh_capability_requirements();
3552
3553 let response = ModuleControlResponseToModule::CatalogUpdate {};
3554 control_response_body_frame(
3555 &frame,
3556 &response,
3557 "ModuleControlResponseToModule::CatalogUpdate",
3558 )
3559 .map(|frame| vec![frame])
3560 }
3561
3562 pub(crate) fn with_activity_ledger_config(
3563 mut self,
3564 run_dir: &Path,
3565 incarnation: String,
3566 config: crate::activity::ActivityConfig,
3567 ) -> Self {
3568 self.activity = Some(crate::activity::ActivityLedger::open_config(
3569 run_dir.join("activity.jsonl"),
3570 incarnation,
3571 config,
3572 ));
3573 self.supervisor.set_activity(self.activity.clone());
3574 self
3575 }
3576
3577 pub(crate) async fn configure_activity(
3578 &self,
3579 config: crate::activity::ActivityConfig,
3580 ) -> Result<(), &'static str> {
3581 match &self.activity {
3582 Some(ledger) => ledger.configure(config).await,
3583 None => Ok(()),
3584 }
3585 }
3586
3587 fn cancel_activity_subscription(
3588 &self,
3589 ctx: &RouteCtx,
3590 frame: &Frame,
3591 ) -> Result<bool, RouterError> {
3592 let task = self
3593 .activity_streams
3594 .lock()
3595 .unwrap_or_else(|p| p.into_inner())
3596 .remove(&(ctx.connection_id, frame.header.corr));
3597 let Some(task) = task else {
3598 return Ok(false);
3599 };
3600 task.abort();
3601 let end = Frame::build_with_version(
3602 response_version(frame),
3603 FrameType::StreamEnd,
3604 control_flags(),
3605 0,
3606 0,
3607 frame.header.corr,
3608 Vec::new(),
3609 )
3610 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3611 let sink = ctx.egress.clone();
3612 tokio::spawn(async move {
3613 let _ = sink.send(end).await;
3614 });
3615 Ok(true)
3616 }
3617
3618 async fn handle_activity_read(
3619 &self,
3620 ctx: &RouteCtx,
3621 frame: Frame,
3622 ) -> Result<Vec<Frame>, RouterError> {
3623 use subc_control::{
3624 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3625 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3626 };
3627 #[derive(Deserialize)]
3628 struct Identity {
3629 #[serde(default)]
3630 consumer_identity: Option<ConsumerIdentity>,
3631 }
3632 let principal = match self
3636 .forwarding
3637 .module_id_for_connection(ctx.connection_id)
3638 .map_err(RouterError::Forwarding)?
3639 {
3640 Some(module_id) => Ok(Principal::Reserved { module_id }),
3641 None => {
3642 let identity: Identity = match serde_json::from_slice(&frame.body) {
3643 Ok(identity) => identity,
3644 Err(_) => {
3645 return Ok(vec![control_error_frame(
3646 &frame,
3647 ACTIVITY_MALFORMED,
3648 "malformed activity read",
3649 )?])
3650 }
3651 };
3652 self.route_open_principal(&frame, identity.consumer_identity)?
3653 }
3654 };
3655 let admitted = matches!(principal, Ok(Principal::Direct))
3656 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3657 if !admitted {
3658 return Ok(vec![control_error_frame(
3659 &frame,
3660 ACTIVITY_READ_FORBIDDEN,
3661 "activity reads require an operator or trusted mirror principal",
3662 )?]);
3663 }
3664 let Some(ledger) = &self.activity else {
3665 return Ok(vec![control_error_frame(
3666 &frame,
3667 ACTIVITY_UNAVAILABLE,
3668 "activity ledger unavailable",
3669 )?]);
3670 };
3671 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3672 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3673 {
3674 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3675 Ok(request) => ledger.query(request).await,
3676 Err(_) => Err(ACTIVITY_MALFORMED),
3677 };
3678 return match result {
3679 Ok(reply) => Ok(vec![control_response_body_frame(
3680 &frame,
3681 &reply,
3682 ops::ACTIVITY_QUERY,
3683 )?]),
3684 Err(code) => Ok(vec![control_error_frame(
3685 &frame,
3686 code,
3687 "activity query refused",
3688 )?]),
3689 };
3690 }
3691 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3692 Ok(request) => request,
3693 Err(_) => {
3694 return Ok(vec![control_error_frame(
3695 &frame,
3696 ACTIVITY_MALFORMED,
3697 "malformed activity subscription",
3698 )?])
3699 }
3700 };
3701 let mut subscription = match ledger.subscribe(request.since).await {
3702 Ok(subscription) => subscription,
3703 Err(code) => {
3704 return Ok(vec![control_error_frame(
3705 &frame,
3706 code,
3707 "activity subscription refused",
3708 )?])
3709 }
3710 };
3711 let key = (ctx.connection_id, frame.header.corr);
3712 let sink = ctx.egress.clone();
3713 let mut streams = self
3714 .activity_streams
3715 .lock()
3716 .unwrap_or_else(|p| p.into_inner());
3717 streams.retain(|_, task| !task.is_finished());
3718 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3719 return Ok(vec![control_error_frame(
3720 &frame,
3721 "duplicate_subscription",
3722 "activity subscription correlation id already in use",
3723 )?]);
3724 }
3725 let task = tokio::spawn(async move {
3726 loop {
3727 let entry = if let Some(entry) = subscription.replay.pop_front() {
3728 Some(entry)
3729 } else {
3730 subscription.receiver.recv().await
3731 };
3732 let Some(entry) = entry else {
3733 break;
3734 };
3735 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3736 return;
3737 };
3738 let Ok(next) = Frame::build_with_version(
3739 response_version(&frame),
3740 FrameType::StreamData,
3741 control_flags(),
3742 0,
3743 0,
3744 frame.header.corr,
3745 body,
3746 ) else {
3747 return;
3748 };
3749 if sink.send(next).await.is_err() {
3750 return;
3751 }
3752 }
3753 if let Some(code) = subscription.terminal_code() {
3756 if let Ok(end) =
3757 control_error_frame(&frame, code, "activity subscriber fell behind")
3758 {
3759 let _ = sink.send(end).await;
3760 }
3761 }
3762 });
3763 streams.insert(key, task.abort_handle());
3764 Ok(Vec::new())
3765 }
3766
3767 async fn handle_activity_record(
3768 &self,
3769 ctx: &RouteCtx,
3770 frame: Frame,
3771 ) -> Result<Vec<Frame>, RouterError> {
3772 use subc_control::{
3773 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3774 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3775 };
3776 let result = async {
3777 let module = self
3778 .forwarding
3779 .module_id_for_connection(ctx.connection_id)
3780 .ok()
3781 .flatten()
3782 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3783 let request: ActivityRecordRequest =
3784 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3785 crate::activity::validate(&module, &request)?;
3786 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3787 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3788 let at_ms = std::time::SystemTime::now()
3789 .duration_since(std::time::UNIX_EPOCH)
3790 .unwrap_or_default()
3791 .as_millis() as u64;
3792 ledger.record(module, request, actor, at_ms).await
3793 }
3794 .await;
3795 match result {
3796 Ok(entry) => Ok(vec![control_response_body_frame(
3797 &frame,
3798 &ActivityRecordResponse { seq: entry.seq },
3799 "activity.record",
3800 )?]),
3801 Err(code) => Ok(vec![control_error_frame(
3802 &frame,
3803 code,
3804 "activity recording refused",
3805 )?]),
3806 }
3807 }
3808
3809 fn activity_actor(
3810 &self,
3811 connection: ConnectionId,
3812 module: &str,
3813 on: Option<&subc_control::ActivityOn>,
3814 ) -> Result<subc_control::ActivityActor, &'static str> {
3815 use crate::forwarding::{DataRoute, DataRouteState};
3816 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3817 let principal = |id: &str| Principal::Reserved {
3818 module_id: id.to_owned(),
3819 };
3820 let mut actor = ActivityActor {
3821 principal: principal(module),
3822 scope_owner: None,
3823 agent_id: None,
3824 run_id: None,
3825 flow_id: None,
3826 project_root: None,
3827 workdir: None,
3828 project_id: None,
3829 };
3830 let scope = match on {
3831 None => return Ok(actor),
3832 Some(ActivityOn::Scope(scope)) => {
3833 let selector = subc_protocol::scope::ScopeSelector::new(
3834 principal(module),
3835 scope.scope_ref.clone(),
3836 Some(scope.epoch),
3837 );
3838 self.scopes
3839 .read()
3840 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3841 .attribute(&principal(module), module, &selector, true)
3842 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3843 }
3844 Some(ActivityOn::Route(route)) => {
3845 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3846 .forwarding
3847 .lookup_data_route(connection, route.channel, route.epoch)
3848 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3849 else {
3850 return Err(ACTIVITY_ACTOR_UNKNOWN);
3851 };
3852 if binding.module_id != module {
3853 return Err(ACTIVITY_ACTOR_UNKNOWN);
3854 }
3855 actor.principal = binding.principal.clone();
3856 actor.project_root = binding
3857 .project_root
3858 .as_ref()
3859 .map(|root| root.as_path().to_path_buf());
3860 let Some(bound) = &binding.scope else {
3861 return Ok(actor);
3862 };
3863 let owner = principal(&bound.owner);
3864 let selector = subc_protocol::scope::ScopeSelector::new(
3865 owner.clone(),
3866 bound.scope_ref.clone(),
3867 Some(bound.tag.scope_epoch),
3868 );
3869 let admitted = self
3870 .scopes
3871 .read()
3872 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3873 .attribute(&owner, module, &selector, true)
3874 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3875 if admitted.tag != bound.tag {
3876 return Err(ACTIVITY_ACTOR_UNKNOWN);
3877 }
3878 admitted
3879 }
3880 };
3881 actor.scope_owner = Some(scope.stamp.owner);
3882 actor.agent_id = scope.stamp.attributes.agent_id;
3883 actor.run_id = scope.stamp.attributes.run_id;
3884 actor.flow_id = scope.stamp.attributes.flow_id;
3885 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3886 actor.project_id = scope.stamp.project_id;
3887 Ok(actor)
3888 }
3889
3890 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3891 self.refresh_capability_requirements();
3892 let mut counters = self.counters.snapshot();
3898 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3899 obj.insert(
3900 "activity_unparseable_lines".into(),
3901 ledger.read_errors().into(),
3902 );
3903 obj.insert(
3904 "activity_lifecycle_append_failures".into(),
3905 ledger.lifecycle_failures().into(),
3906 );
3907 obj.insert(
3908 "activity_started_digest_skips".into(),
3909 ledger.lifecycle_digest_skips().into(),
3910 );
3911 }
3912 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3913 self.forwarding.client_route_concentration(),
3914 counters.as_object_mut(),
3915 ) {
3916 obj.insert(
3917 "client_connections_with_routes".into(),
3918 connections_with_routes.into(),
3919 );
3920 obj.insert("max_routes_on_one_connection".into(), max.into());
3921 }
3922 if let (Some(open_breakers), Some(obj)) = (
3928 self.route_bind_breakers.open_snapshot(),
3929 counters.as_object_mut(),
3930 ) {
3931 obj.insert("route_bind_breakers_open".into(), open_breakers);
3932 }
3933 let response = ClientControlResponse::ServerDescribe {
3934 protocol_ver: PROTOCOL_VERSION,
3935 subc_ops: subc_ops(),
3936 capabilities: self.describe_capabilities(),
3937 connected_clients: self.connected_clients.count(),
3938 counters: Some(counters),
3939 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3940 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3941 capability_requirements: self.capability_requirement_statuses(),
3942 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3943 config_path: self
3944 .rescan
3945 .as_ref()
3946 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3947 };
3948 Ok(vec![control_response_body_frame(
3949 &frame,
3950 &response,
3951 "ClientControlResponse::ServerDescribe",
3952 )?])
3953 }
3954
3955 fn handle_catalog_list(
3956 &self,
3957 frame: Frame,
3958 module_id: Option<String>,
3959 ) -> Result<Vec<Frame>, RouterError> {
3960 let (generation, modules) = self
3961 .registry
3962 .list_modules_with_nonce_verification()
3963 .map_err(|err| {
3964 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3965 })?;
3966 let entries = modules
3967 .into_iter()
3968 .filter(|(registration, _)| {
3969 module_id
3970 .as_deref()
3971 .map(|wanted| registration.manifest.module_id == wanted)
3972 .unwrap_or(true)
3973 })
3974 .map(|(registration, nonce_verified)| {
3975 let not_ready = self.not_ready_reason(®istration);
3976 let roles = registration.manifest.provides;
3977 CatalogEntry::new(
3978 registration.manifest.module_id,
3979 roles,
3980 registration.control_ops,
3981 )
3982 .with_ready(not_ready.is_none())
3983 .with_not_ready(not_ready)
3984 .with_module_version(Some(registration.manifest.module_version))
3985 .with_capabilities(registration.manifest.capabilities)
3986 .with_self_signals(registration.manifest.self_signals)
3987 .with_events(registration.manifest.events)
3988 .with_operation_delivery(registration.manifest.operation_delivery)
3989 .with_rich_ui(registration.manifest.rich_ui)
3990 .with_nonce_verified(nonce_verified)
3991 })
3992 .collect();
3993 let response = ClientControlResponse::CatalogList {
3994 generation,
3995 modules: entries,
3996 subc_ops: subc_ops(),
3997 };
3998 Ok(vec![control_response_body_frame(
3999 &frame,
4000 &response,
4001 "ClientControlResponse::CatalogList",
4002 )?])
4003 }
4004
4005 fn route_open_principal(
4006 &self,
4007 frame: &Frame,
4008 consumer_identity: Option<ConsumerIdentity>,
4009 ) -> Result<Result<Principal, Frame>, RouterError> {
4010 let Some(consumer_identity) = consumer_identity else {
4011 return Ok(Ok(Principal::Direct));
4012 };
4013
4014 if self.supervisor.spawned_consumer_authorized(
4015 &consumer_identity.module_id,
4016 &consumer_identity.launch_nonce,
4017 ) {
4018 return Ok(Ok(Principal::Reserved {
4019 module_id: consumer_identity.module_id,
4020 }));
4021 }
4022
4023 Ok(Err(control_error_frame(
4024 frame,
4025 "bad_consumer_identity",
4026 format!(
4027 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4028 consumer_identity.module_id
4029 ),
4030 )?))
4031 }
4032
4033 fn route_open_refusal_frame(
4040 &self,
4041 ctx: &RouteCtx,
4042 frame: &Frame,
4043 module_id: &str,
4044 reason: &'static str,
4045 code: &'static str,
4046 message: impl Into<String>,
4047 ) -> Result<Frame, RouterError> {
4048 self.observe_route_open_refusal(ctx, module_id, reason, code);
4049 control_error_frame(frame, code, message.into())
4050 }
4051
4052 fn route_open_breaker_refusal_frame(
4070 &self,
4071 ctx: &RouteCtx,
4072 frame: &Frame,
4073 module_id: &str,
4074 consecutive_timeouts: u32,
4075 retry_in: Duration,
4076 probe_in_flight: bool,
4077 ) -> Result<Frame, RouterError> {
4078 self.counters
4079 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4080 debug!(
4081 target: "control",
4082 code = "module_timeout",
4083 module_id = ?module_id,
4084 connection_id = ctx.connection_id.get(),
4085 consecutive_timeouts,
4086 retry_in_ms = retry_in.as_millis() as u64,
4087 probe_in_flight,
4088 "route.open refused by open bind-relay breaker"
4089 );
4090 let detail = if probe_in_flight {
4097 "one test route is already being tried; retry once it settles".to_string()
4098 } else {
4099 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4100 };
4101 control_error_frame(
4102 frame,
4103 "module_timeout",
4104 format!(
4105 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4106 timed out in a row); {detail}; its established routes are unaffected"
4107 ),
4108 )
4109 }
4110
4111 fn observe_route_open_refusal(
4124 &self,
4125 ctx: &RouteCtx,
4126 module_id: &str,
4127 reason: &'static str,
4128 code: &'static str,
4129 ) {
4130 self.counters.increment_route_open_refused(code);
4131 info!(
4132 target: "control",
4133 code,
4134 reason,
4135 module_id = ?module_id,
4136 connection_id = ctx.connection_id.get(),
4137 "route.open refused"
4138 );
4139 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4140 self.route_outages.record_not_serving(module_id, reason);
4141 }
4142 }
4143
4144 fn observe_route_open_accept(
4189 &self,
4190 ctx: &RouteCtx,
4191 module_id: &str,
4192 principal: &str,
4193 scope: &str,
4194 ) {
4195 self.route_outages.record_accepted(module_id);
4196 self.counters.increment_route_open_accepted(principal);
4197 info!(
4198 target: "control",
4199 principal,
4200 module_id,
4201 connection_id = ctx.connection_id.get(),
4202 scope = %scope,
4203 "route.open accepted"
4204 );
4205 }
4206
4207 fn supervised_absent_route_open_refusal_frame(
4208 &self,
4209 ctx: &RouteCtx,
4210 frame: &Frame,
4211 module_id: &str,
4212 code: &'static str,
4213 status: &crate::supervise::ModuleStatus,
4214 ) -> Result<Frame, RouterError> {
4215 self.counters.increment_route_open_refused(code);
4216 info!(
4217 target: "control",
4218 code,
4219 reason = "supervised_not_registered",
4220 module_id = ?module_id,
4221 connection_id = ctx.connection_id.get(),
4222 state = %status.state,
4223 enabled = status.enabled,
4224 live = status.live,
4225 "route.open refused"
4226 );
4227 self.route_outages
4231 .record_not_serving(module_id, "supervised_not_registered");
4232 let message = if let Some(pid) = status.waiting_for_previous_pid {
4233 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4234 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4235 })
4236 } else {
4237 format!(
4238 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4239 status.state, status.enabled, status.live
4240 )
4241 };
4242 control_error_frame(frame, code, message)
4243 }
4244
4245 async fn handle_route_open(
4246 &self,
4247 ctx: &RouteCtx,
4248 frame: Frame,
4249 request: RouteOpenRequest,
4250 timings: &mut RouteOpenTimings,
4251 ) -> Result<Vec<Frame>, RouterError> {
4252 let RouteOpenRequest {
4253 target,
4254 mut identity,
4255 consumer_identity,
4256 consumer_capabilities,
4257 role_versions,
4258 admission_facts,
4259 scope,
4260 } = request;
4261 let target_module_id = target_module_id(&target).to_string();
4262 timings.module_id.clone_from(&target_module_id);
4263 if self
4264 .registry
4265 .get_module_by_connection(ctx.connection_id)
4266 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4267 .is_some()
4268 {
4269 return Ok(vec![control_error_frame(
4270 &frame,
4271 "invalid_request",
4272 "module connections cannot open client routes",
4273 )?]);
4274 }
4275 debug!(
4276 connection_id = ctx.connection_id.get(),
4277 corr = frame.header.corr,
4278 module_id = %target_module_id,
4279 "handling route.open"
4280 );
4281
4282 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4288 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4289 self.observe_route_open_refusal(
4290 ctx,
4291 &target_module_id,
4292 "invalid_role_versions",
4293 error_codes::INVALID_REQUEST,
4294 );
4295 return Ok(vec![control_error_body_frame(
4296 &frame,
4297 ErrorBody {
4298 code: error_codes::INVALID_REQUEST.to_string(),
4299 message: error.to_string(),
4300 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4301 },
4302 )?]);
4303 }
4304
4305 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4306 if status.waiting_for_previous_pid.is_some() {
4307 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4308 ctx,
4309 &frame,
4310 &target_module_id,
4311 "module_warming",
4312 &status,
4313 )?]);
4314 }
4315 }
4316
4317 let Some(registration) = self
4336 .registry
4337 .get_module(&target_module_id)
4338 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4339 else {
4340 if let Some((status, warming)) =
4341 self.supervisor_status(&target_module_id, frame.header.corr)?
4342 {
4343 if status.protocol == ModuleProtocol::None {
4350 return Ok(vec![self.route_open_refusal_frame(
4351 ctx,
4352 &frame,
4353 &target_module_id,
4354 "protocol_none",
4355 error_codes::MODULE_NO_PROTOCOL,
4356 format!(
4357 "module_id '{target_module_id}' is declared protocol: none; \
4358 it speaks no subc wire and serves no routes"
4359 ),
4360 )?]);
4361 }
4362 let code = if warming {
4363 "module_warming"
4364 } else {
4365 "target_unavailable"
4366 };
4367 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4368 ctx,
4369 &frame,
4370 &target_module_id,
4371 code,
4372 &status,
4373 )?]);
4374 }
4375 if let Some(removed_ago_ms) =
4376 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4377 {
4378 return Ok(vec![self.route_open_refusal_frame(
4379 ctx,
4380 &frame,
4381 &target_module_id,
4382 "removed",
4383 error_codes::MODULE_REMOVED,
4384 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4385 )?]);
4386 }
4387 return Ok(vec![self.route_open_refusal_frame(
4388 ctx,
4389 &frame,
4390 &target_module_id,
4391 "not_registered",
4392 error_codes::UNKNOWN_MODULE,
4393 format!("module_id '{target_module_id}' is not registered"),
4394 )?]);
4395 };
4396
4397 if !registration.ready {
4401 self.counters
4402 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4403 info!(
4404 target: "control",
4405 code = error_codes::MODULE_WARMING,
4406 module_id = ?target_module_id,
4407 connection_id = ctx.connection_id.get(),
4408 reason = "declared_not_ready",
4409 "route.open refused"
4410 );
4411 self.route_outages
4414 .record_not_serving(&target_module_id, "declared_not_ready");
4415 return Ok(vec![control_error_body_frame(
4416 &frame,
4417 ErrorBody {
4418 code: error_codes::MODULE_WARMING.to_string(),
4419 message: format!(
4420 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4421 ),
4422 detail: Some(serde_json::json!({
4423 "reason": "declared_not_ready"
4424 })),
4425 },
4426 )?]);
4427 }
4428
4429 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4448 self.counters
4449 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4450 info!(
4451 target: "control",
4452 code = error_codes::MODULE_WARMING,
4453 module_id = ?target_module_id,
4454 connection_id = ctx.connection_id.get(),
4455 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4456 capability = %capability,
4457 "route.open refused"
4458 );
4459 return Ok(vec![control_error_body_frame(
4460 &frame,
4461 ErrorBody {
4462 code: error_codes::MODULE_WARMING.to_string(),
4463 message: format!(
4464 "module_id '{target_module_id}' requires capability '{capability}', \
4465 which no registered module provides; retry"
4466 ),
4467 detail: Some(serde_json::json!({
4468 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4469 "capability": capability,
4470 })),
4471 },
4472 )?]);
4473 }
4474
4475 if !target_has_required_role(&target, ®istration.manifest.provides) {
4476 return Ok(vec![self.route_open_refusal_frame(
4477 ctx,
4478 &frame,
4479 &target_module_id,
4480 "role_not_provided",
4481 "target_unavailable",
4482 format!("module_id '{target_module_id}' does not provide the requested target"),
4483 )?]);
4484 }
4485
4486 if registration.state != ChannelState::Active {
4487 return Ok(vec![self.route_open_refusal_frame(
4488 ctx,
4489 &frame,
4490 &target_module_id,
4491 "registration_not_active",
4492 "target_unavailable",
4493 format!("module_id '{target_module_id}' is not active"),
4494 )?]);
4495 }
4496
4497 if self
4498 .forwarding
4499 .module_is_draining(&target_module_id)
4500 .map_err(RouterError::Forwarding)?
4501 {
4502 return Ok(vec![self.route_open_refusal_frame(
4503 ctx,
4504 &frame,
4505 &target_module_id,
4506 "reloading",
4507 "module_reloading",
4508 format!("module_id '{target_module_id}' is reloading"),
4509 )?]);
4510 }
4511
4512 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4513 process_liveness.process_live(&target_module_id) == Some(false)
4514 }) {
4515 if process_liveness.process_replacing(&target_module_id) {
4522 return Ok(vec![self.route_open_refusal_frame(
4523 ctx,
4524 &frame,
4525 &target_module_id,
4526 "reloading",
4527 "module_reloading",
4528 format!("module_id '{target_module_id}' is reloading"),
4529 )?]);
4530 }
4531 return Ok(vec![self.route_open_refusal_frame(
4532 ctx,
4533 &frame,
4534 &target_module_id,
4535 "supervisor_not_live",
4536 "target_unavailable",
4537 format!("module_id '{target_module_id}' is not live"),
4538 )?]);
4539 }
4540
4541 if !self
4542 .forwarding
4543 .has_live_module_connection(&target_module_id)
4544 .map_err(RouterError::Forwarding)?
4545 {
4546 return Ok(vec![self.route_open_refusal_frame(
4547 ctx,
4548 &frame,
4549 &target_module_id,
4550 "no_forwarding_connection",
4551 "target_unavailable",
4552 format!("module_id '{target_module_id}' has no live forwarding connection"),
4553 )?]);
4554 }
4555
4556 if let Some(error) =
4557 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4558 {
4559 self.observe_route_open_refusal(
4560 ctx,
4561 &target_module_id,
4562 "op_not_allowed",
4563 "op_not_allowed",
4564 );
4565 return Ok(vec![error]);
4566 }
4567
4568 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4569 Ok(principal) => principal,
4570 Err(error) => {
4571 self.observe_route_open_refusal(
4572 ctx,
4573 &target_module_id,
4574 "bad_consumer_identity",
4575 "bad_consumer_identity",
4576 );
4577 return Ok(vec![error]);
4578 }
4579 };
4580 timings.principal = principal_label(&principal);
4581
4582 if let Principal::Reserved {
4586 module_id: opening_module_id,
4587 } = &principal
4588 {
4589 if let Some(opening_registration) = self
4590 .registry
4591 .get_module(opening_module_id)
4592 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4593 {
4594 if let Some(capability) =
4595 denied_capability(&opening_registration.manifest, ®istration.manifest)
4596 {
4597 warn!(
4598 opening_module_id,
4599 target_module_id,
4600 capability,
4601 "refusing route.open because an attested capability deny edge matches"
4602 );
4603 return Ok(vec![self.route_open_refusal_frame(
4604 ctx,
4605 &frame,
4606 &target_module_id,
4607 "capability_deny_edge",
4608 "capability_forbidden",
4609 format!(
4610 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4611 ),
4612 )?]);
4613 }
4614 }
4615 }
4616
4617 if admission_facts.is_some() {
4618 let carrier_matches = matches!(
4619 &principal,
4620 Principal::Reserved { module_id }
4621 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4622 );
4623 if !carrier_matches {
4624 return Ok(vec![self.route_open_refusal_frame(
4625 ctx,
4626 &frame,
4627 &target_module_id,
4628 "admission_facts_carrier_not_permitted",
4629 "admission_facts_not_permitted",
4630 "admission facts may only be carried by the configured reserved module",
4631 )?]);
4632 }
4633
4634 let target_allowed = self
4635 .admission_facts_targets
4636 .as_ref()
4637 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4638 if !target_allowed {
4639 return Ok(vec![self.route_open_refusal_frame(
4640 ctx,
4641 &frame,
4642 &target_module_id,
4643 "admission_facts_target_not_listed",
4644 "admission_facts_target_not_allowed",
4645 format!(
4646 "admission facts are not permitted for target module_id '{target_module_id}'"
4647 ),
4648 )?]);
4649 }
4650
4651 }
4655
4656 let (bound_scope, mut scope_stamp) = match &scope {
4662 None => (None, None),
4663 Some(selector) => {
4664 let owner_configured = match &selector.owner {
4665 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4671 _ => false,
4672 };
4673 let mut scopes = self
4674 .scopes
4675 .write()
4676 .unwrap_or_else(|poisoned| poisoned.into_inner());
4677 let admitted = scopes
4678 .admit(&principal, &target_module_id, selector, owner_configured)
4679 .and_then(|admission| {
4680 crate::scopes::check_target_flow_support(
4681 &admission.stamp,
4682 &target_module_id,
4683 registration.manifest.capabilities.as_ref(),
4684 )?;
4685 crate::scopes::check_target_agent_run_support(
4686 &admission.stamp,
4687 &target_module_id,
4688 registration.manifest.capabilities.as_ref(),
4689 )?;
4690 let bound = scopes.reserve_bind(
4691 &admission,
4692 &principal,
4693 ctx.connection_id,
4694 selector.hold_presence,
4695 );
4696 Ok((admission, bound))
4697 });
4698 match admitted {
4699 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4700 Err(refusal) => {
4701 return Ok(vec![self.route_open_refusal_frame(
4702 ctx,
4703 &frame,
4704 &target_module_id,
4705 refusal.code,
4706 refusal.code,
4707 refusal.message,
4708 )?]);
4709 }
4710 }
4711 }
4712 };
4713
4714 timings.enter(RouteOpenPhase::ProjectRoot);
4736 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4737 Ok(project_root) => project_root,
4738 Err(err) => {
4739 return Ok(vec![control_error_frame(
4740 &frame,
4741 "invalid_project_root",
4742 err.to_string(),
4743 )?])
4744 }
4745 };
4746 timings.enter(RouteOpenPhase::Admission);
4747 identity.project_root = project_root.as_path().to_path_buf();
4748
4749 let _concurrency_guard = match self
4760 .route_bind_concurrency
4761 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4762 {
4763 Ok(guard) => guard,
4764 Err(in_flight) => {
4765 return Ok(vec![self.route_open_target_capacity_refusal(
4766 ctx,
4767 &frame,
4768 &target_module_id,
4769 in_flight,
4770 )?]);
4771 }
4772 };
4773
4774 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4777 RouteBindAdmission::Admitted { guard, probe } => {
4778 if probe {
4779 info!(
4780 module_id = %target_module_id,
4781 connection_id = ctx.connection_id.get(),
4782 "route.bind breaker half-open: admitting one probe"
4783 );
4784 }
4785 guard
4786 }
4787 RouteBindAdmission::Refused {
4788 consecutive_timeouts,
4789 retry_in,
4790 probe_in_flight,
4791 } => {
4792 return Ok(vec![self.route_open_breaker_refusal_frame(
4793 ctx,
4794 &frame,
4795 &target_module_id,
4796 consecutive_timeouts,
4797 retry_in,
4798 probe_in_flight,
4799 )?]);
4800 }
4801 };
4802
4803 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4808 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4809 timings.enter(RouteOpenPhase::EgressReservation);
4810 let pending = match self
4811 .forwarding
4812 .begin_route_bind_relay_for(
4813 ctx.connection_id,
4814 ctx.egress.clone(),
4815 response_version(&frame),
4816 frame.header.corr,
4817 &target_module_id,
4818 principal.clone(),
4819 bound_scope,
4820 Some(project_root),
4821 relay_deadline,
4822 )
4823 .await
4824 {
4825 Ok(pending) => pending,
4826 Err(err) => {
4827 return Ok(vec![self.route_open_refusal_frame(
4828 ctx,
4829 &frame,
4830 &target_module_id,
4831 "relay_reservation_failed",
4832 forwarding_error_code(&err),
4833 err.to_string(),
4834 )?])
4835 }
4836 };
4837 timings.enter(RouteOpenPhase::Admission);
4838 let crate::forwarding::PendingRouteBindRelay {
4839 endpoint,
4840 module_sink,
4841 negotiated_ver,
4842 client_channel,
4843 client_epoch,
4844 module_channel,
4845 module_epoch,
4846 corr: relay_corr,
4847 receiver,
4848 } = pending;
4849 let mut reservation =
4850 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4851
4852 let relay_registration = self
4857 .registry
4858 .get_module_by_connection(endpoint.connection_id)
4859 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4860 let relay_capabilities = relay_registration
4861 .as_ref()
4862 .and_then(|registration| registration.manifest.capabilities.as_ref());
4863 if let Some(selector) = &scope {
4864 if let Err(refusal) = self
4867 .scopes
4868 .read()
4869 .unwrap_or_else(|poisoned| poisoned.into_inner())
4870 .check_bind_proof(&principal, selector)
4871 {
4872 reservation.release_and_disarm();
4873 return Ok(vec![self.route_open_refusal_frame(
4874 ctx,
4875 &frame,
4876 &target_module_id,
4877 refusal.code,
4878 refusal.code,
4879 refusal.message,
4880 )?]);
4881 }
4882 }
4883 if let Some(stamp) = scope_stamp
4884 .as_ref()
4885 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4886 {
4887 if let Err(refusal) = crate::scopes::check_target_flow_support(
4888 stamp,
4889 &target_module_id,
4890 relay_capabilities,
4891 )
4892 .and_then(|()| {
4893 crate::scopes::check_target_agent_run_support(
4894 stamp,
4895 &target_module_id,
4896 relay_capabilities,
4897 )
4898 }) {
4899 reservation.release_and_disarm();
4900 return Ok(vec![self.route_open_refusal_frame(
4901 ctx,
4902 &frame,
4903 &target_module_id,
4904 refusal.code,
4905 refusal.code,
4906 refusal.message,
4907 )?]);
4908 }
4909 }
4910
4911 if !relay_capabilities.is_some_and(|caps| {
4914 caps.provides
4915 .iter()
4916 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
4917 }) {
4918 if let Some(stamp) = scope_stamp.as_mut() {
4919 stamp.workdir = None;
4920 stamp.project_id = None;
4921 }
4922 }
4923 if !relay_capabilities.is_some_and(|caps| {
4924 caps.provides
4925 .iter()
4926 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4927 }) {
4928 if let Some(stamp) = scope_stamp.as_mut() {
4929 stamp.attributes.on_behalf_of_agent = None;
4930 }
4931 }
4932 let call_mode = if relay_capabilities
4933 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4934 {
4935 Some(derive_call_mode(scope_stamp.as_ref()))
4936 } else {
4937 if let Some(stamp) = scope_stamp.as_mut() {
4938 stamp.expires_at_ms = None;
4939 }
4940 None
4941 };
4942
4943 debug!(
4944 connection_id = ctx.connection_id.get(),
4945 client_channel,
4946 client_epoch,
4947 module_channel,
4948 module_epoch,
4949 "reserved route handle pair"
4950 );
4951 let route_principal_label = principal_label(&principal);
4954 let scope_label = scope_stamp
4955 .as_ref()
4956 .map_or_else(|| "none".to_string(), route_open_scope_label);
4957 let relay = ModuleControlRequest::RouteBind {
4958 route_channel: module_channel,
4959 epoch: module_epoch,
4960 target,
4961 identity,
4962 principal: Some(principal),
4963 consumer_capabilities,
4964 role_versions,
4965 admission_facts,
4966 scope: scope_stamp,
4967 call_mode,
4968 };
4969 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4970 RouterError::backend(
4971 0,
4972 frame.header.corr,
4973 format!("failed to encode route.bind request: {err}"),
4974 )
4975 })?;
4976 let relay_frame = Frame::build_with_version(
4977 negotiated_ver,
4978 FrameType::Request,
4979 control_flags(),
4980 0,
4981 0,
4982 relay_corr,
4983 relay_body,
4984 )
4985 .map_err(RouterError::FrameBuild)?;
4986
4987 timings.enter(RouteOpenPhase::BindSend);
4988 let sent = module_sink.send(relay_frame).await;
4989 timings.enter(RouteOpenPhase::Admission);
4990 if let Err(err) = sent {
4991 reservation.release_and_disarm();
4992 return Ok(vec![self.route_open_refusal_frame(
4993 ctx,
4994 &frame,
4995 &target_module_id,
4996 "relay_send_failed",
4997 "target_unavailable",
4998 err.to_string(),
4999 )?]);
5000 }
5001
5002 #[cfg(test)]
5003 if let Some(hold) = &self.route_bind_relay_sent_hold {
5004 hold.wait().await;
5008 hold.wait().await;
5009 }
5010
5011 if self
5016 .forwarding
5017 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5018 .map_err(RouterError::Forwarding)?
5019 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5020 {
5021 self.send_abandoned_route_bind_goodbye(
5022 &module_sink,
5023 negotiated_ver,
5024 module_channel,
5025 module_epoch,
5026 );
5027 }
5028
5029 timings.enter(RouteOpenPhase::BindAckWait);
5030 let outcome = timeout_at(relay_deadline, receiver).await;
5031 timings.enter(RouteOpenPhase::Admission);
5032 match outcome {
5033 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5034 reservation.disarm();
5035 if breaker.record_accepted() {
5036 info!(
5037 module_id = %target_module_id,
5038 "route.bind breaker closed: the probe was accepted"
5039 );
5040 }
5041 self.observe_route_open_accept(
5042 ctx,
5043 &target_module_id,
5044 &route_principal_label,
5045 &scope_label,
5046 );
5047 Ok(Vec::new())
5048 }
5049 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5050 reservation.release_and_disarm();
5051 breaker.record_inconclusive();
5055 let scope_code = match body.code.as_str() {
5059 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5060 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5061 _ => None,
5062 };
5063 if let Some(code) = scope_code {
5064 self.observe_route_open_refusal(
5065 ctx,
5066 &target_module_id,
5067 "scope_changed_before_commit",
5068 code,
5069 );
5070 return Ok(vec![control_error_body_frame(&frame, body)?]);
5071 }
5072 self.counters
5073 .increment_route_open_refused("module_rejected");
5074 info!(
5075 target: "control",
5076 code = "module_rejected",
5077 module_code = ?body.code,
5078 module_id = ?target_module_id,
5079 connection_id = ctx.connection_id.get(),
5080 "route.open refused"
5081 );
5082 Ok(vec![control_error_body_frame(&frame, body)?])
5083 }
5084 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5085 reservation.release_and_disarm();
5086 breaker.record_inconclusive();
5087 tracing::warn!(
5094 module_id = %target_module_id,
5095 "route.bind relay abandoned: {message}"
5096 );
5097 Ok(vec![self.route_open_refusal_frame(
5098 ctx,
5099 &frame,
5100 &target_module_id,
5101 "relay_abandoned",
5102 "target_unavailable",
5103 message,
5104 )?])
5105 }
5106 Ok(Err(_)) => {
5107 reservation.release_and_disarm();
5108 breaker.record_inconclusive();
5109 Ok(vec![self.route_open_refusal_frame(
5110 ctx,
5111 &frame,
5112 &target_module_id,
5113 "relay_waiter_canceled",
5114 "target_unavailable",
5115 "route.bind relay waiter was canceled before the module responded",
5116 )?])
5117 }
5118 Err(_) => {
5119 reservation.release_and_disarm();
5120 if let Some(opened) = breaker.record_timeout(
5124 self.route_bind_breaker_threshold,
5125 self.route_bind_breaker_cooldown,
5126 ) {
5127 warn!(
5128 module_id = %target_module_id,
5129 consecutive_timeouts = opened.consecutive_timeouts,
5130 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5131 reopened_after_probe = opened.reopened_after_probe,
5132 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5133 );
5134 }
5135 tracing::warn!(
5143 module_id = %target_module_id,
5144 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5145 "route.bind relay timed out: module did not ack within budget"
5146 );
5147 Ok(vec![self.route_open_refusal_frame(
5148 ctx,
5149 &frame,
5150 &target_module_id,
5151 "relay_timed_out",
5152 "module_timeout",
5153 format!(
5154 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5155 route_bind_relay_timeout
5156 ),
5157 )?])
5158 }
5159 }
5160 }
5161
5162 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5163 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5164 snapshot: self.supervisor.spawn_snapshot(),
5165 };
5166 Ok(vec![control_response_body_frame(
5167 &frame,
5168 &response,
5169 "ClientControlResponse::SupervisorSpawnSnapshot",
5170 )?])
5171 }
5172
5173 fn handle_supervisor_spawn_subscribe(
5174 &self,
5175 ctx: &RouteCtx,
5176 frame: Frame,
5177 since: Option<SpawnCursor>,
5178 ) -> Result<Vec<Frame>, RouterError> {
5179 match self.supervisor.subscribe_spawns(
5180 ctx.connection_id,
5181 frame.header.corr,
5182 response_version(&frame),
5183 since,
5184 ctx.egress.clone(),
5185 ) {
5186 Ok(()) => Ok(Vec::new()),
5187 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5188 Ok(vec![control_error_body_frame(
5189 &frame,
5190 ErrorBody {
5191 code: "spawn_cursor_incarnation_mismatch".to_string(),
5192 message: "spawn cursor belongs to a different daemon incarnation"
5193 .to_string(),
5194 detail: Some(serde_json::json!({
5195 "current_daemon_incarnation": current
5196 })),
5197 },
5198 )?])
5199 }
5200 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5201 &frame,
5202 ErrorBody {
5203 code: "spawn_cursor_too_old".to_string(),
5204 message: "spawn cursor predates the retained event ring".to_string(),
5205 detail: Some(serde_json::json!({
5206 "oldest_retained_cursor": oldest
5207 })),
5208 },
5209 )?]),
5210 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5211 0,
5212 frame.header.corr,
5213 format!("failed to open supervisor spawn subscription: {error}"),
5214 )),
5215 }
5216 }
5217
5218 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5219 let generation = self
5220 .registry
5221 .generation()
5222 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5223 let mut modules = Vec::new();
5224 for module in self.supervisor.list() {
5225 let status = module.status_for_control("list").map_err(|err| {
5226 RouterError::backend(
5227 0,
5228 frame.header.corr,
5229 format!("failed to read supervisor status: {err}"),
5230 )
5231 })?;
5232 let (configured, _) = module.configuration().map_err(|err| {
5233 RouterError::backend(
5234 0,
5235 frame.header.corr,
5236 format!("failed to read module configuration: {err}"),
5237 )
5238 })?;
5239 let image = module.running_image_agreement().await;
5241 let resources = Some(module.child_resource_usage());
5244 let mut verdict =
5245 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5246 verdict.spec = Some(launch_spec_agreement(
5247 &configured,
5248 status.process_alive,
5249 status.launch_spec_digest.as_ref(),
5250 ));
5251 let pending_reload = Some(verdict);
5252 modules.push(
5255 SupervisorEntry::new(
5256 status.module_id,
5257 status.state.to_string(),
5258 status.enabled,
5259 status.live,
5260 status.health.status,
5261 )
5262 .with_launch_nonce_env(Some(!cfg!(unix)))
5263 .with_launch_nonce_source(status.launch_nonce_source)
5264 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5265 .with_protocol(status.protocol)
5266 .with_pending_reload(pending_reload)
5267 .with_last_probe_ms(status.health.last_probe_ms)
5268 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5269 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5270 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5271 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5272 .with_restart_count(Some(status.restart_count))
5273 .with_max_restarts(Some(status.max_restarts))
5274 .with_lifetime_restarts(Some(status.lifetime_restarts))
5275 .with_spawn_generation(Some(status.spawn_generation))
5276 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5277 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5278 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5279 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5280 .with_resources(resources),
5281 );
5282 }
5283 let response = ClientControlResponse::SupervisorList {
5284 generation,
5285 modules,
5286 };
5287 Ok(vec![control_response_body_frame(
5288 &frame,
5289 &response,
5290 "ClientControlResponse::SupervisorList",
5291 )?])
5292 }
5293
5294 fn handle_supervisor_stderr_tail(
5295 &self,
5296 frame: Frame,
5297 module_id: String,
5298 max_lines: Option<u32>,
5299 max_bytes: Option<u32>,
5300 ) -> Result<Vec<Frame>, RouterError> {
5301 let Some(module) = self.supervisor.get(&module_id) else {
5302 return Ok(vec![control_error_frame(
5303 &frame,
5304 "unknown_module",
5305 format!("module_id '{module_id}' is not supervised"),
5306 )?]);
5307 };
5308
5309 let snapshot = module.stderr_tail(
5310 max_lines.map(|value| value as usize),
5311 max_bytes.map(|value| value as usize),
5312 );
5313
5314 let response = ClientControlResponse::SupervisorStderrTail {
5315 module_id,
5316 tail: StderrTail {
5317 capture: match snapshot.capture {
5318 CaptureState::Captured => StderrCaptureState::Captured,
5319 CaptureState::Incomplete { reason } => {
5320 StderrCaptureState::Incomplete { reason }
5321 }
5322 CaptureState::NotCaptured { reason } => {
5323 StderrCaptureState::NotCaptured { reason }
5324 }
5325 },
5326 entries: snapshot
5327 .entries
5328 .into_iter()
5329 .map(|entry| match entry {
5330 TailEntry::Line {
5331 text,
5332 truncated,
5333 at_ms,
5334 } => StderrTailEntry::Line {
5335 text,
5336 truncated,
5337 at_ms,
5338 },
5339 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5340 })
5341 .collect(),
5342 dropped_lines: snapshot.dropped_lines,
5343 },
5344 };
5345 Ok(vec![control_response_body_frame(
5346 &frame,
5347 &response,
5348 "ClientControlResponse::SupervisorStderrTail",
5349 )?])
5350 }
5351
5352 async fn handle_supervisor_terminals(
5353 &self,
5354 frame: Frame,
5355 module_id: String,
5356 ) -> Result<Vec<Frame>, RouterError> {
5357 let Some(module) = self.supervisor.get(&module_id) else {
5358 return Ok(vec![control_error_frame(
5359 &frame,
5360 "unknown_module",
5361 format!("module_id '{module_id}' is not supervised"),
5362 )?]);
5363 };
5364
5365 let terminals = module
5368 .read_durable_terminal_history()
5369 .await
5370 .map_err(|error| {
5371 RouterError::backend(
5372 0,
5373 frame.header.corr,
5374 format!("failed to read terminal history: {error}"),
5375 )
5376 })?;
5377 let response = ClientControlResponse::SupervisorTerminals {
5378 module_id,
5379 terminals,
5380 };
5381 Ok(vec![control_response_body_frame(
5382 &frame,
5383 &response,
5384 "ClientControlResponse::SupervisorTerminals",
5385 )?])
5386 }
5387
5388 fn handle_supervisor_routes(
5389 &self,
5390 frame: Frame,
5391 module_id: Option<String>,
5392 ) -> Result<Vec<Frame>, RouterError> {
5393 let modules = self
5394 .forwarding
5395 .route_census(module_id.as_deref())
5396 .map_err(RouterError::Forwarding)?
5397 .into_iter()
5398 .map(|(module_id, routes)| SupervisorRouteModule {
5399 module_id,
5400 routes: routes
5401 .into_iter()
5402 .map(|route| SupervisorRoute {
5403 consumer: match route.principal {
5404 Principal::Reserved { module_id } => {
5405 SupervisorRouteConsumer::Reserved { module_id }
5406 }
5407 Principal::Direct | Principal::Unverified => {
5408 SupervisorRouteConsumer::Direct {
5409 connection_id: route.goodbye_target.connection_id.get(),
5410 }
5411 }
5412 },
5413 age_ms: Instant::now()
5414 .saturating_duration_since(route.bound_at)
5415 .as_millis()
5416 .try_into()
5417 .unwrap_or(u64::MAX),
5418 draining: route.draining,
5419 drain_reason: route.drain_reason,
5420 })
5421 .collect(),
5422 })
5423 .collect();
5424 let response = ClientControlResponse::SupervisorRoutes { modules };
5425 Ok(vec![control_response_body_frame(
5426 &frame,
5427 &response,
5428 "ClientControlResponse::SupervisorRoutes",
5429 )?])
5430 }
5431
5432 async fn handle_supervisor_provenance(
5433 &self,
5434 frame: Frame,
5435 module_id: Option<String>,
5436 ) -> Result<Vec<Frame>, RouterError> {
5437 let mut selected = if let Some(module_id) = module_id {
5438 let Some(module) = self.supervisor.get(&module_id) else {
5439 return Ok(vec![control_error_frame(
5440 &frame,
5441 "unknown_module",
5442 format!("module_id '{module_id}' is not supervised"),
5443 )?]);
5444 };
5445 vec![module]
5446 } else {
5447 self.supervisor.list()
5448 };
5449
5450 let mut modules = Vec::with_capacity(selected.len());
5451 for module in selected.drain(..) {
5452 let (status, observed_image) = module
5453 .status_and_running_image_agreement()
5454 .await
5455 .map_err(|err| {
5456 RouterError::backend(
5457 0,
5458 frame.header.corr,
5459 format!("failed to read supervisor status: {err}"),
5460 )
5461 })?;
5462 let module_declared = self
5463 .registry
5464 .get_module(&status.module_id)
5465 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5466 .and_then(|registration| registration.manifest.provenance)
5467 .map(|build| ModuleDeclaredProvenance::Reported { build })
5468 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5469 #[cfg(test)]
5470 let running_image = match &self.provenance_probe_override {
5471 Some(result) => result.clone(),
5472 None => observed_image,
5473 };
5474 #[cfg(not(test))]
5475 let running_image = observed_image;
5476 modules.push(SupervisorModuleProvenance {
5477 module_id: status.module_id,
5478 module_declared,
5479 daemon_observed: SupervisorObservedProcess {
5480 launch_nonce_source: status.launch_nonce_source,
5481 pid: status.pid,
5482 spawned_at_ms: status.spawned_at_ms,
5483 spawned_from: status.spawned_from,
5484 running_image,
5485 },
5486 });
5487 }
5488 let daemon = SupervisorDaemonProvenance {
5489 daemon_build: self.daemon_provenance.build.clone(),
5490 daemon_observed: DaemonObservedProcess {
5491 pid: self.daemon_provenance.pid,
5492 started_at_ms: self
5493 .daemon_provenance
5494 .start_clock
5495 .map(|clock| clock.started_at_ms())
5496 .or(self.daemon_provenance.started_at_ms),
5497 running_image: self
5498 .daemon_provenance
5499 .probe
5500 .observe(
5501 self.daemon_provenance.pid,
5502 self.daemon_provenance.executable_path.as_deref(),
5503 self.daemon_provenance.executable_identity.clone(),
5504 self.daemon_provenance.process_start_time,
5505 )
5506 .await,
5507 },
5508 };
5509 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5510 Ok(vec![control_response_body_frame(
5511 &frame,
5512 &response,
5513 "ClientControlResponse::SupervisorProvenance",
5514 )?])
5515 }
5516
5517 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5518 self.refresh_capability_requirements();
5519 let generation = self
5520 .registry
5521 .generation()
5522 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5523 let modules = self
5524 .supervisor
5525 .list()
5526 .into_iter()
5527 .map(|module| {
5528 let status = module.status_for_control("health").map_err(|err| {
5529 RouterError::backend(
5530 0,
5531 frame.header.corr,
5532 format!("failed to read supervisor health: {err}"),
5533 )
5534 })?;
5535 let module_id = status.module_id;
5536 let capability_detail = self
5537 .capability_evaluator
5538 .required_problem_detail(&module_id);
5539 Ok(SupervisorHealthEntry {
5540 module_id,
5541 status: status.health.status,
5542 detail: append_capability_problem_detail(
5543 status.health.detail,
5544 capability_detail,
5545 ),
5546 metrics: status.health.metrics,
5547 consecutive_failures: status.health.consecutive_failures,
5548 late_answer_count: status.health.late_answer_count,
5549 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5550 last_action: status.health.last_action,
5551 last_action_ms: status.health.last_action_ms,
5552 last_probe_ms: status.health.last_probe_ms,
5553 })
5554 })
5555 .collect::<Result<Vec<_>, RouterError>>()?;
5556 let response = ClientControlResponse::SupervisorHealth {
5557 generation,
5558 modules,
5559 };
5560 Ok(vec![control_response_body_frame(
5561 &frame,
5562 &response,
5563 "ClientControlResponse::SupervisorHealth",
5564 )?])
5565 }
5566
5567 async fn handle_supervisor_restart(
5568 &self,
5569 frame: Frame,
5570 module_id: String,
5571 drain_timeout_ms: Option<u64>,
5572 requester: Principal,
5573 ) -> Result<Vec<Frame>, RouterError> {
5574 let operation_lock = self.supervisor.operation_lock();
5575 let _operation_guard = operation_lock.lock().await;
5576 let Some(module) = self.supervisor.get(&module_id) else {
5577 return Ok(vec![control_error_frame(
5578 &frame,
5579 "unknown_module",
5580 format!("module_id '{module_id}' is not supervised"),
5581 )?]);
5582 };
5583
5584 self.route_outages.mark_operator_action(&module_id);
5585 if let Err(err) = module
5586 .restart_requested(drain_timeout_ms, Some(requester))
5587 .await
5588 {
5589 self.route_outages
5590 .operator_action_ended_unrefused(&module_id);
5591 let (code, message) = match err {
5592 crate::supervise::SuperviseError::Disabled { .. } => {
5593 ("module_disabled", err.to_string())
5594 }
5595 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5596 ("swap_in_progress", err.to_string())
5597 }
5598 _ => (
5599 "target_unavailable",
5600 format!("failed to restart module_id '{module_id}': {err}"),
5601 ),
5602 };
5603 return Ok(vec![control_error_frame(&frame, code, message)?]);
5604 }
5605
5606 let response = ClientControlResponse::SupervisorAck {
5607 module_id,
5608 applied: true,
5609 };
5610 Ok(vec![control_response_body_frame(
5611 &frame,
5612 &response,
5613 "ClientControlResponse::SupervisorAck",
5614 )?])
5615 }
5616
5617 async fn handle_supervisor_swap(
5621 &self,
5622 frame: Frame,
5623 module_id: String,
5624 ready_timeout_ms: Option<u64>,
5625 requester: Principal,
5626 ) -> Result<Vec<Frame>, RouterError> {
5627 let module = {
5634 let operation_lock = self.supervisor.operation_lock();
5635 let _operation_guard = operation_lock.lock().await;
5636 self.supervisor.get(&module_id)
5637 };
5638 let Some(module) = module else {
5639 return Ok(vec![control_error_frame(
5640 &frame,
5641 "unknown_module",
5642 format!("module_id '{module_id}' is not supervised"),
5643 )?]);
5644 };
5645
5646 self.route_outages.mark_operator_action(&module_id);
5647 if let Err(err) = module
5648 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5649 .await
5650 {
5651 self.route_outages
5652 .operator_action_ended_unrefused(&module_id);
5653 use crate::supervise::SuperviseError;
5654 let message = err.to_string();
5655 let error = match err {
5656 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5657 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5658 code: "swap_refused".to_string(),
5659 message,
5660 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5661 },
5662 SuperviseError::SwapFailed {
5663 arm,
5664 candidate_exit,
5665 ..
5666 } => ErrorBody {
5667 code: "swap_failed".to_string(),
5668 message,
5669 detail: Some(serde_json::json!({
5670 "arm": arm.as_str(),
5671 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5672 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5673 })),
5674 },
5675 _ => ErrorBody::new(
5676 "target_unavailable",
5677 format!("failed to swap module_id '{module_id}': {message}"),
5678 ),
5679 };
5680 return Ok(vec![control_error_body_frame(&frame, error)?]);
5681 }
5682 self.route_outages
5686 .operator_action_ended_unrefused(&module_id);
5687
5688 let response = ClientControlResponse::SupervisorAck {
5689 module_id,
5690 applied: true,
5691 };
5692 Ok(vec![control_response_body_frame(
5693 &frame,
5694 &response,
5695 "ClientControlResponse::SupervisorAck",
5696 )?])
5697 }
5698
5699 async fn handle_supervisor_reload(
5700 &self,
5701 frame: Frame,
5702 module_id: String,
5703 requester: Principal,
5704 ) -> Result<Vec<Frame>, RouterError> {
5705 let operation_lock = self.supervisor.operation_lock();
5706 let _operation_guard = operation_lock.lock().await;
5707 let Some(module) = self.supervisor.get(&module_id) else {
5708 return Ok(vec![control_error_frame(
5709 &frame,
5710 "unknown_module",
5711 format!("module_id '{module_id}' is not supervised"),
5712 )?]);
5713 };
5714
5715 self.route_outages.mark_operator_action(&module_id);
5716 if let Err(err) = module.reload_requested(Some(requester)).await {
5717 self.route_outages
5718 .operator_action_ended_unrefused(&module_id);
5719 let (code, message) = match err {
5720 crate::supervise::SuperviseError::Disabled { .. } => {
5721 ("module_disabled", err.to_string())
5722 }
5723 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5724 ("swap_in_progress", err.to_string())
5725 }
5726 _ => (
5727 "reload_failed",
5728 format!("failed to reload module_id '{module_id}': {err}"),
5729 ),
5730 };
5731 return Ok(vec![control_error_frame(&frame, code, message)?]);
5732 }
5733
5734 let response = ClientControlResponse::SupervisorAck {
5735 module_id,
5736 applied: true,
5737 };
5738 Ok(vec![control_response_body_frame(
5739 &frame,
5740 &response,
5741 "ClientControlResponse::SupervisorAck",
5742 )?])
5743 }
5744
5745 async fn handle_supervisor_rescan(
5749 &self,
5750 frame: Frame,
5751 mode: RescanMode,
5752 requester: Principal,
5753 ) -> Result<Vec<Frame>, RouterError> {
5754 let preview = mode.previews();
5755 let Some(context) = self.rescan.clone() else {
5756 return Ok(vec![control_error_frame(
5757 &frame,
5758 "rescan_unavailable",
5759 "the daemon was not started with a reloadable config path".to_string(),
5760 )?]);
5761 };
5762
5763 let operation_lock = self.supervisor.operation_lock();
5764 let _operation_guard = operation_lock.lock().await;
5765 let loaded = match self.load_rescan_config(&mode, &context) {
5769 Ok(config) => config,
5770 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5771 };
5772 let Some((config, activity_config)) = loaded else {
5780 return Ok(vec![control_error_frame(
5781 &frame,
5782 "invalid_daemon_config",
5783 format!(
5784 "daemon config not found at {}; refusing to rescan (an absent config would \
5785 retire every supervised module)",
5786 context.config_path.display()
5787 ),
5788 )?]);
5789 };
5790 let (
5791 configured_port,
5792 storage_config,
5793 admission_facts_carrier_module_id,
5794 admission_facts_targets,
5795 scope_authority_owners,
5796 privileged_operator_prompt,
5797 modules,
5798 reserved_capabilities,
5799 ) = (
5800 config.port,
5801 config.storage,
5802 config.admission_facts_carrier_module_id,
5803 config.admission_facts_targets,
5804 config.scope_authority_owners,
5805 config.privileged_operator_prompt,
5806 config.modules,
5807 config.reserved_capabilities,
5808 );
5809
5810 let mut restart_required = Vec::new();
5819 for section in RestartRequiredSection::ALL {
5820 let changed = match section {
5821 RestartRequiredSection::Port => configured_port != context.configured_port,
5822 RestartRequiredSection::Storage => storage_config != context.storage_config,
5823 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5824 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5825 }
5826 RestartRequiredSection::AdmissionFactsTargets => {
5827 admission_facts_targets != context.admission_facts_targets
5828 }
5829 RestartRequiredSection::ScopeAuthorityOwners => {
5830 scope_authority_owners != context.scope_authority_owners
5831 }
5832 RestartRequiredSection::PrivilegedOperatorPrompt => {
5833 privileged_operator_prompt != context.privileged_operator_prompt
5834 }
5835 };
5836 if changed {
5837 restart_required.push(section.label().to_string());
5838 }
5839 }
5840 if !restart_required.is_empty() {
5841 warn!(
5842 config_path = %context.config_path.display(),
5843 sections = %restart_required.join(", "),
5844 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5845 );
5846 }
5847
5848 for configured in &modules {
5849 if let Err(err) = validate_spec(&configured.module_spec()) {
5850 return Ok(vec![control_error_frame(
5851 &frame,
5852 "invalid_daemon_config",
5853 format!("supervisor rescan rejected daemon config: {err}"),
5854 )?]);
5855 }
5856 }
5857
5858 let configured_capabilities = modules
5859 .iter()
5860 .map(|module| (module.module_id.clone(), module.enabled))
5861 .collect::<Vec<_>>();
5862 let preview_capability_warnings = if preview {
5863 let (_, registrations) = self.runtime_capability_snapshot()?;
5864 let current_modules = self
5865 .supervisor
5866 .list()
5867 .into_iter()
5868 .map(|module| module.module_id().to_string())
5869 .collect::<BTreeSet<_>>();
5870 let resulting_modules = configured_capabilities.clone();
5871 let removed = current_modules
5872 .into_iter()
5873 .filter(|module_id| {
5874 !resulting_modules
5875 .iter()
5876 .any(|(configured_id, _)| configured_id == module_id)
5877 })
5878 .collect::<Vec<_>>();
5879 let disabled = match &mode {
5882 RescanMode::Candidate(_) => resulting_modules
5883 .iter()
5884 .filter(|(_, enabled)| !enabled)
5885 .map(|(module_id, _)| module_id.clone())
5886 .collect::<Vec<_>>(),
5887 _ => Vec::new(),
5888 };
5889 self.capability_evaluator.preview_warnings(
5890 resulting_modules,
5891 &removed,
5892 &disabled,
5893 ®istrations,
5894 )
5895 } else {
5896 Vec::new()
5897 };
5898 let reconciled = match &mode {
5899 RescanMode::Candidate(candidate) => self
5900 .preview_candidate(&context, modules, candidate)
5901 .map(|result| (result, Vec::new())),
5902 _ => {
5903 self.reconcile_supervised_modules(
5904 &context.supervisor,
5905 modules,
5906 preview,
5907 requester.clone(),
5908 )
5909 .await
5910 }
5911 };
5912 let (result, changed) = match reconciled {
5913 Ok(result) => result,
5914 Err(message) => {
5915 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5916 }
5917 };
5918 if !preview {
5919 if let Some(ledger) = &self.activity {
5920 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
5921 ledger.config_changed(&result.added, &result.removed, &changed, requester);
5922 }
5923 if let Err(code) = self.configure_activity(activity_config).await {
5924 warn!(
5925 code,
5926 "activity retention could not be applied during rescan"
5927 );
5928 }
5929 self.capability_evaluator
5930 .configure(configured_capabilities, reserved_capabilities);
5931 self.capability_evaluator.wake_deadline_loop();
5932 self.refresh_capability_requirements();
5933 }
5934 let mut result = result;
5935 result.restart_required = restart_required;
5936 result.capability_warnings = preview_capability_warnings;
5937 let response = ClientControlResponse::SupervisorRescan { result };
5938 Ok(vec![control_response_body_frame(
5939 &frame,
5940 &response,
5941 "ClientControlResponse::SupervisorRescan",
5942 )?])
5943 }
5944
5945 async fn handle_supervisor_release_reserved(
5946 &self,
5947 frame: Frame,
5948 module_id: String,
5949 ) -> Result<Vec<Frame>, RouterError> {
5950 let Some(context) = self.rescan.clone() else {
5951 return Ok(vec![control_error_frame(
5952 &frame,
5953 "release_unavailable",
5954 "reserved-id release requires a daemon started with a reloadable config path",
5955 )?]);
5956 };
5957 let operation_lock = self.supervisor.operation_lock();
5958 let _operation_guard = operation_lock.lock().await;
5959 let loaded = match crate::daemon_config::load(&context.config_path) {
5960 Ok(Some(config)) => config,
5961 Ok(None) => {
5962 return Ok(vec![control_error_frame(
5963 &frame,
5964 "invalid_daemon_config",
5965 format!(
5966 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5967 context.config_path.display()
5968 ),
5969 )?])
5970 }
5971 Err(err) => {
5972 return Ok(vec![control_error_frame(
5973 &frame,
5974 "invalid_daemon_config",
5975 format!("unable to verify reserved-id release against daemon config: {err}"),
5976 )?])
5977 }
5978 };
5979 if loaded
5980 .modules
5981 .iter()
5982 .any(|configured| configured.module_id == module_id)
5983 {
5984 return Ok(vec![control_error_frame(
5985 &frame,
5986 "reserved_module_configured",
5987 format!(
5988 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5989 ),
5990 )?]);
5991 }
5992 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5993 return Ok(vec![control_error_frame(
5994 &frame,
5995 "reserved_gate_not_retained",
5996 format!(
5997 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5998 ),
5999 )?]);
6000 }
6001
6002 let response = ClientControlResponse::SupervisorAck {
6003 module_id,
6004 applied: true,
6005 };
6006 Ok(vec![control_response_body_frame(
6007 &frame,
6008 &response,
6009 "ClientControlResponse::SupervisorAck",
6010 )?])
6011 }
6012
6013 fn plan_rescan(
6017 &self,
6018 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6019 ) -> Result<RescanPlan, String> {
6020 let mut current = BTreeMap::new();
6021 for module in self.supervisor.list() {
6022 let (spec, health) = module.configuration().map_err(|err| {
6023 format!(
6024 "failed to read configuration for module_id '{}': {err}",
6025 module.module_id()
6026 )
6027 })?;
6028 let enabled = module
6029 .status()
6030 .map_err(|err| {
6031 format!(
6032 "failed to read status for module_id '{}': {err}",
6033 module.module_id()
6034 )
6035 })?
6036 .enabled;
6037 current.insert(
6038 module.module_id().to_string(),
6039 (module, spec, health, enabled),
6040 );
6041 }
6042 let configured = configured_modules
6043 .into_iter()
6044 .map(|module| (module.module_id.clone(), module))
6045 .collect::<BTreeMap<_, _>>();
6046
6047 let added = configured
6048 .keys()
6049 .filter(|module_id| !current.contains_key(*module_id))
6050 .cloned()
6051 .collect::<Vec<_>>();
6052 let removed = current
6053 .keys()
6054 .filter(|module_id| !configured.contains_key(*module_id))
6055 .cloned()
6056 .collect::<Vec<_>>();
6057 let mut changed_pending_reload = Vec::new();
6058 let mut configuration_changes = BTreeSet::new();
6059 let mut enabled_changes = BTreeSet::new();
6060 let mut unchanged = 0_u32;
6061
6062 for (module_id, configured_module) in &configured {
6063 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6064 else {
6065 continue;
6066 };
6067 let launch_changed = *current_spec != configured_module.module_spec();
6071 let configuration_changed =
6072 launch_changed || *current_health != configured_module.health;
6073 let enabled_changed = *current_enabled != configured_module.enabled;
6074 if configuration_changed {
6075 configuration_changes.insert(module_id.clone());
6076 }
6077 if launch_changed {
6078 changed_pending_reload.push(module_id.clone());
6079 }
6080 if enabled_changed {
6081 enabled_changes.insert(module_id.clone());
6082 }
6083 if !configuration_changed && !enabled_changed {
6084 unchanged = unchanged.saturating_add(1);
6085 }
6086 }
6087 Ok(RescanPlan {
6088 current,
6089 configured,
6090 added,
6091 removed,
6092 changed_pending_reload,
6093 configuration_changes,
6094 enabled_changes,
6095 unchanged,
6096 })
6097 }
6098
6099 async fn reconcile_supervised_modules(
6108 &self,
6109 supervisor: &Supervisor,
6110 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6111 preview: bool,
6112 requester: Principal,
6113 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6114 let plan = self.plan_rescan(configured_modules)?;
6115
6116 if preview {
6122 return Ok((plan.preview_result(), Vec::new()));
6123 }
6124 let changed = plan
6125 .configuration_changes
6126 .union(&plan.enabled_changes)
6127 .cloned()
6128 .collect::<Vec<_>>();
6129 let RescanPlan {
6130 current,
6131 configured,
6132 added,
6133 removed,
6134 changed_pending_reload,
6135 configuration_changes,
6136 enabled_changes,
6137 unchanged,
6138 } = plan;
6139 if !(added.is_empty()
6144 && removed.is_empty()
6145 && configuration_changes.is_empty()
6146 && enabled_changes.is_empty())
6147 {
6148 if let Some(context) = &self.rescan {
6149 context
6150 .generation
6151 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6152 }
6153 }
6154
6155 for module_id in &removed {
6156 let module = ¤t
6157 .get(module_id)
6158 .expect("removed module came from current supervisor state")
6159 .0;
6160 module
6161 .retire_requested(Some(requester.clone()))
6162 .await
6163 .map_err(|err| {
6164 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6165 })?;
6166 self.supervisor.record_rescan_removal(module_id);
6200 self.supervisor.retire(module_id);
6201 self.route_outages.forget(module_id);
6202 }
6203
6204 for module_id in configured.keys() {
6205 let Some((module, _, _, _)) = current.get(module_id) else {
6206 continue;
6207 };
6208 let configured_module = configured
6209 .get(module_id)
6210 .expect("configured module id came from configured map");
6211 if configuration_changes.contains(module_id) {
6212 module
6213 .update_configuration(
6214 configured_module.module_spec(),
6215 configured_module.health.clone(),
6216 configured_module.drain_timeout_ms,
6217 )
6218 .await
6219 .map_err(|err| {
6220 format!(
6221 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6222 )
6223 })?;
6224 }
6225 if enabled_changes.contains(module_id) {
6226 self.route_outages.mark_operator_action(module_id);
6229 module
6230 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6231 .await
6232 .map_err(|err| {
6233 self.route_outages.operator_action_ended_unrefused(module_id);
6234 format!(
6235 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6236 configured_module.enabled
6237 )
6238 })?;
6239 }
6240 }
6241
6242 for module_id in &added {
6243 let configured_module = configured
6244 .get(module_id)
6245 .expect("added module id came from configured map");
6246 supervisor
6247 .supervise_configured_requested(
6248 configured_module.module_spec(),
6249 configured_module.enabled,
6250 configured_module.health.clone(),
6251 configured_module.drain_timeout_ms,
6252 configured_module.restart,
6253 Some(requester.clone()),
6254 )
6255 .map_err(|err| {
6256 format!("failed to add module_id '{module_id}' during rescan: {err}")
6257 })?;
6258 }
6259
6260 Ok((
6261 SupervisorRescanResult {
6262 added,
6263 removed,
6264 changed_pending_reload,
6265 enabled_changes: enabled_changes.iter().cloned().collect(),
6266 unchanged,
6267 preview: false,
6268 restart_required: Vec::new(),
6271 capability_warnings: Vec::new(),
6272 module_actions: Vec::new(),
6273 earlier_pending: Vec::new(),
6274 expect: None,
6275 },
6276 changed,
6277 ))
6278 }
6279
6280 async fn handle_supervisor_set_enabled(
6281 &self,
6282 frame: Frame,
6283 module_id: String,
6284 enabled: bool,
6285 requester: Principal,
6286 ) -> Result<Vec<Frame>, RouterError> {
6287 let operation_lock = self.supervisor.operation_lock();
6288 let _operation_guard = operation_lock.lock().await;
6289 let Some(module) = self.supervisor.get(&module_id) else {
6290 return Ok(vec![control_error_frame(
6291 &frame,
6292 "unknown_module",
6293 format!("module_id '{module_id}' is not supervised"),
6294 )?]);
6295 };
6296
6297 self.route_outages.mark_operator_action(&module_id);
6300 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6301 Ok(applied) => applied,
6302 Err(err) => {
6303 self.route_outages
6304 .operator_action_ended_unrefused(&module_id);
6305 return Ok(vec![control_error_frame(
6306 &frame,
6307 "target_unavailable",
6308 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6309 )?]);
6310 }
6311 };
6312 if !applied {
6313 self.route_outages
6316 .operator_action_ended_unrefused(&module_id);
6317 }
6318
6319 self.capability_evaluator.wake_deadline_loop();
6320 self.refresh_capability_requirements();
6321 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6322 Ok(vec![control_response_body_frame(
6323 &frame,
6324 &response,
6325 "ClientControlResponse::SupervisorAck",
6326 )?])
6327 }
6328
6329 async fn handle_supervisor_health_probe(
6330 &self,
6331 frame: Frame,
6332 module_id: String,
6333 ) -> Result<Vec<Frame>, RouterError> {
6334 self.refresh_capability_requirements();
6335 let Some(registration) = self
6336 .registry
6337 .get_module(&module_id)
6338 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6339 else {
6340 return Ok(vec![control_error_frame(
6341 &frame,
6342 "unknown_module",
6343 format!("module_id '{module_id}' is not registered"),
6344 )?]);
6345 };
6346
6347 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6360 {
6361 return Ok(vec![control_error_frame(
6362 &frame,
6363 "health_not_advertised",
6364 format!("module_id '{module_id}' did not advertise health.check"),
6365 )?]);
6366 }
6367
6368 let deadline = Instant::now() + self.health_probe_timeout;
6369 let pending = match self.forwarding.begin_module_control_rpc_for(
6370 &module_id,
6371 MODULE_CONTROL_OP_HEALTH_CHECK,
6372 deadline,
6373 ) {
6374 Ok(pending) => pending,
6375 Err(err) => {
6376 return Ok(vec![control_error_frame(
6377 &frame,
6378 forwarding_error_code(&err),
6379 err.to_string(),
6380 )?])
6381 }
6382 };
6383
6384 let PendingModuleControlRpc {
6385 endpoint,
6386 module_sink,
6387 negotiated_ver,
6388 corr: probe_corr,
6389 receiver,
6390 } = pending;
6391 let mut guard =
6392 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6393 let probe_body =
6394 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6395 RouterError::backend(
6396 0,
6397 frame.header.corr,
6398 format!("failed to encode health.check request: {err}"),
6399 )
6400 })?;
6401 let probe_frame = Frame::build_with_version(
6402 negotiated_ver,
6403 FrameType::Request,
6404 control_flags(),
6405 0,
6406 0,
6407 probe_corr,
6408 probe_body,
6409 )
6410 .map_err(RouterError::FrameBuild)?;
6411
6412 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6415 Ok(Ok(())) => {}
6416 Ok(Err(err)) => {
6417 return Ok(vec![control_error_frame(
6418 &frame,
6419 "target_unavailable",
6420 err.to_string(),
6421 )?]);
6422 }
6423 Err(_) => {
6424 return Ok(vec![control_error_frame(
6425 &frame,
6426 "module_timeout",
6427 format!(
6428 "module_id '{module_id}' did not answer health.check within {:?}",
6429 self.health_probe_timeout
6430 ),
6431 )?]);
6432 }
6433 }
6434
6435 match timeout_at(deadline, receiver).await {
6436 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6437 guard.disarm();
6438 let Some(report) = response.health_report() else {
6439 return Ok(vec![control_error_frame(
6440 &frame,
6441 "invalid_control_body",
6442 "health.check RPC returned a non-health response",
6443 )?]);
6444 };
6445 let HealthReport {
6450 status,
6451 detail,
6452 metrics,
6453 } = report;
6454 let capability_detail = self
6455 .capability_evaluator
6456 .required_problem_detail(&module_id);
6457 let response = ClientControlResponse::SupervisorHealthProbe {
6458 module_id,
6459 status,
6460 detail: append_capability_problem_detail(detail, capability_detail),
6461 metrics,
6462 };
6463 Ok(vec![control_response_body_frame(
6464 &frame,
6465 &response,
6466 "ClientControlResponse::SupervisorHealthProbe",
6467 )?])
6468 }
6469 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6470 guard.disarm();
6471 Ok(vec![control_error_body_frame(&frame, body)?])
6472 }
6473 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6474 guard.disarm();
6475 Ok(vec![control_error_frame(
6476 &frame,
6477 "target_unavailable",
6478 message,
6479 )?])
6480 }
6481 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6482 guard.disarm();
6483 Ok(vec![control_error_frame(
6484 &frame,
6485 "invalid_control_body",
6486 message,
6487 )?])
6488 }
6489 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6490 guard.disarm();
6491 Ok(vec![control_error_frame(
6492 &frame,
6493 "invalid_control_body",
6494 format!("expected module-control op '{expected}', got '{actual}'"),
6495 )?])
6496 }
6497 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6498 guard.disarm();
6499 Ok(vec![control_error_frame(
6500 &frame,
6501 "module_timeout",
6502 format!(
6503 "module_id '{module_id}' answered health.check after {:?}",
6504 self.health_probe_timeout
6505 ),
6506 )?])
6507 }
6508 Ok(Err(_)) => Ok(vec![control_error_frame(
6509 &frame,
6510 "target_unavailable",
6511 "health.check waiter was canceled before the module responded",
6512 )?]),
6513 Err(_) => Ok(vec![control_error_frame(
6514 &frame,
6515 "module_timeout",
6516 format!(
6517 "module_id '{module_id}' did not answer health.check within {:?}",
6518 self.health_probe_timeout
6519 ),
6520 )?]),
6521 }
6522 }
6523
6524 fn supervisor_status(
6525 &self,
6526 module_id: &str,
6527 corr: u64,
6528 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6529 self.supervisor
6530 .get(module_id)
6531 .map(|module| {
6532 let warming = module.is_warming_for_control("status").map_err(|err| {
6533 RouterError::backend(
6534 0,
6535 corr,
6536 format!(
6537 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6538 ),
6539 )
6540 })?;
6541 module.status_for_control("status").map_err(|err| {
6542 RouterError::backend(
6543 0,
6544 corr,
6545 format!(
6546 "failed to read supervisor status for module_id '{module_id}': {err}"
6547 ),
6548 )
6549 }).map(|status| (status, warming))
6550 })
6551 .transpose()
6552 }
6553
6554 fn guard_module_control_op(
6555 &self,
6556 frame: &Frame,
6557 module_id: &str,
6558 op: &str,
6559 ) -> Result<Option<Frame>, RouterError> {
6560 if self.module_grants_op(module_id, op, frame.header.corr)? {
6561 return Ok(None);
6562 }
6563
6564 Ok(Some(control_error_frame(
6565 frame,
6566 "op_not_allowed",
6567 format!("module_id '{module_id}' did not grant control op '{op}'"),
6568 )?))
6569 }
6570
6571 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6572 let Some(registration) = self
6573 .registry
6574 .get_module(module_id)
6575 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6576 else {
6577 return Ok(false);
6578 };
6579 Ok(module_registration_grants_op(®istration.control_ops, op))
6580 }
6581
6582 fn handle_status_update(
6583 &self,
6584 endpoint: ModuleEndpointId,
6585 frame: Frame,
6586 ) -> Result<Vec<Frame>, RouterError> {
6587 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6588 Ok(update) => update,
6589 Err(err) => {
6590 if is_known_module_push_op(&frame.body) {
6595 return Ok(vec![control_error_frame(
6596 &frame,
6597 "invalid_control_body",
6598 format!("malformed module control push body: {err}"),
6599 )?]);
6600 }
6601 return Ok(Vec::new());
6602 }
6603 };
6604
6605 match update {
6606 ModuleControlPush::RouteStatus {
6607 route_channel,
6608 route_epoch,
6609 status,
6610 } => {
6611 self.forwarding
6612 .cache_status(endpoint, route_channel, route_epoch, status)
6613 .map_err(RouterError::Forwarding)?;
6614 }
6615 }
6616 Ok(Vec::new())
6617 }
6618
6619 fn handle_route_poll(
6620 &self,
6621 ctx: &RouteCtx,
6622 frame: Frame,
6623 route_channel: u16,
6624 route_epoch: u32,
6625 kind: PollKind,
6626 ) -> Result<Vec<Frame>, RouterError> {
6627 let snapshot = self
6628 .forwarding
6629 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6630 .map_err(RouterError::Forwarding)?;
6631 let response = match (kind, snapshot) {
6632 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6633 ClientControlResponse::RoutePoll {
6634 route_channel,
6635 route_epoch,
6636 status,
6637 live: None,
6638 }
6639 }
6640 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6641 route_channel,
6642 route_epoch,
6643 status: None,
6644 live: None,
6645 },
6646 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6647 let live = self
6673 .process_liveness
6674 .as_ref()
6675 .and_then(|source| source.process_live(&module_id))
6676 .unwrap_or(true);
6677 ClientControlResponse::RoutePoll {
6678 route_channel,
6679 route_epoch,
6680 status: None,
6681 live: Some(live),
6682 }
6683 }
6684 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6685 route_channel,
6686 route_epoch,
6687 status: None,
6688 live: Some(false),
6689 },
6690 };
6691
6692 Ok(vec![control_response_body_frame(
6693 &frame,
6694 &response,
6695 "ClientControlResponse::RoutePoll",
6696 )?])
6697 }
6698
6699 pub(crate) fn observe_module_control_completion(
6700 &self,
6701 completion: ModuleControlRpcCompletion,
6702 ) -> bool {
6703 match completion {
6704 ModuleControlRpcCompletion::Unknown => false,
6705 ModuleControlRpcCompletion::Settled => true,
6706 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6707 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6708 info!(
6709 module_id = %module_id,
6710 latency_ms,
6711 "late health.check answer proves the module is alive"
6712 );
6713 match self
6714 .supervisor
6715 .record_late_health_answer(&module_id, latency_ms)
6716 {
6717 Ok(true) => {}
6718 Ok(false) => debug!(
6719 module_id = %module_id,
6720 latency_ms,
6721 "late health.check answer has no active supervisor snapshot"
6722 ),
6723 Err(err) => warn!(
6724 module_id = %module_id,
6725 latency_ms,
6726 error = %err,
6727 "failed to record late health.check answer"
6728 ),
6729 }
6730 true
6731 }
6732 }
6733 }
6734
6735 fn refuse_to_end_module_connection_for_a_client(
6756 &self,
6757 module_connection_id: ConnectionId,
6758 corr: u64,
6759 err: ForwardingError,
6760 ) -> Result<(), RouterError> {
6761 if let ForwardingError::ConnectionClosing { connection_id } = err {
6762 if connection_id != module_connection_id {
6763 warn!(
6764 module_connection_id = module_connection_id.get(),
6765 client_connection_id = connection_id.get(),
6766 corr,
6767 "dropping a route.bind response for a closing client; the module connection keeps serving"
6768 );
6769 return Ok(());
6770 }
6771 }
6772 Err(RouterError::Forwarding(err))
6773 }
6774
6775 fn handle_module_relay_response(
6776 &self,
6777 connection_id: ConnectionId,
6778 frame: Frame,
6779 ) -> Result<Vec<Frame>, RouterError> {
6780 let mut secondary_error = None;
6781 let outcome = match frame.header.ty {
6782 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6783 Ok(probe) if probe.op == "route.bind" => {
6784 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6785 Ok(ModuleControlResponse::RouteBindAck {}) => {
6786 RouteBindRelayOutcome::Accepted
6787 }
6788 Ok(other) => {
6789 let message =
6790 format!("route.bind response carried unexpected body: {other:?}");
6791 secondary_error = Some(control_error_frame(
6792 &frame,
6793 "invalid_control_body",
6794 message.clone(),
6795 )?);
6796 RouteBindRelayOutcome::ModuleGone(message)
6797 }
6798 Err(err) => {
6799 let message = format!("malformed route.bind response body: {err}");
6800 secondary_error = Some(control_error_frame(
6801 &frame,
6802 "invalid_control_body",
6803 message.clone(),
6804 )?);
6805 RouteBindRelayOutcome::ModuleGone(message)
6806 }
6807 }
6808 }
6809 Ok(probe) => {
6810 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6811 {
6812 Ok(response) => ModuleControlRpcOutcome::Response(response),
6813 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6814 "malformed {} response body: {err}",
6815 probe.op
6816 )),
6817 };
6818 let completion = self
6819 .forwarding
6820 .complete_module_control_rpc(
6821 connection_id,
6822 frame.header.corr,
6823 Some(&probe.op),
6824 outcome,
6825 )
6826 .map_err(RouterError::Forwarding)?;
6827 if !self.observe_module_control_completion(completion) {
6828 debug!(
6829 connection_id = connection_id.get(),
6830 corr = frame.header.corr,
6831 op = %probe.op,
6832 "dropping late or unknown module-control RPC response"
6833 );
6834 }
6835 return Ok(Vec::new());
6836 }
6837 Err(err) => {
6838 if let Some(expected_op) = self
6839 .forwarding
6840 .pending_module_control_op(connection_id, frame.header.corr)
6841 .map_err(RouterError::Forwarding)?
6842 {
6843 let completion = self
6844 .forwarding
6845 .complete_module_control_rpc(
6846 connection_id,
6847 frame.header.corr,
6848 None,
6849 ModuleControlRpcOutcome::MalformedResponse(format!(
6850 "malformed {expected_op} response body: {err}"
6851 )),
6852 )
6853 .map_err(RouterError::Forwarding)?;
6854 if !self.observe_module_control_completion(completion) {
6855 debug!(
6856 connection_id = connection_id.get(),
6857 corr = frame.header.corr,
6858 "dropping late malformed module-control RPC response"
6859 );
6860 }
6861 return Ok(Vec::new());
6862 }
6863 let message = format!("malformed route.bind response body: {err}");
6864 secondary_error = Some(control_error_frame(
6865 &frame,
6866 "invalid_control_body",
6867 message.clone(),
6868 )?);
6869 RouteBindRelayOutcome::ModuleGone(message)
6870 }
6871 },
6872 FrameType::Error => {
6873 if self
6874 .forwarding
6875 .pending_module_control_op(connection_id, frame.header.corr)
6876 .map_err(RouterError::Forwarding)?
6877 .is_some()
6878 {
6879 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6880 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6881 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6882 "malformed module-control ERROR body: {err}"
6883 )),
6884 };
6885 let completion = self
6886 .forwarding
6887 .complete_module_control_rpc(
6888 connection_id,
6889 frame.header.corr,
6890 None,
6891 outcome,
6892 )
6893 .map_err(RouterError::Forwarding)?;
6894 if !self.observe_module_control_completion(completion) {
6895 debug!(
6896 connection_id = connection_id.get(),
6897 corr = frame.header.corr,
6898 "dropping late or unknown module-control RPC error"
6899 );
6900 }
6901 return Ok(Vec::new());
6902 }
6903 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6904 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6905 Err(err) => {
6906 let message = format!("malformed route.bind ERROR body: {err}");
6907 secondary_error = Some(control_error_frame(
6908 &frame,
6909 "invalid_control_body",
6910 message.clone(),
6911 )?);
6912 RouteBindRelayOutcome::ModuleGone(message)
6913 }
6914 }
6915 }
6916 ty => {
6917 return Ok(vec![control_error_frame(
6918 &frame,
6919 "unsupported_control_frame",
6920 format!("unsupported module channel-0 frame {ty:?}"),
6921 )?])
6922 }
6923 };
6924
6925 let settled =
6926 self.forwarding
6927 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6928 let completion = match settled {
6929 Ok(completion) => completion,
6930 Err(err) => {
6931 self.refuse_to_end_module_connection_for_a_client(
6932 connection_id,
6933 frame.header.corr,
6934 err,
6935 )?;
6936 return Ok(secondary_error.into_iter().collect());
6937 }
6938 };
6939 if let Some(target) = completion.abandoned.as_ref() {
6940 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6941 }
6942 if !completion.settled {
6943 debug!(
6944 connection_id = connection_id.get(),
6945 corr = frame.header.corr,
6946 frame_type = ?frame.header.ty,
6947 "dropping late or unknown route.bind relay response"
6948 );
6949 }
6950 Ok(secondary_error.into_iter().collect())
6951 }
6952
6953 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6954 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6955 self.cleanup_connection_with_end_reason(
6959 connection_id,
6960 RegistrationEndReason::ExplicitGoodbye,
6961 )
6962 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6963 Ok(Vec::new())
6964 }
6965}
6966
6967impl Default for ControlHandler {
6968 fn default() -> Self {
6969 Self::new(Arc::new(Registry::default()))
6970 }
6971}
6972
6973impl crate::supervise::SwapPromotionObserver for ControlHandler {
6974 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6975 self.apply_registration_capabilities(registration);
6976 }
6977}
6978
6979fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6980 CapabilityRequirementStatus {
6981 consumer: status.consumer,
6982 capability: status.capability,
6983 need: match status.need {
6984 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6985 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6986 },
6987 verdict: status.verdict.as_str().to_string(),
6988 episode_seq: status.episode_seq,
6989 config_satisfiable: status.config_satisfiable,
6990 runtime_available: status.runtime_available,
6991 detail: status.detail,
6992 }
6993}
6994
6995fn append_capability_problem_detail(
6996 detail: Option<String>,
6997 capability_detail: Option<String>,
6998) -> Option<String> {
6999 match (detail, capability_detail) {
7000 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7001 (Some(detail), None) => Some(detail),
7002 (None, Some(capability_detail)) => Some(capability_detail),
7003 (None, None) => None,
7004 }
7005}
7006
7007fn subc_ops() -> Vec<String> {
7008 SUBC_CONTROL_OPS
7009 .iter()
7010 .map(|op| (*op).to_string())
7011 .collect()
7012}
7013
7014fn module_subc_ops() -> Vec<String> {
7015 SUBC_CONTROL_OPS
7016 .iter()
7017 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7018 .map(|op| (*op).to_string())
7019 .collect()
7020}
7021
7022#[cfg(test)]
7023fn module_baseline_control_ops() -> Vec<String> {
7024 MODULE_BASELINE_CONTROL_OPS
7025 .iter()
7026 .map(|op| (*op).to_string())
7027 .collect()
7028}
7029
7030fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7031 let mut seen = HashSet::new();
7032 let mut effective = Vec::new();
7033 for op in MODULE_BASELINE_CONTROL_OPS {
7034 if seen.insert((*op).to_string()) {
7035 effective.push((*op).to_string());
7036 }
7037 }
7038 for op in declared.unwrap_or_default() {
7039 if seen.insert(op.clone()) {
7040 effective.push(op);
7041 }
7042 }
7043 effective
7044}
7045
7046fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7047 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7048}
7049
7050fn target_module_id(target: &RouteTarget) -> &str {
7051 match target {
7052 RouteTarget::ToolProvider { module_id }
7053 | RouteTarget::ManagementSurface { module_id }
7054 | RouteTarget::InternalService { module_id, .. } => module_id,
7055 }
7056}
7057
7058fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7059 roles.iter().any(|role| match (target, role) {
7060 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7061 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7062 (
7063 RouteTarget::InternalService { service_id, .. },
7064 ProviderRole::InternalService {
7065 service_id: provided,
7066 ..
7067 },
7068 ) => service_id == provided,
7069 _ => false,
7070 })
7071}
7072
7073fn is_routable_role(role: &ProviderRole) -> bool {
7074 matches!(
7075 role,
7076 ProviderRole::ToolProvider { .. }
7077 | ProviderRole::ManagementSurface { .. }
7078 | ProviderRole::InternalService { .. }
7079 )
7080}
7081
7082#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7083enum ControlRequestBodyError {
7084 UnknownOp,
7085 InvalidBody,
7086}
7087
7088#[derive(Debug, Deserialize)]
7089struct ControlOpProbe {
7090 op: String,
7091}
7092
7093const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7096
7097fn is_known_module_push_op(body: &[u8]) -> bool {
7098 serde_json::from_slice::<ControlOpProbe>(body)
7099 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7100 .unwrap_or(false)
7101}
7102
7103fn is_known_module_request_op(body: &[u8]) -> bool {
7104 serde_json::from_slice::<ControlOpProbe>(body)
7105 .map(|probe| is_module_to_subc_op(&probe.op))
7106 .unwrap_or(false)
7107}
7108
7109fn is_module_to_subc_op(op: &str) -> bool {
7110 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7111}
7112
7113fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7114 debug!(
7115 op = %op,
7116 connection_id = connection_id.get(),
7117 corr,
7118 "control dispatch"
7119 );
7120}
7121
7122fn log_slow_control_dispatch(
7123 dispatch_started_at: Option<StdInstant>,
7124 op: &'static str,
7125 connection_id: ConnectionId,
7126 corr: u64,
7127 route_open_timings: Option<&RouteOpenTimings>,
7128) {
7129 let Some(dispatch_started_at) = dispatch_started_at else {
7130 return;
7131 };
7132 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7134 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7135 if let Some(timings) = route_open_timings {
7136 warn!(
7137 op = %op,
7138 connection_id = connection_id.get(),
7139 corr,
7140 elapsed_ms = elapsed.as_millis() as u64,
7141 module_id = ?timings.module_id,
7142 principal = %timings.principal,
7143 admission_ms = timings.admission.as_millis() as u64,
7144 project_root_ms = timings.project_root.as_millis() as u64,
7145 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7146 bind_send_ms = timings.bind_send.as_millis() as u64,
7147 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7148 "slow control dispatch"
7149 );
7150 return;
7151 }
7152 warn!(
7153 op = %op,
7154 connection_id = connection_id.get(),
7155 corr,
7156 elapsed_ms = elapsed.as_millis() as u64,
7157 "slow control dispatch"
7158 );
7159 }
7160}
7161
7162#[cfg(test)]
7164type TestCallerLookup =
7165 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7166
7167fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7169 match request {
7170 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7171 Some(PrivilegedOperation::Restart { module_id })
7172 }
7173 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7174 Some(PrivilegedOperation::Swap { module_id })
7175 }
7176 ClientControlRequest::SupervisorReload { module_id } => {
7177 Some(PrivilegedOperation::Reload { module_id })
7178 }
7179 ClientControlRequest::SupervisorRescan { preview } => {
7180 if *preview {
7181 None
7182 } else {
7183 Some(PrivilegedOperation::Rescan)
7184 }
7185 }
7186 ClientControlRequest::SupervisorRescanChecked {
7191 preview,
7192 candidate,
7193 expect,
7194 } => (!*preview && candidate.is_none() && expect.is_some())
7195 .then_some(PrivilegedOperation::Rescan),
7196 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7197 Some(PrivilegedOperation::SetEnabled {
7198 module_id,
7199 enabled: *enabled,
7200 })
7201 }
7202 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7203 Some(PrivilegedOperation::ReleaseReserved { module_id })
7204 }
7205 ClientControlRequest::ServerDescribe {}
7206 | ClientControlRequest::CatalogList { .. }
7207 | ClientControlRequest::RouteOpen { .. }
7208 | ClientControlRequest::RoutePoll { .. }
7209 | ClientControlRequest::SupervisorList {}
7210 | ClientControlRequest::SupervisorHealth {}
7211 | ClientControlRequest::SupervisorHealthProbe { .. }
7212 | ClientControlRequest::SupervisorRoutes { .. }
7213 | ClientControlRequest::SupervisorProvenance { .. }
7214 | ClientControlRequest::SupervisorStderrTail { .. }
7215 | ClientControlRequest::SupervisorTerminals { .. }
7216 | ClientControlRequest::SupervisorSpawnSnapshot {}
7217 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7218 }
7219}
7220
7221fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7222 match request {
7223 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7224 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7225 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7226 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7227 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7228 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7229 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7230 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7231 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7232 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7233 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7234 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7235 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7236 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7237 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7238 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7239 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7240 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7241 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7242 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7243 }
7244}
7245
7246fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7252 use subc_protocol::scope::ScopeKind;
7253 const MAX_REF_CHARS: usize = 64;
7254 let kind = match stamp.kind {
7255 ScopeKind::Head => "head",
7256 ScopeKind::Worker => "worker",
7257 ScopeKind::Ephemeral => "ephemeral",
7258 };
7259 let mut scope_ref: String = stamp.scope_ref.chars().take(MAX_REF_CHARS).collect();
7260 if stamp.scope_ref.chars().nth(MAX_REF_CHARS).is_some() {
7261 scope_ref.push('…');
7262 }
7263 format!(
7264 "{kind}/{}/{scope_ref:?}@{}",
7265 principal_label(&stamp.owner),
7266 stamp.scope_epoch
7267 )
7268}
7269
7270fn principal_label(principal: &Principal) -> String {
7271 match principal {
7272 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7273 Principal::Direct => "direct".to_string(),
7274 other => format!("{other:?}"),
7275 }
7276}
7277
7278fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7279 match request {
7280 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7281 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7282 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7283 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7284 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7285 }
7286}
7287
7288fn parse_client_control_request(
7289 body: &[u8],
7290) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7291 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7292 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7293 Ok(probe)
7294 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7295 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7296 {
7297 ControlRequestBodyError::InvalidBody
7298 }
7299 Ok(_) => ControlRequestBodyError::UnknownOp,
7300 Err(_) => ControlRequestBodyError::InvalidBody,
7301 };
7302 (err, classification)
7303 })
7304}
7305
7306fn parse_module_control_request_from_module(
7307 body: &[u8],
7308) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7309 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7310 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7311 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7312 Ok(_) => ControlRequestBodyError::UnknownOp,
7313 Err(_) => ControlRequestBodyError::InvalidBody,
7314 };
7315 (err, classification)
7316 })
7317}
7318
7319#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7320enum ProviderRoleKind {
7321 ToolProvider,
7322 PipelineStage,
7323 ManagementSurface,
7324 InternalService,
7325}
7326
7327fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7328 match role {
7329 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7330 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7331 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7332 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7333 }
7334}
7335
7336fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7337 roles.iter().map(provider_role_kind).collect()
7338}
7339
7340const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7343
7344#[derive(Debug, Default, PartialEq, Eq)]
7346struct ScopeOutcomeCounts {
7347 created: usize,
7348 replaced: usize,
7349 updated: usize,
7350 unchanged: usize,
7351 refused: usize,
7352}
7353
7354impl ScopeOutcomeCounts {
7355 fn of(results: &[ScopeRecordResult]) -> Self {
7356 let mut counts = Self::default();
7357 for result in results {
7358 let slot = match result.outcome {
7359 ScopeRecordOutcome::Created => &mut counts.created,
7360 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7361 ScopeRecordOutcome::Updated => &mut counts.updated,
7362 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7363 ScopeRecordOutcome::Refused => &mut counts.refused,
7364 };
7365 *slot += 1;
7366 }
7367 counts
7368 }
7369}
7370
7371#[cfg(test)]
7372mod scope_outcome_count_tests {
7373 use super::*;
7374
7375 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7376 ScopeRecordResult {
7377 scope_ref: "r".to_string(),
7378 scope_epoch: 1,
7379 outcome,
7380 code: None,
7381 message: None,
7382 version: None,
7383 parent_state: None,
7384 }
7385 }
7386
7387 #[test]
7390 fn every_outcome_is_counted_in_its_own_field() {
7391 let results = [
7392 result(ScopeRecordOutcome::Created),
7393 result(ScopeRecordOutcome::Created),
7394 result(ScopeRecordOutcome::Replaced),
7395 result(ScopeRecordOutcome::Updated),
7396 result(ScopeRecordOutcome::Unchanged),
7397 result(ScopeRecordOutcome::Refused),
7398 result(ScopeRecordOutcome::Refused),
7399 result(ScopeRecordOutcome::Refused),
7400 ];
7401 assert_eq!(
7402 ScopeOutcomeCounts::of(&results),
7403 ScopeOutcomeCounts {
7404 created: 2,
7405 replaced: 1,
7406 updated: 1,
7407 unchanged: 1,
7408 refused: 3,
7409 }
7410 );
7411 }
7412}
7413
7414fn capability_census_trigger(
7418 old: Option<&CapabilityDeclarations>,
7419 new: Option<&CapabilityDeclarations>,
7420) -> bool {
7421 let old_provides = old
7422 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7423 .unwrap_or_default();
7424 let old_denies = old
7425 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7426 .unwrap_or_default();
7427 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7428 provides: Vec::new(),
7429 requires: Vec::new(),
7430 must_never_reach: Vec::new(),
7431 });
7432
7433 new.provides
7434 .iter()
7435 .any(|capability| !old_provides.contains(capability))
7436 || new
7437 .must_never_reach
7438 .iter()
7439 .any(|capability| !old_denies.contains(capability))
7440}
7441
7442fn denied_capability<'a>(
7445 opening_manifest: &'a ModuleManifest,
7446 target_manifest: &ModuleManifest,
7447) -> Option<&'a str> {
7448 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7449 let target_capabilities = target_manifest.capabilities.as_ref()?;
7450 opening_capabilities
7451 .must_never_reach
7452 .iter()
7453 .find(|denied| {
7454 target_capabilities
7455 .provides
7456 .iter()
7457 .any(|provided| provided == *denied)
7458 })
7459 .map(String::as_str)
7460}
7461
7462fn catalog_update_frozen_field_message(
7463 registered: &ModuleManifest,
7464 provides: &[ProviderRole],
7465) -> Option<String> {
7466 let old_has_provides = !registered.provides.is_empty();
7467 let new_has_provides = !provides.is_empty();
7468 if old_has_provides != new_has_provides {
7469 return Some(format!(
7470 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7471 registered.module_id
7472 ));
7473 }
7474
7475 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7476 return Some(format!(
7477 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7478 registered.module_id
7479 ));
7480 }
7481
7482 let registered_concurrency = manifest_concurrency(registered);
7483 let mut candidate = registered.clone();
7484 candidate.provides = provides.to_vec();
7485 let candidate_concurrency = manifest_concurrency(&candidate);
7486 if candidate_concurrency != registered_concurrency {
7487 return Some(format!(
7488 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7489 registered.module_id, registered_concurrency, candidate_concurrency
7490 ));
7491 }
7492
7493 None
7496}
7497
7498fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7499 manifest.provides.iter().any(is_routable_role)
7500}
7501
7502fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7508 manifest
7509 .provides
7510 .iter()
7511 .find_map(|provider| match provider {
7512 ProviderRole::ToolProvider { concurrency, .. }
7513 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7514 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7515 })
7516 .unwrap_or(Concurrency::ModuleManaged)
7517}
7518
7519fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7526 let has_management_surface = manifest
7527 .provides
7528 .iter()
7529 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7530 if !has_management_surface {
7531 return false;
7532 }
7533 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7534 return false;
7535 };
7536 let Some(provides) = raw
7537 .get("manifest")
7538 .and_then(|manifest| manifest.get("provides"))
7539 .and_then(serde_json::Value::as_array)
7540 else {
7541 return false;
7542 };
7543 provides.iter().any(|role| {
7548 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7549 && role.get("concurrency").is_none()
7550 })
7551}
7552
7553fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7554 if peer_version != PROTOCOL_VERSION {
7555 return Err(format!(
7556 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7557 ));
7558 }
7559 Ok(PROTOCOL_VERSION)
7560}
7561
7562fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7563 Frame::build_with_version(
7564 response_version(frame),
7565 FrameType::Pong,
7566 frame.header.flags,
7567 0,
7568 0,
7569 frame.header.corr,
7570 Vec::new(),
7571 )
7572 .map_err(RouterError::FrameBuild)
7573}
7574
7575pub(crate) fn control_error_frame(
7576 frame: &Frame,
7577 code: &'static str,
7578 message: impl Into<String>,
7579) -> Result<Frame, RouterError> {
7580 control_error_body_frame(
7581 frame,
7582 ErrorBody {
7583 code: code.to_string(),
7584 message: message.into(),
7585 detail: None,
7586 },
7587 )
7588}
7589
7590fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7591 let body = serde_json::to_vec(&error).map_err(|err| {
7592 RouterError::backend(
7593 0,
7594 frame.header.corr,
7595 format!("failed to encode control ERROR: {err}"),
7596 )
7597 })?;
7598
7599 Frame::build_with_version(
7600 response_version(frame),
7601 FrameType::Error,
7602 control_flags(),
7603 0,
7604 0,
7605 frame.header.corr,
7606 body,
7607 )
7608 .map_err(RouterError::FrameBuild)
7609}
7610
7611fn control_response_body_frame<T: Serialize>(
7612 frame: &Frame,
7613 reply: &T,
7614 label: &'static str,
7615) -> Result<Frame, RouterError> {
7616 let body = serde_json::to_vec(reply).map_err(|err| {
7617 RouterError::backend(
7618 0,
7619 frame.header.corr,
7620 format!("failed to encode {label}: {err}"),
7621 )
7622 })?;
7623
7624 Frame::build_with_version(
7625 response_version(frame),
7626 FrameType::Response,
7627 control_flags(),
7628 0,
7629 0,
7630 frame.header.corr,
7631 body,
7632 )
7633 .map_err(RouterError::FrameBuild)
7634}
7635
7636fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7666 match err {
7667 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7668 ForwardingError::NoModuleConnection => "target_unavailable",
7669 ForwardingError::ModuleReloading { .. } => "module_reloading",
7670 ForwardingError::ClientRouteChannelExhausted { .. }
7671 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7672 ForwardingError::StaleModuleEndpoint
7673 | ForwardingError::UnknownReservation { .. }
7674 | ForwardingError::ConnectionClosing { .. }
7675 | ForwardingError::ClientEgressClosed { .. }
7676 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7677 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7680 ForwardingError::RelayCorrelationExhausted
7681 | ForwardingError::RouteOpenBuild(_)
7682 | ForwardingError::Poisoned => "forwarding_error",
7683 }
7684}
7685
7686fn response_version(frame: &Frame) -> u8 {
7687 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7688 frame.header.ver
7689 } else {
7690 PROTOCOL_VERSION
7691 }
7692}
7693
7694fn control_flags() -> Flags {
7695 Flags::new(false, Priority::Passive, false)
7696}
7697
7698fn send_goodbye_target_best_effort(
7703 counters: &DaemonCounters,
7704 target: &GoodbyeTarget,
7705 context: &'static str,
7706) {
7707 let Ok(frame) = Frame::build_with_version(
7708 target.negotiated_ver,
7709 FrameType::Goodbye,
7710 control_flags(),
7711 target.channel,
7712 target.epoch,
7713 0,
7714 Vec::new(),
7715 ) else {
7716 return;
7717 };
7718 crate::forwarding::send_module_route_goodbye(
7719 counters,
7720 &target.sink,
7721 frame,
7722 target.module_id.as_deref(),
7723 context,
7724 );
7725}
7726
7727pub(crate) fn send_route_control_pushes(
7728 forwarding: &ForwardingTable,
7729 routes: Vec<EndpointRoute>,
7730 push: ClientControlPush,
7731) {
7732 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7733 for route in routes {
7734 let target = route.goodbye_target;
7735 if let Some((existing, channels)) = targets
7736 .iter_mut()
7737 .find(|(existing, _)| existing.connection_id == target.connection_id)
7738 {
7739 debug_assert_eq!(
7740 existing.negotiated_ver, target.negotiated_ver,
7741 "one connection cannot negotiate multiple frame versions"
7742 );
7743 if !channels.contains(&target.channel) {
7744 channels.push(target.channel);
7745 }
7746 continue;
7747 }
7748 let channel = target.channel;
7749 targets.push((target, vec![channel]));
7750 }
7751 for (target, mut channels) in targets {
7752 channels.sort_unstable();
7753 let mut push = push.clone();
7754 match &mut push {
7755 ClientControlPush::RouteClosing {
7756 channels: covered, ..
7757 }
7758 | ClientControlPush::RouteClosed {
7759 channels: covered, ..
7760 } => *covered = channels,
7761 }
7762 let body = match serde_json::to_vec(&push) {
7763 Ok(body) => body,
7764 Err(err) => {
7765 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7766 continue;
7767 }
7768 };
7769 let frame = match Frame::build_with_version(
7770 target.negotiated_ver,
7771 FrameType::Push,
7772 control_flags(),
7773 0,
7774 0,
7775 0,
7776 body.clone(),
7777 ) {
7778 Ok(frame) => frame,
7779 Err(err) => {
7780 warn!(
7781 route_channel = target.channel,
7782 error = %err,
7783 "failed to build route lifecycle control PUSH frame"
7784 );
7785 continue;
7786 }
7787 };
7788 if let Err(err) = target.sink.try_send(frame) {
7789 if target.close_on_delivery_failure() {
7790 warn!(
7791 target_connection_id = target.connection_id.get(),
7792 route_channel = target.channel,
7793 error = %err,
7794 "route lifecycle control PUSH was not delivered to client; closing target connection"
7795 );
7796 let _ = forwarding.escalate_client_delivery_failure(
7797 target.connection_id,
7798 target.channel,
7799 target.epoch,
7800 CloseReason::new(
7801 "route_lifecycle_push_delivery_failed",
7802 format!(
7803 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7804 target.channel
7805 ),
7806 ),
7807 crate::forwarding::UndeliveredFrame {
7808 module_id: target.module_id.as_deref(),
7809 sink: &target.sink,
7810 },
7811 );
7812 }
7813 }
7814 }
7815}
7816
7817#[cfg(test)]
7818mod tests {
7819 mod activity_read {
7820 include!("activity_read_tests.rs");
7821 }
7822
7823 mod lifecycle_activity {
7824 include!("control_lifecycle_tests.rs");
7825 }
7826 #[cfg(unix)]
7827 #[tokio::test]
7828 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7829 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7830 let path = dir.join("subc.jsonc");
7831 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7832 "program":"/bin/sleep","args":["60"],"protocol":"none",
7833 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7834 }}}).to_string()).unwrap();
7835 let mut configured = crate::daemon_config::load(&path)
7836 .unwrap()
7837 .unwrap()
7838 .modules
7839 .pop()
7840 .unwrap();
7841 let registry = std::sync::Arc::new(crate::Registry::default());
7842 let handle = crate::SupervisorHandle::new();
7843 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7844 .with_handle(handle.clone());
7845 let module = supervisor
7846 .supervise_configured_with_health(
7847 configured.module_spec(),
7848 true,
7849 configured.health.clone(),
7850 None,
7851 configured.restart,
7852 )
7853 .unwrap();
7854 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7855 let before = module.status().unwrap().pid;
7856 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7857 configured.health.cadence = std::time::Duration::from_secs(3600);
7858 let health_only = handler
7859 .reconcile_supervised_modules(
7860 &supervisor,
7861 vec![configured.clone()],
7862 false,
7863 Principal::Direct,
7864 )
7865 .await
7866 .unwrap();
7867 assert!(
7868 health_only.0.changed_pending_reload.is_empty(),
7869 "health policy is already applied live"
7870 );
7871 assert_eq!(module.status().unwrap().pid, before);
7872 assert_eq!(
7873 module.configuration().unwrap().1.http,
7874 configured.health.http
7875 );
7876 configured.args = vec!["61".into()];
7877 let launch = handler
7878 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7879 .await
7880 .unwrap();
7881 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
7882 assert_eq!(
7883 module.status().unwrap().pid,
7884 before,
7885 "a launch edit is stored until reload"
7886 );
7887 module.drain().await.unwrap();
7888 }
7889 use cortexkit_test_support::ScratchDir;
7890 use std::{
7891 collections::BTreeMap,
7892 fmt,
7893 path::PathBuf,
7894 sync::{Arc, Mutex},
7895 time::Duration,
7896 };
7897
7898 use serde_json::{json, Value};
7899 use subc_protocol::{
7900 manifest::{
7901 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7902 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7903 },
7904 session::HealthStatus,
7905 FrameType,
7906 };
7907
7908 use super::*;
7909 use crate::{
7910 forwarding::{DataRoute, DataRouteState},
7911 registry::ChannelState,
7912 router::FrameSink,
7913 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7914 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7915 RouteCtx, Router,
7916 };
7917 use tokio::{
7918 sync::mpsc,
7919 time::{sleep, Instant},
7920 };
7921 use tracing::{
7922 field::{Field, Visit},
7923 Event, Subscriber,
7924 };
7925 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7926
7927 fn fake_aft_stub_path() -> PathBuf {
7946 let mut path = std::env::current_exe().expect("current_exe available in tests");
7947 path.pop(); path.pop(); path.push(if cfg!(windows) {
7950 "fake-aft-stub.exe"
7951 } else {
7952 "fake-aft-stub"
7953 });
7954 assert!(
7955 path.exists(),
7956 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7957 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7958 path.display()
7959 );
7960 path
7961 }
7962
7963 fn client_retries(code: &str) -> bool {
7968 subc_protocol::error_codes::is_retryable_route_open(code)
7969 }
7970
7971 #[test]
7982 fn retryability_of_forwarding_codes_matches_the_failure() {
7983 let transient = [
7986 ForwardingError::NoModuleConnection,
7987 ForwardingError::ModuleReloading {
7988 module_id: "m".into(),
7989 },
7990 ForwardingError::StaleModuleEndpoint,
7991 ForwardingError::UnknownReservation {
7992 client_channel: 1,
7993 module_channel: 1,
7994 },
7995 ForwardingError::ConnectionClosing {
7996 connection_id: ConnectionId::new(1),
7997 },
7998 ForwardingError::ClientEgressClosed {
7999 connection_id: ConnectionId::new(1),
8000 },
8001 ForwardingError::ModuleEgressUnavailable {
8002 connection_id: ConnectionId::new(1),
8003 },
8004 ];
8005 for err in transient {
8006 let code = forwarding_error_code(&err);
8007 assert!(
8008 client_retries(code),
8009 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8010 );
8011 }
8012
8013 let permanent = [
8018 ForwardingError::ConnectionRoleConflict {
8019 connection_id: ConnectionId::new(1),
8020 },
8021 ForwardingError::ClientRouteChannelExhausted {
8022 connection_id: ConnectionId::new(1),
8023 },
8024 ForwardingError::ModuleRouteChannelExhausted {
8025 endpoint: ModuleEndpointId {
8026 connection_id: ConnectionId::new(1),
8027 generation: 1,
8028 },
8029 },
8030 ForwardingError::RelayCorrelationExhausted,
8031 ForwardingError::RouteOpenBuild("x".into()),
8032 ForwardingError::Poisoned,
8033 ];
8034 for err in permanent {
8035 let code = forwarding_error_code(&err);
8036 assert!(
8037 !client_retries(code),
8038 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8039 );
8040 }
8041 }
8042
8043 #[tokio::test]
8071 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8072 let handler = ControlHandler::default();
8073 let frame =
8074 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8075
8076 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8080 assert_eq!(
8081 stamped,
8082 Principal::Direct,
8083 "a caller that proved nothing must not be stamped as a supervised module"
8084 );
8085
8086 let forged = handler
8091 .route_open_principal(
8092 &frame,
8093 Some(ConsumerIdentity {
8094 module_id: "aft".to_string(),
8095 launch_nonce: "not-a-real-nonce".to_string(),
8096 }),
8097 )
8098 .unwrap();
8099 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8100 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8101 }
8102
8103 #[test]
8120 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8121 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8122 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8123 let ClientControlRequest::RouteOpen {
8124 consumer_identity, ..
8125 } = parsed
8126 else {
8127 panic!("route.open body must parse as RouteOpen");
8128 };
8129 assert_eq!(
8130 consumer_identity,
8131 Some(ConsumerIdentity {
8132 module_id: "aft".to_string(),
8133 launch_nonce: "n".to_string(),
8134 }),
8135 "the wire field name must reach the value route_open_principal reads"
8136 );
8137 }
8138
8139 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8140 ModuleManifest::builder(module_id, "0.1.0")
8141 .protocol_ver(protocol_ver)
8142 .provides(vec![ProviderRole::ToolProvider {
8143 tools: vec![Tool {
8144 name: "read".to_string(),
8145 description: None,
8146 execution_mode: ExecutionMode::Pure,
8147 schema: json!({"type": "object"}),
8148 }],
8149 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8150 concurrency: Concurrency::ModuleManaged,
8151 emits_push: true,
8152 sub_supervises: true,
8153 }])
8154 .build()
8155 }
8156
8157 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8158 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8159 }
8160
8161 fn hello_frame_with_control_ops(
8162 module_id: &str,
8163 protocol_ver: u8,
8164 corr: u64,
8165 control_ops: Option<Vec<String>>,
8166 ) -> Frame {
8167 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8168 }
8169
8170 fn hello_frame_with_nonce(
8171 module_id: &str,
8172 protocol_ver: u8,
8173 corr: u64,
8174 launch_nonce: Option<&str>,
8175 ) -> Frame {
8176 hello_frame_full(
8177 module_id,
8178 protocol_ver,
8179 corr,
8180 None,
8181 launch_nonce.map(ToOwned::to_owned),
8182 )
8183 }
8184
8185 fn hello_frame_full(
8186 module_id: &str,
8187 protocol_ver: u8,
8188 corr: u64,
8189 control_ops: Option<Vec<String>>,
8190 launch_nonce: Option<String>,
8191 ) -> Frame {
8192 let body = serde_json::to_vec(&ModuleHelloBody {
8193 manifest: manifest(module_id, protocol_ver),
8194 protocol_ver,
8195 control_ops,
8196 launch_nonce,
8197 })
8198 .unwrap();
8199 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8200 }
8201
8202 fn hello_frame_with_rich_ui(
8203 module_id: &str,
8204 protocol_ver: u8,
8205 corr: u64,
8206 launch_nonce: Option<&str>,
8207 rich_ui: Value,
8208 ) -> Frame {
8209 let mut body = serde_json::to_value(ModuleHelloBody {
8210 manifest: manifest(module_id, protocol_ver),
8211 protocol_ver,
8212 control_ops: None,
8213 launch_nonce: launch_nonce.map(str::to_string),
8214 })
8215 .unwrap();
8216 body["manifest"]["rich_ui"] = rich_ui;
8217 Frame::build(
8218 FrameType::Hello,
8219 control_flags(),
8220 0,
8221 0,
8222 corr,
8223 serde_json::to_vec(&body).unwrap(),
8224 )
8225 .unwrap()
8226 }
8227
8228 fn non_routable_hello_frame_with_control_ops(
8229 module_id: &str,
8230 corr: u64,
8231 control_ops: Option<Vec<String>>,
8232 ) -> Frame {
8233 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8234 manifest.provides.clear();
8235 let body = serde_json::to_vec(&ModuleHelloBody {
8236 manifest,
8237 protocol_ver: PROTOCOL_VERSION,
8238 control_ops,
8239 launch_nonce: None,
8240 })
8241 .unwrap();
8242 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8243 }
8244
8245 fn capability_grammar_hello_frame(
8246 capabilities: Value,
8247 runtime_computed: Option<Value>,
8248 corr: u64,
8249 ) -> Frame {
8250 let mut body = serde_json::to_value(ModuleHelloBody {
8251 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8252 protocol_ver: PROTOCOL_VERSION,
8253 control_ops: None,
8254 launch_nonce: None,
8255 })
8256 .expect("HELLO body serializes");
8257 body["manifest"]["capabilities"] = capabilities;
8258 if let Some(runtime_computed) = runtime_computed {
8259 body["runtime_computed"] = runtime_computed;
8260 }
8261 Frame::build(
8262 FrameType::Hello,
8263 control_flags(),
8264 0,
8265 0,
8266 corr,
8267 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8268 )
8269 .expect("HELLO frame builds")
8270 }
8271
8272 fn channel_request(channel: u16, corr: u64) -> Frame {
8273 Frame::build(
8274 FrameType::Request,
8275 Flags::new(true, Priority::Interactive, false),
8276 channel,
8277 0,
8278 corr,
8279 b"opaque".to_vec(),
8280 )
8281 .unwrap()
8282 }
8283
8284 fn route_ctx(
8285 connection_id: ConnectionId,
8286 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8287 let (tx, rx) = mpsc::channel(8);
8288 (
8289 RouteCtx {
8290 connection_id,
8291 egress: FrameSink::new(tx),
8292 },
8293 rx,
8294 )
8295 }
8296
8297 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8298 serde_json::from_slice(&frame.body).unwrap()
8299 }
8300
8301 async fn hello_via_sink(
8306 handler: &ControlHandler,
8307 ctx: &RouteCtx,
8308 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8309 hello: Frame,
8310 ) -> Frame {
8311 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8312 assert!(
8313 replies.is_empty(),
8314 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8315 );
8316 let ack = rx
8317 .try_recv()
8318 .expect("HELLO_ACK is queued on the module sink")
8319 .frame;
8320 assert_eq!(ack.header.ty, FrameType::HelloAck);
8321 ack
8322 }
8323
8324 fn parse_error(frame: &Frame) -> Value {
8325 serde_json::from_slice(&frame.body).unwrap()
8326 }
8327
8328 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8329 serde_json::from_slice(&frame.body).unwrap()
8330 }
8331
8332 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8333 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8334 route_channel,
8335 route_epoch: 0,
8336 kind,
8337 })
8338 .unwrap();
8339 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8340 }
8341
8342 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8343 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8344 module_id: module_id.to_string(),
8345 })
8346 .unwrap();
8347 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8348 }
8349
8350 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8351 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8352 }
8353
8354 fn route_open_frame_with_consumer_capabilities(
8355 corr: u64,
8356 module_id: &str,
8357 project_root: ScratchDir,
8358 consumer_capabilities: Option<Vec<String>>,
8359 ) -> Frame {
8360 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8361 target: RouteTarget::ToolProvider {
8362 module_id: module_id.to_string(),
8363 },
8364 identity: BindIdentity::new(
8365 project_root.path().to_path_buf(),
8366 "unit".to_string(),
8367 "session".to_string(),
8368 ),
8369 consumer_identity: None,
8370 consumer_capabilities,
8371 role_versions: None,
8372 admission_facts: None,
8373 scope: None,
8374 })
8375 .unwrap();
8376 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8377 }
8378
8379 fn route_open_frame_with_role_versions(
8380 corr: u64,
8381 module_id: &str,
8382 project_root: ScratchDir,
8383 role_versions: Option<BTreeMap<String, String>>,
8384 ) -> Frame {
8385 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8386 target: RouteTarget::ToolProvider {
8387 module_id: module_id.to_string(),
8388 },
8389 identity: BindIdentity::new(
8390 project_root.path().to_path_buf(),
8391 "unit".to_string(),
8392 format!("session-{corr}"),
8393 ),
8394 consumer_identity: None,
8395 consumer_capabilities: None,
8396 role_versions,
8397 admission_facts: None,
8398 scope: None,
8399 })
8400 .unwrap();
8401 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8402 }
8403
8404 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8405 entries
8406 .iter()
8407 .map(|(role, version)| (role.to_string(), version.to_string()))
8408 .collect()
8409 }
8410
8411 fn route_open_frame_with_admission_facts(
8412 corr: u64,
8413 module_id: &str,
8414 project_root: ScratchDir,
8415 consumer_identity: Option<subc_control::ConsumerIdentity>,
8416 facts: Option<Value>,
8417 ) -> Frame {
8418 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8419 target: RouteTarget::ToolProvider {
8420 module_id: module_id.to_string(),
8421 },
8422 identity: BindIdentity::new(
8423 project_root.path().to_path_buf(),
8424 "unit".to_string(),
8425 format!("session-{corr}"),
8426 ),
8427 consumer_identity,
8428 consumer_capabilities: None,
8429 role_versions: None,
8430 admission_facts: facts,
8431 scope: None,
8432 })
8433 .unwrap();
8434 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8435 }
8436
8437 #[derive(Clone, Default)]
8438 struct EventCapture {
8439 events: Arc<Mutex<Vec<CapturedEvent>>>,
8440 }
8441
8442 #[derive(Clone, Debug)]
8443 struct CapturedEvent {
8444 target: String,
8445 level: tracing::Level,
8446 fields: BTreeMap<String, String>,
8447 }
8448
8449 impl EventCapture {
8450 fn events(&self) -> Vec<CapturedEvent> {
8451 self.events.lock().unwrap().clone()
8452 }
8453 }
8454
8455 impl<S> Layer<S> for EventCapture
8456 where
8457 S: Subscriber,
8458 {
8459 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8460 let mut visitor = EventFieldVisitor::default();
8461 event.record(&mut visitor);
8462 self.events.lock().unwrap().push(CapturedEvent {
8463 target: event.metadata().target().to_string(),
8464 level: *event.metadata().level(),
8465 fields: visitor.fields,
8466 });
8467 }
8468 }
8469
8470 #[derive(Default)]
8471 struct EventFieldVisitor {
8472 fields: BTreeMap<String, String>,
8473 }
8474
8475 impl Visit for EventFieldVisitor {
8476 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8477 self.fields
8478 .insert(field.name().to_string(), format!("{value:?}"));
8479 }
8480 }
8481
8482 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8483 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8484 status,
8485 detail: Some("warming".to_string()),
8486 metrics: Some(json!({"queue_depth": 3})),
8487 })
8488 .unwrap();
8489 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8490 }
8491
8492 fn route_bind_ack(corr: u64) -> Frame {
8493 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8494 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8495 }
8496
8497 fn unique_project_root(label: &str) -> ScratchDir {
8498 ScratchDir::new(label)
8499 }
8500
8501 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8502 match parse_route_poll(frame) {
8503 ClientControlResponse::RoutePoll {
8504 status: None,
8505 live: Some(live),
8506 ..
8507 } => assert_eq!(live, expected_live),
8508 other => panic!("unexpected route.poll response: {other:?}"),
8509 }
8510 }
8511
8512 fn bind_liveness_route(
8513 registry: &Registry,
8514 forwarding: &ForwardingTable,
8515 module_id: &str,
8516 ) -> (RouteCtx, u16, u32) {
8517 let module_connection = ConnectionId::new(101);
8518 let client_connection = ConnectionId::new(202);
8519 let registration = registry
8520 .register_with_control_ops(
8521 manifest(module_id, PROTOCOL_VERSION),
8522 PROTOCOL_VERSION,
8523 module_connection,
8524 module_baseline_control_ops(),
8525 )
8526 .unwrap();
8527 let (module_tx, _module_rx) = mpsc::channel(8);
8528 let endpoint = forwarding
8529 .register_module_connection(
8530 module_connection,
8531 module_id.to_string(),
8532 PROTOCOL_VERSION,
8533 manifest_concurrency(®istration.manifest),
8534 FrameSink::new(module_tx),
8535 )
8536 .unwrap();
8537 let (client_ctx, _client_rx) = route_ctx(client_connection);
8538 let pending = forwarding
8539 .begin_route_bind_relay_for_test(
8540 client_connection,
8541 client_ctx.egress.clone(),
8542 1,
8543 module_id,
8544 )
8545 .unwrap();
8546 assert_eq!(pending.endpoint, endpoint);
8547 let route_channel = pending.client_channel;
8548 let route_epoch = pending.client_epoch;
8549 forwarding
8550 .complete_pending_relay(
8551 module_connection,
8552 pending.corr,
8553 RouteBindRelayOutcome::Accepted,
8554 )
8555 .unwrap();
8556 (client_ctx, route_channel, route_epoch)
8557 }
8558
8559 struct FakeProcessLiveness {
8560 live: Option<bool>,
8561 }
8562
8563 impl ModuleProcessLiveness for FakeProcessLiveness {
8564 fn process_live(&self, _module_id: &str) -> Option<bool> {
8565 self.live
8566 }
8567 }
8568
8569 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8570 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8571 {
8572 let registry = Arc::new(Registry::default());
8573 let supervisor_handle = SupervisorHandle::new();
8574 let supervisor = Supervisor::new_for_test(
8575 Arc::clone(®istry),
8576 RestartPolicy::new(1, Duration::from_millis(10)),
8577 )
8578 .with_handle(supervisor_handle.clone());
8579 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8580 let module = supervisor
8581 .spawn(ModuleSpec {
8582 module_id: "stderr-tail-wire".to_string(),
8583 program: fake_aft_stub_path(),
8584 args: Vec::new(),
8585 env: vec![
8586 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8587 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8588 ],
8589 reserved: false,
8590 reserved_prefixes: Vec::new(),
8591 protocol: ModuleProtocol::Subc,
8592 overlap: Default::default(),
8593 })
8594 .unwrap();
8595
8596 let deadline = Instant::now() + Duration::from_secs(5);
8597 loop {
8598 let tail = module.stderr_tail(None, None);
8599 if tail
8600 .entries
8601 .iter()
8602 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8603 && tail.entries.iter().any(|entry| {
8604 matches!(
8605 entry,
8606 TailEntry::Line {
8607 truncated: true,
8608 ..
8609 }
8610 )
8611 })
8612 {
8613 break;
8614 }
8615 assert!(
8616 Instant::now() < deadline,
8617 "module did not produce a truncated line and restart boundary: {tail:?}"
8618 );
8619 sleep(Duration::from_millis(10)).await;
8620 }
8621
8622 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8623 let request = ClientControlRequest::SupervisorStderrTail {
8624 module_id: "stderr-tail-wire".to_string(),
8625 max_lines: None,
8626 max_bytes: None,
8627 };
8628 let frame = Frame::build(
8629 FrameType::Request,
8630 control_flags(),
8631 0,
8632 0,
8633 1,
8634 serde_json::to_vec(&request).unwrap(),
8635 )
8636 .unwrap();
8637 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8638 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8639 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8640 serde_json::from_slice(&responses[0].body).unwrap()
8641 else {
8642 panic!("expected supervisor.stderr_tail response");
8643 };
8644
8645 assert!(
8646 tail.entries
8647 .iter()
8648 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8649 "the control response lost the restart boundary"
8650 );
8651 let Some(StderrTailEntry::Line {
8652 text,
8653 truncated,
8654 at_ms,
8655 }) = tail.entries.iter().find(|entry| {
8656 matches!(
8657 entry,
8658 StderrTailEntry::Line {
8659 truncated: true,
8660 ..
8661 }
8662 )
8663 })
8664 else {
8665 panic!("the control response lost the truncated line");
8666 };
8667 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8668 assert!(*truncated);
8669 assert!(
8670 at_ms.is_some(),
8671 "the control response lost the line's capture time"
8672 );
8673 }
8674
8675 #[tokio::test(flavor = "current_thread")]
8680 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8681 let dir = ScratchDir::new("terminals-off-worker");
8682 let journal_path = dir.join("terminals.jsonl");
8683 let registry = Arc::new(Registry::default());
8684 let supervisor_handle = SupervisorHandle::new();
8685 let supervisor =
8686 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8687 .with_handle(supervisor_handle.clone())
8688 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8689 let module = supervisor
8690 .spawn(ModuleSpec {
8691 module_id: "terminal-off-worker".to_string(),
8692 program: fake_aft_stub_path(),
8693 args: Vec::new(),
8694 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8695 reserved: false,
8696 reserved_prefixes: Vec::new(),
8697 protocol: ModuleProtocol::Subc,
8698 overlap: Default::default(),
8699 })
8700 .unwrap();
8701 let deadline = Instant::now() + Duration::from_secs(5);
8702 while module.terminal_history().entries.len() != 2 {
8703 assert!(Instant::now() < deadline, "module did not record two exits");
8704 sleep(Duration::from_millis(10)).await;
8705 }
8706
8707 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8708 let handler =
8709 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8710 let frame = Frame::build(
8711 FrameType::Request,
8712 control_flags(),
8713 0,
8714 0,
8715 1,
8716 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8717 module_id: "terminal-off-worker".to_string(),
8718 })
8719 .unwrap(),
8720 )
8721 .unwrap();
8722 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8723 let spawned_at = std::time::Instant::now();
8724 let read = tokio::spawn({
8725 let handler = Arc::clone(&handler);
8726 async move { handler.handle_control_frame(&ctx, frame).await }
8727 });
8728 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8731 .await
8732 .unwrap()
8733 .expect("the history read reached its pause");
8734 let elapsed = spawned_at.elapsed();
8735 assert!(
8736 elapsed < Duration::from_secs(2) && !read.is_finished(),
8737 "this task could not run while the history read was paused \
8738 (resumed after {elapsed:?}, read finished: {})",
8739 read.is_finished()
8740 );
8741
8742 drop(release);
8743 let responses = read.await.unwrap().unwrap();
8744 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8745 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8746 panic!("expected supervisor.terminals response");
8747 };
8748 assert_eq!(terminals.entries.len(), 2);
8749 assert_eq!(terminals.journal_skipped_lines, 0);
8750 assert_eq!(terminals.journal_read_errors, 0);
8751 }
8752
8753 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8754 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8755 let registry = Arc::new(Registry::default());
8756 let supervisor_handle = SupervisorHandle::new();
8757 let supervisor =
8758 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8759 .with_handle(supervisor_handle.clone());
8760 let module = supervisor
8761 .spawn(ModuleSpec {
8762 module_id: "terminal-golden".to_string(),
8763 program: fake_aft_stub_path(),
8764 args: Vec::new(),
8765 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8766 reserved: false,
8767 reserved_prefixes: Vec::new(),
8768 protocol: ModuleProtocol::Subc,
8769 overlap: Default::default(),
8770 })
8771 .unwrap();
8772
8773 let deadline = Instant::now() + Duration::from_secs(5);
8774 while module.terminal_history().entries.len() != 2 {
8775 assert!(
8776 Instant::now() < deadline,
8777 "module did not retain two terminal exits: {:?}",
8778 module.terminal_history()
8779 );
8780 sleep(Duration::from_millis(10)).await;
8781 }
8782
8783 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8784 let request = ClientControlRequest::SupervisorTerminals {
8785 module_id: "terminal-golden".to_string(),
8786 };
8787 let frame = Frame::build(
8788 FrameType::Request,
8789 control_flags(),
8790 0,
8791 0,
8792 1,
8793 serde_json::to_vec(&request).unwrap(),
8794 )
8795 .unwrap();
8796 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8797 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8798 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8799 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8800 panic!("expected supervisor.terminals response");
8801 };
8802 assert_eq!(terminals.entries.len(), 2);
8803 assert_eq!(terminals.dropped, 0);
8804
8805 let mut rendered = serde_json::to_value(response).unwrap();
8806 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8809 for (index, entry) in rendered["entries"]
8810 .as_array_mut()
8811 .expect("terminal response entries array")
8812 .iter_mut()
8813 .enumerate()
8814 {
8815 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8816 }
8817
8818 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8819 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8820 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8821 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8822 std::fs::write(&golden_path, &serialized).unwrap();
8823 }
8824 let expected: Value =
8825 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8826 assert_eq!(rendered, expected);
8827 }
8828
8829 #[test]
8830 fn hello_registers_manifest_and_returns_ack() {
8831 let registry = Arc::new(Registry::default());
8832 let handler = ControlHandler::new(Arc::clone(®istry));
8833 let conn = ConnectionId::new(1);
8834
8835 let responses = handler
8836 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8837 .unwrap();
8838
8839 assert_eq!(responses.len(), 1);
8840 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8841 assert_eq!(responses[0].header.channel, 0);
8842 assert_eq!(responses[0].header.corr, 7);
8843 let ack = parse_ack(&responses[0]);
8844 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8845 assert!(ack
8846 .subc_capabilities
8847 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8848 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8849 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8850 assert!(ack
8851 .subc_ops
8852 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8853 assert!(ack
8854 .subc_ops
8855 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8856
8857 let registration = registry.get_module("aft").unwrap().unwrap();
8858 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8859 assert_eq!(registration.state, ChannelState::Active);
8860 assert_eq!(registration.connection_id, conn);
8861 assert_eq!(registration.control_ops, module_baseline_control_ops());
8862 }
8863
8864 #[test]
8865 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8866 let invalid_identifiers = [
8867 ("case_change", "credentials-Provider/v1"),
8868 ("leading_zero", "credentials-provider/v01"),
8869 ("trailing_hyphen", "credentials-provider-/v1"),
8870 ("consecutive_hyphens", "credentials--provider/v1"),
8871 ("uppercase", "Credentials-provider/v1"),
8872 ("missing_v", "credentials-provider/1"),
8873 ("whitespace", "credentials provider/v1"),
8874 ("zero_version", "credentials-provider/v0"),
8875 ("out_of_range_version", "credentials-provider/v4294967296"),
8876 (
8877 "overlength_name",
8878 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8879 ),
8880 ];
8881 let mut cases = invalid_identifiers
8882 .into_iter()
8883 .map(|(name, identifier)| {
8884 (
8885 format!("identifier_{name}"),
8886 "capabilities.provides[0]".to_string(),
8887 identifier.to_string(),
8888 json!({ "provides": [identifier] }),
8889 None,
8890 )
8891 })
8892 .collect::<Vec<_>>();
8893 cases.extend([
8894 (
8895 "unknown_need".to_string(),
8896 "capabilities.requires[0].need".to_string(),
8897 "deferred".to_string(),
8898 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8899 None,
8900 ),
8901 (
8902 "duplicate_provides".to_string(),
8903 "capabilities.provides[1]".to_string(),
8904 "credentials-provider/v1".to_string(),
8905 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8906 None,
8907 ),
8908 (
8909 "duplicate_must_never_reach".to_string(),
8910 "capabilities.must_never_reach[1]".to_string(),
8911 "credentials-provider/v1".to_string(),
8912 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8913 None,
8914 ),
8915 (
8916 "duplicate_requires_same_need".to_string(),
8917 "capabilities.requires[1]".to_string(),
8918 "credentials-provider/v1".to_string(),
8919 json!({ "requires": [
8920 { "capability": "credentials-provider/v1", "need": "required" },
8921 { "capability": "credentials-provider/v1", "need": "required" }
8922 ] }),
8923 None,
8924 ),
8925 (
8926 "duplicate_requires_conflicting_need".to_string(),
8927 "capabilities.requires[1]".to_string(),
8928 "credentials-provider/v1".to_string(),
8929 json!({ "requires": [
8930 { "capability": "credentials-provider/v1", "need": "required" },
8931 { "capability": "credentials-provider/v1", "need": "optional" }
8932 ] }),
8933 None,
8934 ),
8935 (
8936 "capabilities_root_pointer".to_string(),
8937 "runtime_computed[0]".to_string(),
8938 "/capabilities".to_string(),
8939 json!({}),
8940 Some(json!(["/capabilities"])),
8941 ),
8942 (
8943 "capabilities_descendant_pointer".to_string(),
8944 "runtime_computed[0]".to_string(),
8945 "/capabilities/provides".to_string(),
8946 json!({}),
8947 Some(json!(["/capabilities/provides"])),
8948 ),
8949 (
8950 "malformed_pointer_without_leading_slash".to_string(),
8951 "runtime_computed[0]".to_string(),
8952 "capabilities".to_string(),
8953 json!({}),
8954 Some(json!(["capabilities"])),
8955 ),
8956 (
8957 "malformed_pointer_escape".to_string(),
8958 "runtime_computed[0]".to_string(),
8959 "/roles/~2/tools".to_string(),
8960 json!({}),
8961 Some(json!(["/roles/~2/tools"])),
8962 ),
8963 (
8964 "unknown_capabilities_field".to_string(),
8965 "capabilities.future".to_string(),
8966 "<array>".to_string(),
8967 json!({ "future": [] }),
8968 None,
8969 ),
8970 ]);
8971
8972 for (index, (name, field, value, capabilities, runtime_computed)) in
8973 cases.into_iter().enumerate()
8974 {
8975 let registry = Arc::new(Registry::default());
8976 let handler = ControlHandler::new(Arc::clone(®istry));
8977 let response = handler
8978 .handle_control(
8979 ConnectionId::new((index + 1) as u64),
8980 capability_grammar_hello_frame(
8981 capabilities,
8982 runtime_computed,
8983 index as u64 + 1,
8984 ),
8985 )
8986 .expect("invalid HELLO returns a refusal");
8987
8988 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8989 let error = parse_error(&response[0]);
8990 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8991 let message = error["message"]
8992 .as_str()
8993 .expect("error message is a string");
8994 assert!(
8995 message.contains(&field),
8996 "{name}: field missing from {message}"
8997 );
8998 assert!(
8999 message.contains(&value),
9000 "{name}: value missing from {message}"
9001 );
9002 assert_eq!(
9003 registry
9004 .active_registration_count()
9005 .expect("registry reads"),
9006 0,
9007 "{name}: refused HELLO must not create a catalog entry"
9008 );
9009 }
9010 }
9011
9012 #[test]
9013 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9014 let registry = Arc::new(Registry::default());
9015 let handler = ControlHandler::new(Arc::clone(®istry));
9016 let capabilities = json!({
9017 "provides": ["credentials-provider/v1"],
9018 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9019 "must_never_reach": ["federation-transport/v1"]
9020 });
9021 let response = handler
9022 .handle_control(
9023 ConnectionId::new(99),
9024 capability_grammar_hello_frame(
9025 capabilities.clone(),
9026 Some(json!(["/roles/0/tools"])),
9027 99,
9028 ),
9029 )
9030 .expect("valid HELLO registers");
9031 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9032
9033 let request = Frame::build(
9034 FrameType::Request,
9035 control_flags(),
9036 0,
9037 0,
9038 100,
9039 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9040 .expect("catalog request serializes"),
9041 )
9042 .expect("catalog request frame builds");
9043 let response = handler
9044 .handle_catalog_list(request, None)
9045 .expect("catalog list succeeds");
9046 let ClientControlResponse::CatalogList { modules, .. } =
9047 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9048 else {
9049 panic!("catalog request must return catalog.list");
9050 };
9051 assert_eq!(modules.len(), 1);
9052 assert_eq!(
9053 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9054 capabilities
9055 );
9056 }
9057
9058 #[test]
9059 fn catalog_list_mirrors_management_operation_description() {
9060 let registry = Arc::new(Registry::default());
9061 let handler = ControlHandler::new(Arc::clone(®istry));
9062 let description = "List managed records and return their identifiers and metadata.";
9063 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9064 manifest.provides = vec![ProviderRole::ManagementSurface {
9065 operations: vec![ManagementOperation {
9066 name: "records.list".to_string(),
9067 kind: ManagementOperationKind::Query,
9068 description: Some(description.to_string()),
9069 }],
9070 config_schema: json!({"type": "object"}),
9071 observability: vec![ObservabilitySurface {
9072 name: "records.stats".to_string(),
9073 kind: ObservabilityKind::Snapshot,
9074 }],
9075 identity_scope: vec![IdentityScope::Project],
9076 concurrency: Concurrency::ModuleManaged,
9077 }];
9078 registry
9079 .register_with_control_ops(
9080 manifest,
9081 PROTOCOL_VERSION,
9082 ConnectionId::new(99),
9083 Vec::new(),
9084 )
9085 .expect("described management manifest registers");
9086
9087 let request = Frame::build(
9088 FrameType::Request,
9089 control_flags(),
9090 0,
9091 0,
9092 100,
9093 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9094 .expect("catalog request serializes"),
9095 )
9096 .expect("catalog request frame builds");
9097 let response = handler
9098 .handle_catalog_list(request, None)
9099 .expect("catalog list succeeds");
9100 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9101 assert_eq!(
9102 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9103 "catalog.list must preserve the declared operation description verbatim"
9104 );
9105 }
9106
9107 #[test]
9108 fn catalog_list_mirrors_registered_operation_delivery() {
9109 let registry = Arc::new(Registry::default());
9110 let handler = ControlHandler::new(Arc::clone(®istry));
9111 let delivery =
9112 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9113 let mut body = serde_json::to_value(ModuleHelloBody {
9114 manifest: manifest("watch-management", PROTOCOL_VERSION),
9115 protocol_ver: PROTOCOL_VERSION,
9116 control_ops: None,
9117 launch_nonce: None,
9118 })
9119 .unwrap();
9120 body["manifest"]["provides"] = json!([{
9121 "role": "management_surface",
9122 "operations": [
9123 {"name": "status", "kind": "query"},
9124 {"name": "future", "kind": "query"},
9125 {"name": "reset", "kind": "mutate"}
9126 ],
9127 "config_schema": {}, "observability": [], "identity_scope": []
9128 }]);
9129 body["manifest"]["operation_delivery"] = delivery.clone();
9130 let hello = Frame::build(
9131 FrameType::Hello,
9132 control_flags(),
9133 0,
9134 0,
9135 99,
9136 serde_json::to_vec(&body).unwrap(),
9137 )
9138 .unwrap();
9139 let response = handler
9140 .handle_control(ConnectionId::new(99), hello)
9141 .unwrap();
9142 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9143
9144 let (_, registrations) = registry.list_modules().unwrap();
9145 let registered = ®istrations[0].manifest;
9146 assert!(registered.is_watch_operation("status"));
9147 assert!(!registered.is_watch_operation("future"));
9148 assert!(!registered.is_watch_operation("reset"));
9149 assert!(!registered.is_watch_operation("missing"));
9150
9151 let request = Frame::build(
9152 FrameType::Request,
9153 control_flags(),
9154 0,
9155 0,
9156 100,
9157 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9158 )
9159 .unwrap();
9160 let response = handler.handle_catalog_list(request, None).unwrap();
9161 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9162 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9163 let ClientControlResponse::CatalogList { modules, .. } =
9164 serde_json::from_slice(&response[0].body).unwrap()
9165 else {
9166 panic!("expected catalog.list");
9167 };
9168 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9169 }
9170
9171 #[test]
9172 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9173 let registry = Arc::new(Registry::default());
9174 let supervisor = SupervisorHandle::new();
9175 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9176 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9177 let descriptor = json!({
9178 "schema": "ckios.richui.manifest/1",
9179 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9180 "bytes": 4096,
9181 "chunks": 4
9182 });
9183 let connection = ConnectionId::new(201);
9184 let hello = handler
9185 .handle_control(
9186 connection,
9187 hello_frame_with_rich_ui(
9188 "vault",
9189 PROTOCOL_VERSION,
9190 201,
9191 Some("the-real-nonce"),
9192 descriptor.clone(),
9193 ),
9194 )
9195 .unwrap();
9196 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9197
9198 let request =
9199 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9200 let provides = registry
9201 .get_module_by_connection(connection)
9202 .unwrap()
9203 .unwrap()
9204 .manifest
9205 .provides;
9206 handler
9207 .handle_catalog_update(connection, request, provides, None, Some(false))
9208 .unwrap();
9209
9210 let request = Frame::build(
9211 FrameType::Request,
9212 control_flags(),
9213 0,
9214 0,
9215 203,
9216 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9217 )
9218 .unwrap();
9219 let response = handler.handle_catalog_list(request, None).unwrap();
9220 let ClientControlResponse::CatalogList { modules, .. } =
9221 serde_json::from_slice(&response[0].body).unwrap()
9222 else {
9223 panic!("expected catalog.list");
9224 };
9225 assert_eq!(
9226 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9227 descriptor
9228 );
9229 assert!(modules[0].nonce_verified);
9230 }
9231
9232 #[test]
9233 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9234 let registry = Arc::new(Registry::default());
9235 let handler = ControlHandler::new(registry.clone());
9236 let descriptor = json!({
9237 "schema": "ckios.richui.manifest/1",
9238 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9239 "bytes": 128,
9240 "chunks": 2
9241 });
9242 let malformed = [
9243 ("schema", json!("")),
9244 ("schema", json!("a".repeat(65))),
9245 ("schema", json!("Bad schema")),
9246 (
9247 "manifest_digest",
9248 json!(format!("sha256:{}", "A".repeat(64))),
9249 ),
9250 ("manifest_digest", json!("sha256:abc")),
9251 ("bytes", json!(0)),
9252 ("bytes", json!(16_777_217)),
9253 ("bytes", json!("128")),
9254 ("chunks", json!(0)),
9255 ("chunks", json!(1025)),
9256 ("chunks", json!("2")),
9257 ];
9258 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9259 let mut invalid = descriptor.clone();
9260 invalid[field] = invalid_value;
9261 let response = handler
9262 .handle_control(
9263 ConnectionId::new(210 + index as u64),
9264 hello_frame_with_rich_ui(
9265 &format!("bad-rich-ui-{index}"),
9266 PROTOCOL_VERSION,
9267 210 + index as u64,
9268 None,
9269 invalid,
9270 ),
9271 )
9272 .unwrap();
9273 assert_eq!(response[0].header.ty, FrameType::Error);
9274 let error = parse_error(&response[0]);
9275 assert_eq!(error["code"], "invalid_manifest");
9276 assert!(error["message"]
9277 .as_str()
9278 .unwrap()
9279 .contains(&format!("rich_ui.{field}")));
9280 assert!(registry
9281 .get_module(&format!("bad-rich-ui-{index}"))
9282 .unwrap()
9283 .is_none());
9284 }
9285 }
9286
9287 #[test]
9288 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9289 let registry = Arc::new(Registry::default());
9290 let handler = ControlHandler::new(Arc::clone(®istry));
9291 let connection = ConnectionId::new(219);
9292 registry
9293 .register_with_control_ops(
9294 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9295 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9296 "bad schema",
9297 format!("sha256:{}", "a".repeat(64)),
9298 128,
9299 2,
9300 )))
9301 .build(),
9302 PROTOCOL_VERSION,
9303 connection,
9304 Vec::new(),
9305 )
9306 .unwrap();
9307 let request =
9308 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9309 let response = handler
9310 .handle_catalog_update(connection, request, Vec::new(), None, None)
9311 .unwrap();
9312 assert_eq!(response[0].header.ty, FrameType::Error);
9313 let error = parse_error(&response[0]);
9314 assert_eq!(error["code"], "invalid_manifest");
9315 assert!(error["message"]
9316 .as_str()
9317 .unwrap()
9318 .contains("rich_ui.schema"));
9319 }
9320
9321 #[test]
9322 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9323 let registry = Arc::new(Registry::default());
9324 let supervisor = SupervisorHandle::new();
9325 supervisor.open_swap("provider", "candidate-nonce".to_string());
9326 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9327 let connection = ConnectionId::new(218);
9328 let hello = handler
9329 .handle_control(
9330 connection,
9331 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9332 )
9333 .unwrap();
9334 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9335 assert!(registry.list_modules().unwrap().1.is_empty());
9336
9337 registry.promote_candidate("provider").unwrap().unwrap();
9338 let request = Frame::build(
9339 FrameType::Request,
9340 control_flags(),
9341 0,
9342 0,
9343 219,
9344 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9345 )
9346 .unwrap();
9347 let response = handler.handle_catalog_list(request, None).unwrap();
9348 let ClientControlResponse::CatalogList { modules, .. } =
9349 serde_json::from_slice(&response[0].body).unwrap()
9350 else {
9351 panic!("expected catalog.list");
9352 };
9353 assert_eq!(modules.len(), 1);
9354 assert!(modules[0].nonce_verified);
9355 }
9356
9357 #[test]
9358 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9359 let registry = Arc::new(Registry::default());
9360 let handler = ControlHandler::new(Arc::clone(®istry));
9361 let hello = Frame::build(
9362 FrameType::Hello,
9363 control_flags(),
9364 0,
9365 0,
9366 220,
9367 serde_json::to_vec(&ModuleHelloBody {
9368 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9369 protocol_ver: PROTOCOL_VERSION,
9370 control_ops: None,
9371 launch_nonce: Some("unverified".to_string()),
9372 })
9373 .unwrap(),
9374 )
9375 .unwrap();
9376 assert_eq!(
9377 handler
9378 .handle_control(ConnectionId::new(220), hello)
9379 .unwrap()[0]
9380 .header
9381 .ty,
9382 FrameType::HelloAck
9383 );
9384 let request = Frame::build(
9385 FrameType::Request,
9386 control_flags(),
9387 0,
9388 0,
9389 221,
9390 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9391 )
9392 .unwrap();
9393 let response = handler.handle_catalog_list(request, None).unwrap();
9394 let ClientControlResponse::CatalogList { modules, .. } =
9395 serde_json::from_slice(&response[0].body).unwrap()
9396 else {
9397 panic!("expected catalog.list");
9398 };
9399 assert_eq!(
9400 serde_json::to_vec(&modules[0]).unwrap(),
9401 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9402 );
9403 assert!(!modules[0].nonce_verified);
9404 }
9405
9406 #[test]
9407 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9408 let registry = Arc::new(Registry::default());
9409 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9410 [("vault".to_string(), true), ("squatter".to_string(), true)],
9411 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9412 );
9413 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9414 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9415 provides: vec!["credentials-provider/v1".to_string()],
9416 requires: Vec::new(),
9417 must_never_reach: Vec::new(),
9418 });
9419 let frame = Frame::build(
9420 FrameType::Hello,
9421 control_flags(),
9422 0,
9423 0,
9424 77,
9425 serde_json::to_vec(&ModuleHelloBody {
9426 manifest: squatter,
9427 protocol_ver: PROTOCOL_VERSION,
9428 control_ops: None,
9429 launch_nonce: None,
9430 })
9431 .expect("HELLO serializes"),
9432 )
9433 .expect("HELLO frame builds");
9434 let response = handler
9435 .handle_control(ConnectionId::new(77), frame)
9436 .expect("reserved claim receives a typed refusal");
9437 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9438 assert_eq!(
9439 registry
9440 .active_registration_count()
9441 .expect("registry reads"),
9442 0,
9443 "a reserved capability refusal must not leave a catalog entry"
9444 );
9445 }
9446
9447 #[test]
9448 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9449 for settlement in ["timeout", "inconclusive", "drop"] {
9450 let breakers = RouteBindBreakers::default();
9451 let RouteBindAdmission::Admitted {
9452 guard: mut old,
9453 probe: false,
9454 } = breakers.admit("prov")
9455 else {
9456 panic!("ordinary relay admitted")
9457 };
9458 let RouteBindAdmission::Admitted {
9459 guard: mut opener, ..
9460 } = breakers.admit("prov")
9461 else {
9462 panic!("second relay admitted")
9463 };
9464 assert!(
9465 !opener
9466 .record_timeout(1, Duration::ZERO)
9467 .unwrap()
9468 .reopened_after_probe
9469 );
9470 let RouteBindAdmission::Admitted {
9471 guard: mut probe,
9472 probe: true,
9473 } = breakers.admit("prov")
9474 else {
9475 panic!("one cooldown probe admitted")
9476 };
9477 match settlement {
9478 "timeout" => assert!(
9479 !old.record_timeout(1, Duration::ZERO)
9480 .unwrap()
9481 .reopened_after_probe
9482 ),
9483 "inconclusive" => old.record_inconclusive(),
9484 "drop" => drop(old),
9485 _ => unreachable!(),
9486 }
9487 assert!(
9488 matches!(
9489 breakers.admit("prov"),
9490 RouteBindAdmission::Refused {
9491 probe_in_flight: true,
9492 ..
9493 }
9494 ),
9495 "{settlement} of a pre-open relay cannot release the real probe"
9496 );
9497 assert!(
9498 probe
9499 .record_timeout(1, Duration::ZERO)
9500 .unwrap()
9501 .reopened_after_probe
9502 );
9503 assert!(matches!(
9504 breakers.admit("prov"),
9505 RouteBindAdmission::Admitted { probe: true, .. }
9506 ));
9507 }
9508 let breakers = RouteBindBreakers::default();
9509 let admit = || match breakers.admit("prov") {
9510 RouteBindAdmission::Admitted { guard, .. } => guard,
9511 _ => panic!("relay admitted"),
9512 };
9513 admit().record_timeout(1, Duration::ZERO);
9514 let mut old_probe = admit();
9515 breakers.reset_for_new_module_connection("prov");
9516 admit().record_timeout(1, Duration::ZERO);
9517 let _new_probe = admit();
9518 old_probe.record_inconclusive();
9519 assert!(matches!(
9520 breakers.admit("prov"),
9521 RouteBindAdmission::Refused {
9522 probe_in_flight: true,
9523 ..
9524 }
9525 ));
9526 }
9527
9528 #[tokio::test]
9529 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9530 for candidate in [false, true] {
9531 let registry = Arc::new(Registry::default());
9532 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9533 [("vault".to_string(), true), ("squatter".to_string(), true)],
9534 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9535 );
9536 let conn = ConnectionId::new(77);
9537 let (ctx, mut rx) = route_ctx(conn);
9538 let initial = capability_manifest("squatter", &[], &[]);
9539 if candidate {
9540 registry
9541 .register_candidate_with_control_ops(
9542 initial.clone(),
9543 PROTOCOL_VERSION,
9544 conn,
9545 module_baseline_control_ops(),
9546 )
9547 .unwrap();
9548 handler
9549 .forwarding
9550 .register_candidate_module_connection(
9551 conn,
9552 "squatter".to_string(),
9553 PROTOCOL_VERSION,
9554 manifest_concurrency(&initial),
9555 ctx.egress.clone(),
9556 )
9557 .unwrap();
9558 } else {
9559 hello_via_sink(
9560 &handler,
9561 &ctx,
9562 &mut rx,
9563 hello_frame_with_manifest(initial.clone(), 1),
9564 )
9565 .await;
9566 }
9567 let response = handler
9568 .handle_control_frame(
9569 &ctx,
9570 catalog_update_with_capabilities_frame(
9571 2,
9572 capability_manifest("squatter", &["credentials-provider/v1"], &[])
9573 .capabilities
9574 .unwrap(),
9575 ),
9576 )
9577 .await
9578 .unwrap();
9579 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9580 assert_eq!(
9581 registry
9582 .get_module_by_connection(conn)
9583 .unwrap()
9584 .unwrap()
9585 .manifest,
9586 initial
9587 );
9588 }
9589 }
9590
9591 #[test]
9592 fn server_describe_surfaces_required_capability_verdict_fields() {
9593 let registry = Arc::new(Registry::default());
9594 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9595 [
9596 ("consumer".to_string(), true),
9597 ("provider".to_string(), false),
9598 ],
9599 BTreeMap::new(),
9600 );
9601 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9602 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9603 provides: Vec::new(),
9604 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9605 capability: "credentials-provider/v1".to_string(),
9606 need: subc_protocol::manifest::CapabilityNeed::Required,
9607 }],
9608 must_never_reach: Vec::new(),
9609 });
9610 let hello = Frame::build(
9611 FrameType::Hello,
9612 control_flags(),
9613 0,
9614 0,
9615 78,
9616 serde_json::to_vec(&ModuleHelloBody {
9617 manifest: consumer,
9618 protocol_ver: PROTOCOL_VERSION,
9619 control_ops: None,
9620 launch_nonce: None,
9621 })
9622 .expect("HELLO serializes"),
9623 )
9624 .expect("HELLO frame builds");
9625 handler
9626 .handle_control(ConnectionId::new(78), hello)
9627 .expect("consumer registers");
9628 let describe = Frame::build(
9629 FrameType::Request,
9630 control_flags(),
9631 0,
9632 0,
9633 79,
9634 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9635 .expect("request serializes"),
9636 )
9637 .expect("describe frame builds");
9638 let response = handler
9639 .handle_server_describe(describe)
9640 .expect("server.describe succeeds");
9641 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9642 let requirement = &rendered["capability_requirements"][0];
9643 assert_eq!(requirement["consumer"], "consumer");
9644 assert_eq!(requirement["verdict"], "never_provided");
9645 assert_eq!(requirement["episode_seq"], 1);
9646 assert_eq!(requirement["config_satisfiable"], false);
9647 assert_eq!(requirement["runtime_available"], false);
9648 assert!(requirement["detail"]
9649 .as_str()
9650 .expect("detail string")
9651 .contains("credentials-provider/v1"));
9652 }
9653
9654 fn deregister_required_provider(
9660 daemon_shutdown_begun: bool,
9661 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
9662 const CAPABILITY: &str = "project-identity/v1";
9663 let registry = Arc::new(Registry::default());
9664 let daemon_shutdown = DaemonShutdownFlag::default();
9665 let handler = ControlHandler::new(Arc::clone(®istry))
9666 .with_capability_config(
9667 [
9668 ("consumer".to_string(), true),
9669 ("provider".to_string(), true),
9670 ],
9671 BTreeMap::new(),
9672 )
9673 .with_daemon_shutdown_flag(daemon_shutdown.clone());
9674 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9675 consumer.capabilities = Some(CapabilityDeclarations {
9676 provides: Vec::new(),
9677 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9678 capability: CAPABILITY.to_string(),
9679 need: subc_protocol::manifest::CapabilityNeed::Required,
9680 }],
9681 must_never_reach: Vec::new(),
9682 });
9683 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
9684 let provider_conn = ConnectionId::new(182);
9685 for (conn, manifest, corr) in [
9686 (ConnectionId::new(181), consumer, 181),
9687 (provider_conn, provider, 182),
9688 ] {
9689 let hello = handler
9690 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
9691 .expect("module registers");
9692 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9693 }
9694 assert_eq!(
9695 handler.capability_evaluator.verdict("consumer", CAPABILITY),
9696 Some(CapabilityVerdict::Provided),
9697 "the registered provider must satisfy the requirement first"
9698 );
9699 if daemon_shutdown_begun {
9700 daemon_shutdown.set_for_test();
9701 }
9702
9703 let capture = EventCapture::default();
9704 let _subscriber =
9705 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9706 handler
9707 .cleanup_connection(provider_conn)
9708 .expect("provider deregisters");
9709 let events = capture
9710 .events()
9711 .into_iter()
9712 .filter(|event| {
9713 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
9714 })
9715 .collect();
9716 (
9717 events,
9718 handler.capability_evaluator.verdict("consumer", CAPABILITY),
9719 )
9720 }
9721
9722 #[test]
9723 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
9724 let (events, verdict) = deregister_required_provider(true);
9725 assert_eq!(
9726 verdict,
9727 Some(CapabilityVerdict::NeverProvided),
9728 "shutdown must not change the verdict route.open refusals read"
9729 );
9730 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9731 let event = &events[0];
9732 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
9733 assert_eq!(
9734 event.fields.get("cause").map(String::as_str),
9735 Some("\"daemon_shutdown\""),
9736 "{event:?}"
9737 );
9738 assert_eq!(
9739 event.fields.get("verdict").map(String::as_str),
9740 Some("\"never_provided\""),
9741 "{event:?}"
9742 );
9743 }
9744
9745 #[test]
9746 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
9747 let (events, verdict) = deregister_required_provider(false);
9748 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
9749 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9750 let event = &events[0];
9751 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
9752 assert_eq!(event.fields.get("cause"), None, "{event:?}");
9753 assert_eq!(
9754 event.fields.get("verdict").map(String::as_str),
9755 Some("\"never_provided\""),
9756 "{event:?}"
9757 );
9758 }
9759
9760 #[test]
9761 fn catalog_list_omits_capabilities_for_legacy_manifest() {
9762 let registry = Arc::new(Registry::default());
9763 let handler = ControlHandler::new(Arc::clone(®istry));
9764 let hello = handler
9765 .handle_control(
9766 ConnectionId::new(101),
9767 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9768 )
9769 .expect("legacy HELLO registers");
9770 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9771
9772 let request = Frame::build(
9773 FrameType::Request,
9774 control_flags(),
9775 0,
9776 0,
9777 102,
9778 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9779 .expect("catalog request serializes"),
9780 )
9781 .expect("catalog request frame builds");
9782 let response = handler
9783 .handle_catalog_list(request, None)
9784 .expect("catalog list succeeds");
9785 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9786 assert!(
9787 body["modules"][0].get("capabilities").is_none(),
9788 "legacy manifest must retain an absent capabilities field on catalog.list"
9789 );
9790 }
9791
9792 #[test]
9793 fn hello_ack_omits_storage_when_no_storage_config() {
9794 let registry = Arc::new(Registry::default());
9795 let handler = ControlHandler::new(Arc::clone(®istry));
9796 let responses = handler
9797 .handle_control(
9798 ConnectionId::new(1),
9799 hello_frame("aft", PROTOCOL_VERSION, 7),
9800 )
9801 .unwrap();
9802 let ack = parse_ack(&responses[0]);
9803 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9804 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9805 }
9806
9807 #[tokio::test]
9808 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9809 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9810 let registry = Arc::new(Registry::default());
9811 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
9812 let responses = handler
9813 .handle_control(
9814 ConnectionId::new(1),
9815 hello_frame("aft", PROTOCOL_VERSION, 7),
9816 )
9817 .unwrap();
9818 let ack = parse_ack(&responses[0]);
9819 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9820
9821 let described = handler
9822 .handle_control_frame(
9823 &route_ctx(ConnectionId::new(2)).0,
9824 Frame::build(
9825 FrameType::Request,
9826 control_flags(),
9827 0,
9828 0,
9829 9,
9830 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9831 )
9832 .unwrap(),
9833 )
9834 .await
9835 .unwrap();
9836 let ClientControlResponse::ServerDescribe { machine_id, .. } =
9837 serde_json::from_slice(&described[0].body).unwrap()
9838 else {
9839 panic!("server.describe answered with another shape");
9840 };
9841 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9842 }
9843
9844 #[test]
9845 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9846 let registry = Arc::new(Registry::default());
9849 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
9850 crate::daemon_config::StorageConfig::Sqlite {
9851 data_home: std::path::PathBuf::from("/data"),
9852 },
9853 ));
9854
9855 let responses = handler
9856 .handle_control(
9857 ConnectionId::new(1),
9858 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9859 )
9860 .unwrap();
9861 let ack = parse_ack(&responses[0]);
9862 assert_eq!(
9863 ack.storage,
9864 Some(serde_json::json!({
9865 "module_id": "alfonso-routing",
9866 "storage_namespace": "default",
9867 "isolation": { "kind": "module" },
9868 "backend": {
9869 "backend": "sqlite",
9870 "path": "/data/cortexkit/alfonso-routing/store.db"
9871 }
9872 })),
9873 "the delivered descriptor is the module's own sqlite store path"
9874 );
9875 }
9876
9877 #[test]
9878 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9879 let registry = Arc::new(Registry::default());
9880 let handler = ControlHandler::new(Arc::clone(®istry));
9881 let conn = ConnectionId::new(1);
9882 let responses = handler
9883 .handle_control(
9884 conn,
9885 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9886 )
9887 .unwrap();
9888 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9889 let registration = registry.get_module("aft").unwrap().unwrap();
9890 assert_eq!(registration.control_ops, module_baseline_control_ops());
9891
9892 let frame =
9893 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9894 assert!(handler
9895 .guard_module_control_op(&frame, "aft", "route.bind")
9896 .unwrap()
9897 .is_none());
9898 let error = handler
9899 .guard_module_control_op(&frame, "aft", "test.synthetic")
9900 .unwrap()
9901 .expect("synthetic ungranted op should be rejected");
9902 assert_eq!(error.header.ty, FrameType::Error);
9903 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9904 }
9905
9906 #[test]
9907 fn hello_control_ops_some_adds_optional_grants() {
9908 let registry = Arc::new(Registry::default());
9909 let handler = ControlHandler::new(Arc::clone(®istry));
9910 handler
9911 .handle_control(
9912 ConnectionId::new(1),
9913 hello_frame_with_control_ops(
9914 "aft",
9915 PROTOCOL_VERSION,
9916 7,
9917 Some(vec![
9918 "future.synthetic".to_string(),
9919 "route.bind".to_string(),
9920 ]),
9921 ),
9922 )
9923 .unwrap();
9924 let registration = registry.get_module("aft").unwrap().unwrap();
9925 assert_eq!(
9926 registration.control_ops,
9927 vec![
9928 "route.bind".to_string(),
9929 "route.status".to_string(),
9930 "future.synthetic".to_string(),
9931 ]
9932 );
9933 let frame =
9934 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9935 assert!(handler
9936 .guard_module_control_op(&frame, "aft", "future.synthetic")
9937 .unwrap()
9938 .is_none());
9939 }
9940
9941 #[tokio::test]
9942 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9943 let registry = Arc::new(Registry::default());
9944 let forwarding = Arc::new(ForwardingTable::default());
9945 let handler =
9946 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9947 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9948 hello_via_sink(
9949 &handler,
9950 &module_ctx,
9951 &mut module_rx,
9952 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9953 )
9954 .await;
9955
9956 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9957 let responses = handler
9958 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9959 .await
9960 .unwrap();
9961 assert_eq!(responses.len(), 1);
9962 assert_eq!(responses[0].header.ty, FrameType::Error);
9963 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9964 assert!(module_rx.try_recv().is_err());
9965 }
9966
9967 #[tokio::test]
9968 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9969 let registry = Arc::new(Registry::default());
9970 let forwarding = Arc::new(ForwardingTable::default());
9971 let handler =
9972 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9973 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9974 hello_via_sink(
9975 &handler,
9976 &module_ctx,
9977 &mut module_rx,
9978 hello_frame_with_control_ops(
9979 "aft",
9980 PROTOCOL_VERSION,
9981 7,
9982 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9983 ),
9984 )
9985 .await;
9986
9987 let project_root = unique_project_root("demux");
9988 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9989 let route_handler = handler.clone();
9990 let route_task = tokio::spawn(async move {
9991 route_handler
9992 .handle_control_frame(
9993 &route_client_ctx,
9994 route_open_frame(100, "aft", project_root),
9995 )
9996 .await
9997 .unwrap()
9998 });
9999 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10000 .await
10001 .unwrap()
10002 .unwrap();
10003 assert!(matches!(
10004 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10005 ModuleControlRequest::RouteBind { .. }
10006 ));
10007
10008 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10009 let health_handler = handler.clone();
10010 let health_task = tokio::spawn(async move {
10011 health_handler
10012 .handle_control_frame(
10013 &health_client_ctx,
10014 supervisor_health_probe_frame(101, "aft"),
10015 )
10016 .await
10017 .unwrap()
10018 });
10019 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10020 .await
10021 .unwrap()
10022 .unwrap();
10023 assert_eq!(
10024 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10025 ModuleControlRequest::HealthCheck {}
10026 );
10027
10028 handler
10029 .handle_control_frame(
10030 &module_ctx,
10031 health_response(health_frame.header.corr, HealthStatus::Degraded),
10032 )
10033 .await
10034 .unwrap();
10035 let health_response = health_task.await.unwrap();
10036 assert_eq!(health_response.len(), 1);
10037 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10038 ClientControlResponse::SupervisorHealthProbe {
10039 module_id,
10040 status,
10041 detail,
10042 metrics,
10043 } => {
10044 assert_eq!(module_id, "aft");
10045 assert_eq!(status, HealthStatus::Degraded);
10046 assert_eq!(detail.as_deref(), Some("warming"));
10047 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10048 }
10049 other => panic!("unexpected health response: {other:?}"),
10050 }
10051
10052 handler
10053 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10054 .await
10055 .unwrap();
10056 let route_response = route_task.await.unwrap();
10057 assert!(route_response.is_empty());
10058 let published = route_client_rx.recv().await.unwrap();
10059 assert!(matches!(
10060 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10061 ClientControlResponse::RouteOpen { .. }
10062 ));
10063 }
10064
10065 async fn relay_route_open(
10070 handler: &ControlHandler,
10071 client_connection: ConnectionId,
10072 client_egress: &FrameSink,
10073 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10074 corr: u64,
10075 module_id: &str,
10076 project_root_label: &str,
10077 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10078 let ctx = RouteCtx {
10079 connection_id: client_connection,
10080 egress: client_egress.clone(),
10081 };
10082 let handler = handler.clone();
10083 let project_root = unique_project_root(project_root_label);
10084 let module_id = module_id.to_string();
10085 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10086 let task = tokio::spawn(async move {
10087 let _guard = tracing::dispatcher::set_default(&dispatch);
10088 handler
10089 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10090 .await
10091 .unwrap()
10092 });
10093 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10094 .await
10095 .expect("module receives the relayed route.bind")
10096 .expect("module egress is open");
10097 (task, bind.frame)
10098 }
10099
10100 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10101 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10102 ModuleControlRequest::RouteBind {
10103 route_channel,
10104 epoch,
10105 ..
10106 } => (route_channel, epoch),
10107 other => panic!("expected a route.bind request, got {other:?}"),
10108 }
10109 }
10110
10111 fn published_route(frame: &Frame) -> (u16, u32) {
10112 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10113 ClientControlResponse::RouteOpen {
10114 route_channel,
10115 route_epoch,
10116 } => (route_channel, route_epoch),
10117 other => panic!("expected a route.open response, got {other:?}"),
10118 }
10119 }
10120
10121 #[tokio::test]
10141 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10142 let registry = Arc::new(Registry::default());
10143 let forwarding = Arc::new(ForwardingTable::default());
10144 let handler =
10145 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10146
10147 let module_connection = ConnectionId::new(30);
10148 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10149 hello_via_sink(
10150 &handler,
10151 &module_ctx,
10152 &mut module_rx,
10153 hello_frame("aft", PROTOCOL_VERSION, 7),
10154 )
10155 .await;
10156
10157 let dying_client = ConnectionId::new(31);
10158 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10159
10160 let (first_task, first_bind) = relay_route_open(
10163 &handler,
10164 dying_client,
10165 &dying_ctx.egress,
10166 &mut module_rx,
10167 100,
10168 "aft",
10169 "closing-first",
10170 )
10171 .await;
10172 handler
10173 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10174 .await
10175 .unwrap();
10176 assert!(first_task.await.unwrap().is_empty());
10177 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10178
10179 let (second_task, second_bind) = relay_route_open(
10181 &handler,
10182 dying_client,
10183 &dying_ctx.egress,
10184 &mut module_rx,
10185 101,
10186 "aft",
10187 "closing-second",
10188 )
10189 .await;
10190 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10191
10192 assert!(forwarding
10195 .escalate_client_delivery_failure(
10196 dying_client,
10197 first_channel,
10198 first_epoch,
10199 CloseReason::new(
10200 "module_to_client_delivery_failed",
10201 "client egress refused a module frame",
10202 ),
10203 crate::forwarding::UndeliveredFrame {
10204 module_id: None,
10205 sink: &dying_ctx.egress,
10206 },
10207 )
10208 .unwrap());
10209 assert!(!dying_ctx.egress.is_closed());
10210
10211 let ack = handler
10213 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10214 .await;
10215 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10216 if module_loop_error.is_some() {
10217 handler.cleanup_connection(module_connection).unwrap();
10221 }
10222 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10226 .await
10227 .ok()
10228 .flatten();
10229
10230 assert!(
10232 registry
10233 .get_module_by_connection(module_connection)
10234 .unwrap()
10235 .is_some(),
10236 "one client's closing connection ended the shared module connection: \
10237 {module_loop_error:?}"
10238 );
10239 let cotenant = ConnectionId::new(32);
10241 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10242 let (cotenant_task, cotenant_bind) = relay_route_open(
10243 &handler,
10244 cotenant,
10245 &cotenant_ctx.egress,
10246 &mut module_rx,
10247 102,
10248 "aft",
10249 "closing-cotenant",
10250 )
10251 .await;
10252 handler
10253 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10254 .await
10255 .unwrap();
10256 assert!(cotenant_task.await.unwrap().is_empty());
10257 let (cotenant_channel, cotenant_epoch) =
10258 published_route(&cotenant_rx.recv().await.unwrap());
10259 assert!(matches!(
10260 forwarding
10261 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10262 .unwrap(),
10263 DataRoute::Client(DataRouteState::Bound(_))
10264 ));
10265
10266 let goodbye = post_ack_module_frame
10269 .expect("module receives a GOODBYE for the abandoned route channel");
10270 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10271 assert_eq!(goodbye.header.channel, abandoned_channel);
10272 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10273
10274 assert!(dying_rx.try_recv().is_err());
10278 let second_response = second_task.await.unwrap();
10279 assert_eq!(second_response.len(), 1);
10280 assert_eq!(
10281 parse_error(&second_response[0])["code"],
10282 "target_unavailable"
10283 );
10284 }
10285
10286 async fn catalog_snapshot(
10289 handler: &ControlHandler,
10290 client_ctx: &RouteCtx,
10291 corr: u64,
10292 ) -> (u64, Vec<(String, bool)>) {
10293 let body =
10294 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10295 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10296 let reply = handler
10297 .handle_control_frame(client_ctx, request)
10298 .await
10299 .unwrap()
10300 .pop()
10301 .unwrap();
10302 let ClientControlResponse::CatalogList {
10303 generation,
10304 modules,
10305 ..
10306 } = serde_json::from_slice(&reply.body).unwrap()
10307 else {
10308 panic!("not a catalog.list reply");
10309 };
10310 (
10311 generation,
10312 modules
10313 .into_iter()
10314 .map(|entry| (entry.module_id, entry.ready))
10315 .collect(),
10316 )
10317 }
10318
10319 fn hello_held_before_forwarding() -> (
10325 ControlHandler,
10326 Arc<std::sync::Barrier>,
10327 std::thread::JoinHandle<Vec<Frame>>,
10328 RouteCtx,
10329 mpsc::Receiver<crate::router::OutboundFrame>,
10330 ) {
10331 let hold = Arc::new(std::sync::Barrier::new(2));
10332 let handler = ControlHandler::with_forwarding(
10333 Arc::new(Registry::default()),
10334 Arc::new(ForwardingTable::default()),
10335 )
10336 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10337 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10338 let hello = {
10339 let handler = handler.clone();
10340 let module_ctx = module_ctx.clone();
10341 std::thread::spawn(move || {
10342 handler
10343 .handle_hello(
10344 module_ctx.connection_id,
10345 Some(module_ctx.egress.clone()),
10346 hello_frame("aft", PROTOCOL_VERSION, 7),
10347 )
10348 .unwrap()
10349 })
10350 };
10351 (handler, hold, hello, module_ctx, module_rx)
10352 }
10353
10354 #[tokio::test]
10363 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10364 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10365 let client = ConnectionId::new(41);
10366 let (client_ctx, mut client_rx) = route_ctx(client);
10367
10368 hold.wait();
10369 assert!(handler.registry.get_module("aft").unwrap().is_some());
10371 assert!(!handler
10372 .forwarding
10373 .has_live_module_connection("aft")
10374 .unwrap());
10375 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10376 assert!(
10377 listed.is_empty(),
10378 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10379 );
10380 let refused = handler
10381 .handle_control_frame(
10382 &client_ctx,
10383 route_open_frame(301, "aft", unique_project_root("listed-window")),
10384 )
10385 .await
10386 .unwrap();
10387 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10388 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10389 let duplicate = handler
10390 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10391 .await
10392 .unwrap();
10393 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10394
10395 hold.wait();
10396 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10397 let ack = module_rx.try_recv().unwrap().frame;
10398 assert_eq!(ack.header.ty, FrameType::HelloAck);
10399 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10400 assert_eq!(listed, vec![("aft".to_string(), true)]);
10401
10402 let (open_task, bind) = relay_route_open(
10403 &handler,
10404 client,
10405 &client_ctx.egress,
10406 &mut module_rx,
10407 303,
10408 "aft",
10409 "listed-after",
10410 )
10411 .await;
10412 assert!(handler
10413 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10414 .await
10415 .unwrap()
10416 .is_empty());
10417 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10418 published_route(&client_rx.try_recv().unwrap().frame);
10419 }
10420
10421 #[tokio::test]
10428 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10429 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10430 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10431 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10432
10433 hold.wait();
10434 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10435 assert!(listed.is_empty());
10436 assert_eq!(
10437 during, before,
10438 "the generation moved while the module was not listed"
10439 );
10440
10441 hold.wait();
10442 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10443 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10444 assert_eq!(listed, vec![("aft".to_string(), true)]);
10445 assert_eq!(
10446 after,
10447 before + 1,
10448 "the module appeared without exactly one generation bump"
10449 );
10450 }
10451
10452 #[tokio::test]
10457 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10458 let registry = Arc::new(Registry::default());
10459 let handler = ControlHandler::with_forwarding(
10460 Arc::clone(®istry),
10461 Arc::new(ForwardingTable::default()),
10462 );
10463 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10464 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10465 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10466 drop(module_rx);
10467
10468 let replies = handler
10469 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10470 .await
10471 .unwrap();
10472 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10473 assert!(registry.get_module("aft").unwrap().is_none());
10474 assert!(registry
10475 .get_module_by_connection(module_ctx.connection_id)
10476 .unwrap()
10477 .is_none());
10478 assert_eq!(
10479 catalog_snapshot(&handler, &client_ctx, 301).await,
10480 (before, Vec::new())
10481 );
10482 }
10483
10484 async fn route_open_held_after_bind_send() -> (
10488 ControlHandler,
10489 Arc<tokio::sync::Barrier>,
10490 RouteCtx,
10491 mpsc::Receiver<crate::router::OutboundFrame>,
10492 ) {
10493 let hold = Arc::new(tokio::sync::Barrier::new(2));
10494 let handler = ControlHandler::with_forwarding(
10495 Arc::new(Registry::default()),
10496 Arc::new(ForwardingTable::default()),
10497 )
10498 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10499 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10500 hello_via_sink(
10501 &handler,
10502 &module_ctx,
10503 &mut module_rx,
10504 hello_frame("aft", PROTOCOL_VERSION, 7),
10505 )
10506 .await;
10507 (handler, hold, module_ctx, module_rx)
10508 }
10509
10510 #[tokio::test]
10517 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10518 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10519 let client = ConnectionId::new(41);
10520 let (client_ctx, mut client_rx) = route_ctx(client);
10521
10522 let (open_task, bind) = relay_route_open(
10523 &handler,
10524 client,
10525 &client_ctx.egress,
10526 &mut module_rx,
10527 200,
10528 "aft",
10529 "acked-before-mark",
10530 )
10531 .await;
10532 let (module_channel, module_epoch) = route_bind_channel(&bind);
10533 hold.wait().await;
10535
10536 assert!(handler
10538 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10539 .await
10540 .unwrap()
10541 .is_empty());
10542 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10543
10544 hold.wait().await;
10545 assert!(
10546 open_task.await.unwrap().is_empty(),
10547 "route.open answers accepted"
10548 );
10549
10550 match module_rx.try_recv() {
10553 Err(mpsc::error::TryRecvError::Empty) => {}
10554 Ok(outbound) => panic!(
10555 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10556 outbound.frame.header.ty,
10557 outbound.frame.header.channel,
10558 outbound.frame.header.epoch
10559 ),
10560 Err(err) => panic!("module egress closed: {err}"),
10561 }
10562
10563 let router = Router::with_control_handler(Arc::new(handler.clone()));
10566 let request = Frame::build(
10567 FrameType::Request,
10568 Flags::new(true, Priority::Interactive, false),
10569 client_channel,
10570 client_epoch,
10571 201,
10572 b"first request".to_vec(),
10573 )
10574 .unwrap();
10575 router
10576 .route_for_connection(&client_ctx, request)
10577 .await
10578 .unwrap();
10579 let delivered = module_rx.try_recv().unwrap().frame;
10580 assert_eq!(delivered.header.ty, FrameType::Request);
10581 assert_eq!(
10582 (delivered.header.channel, delivered.header.epoch),
10583 (module_channel, module_epoch)
10584 );
10585 assert_eq!(delivered.body, b"first request");
10586 }
10587
10588 #[tokio::test]
10594 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10595 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10596 let client = ConnectionId::new(42);
10597 let (client_ctx, _client_rx) = route_ctx(client);
10598
10599 let (open_task, bind) = relay_route_open(
10600 &handler,
10601 client,
10602 &client_ctx.egress,
10603 &mut module_rx,
10604 300,
10605 "aft",
10606 "released-before-mark",
10607 )
10608 .await;
10609 let (module_channel, module_epoch) = route_bind_channel(&bind);
10610 hold.wait().await;
10611
10612 handler.cleanup_connection(client).unwrap();
10613 assert!(
10614 module_rx.try_recv().is_err(),
10615 "client teardown must not GOODBYE a bind route.open is still queueing"
10616 );
10617
10618 hold.wait().await;
10619 let response = open_task.await.unwrap();
10620 assert_eq!(response.len(), 1);
10621 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10622
10623 let goodbye = module_rx
10624 .try_recv()
10625 .expect("module is told to drop the binding the daemon abandoned")
10626 .frame;
10627 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10628 assert_eq!(
10629 (goodbye.header.channel, goodbye.header.epoch),
10630 (module_channel, module_epoch)
10631 );
10632 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10633 }
10634
10635 #[tokio::test]
10640 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10641 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10642 let client = ConnectionId::new(43);
10643 let (client_ctx, client_rx) = route_ctx(client);
10644
10645 let (open_task, bind) = relay_route_open(
10646 &handler,
10647 client,
10648 &client_ctx.egress,
10649 &mut module_rx,
10650 400,
10651 "aft",
10652 "gone-client-before-mark",
10653 )
10654 .await;
10655 let (module_channel, module_epoch) = route_bind_channel(&bind);
10656 hold.wait().await;
10657
10658 drop(client_rx);
10659 assert!(client_ctx.egress.is_closed());
10660 handler
10661 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10662 .await
10663 .unwrap();
10664
10665 hold.wait().await;
10666 let response = open_task.await.unwrap();
10667 assert_eq!(response.len(), 1);
10668 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10669
10670 let goodbye = module_rx
10671 .try_recv()
10672 .expect("module is told to drop the binding it acked")
10673 .frame;
10674 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10675 assert_eq!(
10676 (goodbye.header.channel, goodbye.header.epoch),
10677 (module_channel, module_epoch)
10678 );
10679 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10680 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10681 }
10682
10683 #[test]
10690 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10691 let handler = ControlHandler::default();
10692 let module_connection = ConnectionId::new(30);
10693 let client_connection = ConnectionId::new(31);
10694
10695 handler
10696 .refuse_to_end_module_connection_for_a_client(
10697 module_connection,
10698 77,
10699 ForwardingError::ConnectionClosing {
10700 connection_id: client_connection,
10701 },
10702 )
10703 .expect("a closing client must never end the module connection");
10704
10705 assert!(matches!(
10706 handler.refuse_to_end_module_connection_for_a_client(
10707 module_connection,
10708 78,
10709 ForwardingError::ConnectionClosing {
10710 connection_id: module_connection,
10711 },
10712 ),
10713 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10714 connection_id
10715 })) if connection_id == module_connection
10716 ));
10717 assert!(matches!(
10718 handler.refuse_to_end_module_connection_for_a_client(
10719 module_connection,
10720 79,
10721 ForwardingError::Poisoned,
10722 ),
10723 Err(RouterError::Forwarding(ForwardingError::Poisoned))
10724 ));
10725 assert!(matches!(
10726 handler.refuse_to_end_module_connection_for_a_client(
10727 module_connection,
10728 80,
10729 ForwardingError::StaleModuleEndpoint,
10730 ),
10731 Err(RouterError::Forwarding(
10732 ForwardingError::StaleModuleEndpoint
10733 ))
10734 ));
10735 }
10736
10737 #[tokio::test]
10750 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10751 let registry = Arc::new(Registry::default());
10752 let forwarding = Arc::new(ForwardingTable::default());
10753 let supervisor = SupervisorHandle::new();
10754 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10755 let handler =
10756 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10757 .with_supervisor(supervisor);
10758
10759 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10760 hello_via_sink(
10761 &handler,
10762 &target_ctx,
10763 &mut target_rx,
10764 hello_frame("target", PROTOCOL_VERSION, 1),
10765 )
10766 .await;
10767
10768 let wrong_nonce = handler
10772 .handle_control_frame(
10773 &route_ctx(ConnectionId::new(91)).0,
10774 route_open_frame_with_admission_facts(
10775 20,
10776 "target",
10777 unique_project_root("admission-facts"),
10778 Some(subc_control::ConsumerIdentity {
10779 module_id: "fed".to_string(),
10780 launch_nonce: "not-the-real-nonce".to_string(),
10781 }),
10782 None,
10783 ),
10784 )
10785 .await
10786 .unwrap();
10787 assert_eq!(
10788 parse_error(&wrong_nonce[0])["code"],
10789 "bad_consumer_identity",
10790 "a mismatched launch nonce must be refused, not stamped Reserved"
10791 );
10792
10793 let never_spawned = handler
10797 .handle_control_frame(
10798 &route_ctx(ConnectionId::new(92)).0,
10799 route_open_frame_with_admission_facts(
10800 21,
10801 "target",
10802 unique_project_root("admission-facts"),
10803 Some(subc_control::ConsumerIdentity {
10804 module_id: "never-spawned".to_string(),
10805 launch_nonce: "any-nonce".to_string(),
10806 }),
10807 None,
10808 ),
10809 )
10810 .await
10811 .unwrap();
10812 assert_eq!(
10813 parse_error(&never_spawned[0])["code"],
10814 "bad_consumer_identity",
10815 "an unspawned module_id must be refused rather than accepted for lack of a record"
10816 );
10817 }
10818
10819 #[tokio::test]
10842 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10843 let registry = Arc::new(Registry::default());
10844 let forwarding = Arc::new(ForwardingTable::default());
10845 let supervisor = SupervisorHandle::new();
10846 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10847 let handler =
10848 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10849 .with_supervisor(supervisor);
10850
10851 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10852 hello_via_sink(
10853 &handler,
10854 &target_ctx,
10855 &mut target_rx,
10856 hello_frame("target", PROTOCOL_VERSION, 1),
10857 )
10858 .await;
10859
10860 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10861 let route_handler = handler.clone();
10862 let route_task = tokio::spawn(async move {
10863 route_handler
10864 .handle_control_frame(
10865 &client_ctx,
10866 route_open_frame_with_admission_facts(
10867 30,
10868 "target",
10869 unique_project_root("admission-facts"),
10870 Some(subc_control::ConsumerIdentity {
10871 module_id: "fed".to_string(),
10872 launch_nonce: "fed-nonce".to_string(),
10873 }),
10874 None,
10875 ),
10876 )
10877 .await
10878 .unwrap()
10879 });
10880
10881 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10888 .await
10889 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10890 .expect("module control channel closed before route.bind");
10891 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10892 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10893 panic!("expected route.bind")
10894 };
10895 assert_eq!(
10896 principal,
10897 Some(Principal::Reserved {
10898 module_id: "fed".to_string()
10899 }),
10900 "a correctly attested consumer must be stamped Reserved for its own id"
10901 );
10902
10903 handler
10904 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10905 .await
10906 .unwrap();
10907 assert!(route_task.await.unwrap().is_empty());
10908 assert!(
10909 matches!(
10910 serde_json::from_slice::<ClientControlResponse>(
10911 &client_rx.recv().await.unwrap().body
10912 )
10913 .unwrap(),
10914 ClientControlResponse::RouteOpen { .. }
10915 ),
10916 "the route must actually open, not merely avoid an error"
10917 );
10918 }
10919
10920 #[tokio::test(start_paused = true)]
10921 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10922 let registry = Arc::new(Registry::default());
10923 let forwarding = Arc::new(ForwardingTable::default());
10924 let supervisor = SupervisorHandle::new();
10925 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10926 let handler =
10927 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10928 .with_supervisor(supervisor);
10929
10930 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10931 hello_via_sink(
10932 &handler,
10933 &target_ctx,
10934 &mut target_rx,
10935 hello_frame("target", PROTOCOL_VERSION, 1),
10936 )
10937 .await;
10938
10939 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10940 let direct_handler = handler.clone();
10941 let direct_open = tokio::spawn(async move {
10942 direct_handler
10943 .handle_control_frame(
10944 &direct_ctx,
10945 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10946 )
10947 .await
10948 .unwrap()
10949 });
10950 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10951 .await
10952 .expect("no direct route.bind within 5s")
10953 .expect("target control channel closed before direct route.bind");
10954 handler
10955 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10956 .await
10957 .unwrap();
10958 assert!(direct_open.await.unwrap().is_empty());
10959 let _ = direct_rx.recv().await.unwrap();
10960
10961 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10962 let reserved_handler = handler.clone();
10963 let reserved_open = tokio::spawn(async move {
10964 reserved_handler
10965 .handle_control_frame(
10966 &reserved_ctx,
10967 route_open_frame_with_admission_facts(
10968 3,
10969 "target",
10970 unique_project_root("admission-facts"),
10971 Some(ConsumerIdentity {
10972 module_id: "fed".to_string(),
10973 launch_nonce: "fed-nonce".to_string(),
10974 }),
10975 None,
10976 ),
10977 )
10978 .await
10979 .unwrap()
10980 });
10981 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10982 .await
10983 .expect("no reserved route.bind within 5s")
10984 .expect("target control channel closed before reserved route.bind");
10985 handler
10986 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10987 .await
10988 .unwrap();
10989 assert!(reserved_open.await.unwrap().is_empty());
10990 let _ = reserved_rx.recv().await.unwrap();
10991
10992 forwarding
10993 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10994 .unwrap();
10995 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10996 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10997 module_id: Some("target".to_string()),
10998 })
10999 .unwrap();
11000 let census_frame =
11001 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11002 let response = handler
11003 .handle_control_frame(&census_ctx, census_frame)
11004 .await
11005 .unwrap()
11006 .pop()
11007 .unwrap();
11008 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11009 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11010 assert!(matches!(
11011 decoded,
11012 ClientControlResponse::SupervisorRoutes { .. }
11013 ));
11014 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11015 assert_eq!(routes.len(), 2);
11016 assert!(routes.iter().all(|route| route["draining"] == true));
11017 assert!(
11020 routes.iter().all(|route| route["drain_reason"] == "reload"),
11021 "draining routes must name the drain's reason: {routes:?}"
11022 );
11023 assert!(routes.iter().any(|route| {
11024 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11025 }));
11026 assert!(routes.iter().any(|route| {
11027 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11028 }));
11029
11030 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11031 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11032 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11033 std::fs::write(
11034 &golden_path,
11035 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11036 )
11037 .unwrap();
11038 }
11039 let expected: Value =
11040 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11041 assert_eq!(actual, expected);
11042 }
11043
11044 async fn query_live_roots(
11045 handler: &ControlHandler,
11046 module_ctx: &RouteCtx,
11047 ) -> ModuleControlResponseToModule {
11048 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11049 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11050 let response = handler
11051 .handle_control_frame(module_ctx, frame)
11052 .await
11053 .unwrap()
11054 .pop()
11055 .unwrap();
11056 serde_json::from_slice(&response.body).unwrap()
11057 }
11058
11059 #[tokio::test(start_paused = true)]
11060 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11061 let registry = Arc::new(Registry::default());
11062 let forwarding = Arc::new(ForwardingTable::default());
11063 let handler = ControlHandler::with_forwarding(registry, forwarding);
11064 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11065 hello_via_sink(
11066 &handler,
11067 &target_ctx,
11068 &mut target_rx,
11069 hello_frame("target", PROTOCOL_VERSION, 1),
11070 )
11071 .await;
11072 let root = unique_project_root("live-roots-known");
11073 let path = ProjectRootId::from_path_allowing_missing(root.path())
11074 .unwrap()
11075 .as_path()
11076 .to_path_buf();
11077 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11078 let open_handler = handler.clone();
11079 let opened = tokio::spawn(async move {
11080 open_handler
11081 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11082 .await
11083 .unwrap()
11084 });
11085 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11086 .await
11087 .unwrap()
11088 .unwrap();
11089 handler
11090 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11091 .await
11092 .unwrap();
11093 assert!(opened.await.unwrap().is_empty());
11094 let _ = client_rx.recv().await.unwrap();
11095
11096 let root = unique_project_root("live-roots-pending");
11097 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11098 .unwrap()
11099 .as_path()
11100 .to_path_buf();
11101 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11102 let open_handler = handler.clone();
11103 let pending = tokio::spawn(async move {
11104 open_handler
11105 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11106 .await
11107 .unwrap()
11108 });
11109 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11110 .await
11111 .unwrap()
11112 .unwrap();
11113 let actual = query_live_roots(&handler, &target_ctx).await;
11114 let ModuleControlResponseToModule::LiveRoots {
11115 roots,
11116 unknown_root_bindings,
11117 total_bindings,
11118 } = actual
11119 else {
11120 panic!("expected live roots")
11121 };
11122 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11123 assert_eq!(unknown_root_bindings, 0);
11124 assert_eq!(
11125 roots.len(),
11126 2,
11127 "root-known arm must retain each canonical root"
11128 );
11129 assert_eq!(
11130 total_bindings,
11131 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11132 );
11133 let counts = roots
11134 .iter()
11135 .map(|root| (root.project_root.clone(), root.bound, root.pending))
11136 .collect::<Vec<_>>();
11137 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11138 expected.sort_by(|a, b| a.0.cmp(&b.0));
11139 assert_eq!(
11140 counts, expected,
11141 "roots must sort by path and count pending separately"
11142 );
11143 handler
11144 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11145 .await
11146 .unwrap();
11147 assert!(pending.await.unwrap().is_empty());
11148 }
11149
11150 #[tokio::test(start_paused = true)]
11151 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11152 let forwarding = Arc::new(ForwardingTable::default());
11153 let handler =
11154 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11155 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11156 hello_via_sink(
11157 &handler,
11158 &target_ctx,
11159 &mut target_rx,
11160 hello_frame("target", PROTOCOL_VERSION, 1),
11161 )
11162 .await;
11163 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11164 let pending = forwarding
11165 .begin_route_bind_relay_for_test(
11166 client_ctx.connection_id,
11167 client_ctx.egress.clone(),
11168 2,
11169 "target",
11170 )
11171 .unwrap();
11172 forwarding
11173 .complete_pending_relay(
11174 target_ctx.connection_id,
11175 pending.corr,
11176 RouteBindRelayOutcome::Accepted,
11177 )
11178 .unwrap();
11179 let actual = query_live_roots(&handler, &target_ctx).await;
11180 let ModuleControlResponseToModule::LiveRoots {
11181 roots,
11182 unknown_root_bindings,
11183 total_bindings,
11184 } = actual
11185 else {
11186 panic!("expected live roots")
11187 };
11188 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11189 assert_eq!(
11190 unknown_root_bindings, 1,
11191 "unknown-root arm must not read as no bindings"
11192 );
11193 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11194 assert_eq!(
11195 total_bindings,
11196 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11197 );
11198 }
11199
11200 #[tokio::test(start_paused = true)]
11208 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11209 let forwarding = Arc::new(ForwardingTable::default());
11210 let handler =
11211 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11212 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11213 let replies = handler
11214 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11215 .await
11216 .unwrap();
11217 let queued_by_hello = module_rx.len();
11218
11219 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11220 let open_handler = handler.clone();
11221 let open = tokio::spawn(async move {
11222 open_handler
11223 .handle_control_frame(
11224 &client_ctx,
11225 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11226 )
11227 .await
11228 .unwrap()
11229 });
11230 let mut spins = 0;
11232 while module_rx.len() == queued_by_hello {
11233 spins += 1;
11234 assert!(spins < 10_000, "route.open never queued a route.bind");
11235 tokio::task::yield_now().await;
11236 }
11237
11238 for reply in replies {
11240 module_ctx.egress.send(reply).await.unwrap();
11241 }
11242
11243 let first = module_rx.recv().await.unwrap().frame;
11244 assert_eq!(
11245 first.header.ty,
11246 FrameType::HelloAck,
11247 "the first frame a registering module reads must be its HELLO_ACK"
11248 );
11249 assert_eq!(first.header.corr, 7);
11250 let second = module_rx.recv().await.unwrap().frame;
11251 assert_eq!(second.header.ty, FrameType::Request);
11252 assert!(
11253 matches!(
11254 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11255 ModuleControlRequest::RouteBind { .. }
11256 ),
11257 "the route.bind follows the ack"
11258 );
11259 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11260
11261 handler
11262 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11263 .await
11264 .unwrap();
11265 assert!(open.await.unwrap().is_empty());
11266 let _ = client_rx.recv().await.unwrap();
11267 }
11268
11269 #[tokio::test(start_paused = true)]
11270 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11271 let handler = ControlHandler::with_forwarding(
11272 Arc::new(Registry::default()),
11273 Arc::new(ForwardingTable::default()),
11274 );
11275 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11276 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11277 hello_via_sink(
11278 &handler,
11279 &first_ctx,
11280 &mut first_rx,
11281 hello_frame("first", PROTOCOL_VERSION, 1),
11282 )
11283 .await;
11284 hello_via_sink(
11285 &handler,
11286 &second_ctx,
11287 &mut second_rx,
11288 hello_frame("second", PROTOCOL_VERSION, 2),
11289 )
11290 .await;
11291 let root = unique_project_root("second-only");
11292 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11293 let cloned = handler.clone();
11294 let open = tokio::spawn(async move {
11295 cloned
11296 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11297 .await
11298 .unwrap()
11299 });
11300 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11301 .await
11302 .unwrap()
11303 .unwrap();
11304 let first = query_live_roots(&handler, &first_ctx).await;
11305 let second = query_live_roots(&handler, &second_ctx).await;
11306 assert!(
11307 matches!(
11308 first,
11309 ModuleControlResponseToModule::LiveRoots {
11310 total_bindings: 0,
11311 ..
11312 }
11313 ),
11314 "cross-module scope must not expose another module's roots"
11315 );
11316 assert!(
11317 matches!(
11318 second,
11319 ModuleControlResponseToModule::LiveRoots {
11320 total_bindings: 1,
11321 ..
11322 }
11323 ),
11324 "second module must see its pending route"
11325 );
11326 handler
11327 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11328 .await
11329 .unwrap();
11330 assert!(open.await.unwrap().is_empty());
11331 }
11332
11333 #[tokio::test(start_paused = true)]
11334 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11335 let handler = ControlHandler::with_forwarding(
11336 Arc::new(Registry::default()),
11337 Arc::new(ForwardingTable::default()),
11338 );
11339 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11340 hello_via_sink(
11341 &handler,
11342 &target_ctx,
11343 &mut target_rx,
11344 hello_frame("target", PROTOCOL_VERSION, 1),
11345 )
11346 .await;
11347 let actual = query_live_roots(&handler, &target_ctx).await;
11348 let ModuleControlResponseToModule::LiveRoots {
11349 roots,
11350 unknown_root_bindings,
11351 total_bindings,
11352 } = actual
11353 else {
11354 panic!("expected live roots")
11355 };
11356 assert!(roots.is_empty());
11357 assert_eq!(unknown_root_bindings, 0);
11358 assert_eq!(total_bindings, 0);
11359 assert_eq!(
11360 total_bindings,
11361 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11362 );
11363 }
11364
11365 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11372 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11373 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11374 let text = std::fs::read_to_string(&path)
11375 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11376 let vectors: Vec<(String, Value)> = text
11377 .lines()
11378 .filter(|line| !line.trim().is_empty())
11379 .map(|line| {
11380 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11381 let id = entry["corpus_id"]
11382 .as_str()
11383 .expect("every vector carries a corpus_id")
11384 .to_string();
11385 (id, entry["package"].clone())
11386 })
11387 .collect();
11388 assert_eq!(
11392 vectors.len(),
11393 3,
11394 "vendored fed corpus changed size; re-sync from subc-federation"
11395 );
11396
11397 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11412 let richest = vectors
11413 .iter()
11414 .filter_map(|(_, package)| package.as_object())
11415 .map(|object| {
11416 object
11417 .keys()
11418 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11419 .count()
11420 })
11421 .max()
11422 .unwrap_or(0);
11423 assert!(
11424 richest >= 2,
11425 "vendored corpus no longer carries a package with unmodelled fields, \
11426 so the relay test can no longer distinguish a verbatim relay from a lossy one"
11427 );
11428
11429 vectors
11430 }
11431
11432 #[tokio::test]
11440 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11441 for (corpus_id, package) in fed_admission_facts_vectors() {
11442 let registry = Arc::new(Registry::default());
11443 let forwarding = Arc::new(ForwardingTable::default());
11444 let supervisor = SupervisorHandle::new();
11445 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11446 let handler =
11447 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11448 .with_supervisor(supervisor)
11449 .with_admission_facts_config(
11450 Some("fed".to_string()),
11451 Some(vec!["target".to_string()]),
11452 );
11453
11454 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11455 hello_via_sink(
11456 &handler,
11457 &target_ctx,
11458 &mut target_rx,
11459 hello_frame("target", PROTOCOL_VERSION, 1),
11460 )
11461 .await;
11462
11463 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11464 let route_handler = handler.clone();
11465 let expected = package.clone();
11466 let route_task = tokio::spawn(async move {
11467 route_handler
11468 .handle_control_frame(
11469 &client_ctx,
11470 route_open_frame_with_admission_facts(
11471 20,
11472 "target",
11473 unique_project_root("admission-facts"),
11474 Some(subc_control::ConsumerIdentity {
11475 module_id: "fed".to_string(),
11476 launch_nonce: "fed-nonce".to_string(),
11477 }),
11478 Some(package),
11479 ),
11480 )
11481 .await
11482 .unwrap()
11483 });
11484
11485 let bind_frame = target_rx.recv().await.unwrap();
11486 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11487 let ModuleControlRequest::RouteBind {
11488 admission_facts, ..
11489 } = bind
11490 else {
11491 panic!("{corpus_id}: expected route.bind")
11492 };
11493 assert_eq!(
11494 admission_facts,
11495 Some(expected),
11496 "{corpus_id}: relay must not add, drop or reshape any field"
11497 );
11498
11499 handler
11500 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11501 .await
11502 .unwrap();
11503 route_task.await.unwrap();
11504 }
11505 }
11506
11507 #[tokio::test]
11508 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11509 let registry = Arc::new(Registry::default());
11510 let forwarding = Arc::new(ForwardingTable::default());
11511 let supervisor = SupervisorHandle::new();
11512 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11513 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11514 let handler =
11515 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11516 .with_supervisor(supervisor)
11517 .with_admission_facts_config(
11518 Some("fed".to_string()),
11519 Some(vec!["target".to_string()]),
11520 );
11521
11522 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11523 hello_via_sink(
11524 &handler,
11525 &target_ctx,
11526 &mut target_rx,
11527 hello_frame("target", PROTOCOL_VERSION, 1),
11528 )
11529 .await;
11530 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11531 hello_via_sink(
11532 &handler,
11533 &other_ctx,
11534 &mut other_rx,
11535 hello_frame("other", PROTOCOL_VERSION, 2),
11536 )
11537 .await;
11538
11539 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11540 let expected_facts = facts.clone();
11541 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11542 let route_handler = handler.clone();
11543 let route_task = tokio::spawn(async move {
11544 route_handler
11545 .handle_control_frame(
11546 &client_ctx,
11547 route_open_frame_with_admission_facts(
11548 10,
11549 "target",
11550 unique_project_root("admission-facts"),
11551 Some(subc_control::ConsumerIdentity {
11552 module_id: "fed".to_string(),
11553 launch_nonce: "fed-nonce".to_string(),
11554 }),
11555 Some(facts.clone()),
11556 ),
11557 )
11558 .await
11559 .unwrap()
11560 });
11561 let bind_frame = target_rx.recv().await.unwrap();
11562 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11563 let ModuleControlRequest::RouteBind {
11564 admission_facts, ..
11565 } = bind
11566 else {
11567 panic!("expected route.bind")
11568 };
11569 assert_eq!(admission_facts, Some(expected_facts));
11570 handler
11571 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11572 .await
11573 .unwrap();
11574 assert!(route_task.await.unwrap().is_empty());
11575 assert!(matches!(
11576 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11577 .unwrap(),
11578 ClientControlResponse::RouteOpen { .. }
11579 ));
11580
11581 let direct = handler
11582 .handle_control_frame(
11583 &route_ctx(ConnectionId::new(73)).0,
11584 route_open_frame_with_admission_facts(
11585 11,
11586 "target",
11587 unique_project_root("admission-facts"),
11588 None,
11589 Some(json!({"x": 1})),
11590 ),
11591 )
11592 .await
11593 .unwrap();
11594 assert_eq!(
11595 parse_error(&direct[0])["code"],
11596 "admission_facts_not_permitted"
11597 );
11598
11599 let different_reserved = handler
11600 .handle_control_frame(
11601 &route_ctx(ConnectionId::new(77)).0,
11602 route_open_frame_with_admission_facts(
11603 15,
11604 "target",
11605 unique_project_root("admission-facts"),
11606 Some(subc_control::ConsumerIdentity {
11607 module_id: "other".to_string(),
11608 launch_nonce: "other-nonce".to_string(),
11609 }),
11610 Some(json!({"x": 1})),
11611 ),
11612 )
11613 .await
11614 .unwrap();
11615 assert_eq!(
11616 parse_error(&different_reserved[0])["code"],
11617 "admission_facts_not_permitted"
11618 );
11619
11620 let other_target = handler
11621 .handle_control_frame(
11622 &route_ctx(ConnectionId::new(74)).0,
11623 route_open_frame_with_admission_facts(
11624 12,
11625 "other",
11626 unique_project_root("admission-facts"),
11627 Some(subc_control::ConsumerIdentity {
11628 module_id: "fed".to_string(),
11629 launch_nonce: "fed-nonce".to_string(),
11630 }),
11631 Some(json!({"x": 1})),
11632 ),
11633 )
11634 .await
11635 .unwrap();
11636 assert_eq!(
11637 parse_error(&other_target[0])["code"],
11638 "admission_facts_target_not_allowed"
11639 );
11640
11641 let nonexistent = handler
11642 .handle_control_frame(
11643 &route_ctx(ConnectionId::new(75)).0,
11644 route_open_frame_with_admission_facts(
11645 13,
11646 "missing",
11647 unique_project_root("admission-facts"),
11648 None,
11649 Some(json!({"x": 1})),
11650 ),
11651 )
11652 .await
11653 .unwrap();
11654 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
11655
11656 let described = handler
11657 .handle_control_frame(
11658 &route_ctx(ConnectionId::new(76)).0,
11659 Frame::build(
11660 FrameType::Request,
11661 control_flags(),
11662 0,
11663 0,
11664 14,
11665 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11666 )
11667 .unwrap(),
11668 )
11669 .await
11670 .unwrap();
11671 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11672 serde_json::from_slice(&described[0].body).unwrap()
11673 else {
11674 panic!("expected server.describe response")
11675 };
11676 assert!(capabilities
11677 .iter()
11678 .any(|cap| cap == "admission_facts_relay_v1"));
11679 }
11680
11681 #[tokio::test]
11682 async fn admission_facts_without_configured_carrier_are_rejected() {
11683 let registry = Arc::new(Registry::default());
11684 let forwarding = Arc::new(ForwardingTable::default());
11685 let handler = ControlHandler::with_forwarding(registry, forwarding);
11686 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11687 hello_via_sink(
11688 &handler,
11689 &target_ctx,
11690 &mut target_rx,
11691 hello_frame("target", PROTOCOL_VERSION, 1),
11692 )
11693 .await;
11694
11695 let responses = handler
11696 .handle_control_frame(
11697 &route_ctx(ConnectionId::new(79)).0,
11698 route_open_frame_with_admission_facts(
11699 16,
11700 "target",
11701 unique_project_root("admission-facts"),
11702 None,
11703 Some(json!({"x": 1})),
11704 ),
11705 )
11706 .await
11707 .unwrap();
11708 assert_eq!(
11709 parse_error(&responses[0])["code"],
11710 "admission_facts_not_permitted"
11711 );
11712 }
11713
11714 #[tokio::test]
11715 async fn route_open_relays_consumer_capabilities_verbatim() {
11716 let registry = Arc::new(Registry::default());
11717 let forwarding = Arc::new(ForwardingTable::default());
11718 let handler =
11719 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11720 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11721 hello_via_sink(
11722 &handler,
11723 &module_ctx,
11724 &mut module_rx,
11725 hello_frame("aft", PROTOCOL_VERSION, 7),
11726 )
11727 .await;
11728
11729 let expected = vec!["elicitation".to_string(), "roots".to_string()];
11730 let expected_for_request = expected.clone();
11731 let project_root = unique_project_root("consumer-capabilities-present");
11732 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11733 let route_handler = handler.clone();
11734 let route_task = tokio::spawn(async move {
11735 route_handler
11736 .handle_control_frame(
11737 &client_ctx,
11738 route_open_frame_with_consumer_capabilities(
11739 401,
11740 "aft",
11741 project_root,
11742 Some(expected_for_request),
11743 ),
11744 )
11745 .await
11746 .unwrap()
11747 });
11748 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11749 .await
11750 .unwrap()
11751 .unwrap();
11752 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11753 let ModuleControlRequest::RouteBind {
11754 consumer_capabilities,
11755 ..
11756 } = bind
11757 else {
11758 panic!("expected route.bind request, got {bind:?}");
11759 };
11760 assert_eq!(consumer_capabilities, Some(expected.clone()));
11761
11762 handler
11763 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11764 .await
11765 .unwrap();
11766 let route_response = route_task.await.unwrap();
11767 assert!(route_response.is_empty());
11768 let published = client_rx.recv().await.unwrap();
11769 assert!(matches!(
11770 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11771 ClientControlResponse::RouteOpen { .. }
11772 ));
11773 }
11774
11775 #[tokio::test]
11776 async fn route_open_without_consumer_capabilities_relays_none() {
11777 let registry = Arc::new(Registry::default());
11778 let forwarding = Arc::new(ForwardingTable::default());
11779 let handler =
11780 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11781 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11782 hello_via_sink(
11783 &handler,
11784 &module_ctx,
11785 &mut module_rx,
11786 hello_frame("aft", PROTOCOL_VERSION, 7),
11787 )
11788 .await;
11789
11790 let project_root = unique_project_root("consumer-capabilities-absent");
11791 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11792 let route_handler = handler.clone();
11793 let route_task = tokio::spawn(async move {
11794 route_handler
11795 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11796 .await
11797 .unwrap()
11798 });
11799 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11800 .await
11801 .unwrap()
11802 .unwrap();
11803 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11804 let ModuleControlRequest::RouteBind {
11805 consumer_capabilities,
11806 ..
11807 } = bind
11808 else {
11809 panic!("expected route.bind request, got {bind:?}");
11810 };
11811 assert_eq!(consumer_capabilities, None);
11812
11813 handler
11814 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11815 .await
11816 .unwrap();
11817 let route_response = route_task.await.unwrap();
11818 assert!(route_response.is_empty());
11819 let published = client_rx.recv().await.unwrap();
11820 assert!(matches!(
11821 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11822 ClientControlResponse::RouteOpen { .. }
11823 ));
11824 }
11825
11826 async fn bind_role_versions_for(
11830 sent: Option<BTreeMap<String, String>>,
11831 connection: u64,
11832 ) -> Option<BTreeMap<String, String>> {
11833 let registry = Arc::new(Registry::default());
11834 let forwarding = Arc::new(ForwardingTable::default());
11835 let handler =
11836 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11837 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11838 hello_via_sink(
11839 &handler,
11840 &module_ctx,
11841 &mut module_rx,
11842 hello_frame("aft", PROTOCOL_VERSION, 7),
11843 )
11844 .await;
11845 let project_root = unique_project_root("role-versions");
11846 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11847 let route_handler = handler.clone();
11848 let route_task = tokio::spawn(async move {
11849 route_handler
11850 .handle_control_frame(
11851 &client_ctx,
11852 route_open_frame_with_role_versions(403, "aft", project_root, sent),
11853 )
11854 .await
11855 .unwrap()
11856 });
11857 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11858 .await
11859 .expect("a well-formed route.open reaches the module as a bind")
11860 .unwrap();
11861 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11862 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11863 panic!("expected route.bind request, got {bind:?}");
11864 };
11865 handler
11866 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11867 .await
11868 .unwrap();
11869 assert!(route_task.await.unwrap().is_empty());
11870 let published = client_rx.recv().await.unwrap();
11871 assert!(matches!(
11872 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11873 ClientControlResponse::RouteOpen { .. }
11874 ));
11875 role_versions
11876 }
11877
11878 #[tokio::test]
11879 async fn route_open_relays_role_versions_verbatim() {
11880 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11881 assert_eq!(
11882 bind_role_versions_for(Some(sent.clone()), 141).await,
11883 Some(sent)
11884 );
11885 }
11886
11887 #[tokio::test]
11890 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11891 assert_eq!(bind_role_versions_for(None, 143).await, None);
11892 assert_eq!(
11893 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11894 None
11895 );
11896 }
11897
11898 #[tokio::test]
11899 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11900 let registry = Arc::new(Registry::default());
11901 let forwarding = Arc::new(ForwardingTable::default());
11902 let handler =
11903 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11904 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11905 hello_via_sink(
11906 &handler,
11907 &module_ctx,
11908 &mut module_rx,
11909 hello_frame("aft", PROTOCOL_VERSION, 7),
11910 )
11911 .await;
11912
11913 let nine: BTreeMap<String, String> = (0..9)
11914 .map(|index| (format!("role-{index}"), "v1".to_string()))
11915 .collect();
11916 for (label, malformed) in [
11917 (
11918 "invalid role name",
11919 role_versions(&[("Tool_Provider", "v1")]),
11920 ),
11921 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11922 ("nine entries", nine),
11923 ] {
11924 let responses = tokio::time::timeout(
11927 Duration::from_secs(1),
11928 handler.handle_control_frame(
11929 &route_ctx(ConnectionId::new(148)).0,
11930 route_open_frame_with_role_versions(
11931 404,
11932 "aft",
11933 unique_project_root("role-versions-malformed"),
11934 Some(malformed),
11935 ),
11936 ),
11937 )
11938 .await
11939 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11940 .unwrap();
11941 assert_eq!(responses.len(), 1, "{label}");
11942 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11943 let error = parse_error(&responses[0]);
11944 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11945 assert_eq!(
11946 error["detail"]["field"], "role_versions",
11947 "{label}: {error}"
11948 );
11949 assert!(
11950 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11951 "{label}: a malformed declaration is terminal"
11952 );
11953 assert!(
11954 module_rx.try_recv().is_err(),
11955 "{label}: the module must never see a bind"
11956 );
11957 }
11958 }
11959
11960 #[tokio::test]
11964 async fn route_role_versions_capability_is_advertised() {
11965 let handler = ControlHandler::new(Arc::new(Registry::default()));
11966 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11967 let ack = hello_via_sink(
11968 &handler,
11969 &ctx,
11970 &mut rx,
11971 hello_frame("m", PROTOCOL_VERSION, 1),
11972 )
11973 .await;
11974 let ack = parse_ack(&ack);
11975 assert!(
11976 ack.subc_capabilities
11977 .iter()
11978 .any(|c| c == "route-role-versions/v1"),
11979 "{:?}",
11980 ack.subc_capabilities
11981 );
11982
11983 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11984 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11985 let reply = handler
11986 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11987 .await
11988 .unwrap()
11989 .pop()
11990 .unwrap();
11991 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11992 serde_json::from_slice(&reply.body).unwrap()
11993 else {
11994 panic!("not a server.describe reply");
11995 };
11996 assert!(
11997 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11998 "{capabilities:?}"
11999 );
12000 }
12001
12002 #[tokio::test]
12006 async fn catalog_events_capability_is_advertised() {
12007 let handler = ControlHandler::new(Arc::new(Registry::default()));
12008 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12009 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12010 let reply = handler
12011 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12012 .await
12013 .unwrap()
12014 .pop()
12015 .unwrap();
12016 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12017 serde_json::from_slice(&reply.body).unwrap()
12018 else {
12019 panic!("not a server.describe reply");
12020 };
12021 assert!(
12022 capabilities
12023 .iter()
12024 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12025 "{capabilities:?}"
12026 );
12027 }
12028
12029 #[tokio::test]
12030 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12031 let handler = ControlHandler::new(Arc::new(Registry::default()));
12032 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12033 let ack = hello_via_sink(
12034 &handler,
12035 &module,
12036 &mut module_rx,
12037 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12038 )
12039 .await;
12040 let ack = parse_ack(&ack);
12041 assert!(
12042 ack.subc_capabilities
12043 .iter()
12044 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12045 "{:?}",
12046 ack.subc_capabilities
12047 );
12048 assert!(ack
12049 .subc_capabilities
12050 .iter()
12051 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12052
12053 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12054 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12055 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12056 let reply = handler
12057 .handle_control_frame(&client, frame)
12058 .await
12059 .unwrap()
12060 .pop()
12061 .unwrap();
12062 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12063 serde_json::from_slice(&reply.body).unwrap()
12064 else {
12065 panic!("not a server.describe reply");
12066 };
12067 assert!(
12068 capabilities
12069 .iter()
12070 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12071 "{capabilities:?}"
12072 );
12073 assert!(capabilities
12074 .iter()
12075 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12076 }
12077
12078 #[tokio::test]
12079 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12080 let handler = ControlHandler::new(Arc::new(Registry::default()));
12081 let (module, mut rx) = route_ctx(ConnectionId::new(156));
12082 let ack = hello_via_sink(
12083 &handler,
12084 &module,
12085 &mut rx,
12086 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12087 )
12088 .await;
12089 assert!(parse_ack(&ack)
12090 .subc_capabilities
12091 .iter()
12092 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12093 assert!(parse_ack(&ack)
12094 .subc_capabilities
12095 .iter()
12096 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12097 let (client, _rx) = route_ctx(ConnectionId::new(157));
12098 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12099 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12100 let reply = handler
12101 .handle_control_frame(&client, frame)
12102 .await
12103 .unwrap()
12104 .pop()
12105 .unwrap();
12106 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12107 serde_json::from_slice(&reply.body).unwrap()
12108 else {
12109 panic!("server.describe reply")
12110 };
12111 assert!(capabilities
12112 .iter()
12113 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12114 assert!(capabilities
12115 .iter()
12116 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12117 }
12118
12119 #[tokio::test]
12120 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12121 let handler = ControlHandler::new(Arc::new(Registry::default()));
12122 let (module, mut rx) = route_ctx(ConnectionId::new(158));
12123 let ack = hello_via_sink(
12124 &handler,
12125 &module,
12126 &mut rx,
12127 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12128 )
12129 .await;
12130 assert!(parse_ack(&ack)
12131 .subc_capabilities
12132 .iter()
12133 .any(|cap| cap == "scope-workdir/v1"));
12134 let (client, _rx) = route_ctx(ConnectionId::new(159));
12135 let frame = Frame::build(
12136 FrameType::Request,
12137 control_flags(),
12138 0,
12139 0,
12140 5,
12141 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12142 )
12143 .unwrap();
12144 let reply = handler
12145 .handle_control_frame(&client, frame)
12146 .await
12147 .unwrap()
12148 .remove(0);
12149 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12150 serde_json::from_slice(&reply.body).unwrap()
12151 else {
12152 panic!("server.describe reply")
12153 };
12154 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12155 }
12156
12157 #[tokio::test]
12158 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12159 let handler = ControlHandler::new(Arc::new(Registry::default()));
12160 let (module, mut rx) = route_ctx(ConnectionId::new(154));
12161 let ack = hello_via_sink(
12162 &handler,
12163 &module,
12164 &mut rx,
12165 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12166 )
12167 .await;
12168 assert!(parse_ack(&ack)
12169 .subc_capabilities
12170 .iter()
12171 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12172 let (client, _rx) = route_ctx(ConnectionId::new(155));
12173 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12174 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12175 let reply = handler
12176 .handle_control_frame(&client, frame)
12177 .await
12178 .unwrap()
12179 .pop()
12180 .unwrap();
12181 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12182 serde_json::from_slice(&reply.body).unwrap()
12183 else {
12184 panic!("not a server.describe reply")
12185 };
12186 assert!(capabilities
12187 .iter()
12188 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12189 }
12190
12191 #[tokio::test]
12192 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12193 let registry = Arc::new(Registry::default());
12194 let forwarding = Arc::new(ForwardingTable::default());
12195 let handler =
12196 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12197 .with_health_probe_timeout(Duration::from_secs(5));
12198 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12199 hello_via_sink(
12200 &handler,
12201 &module_ctx,
12202 &mut module_rx,
12203 non_routable_hello_frame_with_control_ops(
12204 "mcp",
12205 300,
12206 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12207 ),
12208 )
12209 .await;
12210 assert!(registry
12211 .get_module("mcp")
12212 .unwrap()
12213 .unwrap()
12214 .manifest
12215 .provides
12216 .is_empty());
12217
12218 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12219 let route_response = handler
12220 .handle_control_frame(
12221 &route_client_ctx,
12222 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12223 )
12224 .await
12225 .unwrap();
12226 assert_eq!(route_response[0].header.ty, FrameType::Error);
12227 assert_eq!(
12228 parse_error(&route_response[0])["code"],
12229 "target_unavailable"
12230 );
12231 assert!(parse_error(&route_response[0])["message"]
12232 .as_str()
12233 .unwrap()
12234 .contains("does not provide the requested target"));
12235 assert!(module_rx.try_recv().is_err());
12236
12237 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12238 let health_handler = handler.clone();
12239 let health_task = tokio::spawn(async move {
12240 health_handler
12241 .handle_control_frame(
12242 &health_client_ctx,
12243 supervisor_health_probe_frame(302, "mcp"),
12244 )
12245 .await
12246 .unwrap()
12247 });
12248 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12249 .await
12250 .unwrap()
12251 .unwrap();
12252 assert_eq!(
12253 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12254 ModuleControlRequest::HealthCheck {}
12255 );
12256 handler
12257 .handle_control_frame(
12258 &module_ctx,
12259 health_response(health_frame.header.corr, HealthStatus::Ok),
12260 )
12261 .await
12262 .unwrap();
12263 let health_response = health_task.await.unwrap();
12264 assert_eq!(health_response[0].header.ty, FrameType::Response);
12265 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12266 ClientControlResponse::SupervisorHealthProbe {
12267 module_id, status, ..
12268 } => {
12269 assert_eq!(module_id, "mcp");
12270 assert_eq!(status, HealthStatus::Ok);
12271 }
12272 other => panic!("unexpected health response: {other:?}"),
12273 }
12274
12275 forwarding
12280 .cleanup_connection(module_ctx.connection_id)
12281 .unwrap();
12282 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12283 let cleanup_response = tokio::time::timeout(
12284 Duration::from_millis(200),
12285 handler.handle_control_frame(
12286 &cleanup_probe_ctx,
12287 supervisor_health_probe_frame(303, "mcp"),
12288 ),
12289 )
12290 .await
12291 .expect("probe should fail immediately when the control lane is gone")
12292 .unwrap();
12293 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12294 assert_eq!(
12295 parse_error(&cleanup_response[0])["code"],
12296 "target_unavailable"
12297 );
12298 assert!(parse_error(&cleanup_response[0])["message"]
12299 .as_str()
12300 .unwrap()
12301 .contains("no module connection"));
12302
12303 handler
12304 .cleanup_connection(module_ctx.connection_id)
12305 .unwrap();
12306 }
12307
12308 #[tokio::test]
12309 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12310 let registry = Arc::new(Registry::default());
12311 let supervisor_handle = SupervisorHandle::new();
12312 let supervisor =
12313 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12314 .with_handle(supervisor_handle.clone())
12315 .with_connection_file_path(
12316 std::env::temp_dir()
12317 .join(format!("subc-route-open-warming-{}", std::process::id())),
12318 );
12319 let module = supervisor
12320 .supervise_configured(
12321 ModuleSpec {
12322 module_id: "warming".to_string(),
12323 program: fake_aft_stub_path(),
12324 args: Vec::new(),
12325 env: Vec::new(),
12326 reserved: false,
12327 reserved_prefixes: Vec::new(),
12328 protocol: ModuleProtocol::Subc,
12329 overlap: Default::default(),
12330 },
12331 true,
12332 )
12333 .unwrap();
12334 assert_eq!(module.state().unwrap(), ModuleState::Running);
12335
12336 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12337 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12338 let response = handler
12339 .handle_control_frame(
12340 &ctx,
12341 route_open_frame(304, "warming", unique_project_root("warming")),
12342 )
12343 .await
12344 .unwrap();
12345 module.stop().await.unwrap();
12346
12347 assert_eq!(response[0].header.ty, FrameType::Error);
12348 let error = parse_error(&response[0]);
12349 assert_eq!(error["code"], "module_warming");
12350 assert!(error["message"]
12351 .as_str()
12352 .unwrap()
12353 .contains("state=running, enabled=true, live=false"));
12354 }
12355
12356 #[test]
12357 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12358 let handler = ControlHandler::new(Arc::new(Registry::default()));
12359 let capture = EventCapture::default();
12360 let _subscriber =
12361 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12362 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12363 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12364 let pending = (0..limit).collect::<Vec<_>>();
12365 let response = handler
12366 .route_open_capacity_refusal(
12367 &ctx,
12368 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12369 "busy",
12370 pending.len(),
12371 limit,
12372 )
12373 .unwrap();
12374 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12375 let event = capture
12376 .events()
12377 .into_iter()
12378 .find(|event| {
12379 event.target == "control"
12380 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12381 })
12382 .expect("connection admission refusal event");
12383 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12384 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12385 }
12386
12387 #[test]
12388 fn route_open_target_cap_logs_admission_reason_and_capacity() {
12389 let handler = ControlHandler::new(Arc::new(Registry::default()));
12390 let capture = EventCapture::default();
12391 let _subscriber =
12392 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12393 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12394 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12395 let guards = (0..limit)
12396 .map(|_| {
12397 handler
12398 .route_bind_concurrency
12399 .try_admit("busy", limit)
12400 .unwrap()
12401 })
12402 .collect::<Vec<_>>();
12403 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12404 Err(in_flight) => in_flight,
12405 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12406 };
12407 let response = handler
12408 .route_open_target_capacity_refusal(
12409 &ctx,
12410 &route_open_frame(397, "busy", unique_project_root("target-cap")),
12411 "busy",
12412 in_flight,
12413 )
12414 .unwrap();
12415 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12416 let event = capture
12417 .events()
12418 .into_iter()
12419 .find(|event| {
12420 event.target == "control"
12421 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12422 })
12423 .expect("target admission refusal event");
12424 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12425 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12426 drop(guards);
12427 }
12428
12429 #[tokio::test(flavor = "current_thread")]
12430 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12431 let handler = ControlHandler::new(Arc::new(Registry::default()));
12432 let capture = EventCapture::default();
12433 let _subscriber =
12434 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12435 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12436 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12437
12438 let _ = handler
12439 .handle_client_control_request(
12440 &ctx,
12441 frame,
12442 ClientControlRequest::SupervisorSetEnabled {
12443 module_id: "broca".to_string(),
12444 enabled: false,
12445 },
12446 &mut None,
12447 )
12448 .await
12449 .unwrap();
12450
12451 let events = capture
12452 .events()
12453 .into_iter()
12454 .filter(|event| {
12455 event.target == "control"
12456 && event.fields.get("message").map(String::as_str)
12457 == Some("supervisor request received")
12458 })
12459 .collect::<Vec<_>>();
12460 assert_eq!(events.len(), 1, "one supervisor request log line");
12461 let event = &events[0];
12462 assert_eq!(event.level, tracing::Level::INFO);
12463 assert_eq!(
12464 event.fields.get("op"),
12465 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12466 );
12467 assert_eq!(
12468 event.fields.get("module_id"),
12469 Some(&"\"broca\"".to_string())
12470 );
12471 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12472 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12473 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12474 }
12475
12476 #[tokio::test(flavor = "current_thread")]
12477 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12478 let registry = Arc::new(Registry::default());
12479 let module_connection = ConnectionId::new(303);
12480 registry
12481 .register_with_control_ops(
12482 manifest("aft", PROTOCOL_VERSION),
12483 PROTOCOL_VERSION,
12484 module_connection,
12485 module_baseline_control_ops(),
12486 )
12487 .unwrap();
12488 let handler = ControlHandler::new(registry);
12489 let capture = EventCapture::default();
12490 let _subscriber =
12491 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12492 let (ctx, _rx) = route_ctx(module_connection);
12493 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12494
12495 let _ = handler
12496 .handle_client_control_request(
12497 &ctx,
12498 frame,
12499 ClientControlRequest::SupervisorSetEnabled {
12500 module_id: "broca".to_string(),
12501 enabled: false,
12502 },
12503 &mut None,
12504 )
12505 .await
12506 .unwrap();
12507
12508 let callers = capture
12509 .events()
12510 .into_iter()
12511 .filter(|event| {
12512 event.fields.get("message").map(String::as_str)
12513 == Some("supervisor request received")
12514 })
12515 .map(|event| event.fields.get("caller").cloned())
12516 .collect::<Vec<_>>();
12517 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12518 }
12519
12520 #[tokio::test(flavor = "current_thread")]
12521 async fn supervisor_list_does_not_log_request_received() {
12522 let handler = ControlHandler::new(Arc::new(Registry::default()));
12523 let capture = EventCapture::default();
12524 let _subscriber =
12525 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12526 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12527 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12528
12529 handler
12530 .handle_client_control_request(
12531 &ctx,
12532 frame,
12533 ClientControlRequest::SupervisorList {},
12534 &mut None,
12535 )
12536 .await
12537 .unwrap();
12538
12539 assert!(capture.events().into_iter().all(|event| {
12540 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12541 }));
12542 }
12543
12544 #[tokio::test(flavor = "current_thread")]
12545 async fn supervisor_rescan_preview_does_not_log_request_received() {
12546 let handler = ControlHandler::new(Arc::new(Registry::default()));
12547 let capture = EventCapture::default();
12548 let _subscriber =
12549 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12550 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
12551 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12552
12553 handler
12554 .handle_client_control_request(
12555 &ctx,
12556 frame,
12557 ClientControlRequest::SupervisorRescan { preview: true },
12558 &mut None,
12559 )
12560 .await
12561 .unwrap();
12562
12563 assert!(capture.events().into_iter().all(|event| {
12564 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12565 }));
12566 }
12567
12568 struct ApproveOperator;
12569 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
12570 fn prompt(
12571 &self,
12572 _: &str,
12573 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
12574 ) -> crate::operator_confirm::ProviderResult {
12575 crate::operator_confirm::ProviderResult::Approved
12576 }
12577 }
12578
12579 #[tokio::test]
12580 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
12581 assert_eq!(
12582 ControlHandler::default().privileged_operator_prompt,
12583 PrivilegedOperatorPrompt::Log
12584 );
12585 for (initial, changed, running) in [
12586 ("log", "enforce", PrivilegedOperatorPrompt::Log),
12587 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
12588 ] {
12589 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
12590 let path = dir.join("subc.jsonc");
12591 std::fs::write(
12592 &path,
12593 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
12594 )
12595 .unwrap();
12596 let config = crate::daemon_config::load(&path).unwrap().unwrap();
12597 let registry = Arc::new(Registry::default());
12598 let handle = SupervisorHandle::new();
12599 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
12600 .with_handle(handle.clone());
12601 let handler = ControlHandler::new(registry)
12602 .with_supervisor(handle)
12603 .with_privileged_operator_prompt(config.privileged_operator_prompt)
12604 .with_supervisor_rescan(supervisor, &path, config.port);
12605 handler.forwarding.operator_confirms().configure(
12606 Arc::new(ApproveOperator),
12607 crate::operator_confirm::OperatorLimits::default(),
12608 );
12609 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
12610
12611 for (disk_mode, needs_restart) in [
12612 (Some(initial), false),
12613 (Some(changed), true),
12614 (Some(changed), true),
12615 (None, running == PrivilegedOperatorPrompt::Enforce),
12616 (Some(initial), false),
12617 ] {
12618 let mut doc = json!({"version":1});
12619 if let Some(mode) = disk_mode {
12620 doc["privileged_operator_prompt"] = json!(mode);
12621 }
12622 std::fs::write(&path, doc.to_string()).unwrap();
12623 for preview in [true, false] {
12624 let request = ClientControlRequest::SupervisorRescan { preview };
12625 let frame = Frame::build(
12626 FrameType::Request,
12627 control_flags(),
12628 0,
12629 0,
12630 1,
12631 serde_json::to_vec(&request).unwrap(),
12632 )
12633 .unwrap();
12634 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12635 assert_eq!(frames.len(), 1);
12636 let ClientControlResponse::SupervisorRescan { result } =
12637 serde_json::from_slice(&frames[0].body).unwrap()
12638 else {
12639 panic!("expected a rescan response");
12640 };
12641 let expected = if needs_restart {
12642 vec!["privileged_operator_prompt"]
12643 } else {
12644 Vec::new()
12645 };
12646 assert_eq!(result.restart_required, expected);
12647 assert_eq!(handler.privileged_operator_prompt, running);
12648 assert_eq!(
12649 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
12650 running
12651 );
12652 }
12653 }
12654 }
12655 }
12656
12657 #[tokio::test]
12658 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
12659 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
12660 let path = dir.join("subc.jsonc");
12661 let registry = Arc::new(Registry::default());
12662 let handle = SupervisorHandle::new();
12663 let supervisor =
12664 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
12665 let handler = ControlHandler::new(registry)
12666 .with_supervisor(handle)
12667 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
12668 .with_supervisor_rescan(supervisor, &path, None);
12669 handler.forwarding.operator_confirms().configure(
12670 Arc::new(ApproveOperator),
12671 crate::operator_confirm::OperatorLimits::default(),
12672 );
12673 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
12674 for preview in [true, false] {
12675 std::fs::write(
12676 &path,
12677 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
12678 )
12679 .unwrap();
12680 let request = ClientControlRequest::SupervisorRescan { preview };
12681 let frame = Frame::build(
12682 FrameType::Request,
12683 control_flags(),
12684 0,
12685 0,
12686 1,
12687 serde_json::to_vec(&request).unwrap(),
12688 )
12689 .unwrap();
12690 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12691 assert_eq!(frames.len(), 1);
12692 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
12693 assert_eq!(
12694 handler.privileged_operator_prompt,
12695 PrivilegedOperatorPrompt::Enforce
12696 );
12697 }
12698 }
12699
12700 #[tokio::test]
12705 async fn route_open_refusal_names_the_check_that_refused() {
12706 let handler = ControlHandler::new(Arc::new(Registry::default()));
12707 let capture = EventCapture::default();
12708 let _subscriber =
12709 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12710 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12711 let response = handler
12712 .handle_control_frame(
12713 &ctx,
12714 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
12715 )
12716 .await
12717 .unwrap();
12718
12719 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12720 let event = capture
12721 .events()
12722 .into_iter()
12723 .find(|event| {
12724 event.target == "control"
12725 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12726 })
12727 .expect("route.open refusal event");
12728 assert_eq!(
12729 event.fields.get("reason"),
12730 Some(&"\"not_registered\"".to_string())
12731 );
12732 }
12733
12734 #[tokio::test]
12735 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12736 let registry = Arc::new(Registry::default());
12737 let supervisor_handle = SupervisorHandle::new();
12738 let supervisor =
12739 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12740 .with_handle(supervisor_handle.clone())
12741 .with_connection_file_path(std::env::temp_dir().join(format!(
12742 "subc-route-open-refusal-info-{}",
12743 std::process::id()
12744 )));
12745 let module = supervisor
12746 .supervise_configured(
12747 ModuleSpec {
12748 module_id: "warming".to_string(),
12749 program: fake_aft_stub_path(),
12750 args: Vec::new(),
12751 env: Vec::new(),
12752 reserved: false,
12753 reserved_prefixes: Vec::new(),
12754 protocol: ModuleProtocol::Subc,
12755 overlap: Default::default(),
12756 },
12757 true,
12758 )
12759 .unwrap();
12760 assert_eq!(module.state().unwrap(), ModuleState::Running);
12761
12762 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12763 assert!(handler
12764 .counters()
12765 .snapshot()
12766 .get("route_open_refused_by_code")
12767 .is_none());
12768 let capture = EventCapture::default();
12769 let _subscriber =
12770 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12771 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
12772 let response = handler
12773 .handle_control_frame(
12774 &ctx,
12775 route_open_frame(394, "warming", unique_project_root("refusal-info")),
12776 )
12777 .await
12778 .unwrap();
12779 module.stop().await.unwrap();
12780
12781 assert_eq!(parse_error(&response[0])["code"], "module_warming");
12782 let event = capture
12783 .events()
12784 .into_iter()
12785 .find(|event| {
12786 event.target == "control"
12787 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
12788 })
12789 .expect("route.open refusal event");
12790 assert_eq!(
12791 event.fields.get("module_id"),
12792 Some(&"\"warming\"".to_string())
12793 );
12794 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
12795 assert_eq!(
12796 event.fields.get("reason"),
12797 Some(&"\"supervised_not_registered\"".to_string())
12798 );
12799 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
12800 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
12801 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
12802 assert_eq!(
12803 handler.counters().snapshot()["route_open_refused_by_code"],
12804 json!({ "module_warming": 1 })
12805 );
12806 }
12807
12808 const OUTAGE_START: &str = "route.open refusing module: not serving";
12809 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
12810
12811 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
12812 capture
12813 .events()
12814 .into_iter()
12815 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
12816 .collect()
12817 }
12818
12819 fn supervise_stub(
12820 registry: &Arc<Registry>,
12821 module_id: &str,
12822 enabled: bool,
12823 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
12824 let supervisor_handle = SupervisorHandle::new();
12825 let supervisor =
12826 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
12827 .with_handle(supervisor_handle.clone())
12828 .with_connection_file_path(std::env::temp_dir().join(format!(
12829 "subc-route-outage-{module_id}-{}",
12830 std::process::id()
12831 )));
12832 let module = supervisor
12833 .supervise_configured(
12834 ModuleSpec {
12835 module_id: module_id.to_string(),
12836 program: fake_aft_stub_path(),
12837 args: Vec::new(),
12838 env: Vec::new(),
12839 reserved: false,
12840 reserved_prefixes: Vec::new(),
12841 protocol: ModuleProtocol::Subc,
12842 overlap: Default::default(),
12843 },
12844 enabled,
12845 )
12846 .unwrap();
12847 (supervisor_handle, module)
12848 }
12849
12850 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12851 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12852 module_id: module_id.to_string(),
12853 drain_timeout_ms: Some(50),
12854 })
12855 .unwrap();
12856 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12857 }
12858
12859 #[test]
12863 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12864 let registry = Arc::new(Registry::default());
12865 let forwarding = Arc::new(ForwardingTable::default());
12866 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12867 let second = ControlHandler::with_forwarding(registry, forwarding);
12868 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12869 }
12870
12871 #[tokio::test(flavor = "current_thread")]
12875 async fn route_open_unknown_module_refusals_add_no_outage_state() {
12876 let handler = ControlHandler::new(Arc::new(Registry::default()));
12877 let capture = EventCapture::default();
12878 let _subscriber =
12879 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12880 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12881 for corr in 0..8 {
12882 let response = handler
12883 .handle_control_frame(
12884 &ctx,
12885 route_open_frame(
12886 380 + corr,
12887 &format!("nobody-{corr}"),
12888 unique_project_root("outage-unknown"),
12889 ),
12890 )
12891 .await
12892 .unwrap();
12893 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12894 }
12895
12896 assert_eq!(handler.route_outages.tracked_module_count(), 0);
12897 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12898 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12899 }
12900
12901 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12906 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12907 let registry = Arc::new(Registry::default());
12908 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
12909 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12910 let capture = EventCapture::default();
12911 let _subscriber =
12912 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12913 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12914 handler.route_outages.record_accepted("outage-restart");
12917
12918 let response = handler
12919 .handle_control_frame(
12920 &ctx,
12921 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12922 )
12923 .await
12924 .unwrap();
12925 assert_eq!(response[0].header.ty, FrameType::Error);
12926 let starts = outage_lines(&capture, OUTAGE_START);
12927 assert_eq!(starts.len(), 1, "{starts:?}");
12928 assert_eq!(starts[0].level, tracing::Level::WARN);
12929 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12930 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12931 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12932 handler.route_outages.record_accepted("outage-restart");
12933 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12934
12935 let restart = handler
12936 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12937 .await
12938 .unwrap();
12939 assert_eq!(
12940 restart[0].header.ty,
12941 FrameType::Response,
12942 "{:?}",
12943 parse_error(&restart[0])
12944 );
12945 handler
12946 .handle_control_frame(
12947 &ctx,
12948 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12949 )
12950 .await
12951 .unwrap();
12952 module.stop().await.unwrap();
12953
12954 let starts = outage_lines(&capture, OUTAGE_START);
12955 assert_eq!(starts.len(), 2, "{starts:?}");
12956 assert_eq!(starts[1].level, tracing::Level::INFO);
12957 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12958 }
12959
12960 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12964 async fn failed_operator_restart_leaves_no_operator_mark() {
12965 let registry = Arc::new(Registry::default());
12966 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
12967 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12968 let capture = EventCapture::default();
12969 let _subscriber =
12970 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12971 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12972 handler.route_outages.record_accepted("outage-disabled");
12973
12974 let restart = handler
12975 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12976 .await
12977 .unwrap();
12978 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12979 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12980
12981 handler
12982 .handle_control_frame(
12983 &ctx,
12984 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12985 )
12986 .await
12987 .unwrap();
12988 let starts = outage_lines(&capture, OUTAGE_START);
12989 assert_eq!(starts.len(), 1, "{starts:?}");
12990 assert_eq!(starts[0].level, tracing::Level::WARN);
12991 }
12992
12993 #[tokio::test(flavor = "current_thread")]
12994 async fn route_open_unknown_module_escapes_target_module_id() {
12995 let handler = ControlHandler::new(Arc::new(Registry::default()));
12996 let capture = EventCapture::default();
12997 let _subscriber =
12998 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12999 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13000 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13001 let response = handler
13002 .handle_control_frame(
13003 &ctx,
13004 route_open_frame(
13005 395,
13006 hostile_module_id,
13007 unique_project_root("hostile-target-module-id"),
13008 ),
13009 )
13010 .await
13011 .unwrap();
13012
13013 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13014 let event = capture
13015 .events()
13016 .into_iter()
13017 .find(|event| {
13018 event.target == "control"
13019 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13020 })
13021 .expect("route.open unknown-module refusal event");
13022 let logged = event.fields.get("module_id").expect("module_id field");
13023 assert!(!logged.bytes().any(|byte| byte < 0x20));
13024 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13025 }
13026
13027 #[tokio::test(flavor = "current_thread")]
13028 async fn route_open_module_rejection_uses_daemon_counter_key() {
13029 let registry = Arc::new(Registry::default());
13030 let forwarding = Arc::new(ForwardingTable::default());
13031 let handler =
13032 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13033 let module_connection = ConnectionId::new(95);
13034 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13035 hello_via_sink(
13036 &handler,
13037 &module_ctx,
13038 &mut module_rx,
13039 hello_frame("aft", PROTOCOL_VERSION, 395),
13040 )
13041 .await;
13042
13043 let client_connection = ConnectionId::new(96);
13044 let (client_ctx, _client_rx) = route_ctx(client_connection);
13045 let capture = EventCapture::default();
13046 let _subscriber =
13047 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13048 let (route_task, bind) = relay_route_open(
13049 &handler,
13050 client_connection,
13051 &client_ctx.egress,
13052 &mut module_rx,
13053 396,
13054 "aft",
13055 "hostile-module-code",
13056 )
13057 .await;
13058 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13059 let rejection = Frame::build(
13060 FrameType::Error,
13061 control_flags(),
13062 0,
13063 0,
13064 bind.header.corr,
13065 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13066 )
13067 .unwrap();
13068 handler
13069 .handle_control_frame(&module_ctx, rejection)
13070 .await
13071 .unwrap();
13072
13073 let response = route_task.await.unwrap();
13074 assert_eq!(parse_error(&response[0])["code"], hostile_code);
13075 let counters = handler.counters().snapshot();
13076 assert_eq!(
13077 counters["route_open_refused_by_code"],
13078 json!({ "module_rejected": 1 })
13079 );
13080 assert!(counters["route_open_refused_by_code"]
13081 .get(hostile_code)
13082 .is_none());
13083
13084 let event = capture
13085 .events()
13086 .into_iter()
13087 .find(|event| {
13088 event.target == "control"
13089 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13090 })
13091 .expect("route.open module-rejection refusal event");
13092 let logged = event.fields.get("module_code").expect("module_code field");
13093 assert!(!logged.bytes().any(|byte| byte < 0x20));
13094 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13095 }
13096
13097 #[tokio::test]
13098 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13099 let registry = Arc::new(Registry::default());
13100 let supervisor_handle = SupervisorHandle::new();
13101 let missing_program = std::env::temp_dir().join(format!(
13102 "subc-route-open-missing-program-{}",
13103 std::process::id()
13104 ));
13105 let supervisor =
13106 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13107 .with_handle(supervisor_handle.clone());
13108 let module = supervisor
13109 .supervise_configured(
13110 ModuleSpec {
13111 module_id: "failed".to_string(),
13112 program: missing_program,
13113 args: Vec::new(),
13114 env: Vec::new(),
13115 reserved: false,
13116 reserved_prefixes: Vec::new(),
13117 protocol: ModuleProtocol::Subc,
13118 overlap: Default::default(),
13119 },
13120 true,
13121 )
13122 .unwrap();
13123 assert_eq!(module.state().unwrap(), ModuleState::Failed);
13124
13125 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13126 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13127 let response = handler
13128 .handle_control_frame(
13129 &ctx,
13130 route_open_frame(305, "failed", unique_project_root("failed")),
13131 )
13132 .await
13133 .unwrap();
13134
13135 assert_eq!(response[0].header.ty, FrameType::Error);
13136 let error = parse_error(&response[0]);
13137 assert_eq!(error["code"], "target_unavailable");
13138 assert!(error["message"]
13139 .as_str()
13140 .unwrap()
13141 .contains("state=failed, enabled=true, live=false"));
13142 }
13143
13144 #[tokio::test]
13145 async fn route_open_role_mismatch_remains_target_unavailable() {
13146 let registry = Arc::new(Registry::default());
13147 let handler = ControlHandler::new(Arc::clone(®istry));
13148 handler
13149 .handle_control(
13150 ConnectionId::new(41),
13151 non_routable_hello_frame_with_control_ops("health-only", 306, None),
13152 )
13153 .unwrap();
13154
13155 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13156 let response = handler
13157 .handle_control_frame(
13158 &ctx,
13159 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13160 )
13161 .await
13162 .unwrap();
13163
13164 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13165 assert!(parse_error(&response[0])["message"]
13166 .as_str()
13167 .unwrap()
13168 .contains("does not provide the requested target"));
13169 }
13170
13171 #[tokio::test]
13172 async fn route_open_inactive_registration_remains_target_unavailable() {
13173 let registry = Arc::new(Registry::default());
13174 let handler = ControlHandler::new(Arc::clone(®istry));
13175 handler
13176 .handle_control(
13177 ConnectionId::new(43),
13178 hello_frame("inactive", PROTOCOL_VERSION, 308),
13179 )
13180 .unwrap();
13181 assert!(registry
13182 .set_module_state_for_test("inactive", ChannelState::Closed)
13183 .unwrap());
13184
13185 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13186 let response = handler
13187 .handle_control_frame(
13188 &ctx,
13189 route_open_frame(309, "inactive", unique_project_root("inactive")),
13190 )
13191 .await
13192 .unwrap();
13193
13194 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13195 assert!(parse_error(&response[0])["message"]
13196 .as_str()
13197 .unwrap()
13198 .contains("is not active"));
13199 }
13200
13201 #[tokio::test]
13202 async fn late_health_reply_is_recorded_through_the_module_response_path() {
13203 let registry = Arc::new(Registry::default());
13204 let forwarding = Arc::new(ForwardingTable::default());
13205 let supervisor_handle = SupervisorHandle::new();
13206 let supervisor =
13207 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
13208 .with_forwarding(Arc::clone(&forwarding))
13209 .with_handle(supervisor_handle.clone());
13210 let module = supervisor
13211 .supervise_configured(
13212 crate::ModuleSpec {
13213 module_id: "late-health-response".to_string(),
13214 program: PathBuf::from("disabled-module"),
13215 args: Vec::new(),
13216 env: Vec::new(),
13217 reserved: false,
13218 reserved_prefixes: Vec::new(),
13219 protocol: ModuleProtocol::Subc,
13220 overlap: Default::default(),
13221 },
13222 false,
13223 )
13224 .unwrap();
13225 let handler =
13226 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13227 .with_supervisor(supervisor_handle);
13228 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13229 handler
13230 .handle_control_frame(
13231 &module_ctx,
13232 hello_frame_with_control_ops(
13233 "late-health-response",
13234 PROTOCOL_VERSION,
13235 7,
13236 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13237 ),
13238 )
13239 .await
13240 .unwrap();
13241 let probe_started_at = Instant::now() - Duration::from_millis(80);
13242 let pending = forwarding
13243 .begin_health_probe_rpc_for(
13244 "late-health-response",
13245 MODULE_CONTROL_OP_HEALTH_CHECK,
13246 probe_started_at,
13247 Instant::now() - Duration::from_millis(1),
13248 )
13249 .unwrap();
13250 assert!(forwarding
13251 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13252 .unwrap());
13253
13254 let responses = handler
13255 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13256 .await
13257 .unwrap();
13258
13259 assert!(responses.is_empty());
13260 let health = module.status().unwrap().health;
13261 assert_eq!(health.late_answer_count, 1);
13262 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13263 }
13264
13265 #[tokio::test]
13266 async fn health_probe_timeout_and_module_death_are_typed() {
13267 let registry = Arc::new(Registry::default());
13268 let forwarding = Arc::new(ForwardingTable::default());
13269 let handler =
13270 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13271 .with_health_probe_timeout(Duration::from_millis(50));
13272 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13273 hello_via_sink(
13274 &handler,
13275 &module_ctx,
13276 &mut module_rx,
13277 hello_frame_with_control_ops(
13278 "aft",
13279 PROTOCOL_VERSION,
13280 7,
13281 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13282 ),
13283 )
13284 .await;
13285
13286 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13287 let responses = handler
13288 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13289 .await
13290 .unwrap();
13291 assert_eq!(responses[0].header.ty, FrameType::Error);
13292 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13293 let _ = module_rx.try_recv();
13294
13295 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13296 let health_handler = handler.clone();
13297 let death_task = tokio::spawn(async move {
13298 health_handler
13299 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13300 .await
13301 .unwrap()
13302 });
13303 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13304 .await
13305 .unwrap()
13306 .unwrap();
13307 handler
13308 .cleanup_connection(module_ctx.connection_id)
13309 .unwrap();
13310 let responses = death_task.await.unwrap();
13311 assert_eq!(responses[0].header.ty, FrameType::Error);
13312 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13313 }
13314
13315 #[test]
13316 fn hello_requires_exact_protocol_version() {
13317 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13318 let registry = Arc::new(Registry::default());
13319 let handler = ControlHandler::new(Arc::clone(®istry));
13320 let responses = handler
13321 .handle_control(
13322 ConnectionId::new(connection),
13323 hello_frame("aft", offered, 9),
13324 )
13325 .unwrap();
13326
13327 assert_eq!(responses.len(), 1);
13328 assert_eq!(responses[0].header.ty, FrameType::Error);
13329 let error = parse_error(&responses[0]);
13330 assert_eq!(error["code"], "version_unsupported");
13331 assert!(registry.get_module("aft").unwrap().is_none());
13332 assert_eq!(registry.active_registration_count().unwrap(), 0);
13333 }
13334 }
13335
13336 #[test]
13337 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13338 let registry = Arc::new(Registry::default());
13339 let forwarding = Arc::new(ForwardingTable::default());
13340 let handler =
13341 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13342 let module_connection = ConnectionId::new(301);
13343 let registration = registry
13344 .register_with_control_ops(
13345 manifest("aft-push", PROTOCOL_VERSION),
13346 PROTOCOL_VERSION,
13347 module_connection,
13348 module_baseline_control_ops(),
13349 )
13350 .unwrap();
13351 let (module_tx, _module_rx) = mpsc::channel(8);
13352 let endpoint = forwarding
13353 .register_module_connection(
13354 module_connection,
13355 "aft-push".to_string(),
13356 PROTOCOL_VERSION,
13357 manifest_concurrency(®istration.manifest),
13358 FrameSink::new(module_tx),
13359 )
13360 .unwrap();
13361
13362 let unknown = Frame::build(
13364 FrameType::Push,
13365 control_flags(),
13366 0,
13367 0,
13368 5,
13369 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13370 )
13371 .unwrap();
13372 let out = handler.handle_status_update(endpoint, unknown).unwrap();
13373 assert!(
13374 out.is_empty(),
13375 "unknown push op must be ignored, got {out:?}"
13376 );
13377
13378 let malformed = Frame::build(
13380 FrameType::Push,
13381 control_flags(),
13382 0,
13383 0,
13384 6,
13385 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13386 )
13387 .unwrap();
13388 let out = handler.handle_status_update(endpoint, malformed).unwrap();
13389 assert_eq!(out.len(), 1);
13390 assert_eq!(out[0].header.ty, FrameType::Error);
13391 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13392 }
13393
13394 #[test]
13395 fn hello_rejected_when_connection_already_owns_client_routes() {
13396 let registry = Arc::new(Registry::default());
13397 let forwarding = Arc::new(ForwardingTable::default());
13398 let handler =
13399 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13400 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
13402 let client_connection = ConnectionId::new(202);
13403
13404 let responses = handler
13407 .handle_control(
13408 client_connection,
13409 hello_frame("aft-second", PROTOCOL_VERSION, 9),
13410 )
13411 .unwrap();
13412 assert_eq!(responses[0].header.ty, FrameType::Error);
13413 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13414 assert!(registry.get_module("aft-second").unwrap().is_none());
13415 }
13416
13417 #[tokio::test]
13418 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13419 let registry = Arc::new(Registry::default());
13420 let forwarding = Arc::new(ForwardingTable::default());
13421 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13422 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13423 hello_via_sink(
13424 &handler,
13425 &module_ctx,
13426 &mut module_rx,
13427 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13428 )
13429 .await;
13430 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13431 let pending = forwarding
13432 .begin_route_bind_relay_for_test(
13433 client_ctx.connection_id,
13434 client_ctx.egress.clone(),
13435 2,
13436 "alpha",
13437 )
13438 .unwrap();
13439 forwarding
13440 .complete_pending_relay(
13441 module_ctx.connection_id,
13442 pending.corr,
13443 RouteBindRelayOutcome::Accepted,
13444 )
13445 .unwrap();
13446 client_rx.try_recv().unwrap();
13447 for module_id in ["beta", "alpha"] {
13448 let replies = handler
13449 .handle_control_frame(
13450 &module_ctx,
13451 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13452 )
13453 .await
13454 .unwrap();
13455 assert_eq!(replies.len(), 1, "second HELLO must be refused");
13456 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13457 }
13458 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13459 assert!(registry.get_module("beta").unwrap().is_none());
13460 assert!(matches!(
13461 forwarding
13462 .lookup_data_route(
13463 client_ctx.connection_id,
13464 pending.client_channel,
13465 pending.client_epoch,
13466 )
13467 .unwrap(),
13468 DataRoute::Client(DataRouteState::Bound(_))
13469 ));
13470 assert!(handler
13471 .hello_launch_nonces
13472 .lock()
13473 .unwrap()
13474 .presented(module_ctx.connection_id, Some("alpha-nonce")));
13475 assert!(module_rx.try_recv().is_err());
13476 }
13477
13478 #[test]
13479 fn reserved_module_hello_requires_matching_launch_nonce() {
13480 let registry = Arc::new(Registry::default());
13481 let supervisor = SupervisorHandle::new();
13482 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13485 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13486
13487 let no_nonce = handler
13489 .handle_control(
13490 ConnectionId::new(1),
13491 hello_frame("vault", PROTOCOL_VERSION, 1),
13492 )
13493 .unwrap();
13494 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13495 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13496 assert!(registry.get_module("vault").unwrap().is_none());
13497
13498 let wrong = handler
13500 .handle_control(
13501 ConnectionId::new(2),
13502 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13503 )
13504 .unwrap();
13505 assert_eq!(wrong[0].header.ty, FrameType::Error);
13506 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13507 assert!(registry.get_module("vault").unwrap().is_none());
13508
13509 let ok = handler
13511 .handle_control(
13512 ConnectionId::new(3),
13513 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13514 )
13515 .unwrap();
13516 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13517 assert!(registry.get_module("vault").unwrap().is_some());
13518 }
13519
13520 #[test]
13521 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13522 let registry = Arc::new(Registry::default());
13523 let supervisor = SupervisorHandle::new();
13524 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13525 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13526 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13527
13528 let squat = handler
13529 .handle_control(
13530 ConnectionId::new(1),
13531 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13532 )
13533 .unwrap();
13534 assert_eq!(squat[0].header.ty, FrameType::Error);
13535 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
13536 assert!(parse_error(&squat[0])["message"]
13537 .as_str()
13538 .unwrap()
13539 .contains("fed:"));
13540
13541 let accepted_peer = handler
13542 .handle_control(
13543 ConnectionId::new(2),
13544 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
13545 )
13546 .unwrap();
13547 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
13548
13549 let accepted_short = handler
13550 .handle_control(
13551 ConnectionId::new(3),
13552 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
13553 )
13554 .unwrap();
13555 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
13556
13557 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
13558 let response = handler
13559 .handle_control(
13560 ConnectionId::new(conn),
13561 hello_frame(module_id, PROTOCOL_VERSION, conn),
13562 )
13563 .unwrap();
13564 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
13565 }
13566 }
13567
13568 #[test]
13569 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
13570 let registry = Arc::new(Registry::default());
13571 let supervisor = SupervisorHandle::new();
13572 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13573 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13574 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
13575 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13576
13577 let owner_nonce = handler
13578 .handle_control(
13579 ConnectionId::new(1),
13580 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
13581 )
13582 .unwrap();
13583 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
13584 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
13585 assert!(registry.get_module("fed:special").unwrap().is_none());
13586
13587 let exact_nonce = handler
13588 .handle_control(
13589 ConnectionId::new(2),
13590 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
13591 )
13592 .unwrap();
13593 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
13594 assert!(registry.get_module("fed:special").unwrap().is_some());
13595 }
13596
13597 #[test]
13598 fn non_reserved_module_ignores_launch_nonce() {
13599 let registry = Arc::new(Registry::default());
13600 let handler = ControlHandler::new(Arc::clone(®istry));
13603 let no_nonce = handler
13604 .handle_control(
13605 ConnectionId::new(1),
13606 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
13607 )
13608 .unwrap();
13609 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
13610 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
13611
13612 let echoed_nonce = handler
13613 .handle_control(
13614 ConnectionId::new(2),
13615 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
13616 )
13617 .unwrap();
13618 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
13619 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
13620 }
13621
13622 #[test]
13623 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
13624 let handler = ControlHandler::default();
13625 let conn = ConnectionId::new(1);
13626 let malformed = Frame::build(
13627 FrameType::Hello,
13628 control_flags(),
13629 0,
13630 0,
13631 3,
13632 b"{not json".to_vec(),
13633 )
13634 .unwrap();
13635
13636 let error = handler.handle_control(conn, malformed).unwrap();
13637 assert_eq!(error[0].header.ty, FrameType::Error);
13638 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
13639
13640 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13641 let pong = handler.handle_control(conn, ping).unwrap();
13642 assert_eq!(pong[0].header.ty, FrameType::Pong);
13643 assert_eq!(pong[0].header.corr, 4);
13644 }
13645
13646 #[test]
13647 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
13648 let registry = Arc::new(Registry::default());
13649 let handler = ControlHandler::new(Arc::clone(®istry));
13650
13651 handler
13652 .handle_control(
13653 ConnectionId::new(1),
13654 hello_frame("aft", PROTOCOL_VERSION, 1),
13655 )
13656 .unwrap();
13657 let duplicate = handler
13658 .handle_control(
13659 ConnectionId::new(2),
13660 hello_frame("aft", PROTOCOL_VERSION, 2),
13661 )
13662 .unwrap();
13663
13664 assert_eq!(duplicate[0].header.ty, FrameType::Error);
13665 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13666 let registration = registry.get_module("aft").unwrap().unwrap();
13667 assert_eq!(registration.connection_id, ConnectionId::new(1));
13668 }
13669
13670 #[test]
13671 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
13672 let registry = Arc::new(Registry::default());
13673 let forwarding = Arc::new(ForwardingTable::default());
13674 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
13675 let handler =
13676 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13677 .with_process_liveness(process_liveness);
13678 let (ctx, route_channel, route_epoch) =
13679 bind_liveness_route(®istry, &forwarding, "aft-dead");
13680 let responses = handler
13681 .handle_route_poll(
13682 &ctx,
13683 route_poll_frame(41, PollKind::Liveness, route_channel),
13684 route_channel,
13685 route_epoch,
13686 PollKind::Liveness,
13687 )
13688 .unwrap();
13689
13690 assert_eq!(responses.len(), 1);
13691 assert_eq!(responses[0].header.ty, FrameType::Response);
13692 assert_route_poll_liveness(&responses[0], false);
13693 }
13694
13695 #[test]
13696 fn liveness_poll_without_process_source_uses_bound_route() {
13697 let registry = Arc::new(Registry::default());
13698 let forwarding = Arc::new(ForwardingTable::default());
13699 let handler =
13700 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13701 let (ctx, route_channel, route_epoch) =
13702 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
13703 let responses = handler
13704 .handle_route_poll(
13705 &ctx,
13706 route_poll_frame(42, PollKind::Liveness, route_channel),
13707 route_channel,
13708 route_epoch,
13709 PollKind::Liveness,
13710 )
13711 .unwrap();
13712
13713 assert_route_poll_liveness(&responses[0], true);
13714 }
13715
13716 #[test]
13717 fn liveness_poll_untracked_process_source_uses_bound_route() {
13718 let registry = Arc::new(Registry::default());
13719 let forwarding = Arc::new(ForwardingTable::default());
13720 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13721 let handler =
13722 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13723 .with_process_liveness(process_liveness);
13724 let (ctx, route_channel, route_epoch) =
13725 bind_liveness_route(®istry, &forwarding, "aft-untracked");
13726 let responses = handler
13727 .handle_route_poll(
13728 &ctx,
13729 route_poll_frame(43, PollKind::Liveness, route_channel),
13730 route_channel,
13731 route_epoch,
13732 PollKind::Liveness,
13733 )
13734 .unwrap();
13735
13736 assert_route_poll_liveness(&responses[0], true);
13737 }
13738
13739 #[tokio::test]
13740 async fn unknown_op_returns_unknown_control_op() {
13741 let handler = ControlHandler::default();
13742 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13743 let request = Frame::build(
13744 FrameType::Request,
13745 control_flags(),
13746 0,
13747 0,
13748 55,
13749 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13750 )
13751 .unwrap();
13752
13753 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13754
13755 assert_eq!(response.len(), 1);
13756 assert_eq!(response[0].header.ty, FrameType::Error);
13757 assert_eq!(response[0].header.corr, 55);
13758 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
13759 }
13760
13761 #[tokio::test]
13762 async fn supervisor_provenance_rejects_unknown_exact_module() {
13763 let handler = ControlHandler::default();
13764 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
13765 let request = Frame::build(
13766 FrameType::Request,
13767 control_flags(),
13768 0,
13769 0,
13770 57,
13771 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
13772 )
13773 .unwrap();
13774
13775 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13776
13777 assert_eq!(response.len(), 1);
13778 assert_eq!(response[0].header.ty, FrameType::Error);
13779 assert_eq!(response[0].header.corr, 57);
13780 let error = parse_error(&response[0]);
13781 assert_eq!(error["code"], "unknown_module");
13782 assert_eq!(error["message"], "module_id 'missing' is not supervised");
13783 }
13784
13785 #[test]
13786 fn provenance_probe_override_keeps_handler_tests_deterministic() {
13787 let expected = subc_control::RunningImageAgreement::Unavailable {
13788 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13789 };
13790 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
13791 assert_eq!(handler.provenance_probe_override, Some(expected));
13792 }
13793
13794 #[test]
13795 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
13796 let verdict = reload_verdict(
13797 std::path::Path::new("/bin/new"),
13798 Some(std::path::Path::new("/bin/old")),
13799 subc_control::RunningImageAgreement::Unavailable {
13800 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13801 },
13802 );
13803 assert!(matches!(
13804 verdict.path,
13805 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
13806 if configured == std::path::Path::new("/bin/new")
13807 && spawned_from == std::path::Path::new("/bin/old")
13808 ));
13809 }
13810
13811 #[test]
13812 fn launch_spec_without_running_digest_is_unavailable_not_match() {
13813 use subc_control::{
13814 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
13815 };
13816 let configured = crate::supervise::ModuleSpec {
13817 module_id: "legacy".into(),
13818 program: "/module".into(),
13819 args: Vec::new(),
13820 env: Vec::new(),
13821 reserved: false,
13822 reserved_prefixes: Vec::new(),
13823 protocol: subc_control::ModuleProtocol::Subc,
13824 overlap: Default::default(),
13825 };
13826 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
13827 assert_eq!(
13828 launch_spec_agreement(&configured, false, Some(&digest)),
13829 Agreement::Unavailable {
13830 reason: Reason::NotRunning
13831 }
13832 );
13833 assert_eq!(
13834 launch_spec_agreement(&configured, true, None),
13835 Agreement::Unavailable {
13836 reason: Reason::SpawnedSpecUnavailable
13837 }
13838 );
13839 assert_eq!(
13840 launch_spec_agreement(&configured, true, Some(&digest)),
13841 Agreement::Match
13842 );
13843 }
13844
13845 #[cfg(windows)]
13846 #[test]
13847 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13848 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13849 let configured = dir.join("module.exe");
13850 std::fs::write(&configured, b"executable path fixture").unwrap();
13851 let spawned = std::fs::canonicalize(&configured).unwrap();
13852 assert_ne!(
13853 configured, spawned,
13854 "fixture must exercise different path spellings"
13855 );
13856 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13857 let lower_case = configured.to_string_lossy().to_lowercase();
13858 for spelling in [
13859 &configured,
13860 Path::new(&forward_slashes),
13861 Path::new(&lower_case),
13862 ] {
13863 let verdict = reload_verdict(
13864 spelling,
13865 Some(&spawned),
13866 subc_control::RunningImageAgreement::Unavailable {
13867 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13868 },
13869 );
13870 assert_eq!(
13871 verdict.path,
13872 ReloadPathAgreement::Match,
13873 "configured {spelling:?}, spawned {spawned:?}"
13874 );
13875 }
13876 }
13877
13878 #[cfg(windows)]
13879 #[test]
13880 fn windows_reload_path_keeps_different_existing_program_pending() {
13881 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13882 let old = dir.join("old.exe");
13883 let new = dir.join("new.exe");
13884 std::fs::write(&old, b"same executable content").unwrap();
13885 std::fs::copy(&old, &new).unwrap();
13886 let spawned = std::fs::canonicalize(&old).unwrap();
13887 let image = subc_control::RunningImageAgreement::Unavailable {
13888 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13889 };
13890 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13891 assert!(
13892 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13893 if configured == new && spawned_from == spawned)
13894 );
13895 let missing = dir.join("missing.exe");
13896 assert!(matches!(
13897 reload_verdict(&missing, Some(&spawned), image).path,
13898 ReloadPathAgreement::Mismatch { .. }
13899 ));
13900 }
13901
13902 #[test]
13903 fn reload_verdict_detects_replaced_image_at_same_path() {
13904 let image = subc_control::RunningImageAgreement::Mismatch {
13905 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13906 digest: "old".into(),
13907 },
13908 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13909 digest: "new".into(),
13910 },
13911 };
13912 let verdict = reload_verdict(
13913 std::path::Path::new("/bin/same"),
13914 Some(std::path::Path::new("/bin/same")),
13915 image.clone(),
13916 );
13917 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13918 assert_eq!(verdict.image, image);
13919 }
13920
13921 #[test]
13922 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13923 let image = subc_control::RunningImageAgreement::Unavailable {
13924 reason: subc_control::RunningImageUnavailableReason::NotRunning,
13925 };
13926 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13927 assert_eq!(
13928 verdict.path,
13929 subc_control::ReloadPathAgreement::Unavailable {
13930 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13931 }
13932 );
13933 assert_eq!(verdict.image, image);
13934
13935 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13936 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13937 };
13938 let verdict = reload_verdict(
13939 std::path::Path::new("/bin/same"),
13940 Some(std::path::Path::new("/bin/same")),
13941 unconfirmed.clone(),
13942 );
13943 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13944 assert_eq!(verdict.image, unconfirmed);
13945 }
13946
13947 #[test]
13948 fn reload_verdict_preserves_each_image_unavailability_reason() {
13949 use subc_control::RunningImageUnavailableReason as Reason;
13950
13951 for reason in [
13952 Reason::NotRunning,
13953 Reason::UnsupportedPlatform,
13954 Reason::RunningExecutableUnreadable,
13955 Reason::SpawnedPathUnreadable,
13956 Reason::HashFailed,
13957 Reason::ProcessIdentityUnconfirmed,
13958 Reason::Unknown("future_probe_reason".to_string()),
13959 ] {
13960 let image = subc_control::RunningImageAgreement::Unavailable {
13961 reason: reason.clone(),
13962 };
13963 let verdict = reload_verdict(
13964 std::path::Path::new("/bin/same"),
13965 Some(std::path::Path::new("/bin/same")),
13966 image.clone(),
13967 );
13968 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13969 assert_eq!(verdict.image, image, "{reason:?}");
13970 }
13971 }
13972
13973 #[tokio::test]
13974 async fn malformed_control_bodies_return_invalid_control_body() {
13975 let handler = ControlHandler::default();
13976 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13977
13978 for (corr, body) in [
13979 (56, br#"{"route_channel":1}"#.as_slice()),
13980 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13981 (
13982 58,
13983 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13984 ),
13985 ] {
13986 let request = Frame::build(
13987 FrameType::Request,
13988 control_flags(),
13989 0,
13990 0,
13991 corr,
13992 body.to_vec(),
13993 )
13994 .unwrap();
13995 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13996
13997 assert_eq!(response.len(), 1);
13998 assert_eq!(response[0].header.ty, FrameType::Error);
13999 assert_eq!(response[0].header.corr, corr);
14000 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14001 }
14002 }
14003
14004 #[tokio::test]
14005 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14006 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14007 let registry = Arc::new(Registry::default());
14008 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14009 let router = Router::with_control_handler(Arc::clone(&control));
14010 let connection = router.begin_connection();
14011 let (ctx, mut rx) = route_ctx(connection.id());
14012
14013 router
14014 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14015 .await
14016 .unwrap();
14017 let response = rx.recv().await.unwrap();
14018 let ack = parse_ack(&response);
14019 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14020 let channel = 1;
14021
14022 let goodbye =
14023 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14024 router.route_for_connection(&ctx, goodbye).await.unwrap();
14025 assert!(rx.try_recv().is_err());
14026 assert!(registry.get_module("aft").unwrap().is_none());
14027
14028 router
14029 .route_for_connection(&ctx, channel_request(channel, 13))
14030 .await
14031 .unwrap();
14032 let error_frame = rx.recv().await.unwrap();
14033 assert_eq!(error_frame.header.ty, FrameType::Error);
14034 assert_eq!(error_frame.header.channel, channel);
14035 let captured = crate::router::test_log::captured_logs(&logs);
14036 assert_eq!(
14037 captured
14038 .lines()
14039 .filter(|line| {
14040 line.contains("module_id=aft")
14041 && line.contains("reason=explicit_goodbye")
14042 && line.contains("module registration ended")
14043 })
14044 .count(),
14045 1,
14046 "unexpected GOODBYE registry log: {captured}"
14047 );
14048 }
14049
14050 #[tokio::test]
14051 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14052 let registry = Arc::new(Registry::default());
14053 let handler = ControlHandler::new(registry).with_capability_config(
14054 [("prov".to_string(), true), ("cons".to_string(), true)],
14055 BTreeMap::new(),
14056 );
14057 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14058 register_capability_manifest(
14059 &handler,
14060 &provider_ctx,
14061 &mut provider_rx,
14062 capability_manifest("prov", &["thing/v1"], &[]),
14063 1,
14064 )
14065 .await;
14066 let mut consumer = capability_manifest("cons", &[], &[]);
14067 consumer.capabilities.as_mut().unwrap().requires.push(
14068 subc_protocol::manifest::CapabilityRequirement {
14069 capability: "thing/v1".to_string(),
14070 need: subc_protocol::manifest::CapabilityNeed::Required,
14071 },
14072 );
14073 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14074 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14075 assert_eq!(
14076 handler.capability_evaluator.verdict("cons", "thing/v1"),
14077 Some(CapabilityVerdict::Provided)
14078 );
14079 let (mut client_rx, _) = open_route_for_capability_test(
14080 &handler,
14081 &provider_ctx,
14082 &mut provider_rx,
14083 703,
14084 3,
14085 "prov",
14086 None,
14087 )
14088 .await;
14089 let goodbye =
14090 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14091 handler
14092 .handle_control_frame(&provider_ctx, goodbye)
14093 .await
14094 .unwrap();
14095 assert_eq!(
14096 handler.capability_evaluator.verdict("cons", "thing/v1"),
14097 Some(CapabilityVerdict::NeverProvided)
14098 );
14099 let closed = client_rx
14100 .try_recv()
14101 .expect("GOODBYE pushes route.closed before route GOODBYE");
14102 assert!(
14103 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14104 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14105 );
14106 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14107 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14108 }
14109
14110 struct RouteClosedFixture {
14111 handler: ControlHandler,
14112 #[cfg(unix)]
14113 supervisor: Supervisor,
14114 module_ctx: RouteCtx,
14115 #[cfg(unix)]
14116 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14117 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14118 route: Arc<crate::forwarding::RouteBinding>,
14119 }
14120
14121 async fn route_closed_fixture() -> RouteClosedFixture {
14122 let registry = Arc::new(Registry::default());
14123 let handler = ControlHandler::new(Arc::clone(®istry));
14124 #[cfg(unix)]
14125 let supervisor = Supervisor::new(registry, RestartPolicy::default())
14126 .with_forwarding(Arc::clone(&handler.forwarding));
14127 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14128 register_capability_manifest(
14129 &handler,
14130 &module_ctx,
14131 &mut module_rx,
14132 capability_manifest("prov", &[], &[]),
14133 1,
14134 )
14135 .await;
14136 let (client_rx, bind) = open_route_for_capability_test(
14137 &handler,
14138 &module_ctx,
14139 &mut module_rx,
14140 702,
14141 2,
14142 "prov",
14143 None,
14144 )
14145 .await;
14146 let ModuleControlRequest::RouteBind {
14147 route_channel: channel,
14148 epoch: route_epoch,
14149 ..
14150 } = bind
14151 else {
14152 panic!("expected route.bind");
14153 };
14154 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14155 handler
14156 .forwarding
14157 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14158 .unwrap()
14159 else {
14160 panic!("the acknowledged route must be bound");
14161 };
14162 RouteClosedFixture {
14163 handler,
14164 #[cfg(unix)]
14165 supervisor,
14166 module_ctx,
14167 #[cfg(unix)]
14168 module_rx,
14169 client_rx,
14170 route,
14171 }
14172 }
14173
14174 impl RouteClosedFixture {
14175 #[cfg(unix)]
14176 async fn cut(&mut self) {
14177 let notices = async {
14178 let mut draining = self.module_rx.recv().await.unwrap();
14179 assert!(matches!(
14180 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14181 &draining.body
14182 )
14183 .unwrap(),
14184 subc_protocol::session::ModuleControlCommand::Draining {
14185 reason: RouteCloseReason::Restart,
14186 ..
14187 }
14188 ));
14189 draining.flushed.take().unwrap().send(()).unwrap();
14190 let mut closing = self.client_rx.recv().await.unwrap();
14191 assert!(matches!(
14192 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14193 ClientControlPush::RouteClosing {
14194 reason: RouteCloseReason::Restart,
14195 ..
14196 }
14197 ));
14198 closing.flushed.take().unwrap().send(()).unwrap();
14199 };
14200 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14201 result.unwrap();
14202 }
14203
14204 fn disconnect(&mut self) -> ClientControlPush {
14205 self.handler
14206 .cleanup_connection(self.module_ctx.connection_id)
14207 .unwrap();
14208 let closed = self.client_rx.try_recv().unwrap();
14209 assert_eq!(closed.header.ty, FrameType::Push);
14210 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14211 assert!(matches!(
14212 &push,
14213 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14214 if channels == &[self.route.client_channel]
14215 ));
14216 assert_eq!(
14217 self.client_rx.try_recv().unwrap().header.ty,
14218 FrameType::Goodbye
14219 );
14220 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14221 push
14222 }
14223 }
14224
14225 #[cfg(unix)]
14226 #[tokio::test(start_paused = true)]
14227 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14228 let mut fixture = route_closed_fixture().await;
14229 fixture.cut().await;
14230 assert!(matches!(
14231 fixture.disconnect(),
14232 ClientControlPush::RouteClosed {
14233 reason: RouteCloseReason::Restart,
14234 drained: true,
14235 ..
14236 }
14237 ));
14238 }
14239
14240 #[cfg(unix)]
14241 #[tokio::test(start_paused = true)]
14242 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14243 let mut fixture = route_closed_fixture().await;
14244 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14245 let started = tokio::time::Instant::now();
14246 fixture.cut().await;
14247 assert!(started.elapsed() >= Duration::from_secs(2));
14248 assert_eq!(fixture.route.flow.in_flight(), 1);
14249 assert!(matches!(
14250 fixture.disconnect(),
14251 ClientControlPush::RouteClosed {
14252 reason: RouteCloseReason::Restart,
14253 drained: false,
14254 ..
14255 }
14256 ));
14257 }
14258
14259 #[tokio::test]
14260 async fn module_connection_crash_reports_crash_and_not_drained() {
14261 let mut fixture = route_closed_fixture().await;
14262 assert!(matches!(
14266 fixture.disconnect(),
14267 ClientControlPush::RouteClosed {
14268 reason: RouteCloseReason::Crash,
14269 drained: false,
14270 ..
14271 }
14272 ));
14273 }
14274
14275 #[tokio::test]
14276 async fn dropping_router_connection_releases_registration() {
14277 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14278 let registry = Arc::new(Registry::default());
14279 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14280 let router = Router::with_control_handler(Arc::clone(&control));
14281 let connection = router.begin_connection();
14282 let connection_id = connection.id();
14283 let (ctx, mut rx) = route_ctx(connection_id);
14284
14285 router
14286 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14287 .await
14288 .unwrap();
14289 let response = rx.recv().await.unwrap();
14290 let ack = parse_ack(&response);
14291 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14292 assert!(registry.get_module("aft").unwrap().is_some());
14293
14294 drop(connection);
14295
14296 assert!(registry.get_module("aft").unwrap().is_none());
14297 assert_eq!(registry.active_registration_count().unwrap(), 0);
14298
14299 control
14300 .cleanup_connection(ConnectionId::new(u64::MAX))
14301 .unwrap();
14302 let captured = crate::router::test_log::captured_logs(&logs);
14303 println!("captured registry lifecycle logs:\n{captured}");
14304 let events: Vec<_> = captured
14305 .lines()
14306 .filter(|line| {
14307 line.contains("module registered module_id=aft ")
14308 || line.contains("module registration ended")
14309 })
14310 .collect();
14311 assert_eq!(
14312 events.len(),
14313 2,
14314 "unexpected registry lifecycle logs: {captured}"
14315 );
14316 assert!(events[0].contains("module registered module_id=aft "));
14317 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14318 assert!(events[1].contains(&format!(
14319 "module_id=aft connection_id={}",
14320 connection_id.get()
14321 )));
14322 assert!(events[1].contains("reason=connection_closed"));
14323 assert!(events[1].contains("module registration ended"));
14324 assert_eq!(
14325 captured
14326 .lines()
14327 .filter(|line| line.contains("module registered module_id=aft "))
14328 .count(),
14329 1,
14330 "legacy registration admission line must appear once: {captured}"
14331 );
14332 }
14333
14334 #[tokio::test]
14335 async fn hello_registration_keeps_legacy_module_registered_line_once() {
14336 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14337 let registry = Arc::new(Registry::default());
14338 let control = ControlHandler::new(Arc::clone(®istry));
14339 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14340 hello_via_sink(
14341 &control,
14342 &ctx,
14343 &mut rx,
14344 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14345 )
14346 .await;
14347
14348 let captured = crate::router::test_log::captured_logs(&logs);
14349 let admissions: Vec<_> = captured
14350 .lines()
14351 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14352 .collect();
14353 assert_eq!(
14354 admissions.len(),
14355 1,
14356 "legacy admission line must remain exactly once: {captured}"
14357 );
14358 assert!(admissions[0].contains("routable_provider=true"));
14359 assert!(admissions[0].contains("connection_id=777"));
14360 }
14361
14362 fn capability_manifest(
14363 module_id: &str,
14364 provides: &[&str],
14365 must_never_reach: &[&str],
14366 ) -> ModuleManifest {
14367 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14368 manifest.capabilities = Some(CapabilityDeclarations {
14369 provides: provides
14370 .iter()
14371 .map(|capability| (*capability).to_string())
14372 .collect(),
14373 requires: Vec::new(),
14374 must_never_reach: must_never_reach
14375 .iter()
14376 .map(|capability| (*capability).to_string())
14377 .collect(),
14378 });
14379 manifest
14380 }
14381
14382 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14383 Frame::build(
14384 FrameType::Hello,
14385 control_flags(),
14386 0,
14387 0,
14388 corr,
14389 serde_json::to_vec(&ModuleHelloBody {
14390 protocol_ver: manifest.protocol_ver,
14391 manifest,
14392 control_ops: None,
14393 launch_nonce: None,
14394 })
14395 .expect("capability test HELLO serializes"),
14396 )
14397 .expect("capability test HELLO frame builds")
14398 }
14399
14400 fn catalog_update_with_capabilities_frame(
14401 corr: u64,
14402 capabilities: CapabilityDeclarations,
14403 ) -> Frame {
14404 Frame::build(
14405 FrameType::Request,
14406 control_flags(),
14407 0,
14408 0,
14409 corr,
14410 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14411 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14412 capabilities: Some(capabilities),
14413 ready: None,
14414 })
14415 .expect("capability catalog.update serializes"),
14416 )
14417 .expect("capability catalog.update frame builds")
14418 }
14419
14420 async fn register_capability_manifest(
14421 handler: &ControlHandler,
14422 ctx: &RouteCtx,
14423 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14424 manifest: ModuleManifest,
14425 corr: u64,
14426 ) {
14427 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14428 }
14429
14430 async fn open_route_for_capability_test(
14431 handler: &ControlHandler,
14432 target_ctx: &RouteCtx,
14433 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14434 client_connection_id: u64,
14435 corr: u64,
14436 target_module_id: &str,
14437 consumer_identity: Option<ConsumerIdentity>,
14438 ) -> (
14439 mpsc::Receiver<crate::router::OutboundFrame>,
14440 ModuleControlRequest,
14441 ) {
14442 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14443 let route_handler = handler.clone();
14444 let target_module_id = target_module_id.to_string();
14445 let route_task = tokio::spawn(async move {
14446 route_handler
14447 .handle_control_frame(
14448 &client_ctx,
14449 route_open_frame_with_admission_facts(
14450 corr,
14451 &target_module_id,
14452 unique_project_root("admission-facts"),
14453 consumer_identity,
14454 None,
14455 ),
14456 )
14457 .await
14458 .expect("capability test route.open succeeds")
14459 });
14460 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14461 .await
14462 .expect("capability test route.open must reach route.bind")
14463 .expect("target control receiver stays open");
14464 let bind_request: ModuleControlRequest =
14465 serde_json::from_slice(&bind.body).expect("route.bind decodes");
14466 handler
14467 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14468 .await
14469 .expect("capability test route.bind ACK succeeds");
14470 assert!(route_task.await.expect("route.open task joins").is_empty());
14471 let opened = client_rx
14472 .recv()
14473 .await
14474 .expect("successful route.open publishes a response");
14475 assert!(matches!(
14476 serde_json::from_slice::<ClientControlResponse>(&opened.body),
14477 Ok(ClientControlResponse::RouteOpen { .. })
14478 ));
14479 (client_rx, bind_request)
14480 }
14481
14482 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
14483 assert_eq!(frame.header.ty, FrameType::Push);
14484 assert_eq!(frame.header.channel, 0);
14485 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14486 .expect("route.closed control push decodes");
14487 let ClientControlPush::RouteClosed { channels, .. } = &push else {
14488 panic!("expected route.closed");
14489 };
14490 assert_eq!(channels.len(), 1, "exactly one violating route closed");
14491 let channels = channels.clone();
14492 assert_eq!(
14493 push,
14494 ClientControlPush::RouteClosed {
14495 module_id: target_module_id.to_string(),
14496 channels,
14497 reason: RouteCloseReason::CapabilityDenied,
14498 drained: false,
14499 abandoned: 0,
14500 excluded_subscriptions: 0,
14501 terminal: Some(false),
14502 }
14503 );
14504 }
14505
14506 #[tokio::test]
14507 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14508 let registry = Arc::new(Registry::default());
14509 let forwarding = Arc::new(ForwardingTable::default());
14510 let supervisor = SupervisorHandle::new();
14511 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14512 let handler =
14513 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14514 .with_supervisor(supervisor);
14515 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14516 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14517 register_capability_manifest(
14518 &handler,
14519 &target_ctx,
14520 &mut target_rx,
14521 capability_manifest("target", &["credentials-provider/v1"], &[]),
14522 1,
14523 )
14524 .await;
14525 register_capability_manifest(
14526 &handler,
14527 &opener_ctx,
14528 &mut opener_rx,
14529 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14530 2,
14531 )
14532 .await;
14533
14534 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14535 let replies = handler
14536 .handle_control_frame(
14537 &client_ctx,
14538 route_open_frame_with_admission_facts(
14539 3,
14540 "target",
14541 unique_project_root("admission-facts"),
14542 Some(ConsumerIdentity {
14543 module_id: "opener".to_string(),
14544 launch_nonce: "opener-nonce".to_string(),
14545 }),
14546 None,
14547 ),
14548 )
14549 .await
14550 .expect("denied route.open returns a typed frame");
14551 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14552 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14553 assert!(
14554 target_rx.try_recv().is_err(),
14555 "forbidden route.open must not relay route.bind"
14556 );
14557 }
14558
14559 #[tokio::test]
14560 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
14561 let registry = Arc::new(Registry::default());
14562 let forwarding = Arc::new(ForwardingTable::default());
14563 let supervisor = SupervisorHandle::new();
14564 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14565 let handler =
14566 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14567 .with_supervisor(supervisor);
14568 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
14569 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
14570 register_capability_manifest(
14571 &handler,
14572 &target_ctx,
14573 &mut target_rx,
14574 capability_manifest("target", &["credentials-provider/v1"], &[]),
14575 1,
14576 )
14577 .await;
14578 register_capability_manifest(
14579 &handler,
14580 &old_opener_ctx,
14581 &mut old_opener_rx,
14582 capability_manifest("opener", &[], &[]),
14583 2,
14584 )
14585 .await;
14586 let (mut client_rx, _) = open_route_for_capability_test(
14587 &handler,
14588 &target_ctx,
14589 &mut target_rx,
14590 712,
14591 3,
14592 "target",
14593 Some(ConsumerIdentity {
14594 module_id: "opener".to_string(),
14595 launch_nonce: "opener-nonce".to_string(),
14596 }),
14597 )
14598 .await;
14599 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14600
14601 handler
14602 .cleanup_connection(old_opener_ctx.connection_id)
14603 .expect("old opener registration cleans up");
14604 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
14605 register_capability_manifest(
14606 &handler,
14607 &new_opener_ctx,
14608 &mut new_opener_rx,
14609 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14610 4,
14611 )
14612 .await;
14613
14614 assert_capability_denied_push(
14615 client_rx
14616 .try_recv()
14617 .expect("HELLO deny addition must emit route.closed")
14618 .frame,
14619 "target",
14620 );
14621 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14622 assert!(matches!(
14623 target_rx.try_recv(),
14624 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14625 ));
14626 }
14627
14628 #[tokio::test]
14629 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
14630 let registry = Arc::new(Registry::default());
14631 let forwarding = Arc::new(ForwardingTable::default());
14632 let supervisor = SupervisorHandle::new();
14633 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14634 let handler =
14635 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14636 .with_supervisor(supervisor);
14637 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
14638 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
14639 register_capability_manifest(
14640 &handler,
14641 &target_ctx,
14642 &mut target_rx,
14643 capability_manifest("target", &[], &[]),
14644 1,
14645 )
14646 .await;
14647 register_capability_manifest(
14648 &handler,
14649 &opener_ctx,
14650 &mut opener_rx,
14651 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14652 2,
14653 )
14654 .await;
14655 let (mut client_rx, _) = open_route_for_capability_test(
14656 &handler,
14657 &target_ctx,
14658 &mut target_rx,
14659 722,
14660 3,
14661 "target",
14662 Some(ConsumerIdentity {
14663 module_id: "opener".to_string(),
14664 launch_nonce: "opener-nonce".to_string(),
14665 }),
14666 )
14667 .await;
14668 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14669
14670 let replies = handler
14671 .handle_control_frame(
14672 &target_ctx,
14673 catalog_update_with_capabilities_frame(
14674 4,
14675 CapabilityDeclarations {
14676 provides: vec!["credentials-provider/v1".to_string()],
14677 requires: Vec::new(),
14678 must_never_reach: Vec::new(),
14679 },
14680 ),
14681 )
14682 .await
14683 .expect("claim catalog.update succeeds");
14684 assert!(matches!(
14685 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
14686 Ok(ModuleControlResponseToModule::CatalogUpdate {})
14687 ));
14688 assert_capability_denied_push(
14689 client_rx
14690 .try_recv()
14691 .expect("claim addition must emit route.closed")
14692 .frame,
14693 "target",
14694 );
14695 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14696 assert!(matches!(
14697 target_rx.try_recv(),
14698 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14699 ));
14700 }
14701
14702 #[tokio::test]
14703 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
14704 let registry = Arc::new(Registry::default());
14705 let forwarding = Arc::new(ForwardingTable::default());
14706 let supervisor = SupervisorHandle::new();
14707 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14708 let handler =
14709 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14710 .with_supervisor(supervisor);
14711 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
14712 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
14713 register_capability_manifest(
14714 &handler,
14715 &target_ctx,
14716 &mut target_rx,
14717 capability_manifest("target", &["credentials-provider/v1"], &[]),
14718 1,
14719 )
14720 .await;
14721 register_capability_manifest(
14722 &handler,
14723 &opener_ctx,
14724 &mut opener_rx,
14725 capability_manifest("opener", &[], &[]),
14726 2,
14727 )
14728 .await;
14729 let (mut client_rx, _) = open_route_for_capability_test(
14730 &handler,
14731 &target_ctx,
14732 &mut target_rx,
14733 732,
14734 3,
14735 "target",
14736 Some(ConsumerIdentity {
14737 module_id: "opener".to_string(),
14738 launch_nonce: "opener-nonce".to_string(),
14739 }),
14740 )
14741 .await;
14742 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14743
14744 handler
14745 .handle_control_frame(
14746 &target_ctx,
14747 catalog_update_with_capabilities_frame(
14748 4,
14749 CapabilityDeclarations {
14750 provides: Vec::new(),
14751 requires: Vec::new(),
14752 must_never_reach: Vec::new(),
14753 },
14754 ),
14755 )
14756 .await
14757 .expect("claim removal catalog.update succeeds");
14758 assert_eq!(
14759 forwarding.active_binding_count().unwrap(),
14760 1,
14761 "removing an attested target claim must leave the route census unchanged"
14762 );
14763 assert!(
14764 client_rx.try_recv().is_err(),
14765 "claim removal must not emit route.closed capability_denied"
14766 );
14767 assert!(
14768 target_rx.try_recv().is_err(),
14769 "claim removal must not send the target a route GOODBYE"
14770 );
14771 }
14772
14773 #[tokio::test]
14776 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
14777 let registry = Arc::new(Registry::default());
14778 let forwarding = Arc::new(ForwardingTable::default());
14779 let supervisor = SupervisorHandle::new();
14780 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14781 let handler =
14782 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14783 .with_supervisor(supervisor);
14784 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
14785 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
14786 register_capability_manifest(
14787 &handler,
14788 &target_ctx,
14789 &mut target_rx,
14790 capability_manifest("target", &["credentials-provider/v1"], &[]),
14791 1,
14792 )
14793 .await;
14794 register_capability_manifest(
14795 &handler,
14796 &opener_ctx,
14797 &mut opener_rx,
14798 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14799 2,
14800 )
14801 .await;
14802
14803 let (_client_rx, bind) = open_route_for_capability_test(
14804 &handler,
14805 &target_ctx,
14806 &mut target_rx,
14807 742,
14808 3,
14809 "target",
14810 None,
14811 )
14812 .await;
14813 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14814 panic!("direct scope-honesty route must bind");
14815 };
14816 assert_eq!(principal, Some(Principal::Direct));
14817 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14818 }
14819
14820 #[tokio::test]
14823 async fn must_never_reach_self_route_is_capability_forbidden() {
14824 let registry = Arc::new(Registry::default());
14825 let forwarding = Arc::new(ForwardingTable::default());
14826 let supervisor = SupervisorHandle::new();
14827 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
14828 let handler =
14829 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14830 .with_supervisor(supervisor);
14831 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
14832 register_capability_manifest(
14833 &handler,
14834 &self_ctx,
14835 &mut self_rx,
14836 capability_manifest(
14837 "self-provider",
14838 &["credentials-provider/v1"],
14839 &["credentials-provider/v1"],
14840 ),
14841 1,
14842 )
14843 .await;
14844
14845 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14846 let replies = handler
14847 .handle_control_frame(
14848 &client_ctx,
14849 route_open_frame_with_admission_facts(
14850 2,
14851 "self-provider",
14852 unique_project_root("admission-facts"),
14853 Some(ConsumerIdentity {
14854 module_id: "self-provider".to_string(),
14855 launch_nonce: "self-nonce".to_string(),
14856 }),
14857 None,
14858 ),
14859 )
14860 .await
14861 .expect("self-route refusal returns a typed frame");
14862 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14863 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14864 assert!(
14865 self_rx.try_recv().is_err(),
14866 "self denial must not relay route.bind"
14867 );
14868 }
14869
14870 #[test]
14871 fn unsupported_channel_zero_frame_returns_error() {
14872 let handler = ControlHandler::default();
14873 let request = Frame::build(
14874 FrameType::Request,
14875 control_flags(),
14876 0,
14877 0,
14878 21,
14879 b"opaque".to_vec(),
14880 )
14881 .unwrap();
14882
14883 let response = handler
14884 .handle_control(ConnectionId::new(1), request)
14885 .unwrap();
14886
14887 assert_eq!(response[0].header.ty, FrameType::Error);
14888 assert_eq!(
14889 parse_error(&response[0])["code"],
14890 "unsupported_control_frame"
14891 );
14892 }
14893
14894 mod swap {
14899 use super::*;
14900
14901 const INCUMBENT: ConnectionId = ConnectionId::new(30);
14902 const CANDIDATE: ConnectionId = ConnectionId::new(40);
14903
14904 struct Swap {
14905 registry: Arc<Registry>,
14906 forwarding: Arc<ForwardingTable>,
14907 handler: ControlHandler,
14908 incumbent_ctx: RouteCtx,
14909 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14910 candidate_ctx: RouteCtx,
14911 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14912 }
14913
14914 async fn swap_with_incumbent() -> Swap {
14915 let registry = Arc::new(Registry::default());
14916 let forwarding = Arc::new(ForwardingTable::default());
14917 let handler =
14918 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14919 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14920 hello_via_sink(
14921 &handler,
14922 &incumbent_ctx,
14923 &mut incumbent_rx,
14924 hello_frame("aft", PROTOCOL_VERSION, 7),
14925 )
14926 .await;
14927 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14928 Swap {
14929 registry,
14930 forwarding,
14931 handler,
14932 incumbent_ctx,
14933 incumbent_rx,
14934 candidate_ctx,
14935 candidate_rx,
14936 }
14937 }
14938
14939 fn register_candidate(swap: &Swap, ready: Option<bool>) {
14940 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14941 candidate_manifest.ready = ready;
14942 let registration = swap
14943 .registry
14944 .register_candidate_with_control_ops(
14945 candidate_manifest,
14946 PROTOCOL_VERSION,
14947 CANDIDATE,
14948 module_baseline_control_ops(),
14949 )
14950 .unwrap();
14951 swap.forwarding
14952 .register_candidate_module_connection(
14953 CANDIDATE,
14954 "aft".to_string(),
14955 PROTOCOL_VERSION,
14956 manifest_concurrency(®istration.manifest),
14957 swap.candidate_ctx.egress.clone(),
14958 )
14959 .unwrap();
14960 }
14961
14962 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14963 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14964 swap.registry.promote_candidate("aft").unwrap().unwrap();
14965 cutover.incumbent.unwrap()
14966 }
14967
14968 #[tokio::test]
14977 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14978 let mut swap = swap_with_incumbent().await;
14979 let hold = Arc::new(std::sync::Barrier::new(2));
14980 let handler = swap
14981 .handler
14982 .clone()
14983 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14984 handler
14985 .supervisor
14986 .open_swap("aft", "candidate-nonce".to_string());
14987 let hello = {
14988 let handler = handler.clone();
14989 let candidate_ctx = swap.candidate_ctx.clone();
14990 std::thread::spawn(move || {
14991 handler
14992 .handle_hello(
14993 candidate_ctx.connection_id,
14994 Some(candidate_ctx.egress.clone()),
14995 hello_frame_with_nonce(
14996 "aft",
14997 PROTOCOL_VERSION,
14998 8,
14999 Some("candidate-nonce"),
15000 ),
15001 )
15002 .unwrap()
15003 })
15004 };
15005
15006 hold.wait();
15007 assert!(swap
15010 .registry
15011 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15012 .unwrap()
15013 .is_some());
15014 assert!(swap
15015 .registry
15016 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15017 .unwrap()
15018 .is_none());
15019 assert!(swap
15020 .forwarding
15021 .module_endpoint_for_connection(CANDIDATE)
15022 .unwrap()
15023 .is_none());
15024
15025 hold.wait();
15026 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15027 assert_eq!(
15028 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15029 FrameType::HelloAck
15030 );
15031 let published = swap
15032 .registry
15033 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15034 .unwrap()
15035 .expect("candidate is published once its HELLO completes");
15036 assert_eq!(published.connection_id, CANDIDATE);
15037 assert!(swap
15038 .forwarding
15039 .module_endpoint_for_connection(CANDIDATE)
15040 .unwrap()
15041 .is_some());
15042 }
15043
15044 fn keyed_total(counters: &Value, key: &str) -> u64 {
15045 counters[key]
15046 .as_object()
15047 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15048 .unwrap_or(0)
15049 }
15050
15051 #[tokio::test]
15052 async fn replacement_logs_old_end_and_new_admission_once() {
15053 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15054 let mut swap = swap_with_incumbent().await;
15055 swap.handler
15056 .supervisor
15057 .open_swap("aft", "candidate-nonce".to_string());
15058 hello_via_sink(
15059 &swap.handler,
15060 &swap.candidate_ctx,
15061 &mut swap.candidate_rx,
15062 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15063 )
15064 .await;
15065 cutover(&swap);
15066 swap.handler.cleanup_connection(INCUMBENT).unwrap();
15067
15068 let captured = crate::router::test_log::captured_logs(&logs);
15069 println!("captured replacement registry logs:\n{captured}");
15070 let new_admissions: Vec<_> = captured
15071 .lines()
15072 .filter(|line| {
15073 line.contains("connection_id=40")
15074 && line.contains("swap candidate registered; not routable until cutover")
15075 })
15076 .collect();
15077 assert_eq!(
15078 new_admissions.len(),
15079 1,
15080 "unexpected admission logs: {captured}"
15081 );
15082 assert!(new_admissions[0].contains("module_id=aft"));
15083 assert!(new_admissions[0].contains("connection_id=40"));
15084 assert!(
15085 new_admissions[0].contains("swap candidate registered; not routable until cutover")
15086 );
15087 assert!(new_admissions[0].contains("ready=true"));
15088
15089 let old_admissions: Vec<_> = captured
15090 .lines()
15091 .filter(|line| {
15092 line.contains("module registered module_id=aft ")
15093 && line.contains("connection_id=30")
15094 })
15095 .collect();
15096 assert_eq!(
15097 old_admissions.len(),
15098 1,
15099 "unexpected incumbent admission: {captured}"
15100 );
15101
15102 let old_ends: Vec<_> = captured
15103 .lines()
15104 .filter(|line| {
15105 line.contains("connection_id=30") && line.contains("module registration ended")
15106 })
15107 .collect();
15108 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15109 assert!(old_ends[0].contains("module_id=aft"));
15110 assert!(old_ends[0].contains("connection_id=30"));
15111 assert!(old_ends[0].contains("reason=replaced"));
15112 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15113 assert_eq!(
15114 captured
15115 .lines()
15116 .filter(|line| line.contains("module registration promoted"))
15117 .count(),
15118 1,
15119 "unexpected promotion logs: {captured}"
15120 );
15121 }
15122
15123 #[tokio::test]
15129 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15130 let mut swap = swap_with_incumbent().await;
15131 let handler = swap.handler.clone();
15132
15133 let cotenant = ConnectionId::new(31);
15135 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15136 let (cotenant_task, cotenant_bind) = relay_route_open(
15137 &handler,
15138 cotenant,
15139 &cotenant_ctx.egress,
15140 &mut swap.incumbent_rx,
15141 100,
15142 "aft",
15143 "swap-cotenant",
15144 )
15145 .await;
15146 handler
15147 .handle_control_frame(
15148 &swap.incumbent_ctx,
15149 route_bind_ack(cotenant_bind.header.corr),
15150 )
15151 .await
15152 .unwrap();
15153 assert!(cotenant_task.await.unwrap().is_empty());
15154 let (cotenant_channel, cotenant_epoch) =
15155 published_route(&cotenant_rx.recv().await.unwrap());
15156
15157 let caller = ConnectionId::new(32);
15159 let (caller_ctx, mut caller_rx) = route_ctx(caller);
15160 let (caller_task, caller_bind) = relay_route_open(
15161 &handler,
15162 caller,
15163 &caller_ctx.egress,
15164 &mut swap.incumbent_rx,
15165 101,
15166 "aft",
15167 "swap-caller",
15168 )
15169 .await;
15170 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15171
15172 register_candidate(&swap, None);
15173 cutover(&swap);
15174
15175 let ack = handler
15177 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15178 .await;
15179 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15180 if module_loop_error.is_some() {
15181 handler.cleanup_connection(INCUMBENT).unwrap();
15184 }
15185
15186 assert!(
15188 cotenant_rx.try_recv().is_err(),
15189 "the co-tenant route on the incumbent was torn down by one late ack: \
15190 {module_loop_error:?}"
15191 );
15192 assert!(matches!(
15193 swap.forwarding
15194 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15195 .unwrap(),
15196 DataRoute::Client(DataRouteState::Bound(_))
15197 ));
15198 assert_eq!(module_loop_error, None);
15199 assert!(swap
15200 .registry
15201 .get_module_by_connection(INCUMBENT)
15202 .unwrap()
15203 .is_some());
15204
15205 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15207 .await
15208 .expect("the incumbent is told to drop the abandoned binding")
15209 .unwrap()
15210 .frame;
15211 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15212 assert_eq!(goodbye.header.channel, abandoned_channel);
15213 assert_eq!(goodbye.header.epoch, abandoned_epoch);
15214 assert!(swap.incumbent_rx.try_recv().is_err());
15215
15216 let response = caller_task.await.unwrap();
15218 assert_eq!(response.len(), 1);
15219 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15220 assert!(caller_rx.try_recv().is_err());
15221
15222 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15226 let counters = handler.counters().snapshot();
15227 assert_eq!(
15228 keyed_total(&counters, "route_open_accepted_by_principal"),
15229 1
15230 );
15231 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15232 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15233 }
15234
15235 #[tokio::test]
15239 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15240 let mut swap = swap_with_incumbent().await;
15241 register_candidate(&swap, None);
15242 let incumbent = cutover(&swap);
15243 swap.forwarding
15244 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15245 .unwrap()
15246 .expect("the incumbent is still registered");
15247
15248 let client = ConnectionId::new(33);
15249 let (client_ctx, mut client_rx) = route_ctx(client);
15250 let route_handler = swap.handler.clone();
15251 let open_ctx = RouteCtx {
15252 connection_id: client,
15253 egress: client_ctx.egress.clone(),
15254 };
15255 let mut route_task = tokio::spawn(async move {
15256 route_handler
15257 .handle_control_frame(
15258 &open_ctx,
15259 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15260 )
15261 .await
15262 .unwrap()
15263 });
15264 let bind = tokio::select! {
15265 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15266 response = &mut route_task => {
15267 let response = response.unwrap();
15268 panic!(
15269 "post-cutover route.open was refused instead of relayed to the candidate: {}",
15270 parse_error(&response[0])["code"]
15271 );
15272 }
15273 };
15274 swap.handler
15275 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15276 .await
15277 .unwrap();
15278 assert!(route_task.await.unwrap().is_empty());
15279 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15280 match swap
15281 .forwarding
15282 .lookup_data_route(client, channel, epoch)
15283 .unwrap()
15284 {
15285 DataRoute::Client(DataRouteState::Bound(route)) => {
15286 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15287 }
15288 other => panic!("expected a bound route on the candidate, got {other:?}"),
15289 }
15290 assert!(swap.incumbent_rx.try_recv().is_err());
15291 }
15292
15293 #[tokio::test]
15298 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15299 let swap = swap_with_incumbent().await;
15300 register_candidate(&swap, Some(false));
15301 let update = Frame::build(
15302 FrameType::Request,
15303 control_flags(),
15304 0,
15305 0,
15306 55,
15307 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15308 provides: manifest("aft", PROTOCOL_VERSION).provides,
15309 capabilities: None,
15310 ready: Some(true),
15311 })
15312 .unwrap(),
15313 )
15314 .unwrap();
15315
15316 let replies = swap
15317 .handler
15318 .handle_control_frame(&swap.candidate_ctx, update)
15319 .await
15320 .unwrap();
15321
15322 assert_eq!(replies.len(), 1);
15323 assert_eq!(
15324 replies[0].header.ty,
15325 FrameType::Response,
15326 "candidate catalog.update was refused: {:?}",
15327 serde_json::from_slice::<Value>(&replies[0].body).ok()
15328 );
15329 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15330 assert_eq!(
15331 swap.registry
15332 .get_module("aft")
15333 .unwrap()
15334 .unwrap()
15335 .connection_id,
15336 INCUMBENT
15337 );
15338 }
15339 }
15340
15341 mod swap_admission {
15345 use super::*;
15346
15347 const INCUMBENT_NONCE: &str = "incumbent-nonce";
15348 const CANDIDATE_NONCE: &str = "candidate-nonce";
15349
15350 fn handler_with_incumbent(
15351 module_id: &str,
15352 reserved: bool,
15353 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15354 let registry = Arc::new(Registry::default());
15355 let supervisor = SupervisorHandle::new();
15356 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15357 if reserved {
15358 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15359 }
15360 let handler =
15361 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
15362 let incumbent = handler
15363 .handle_control(
15364 ConnectionId::new(1),
15365 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15366 )
15367 .unwrap();
15368 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15369 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15370 (registry, supervisor, handler)
15371 }
15372
15373 #[test]
15379 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15380 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15381
15382 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15383 let replies = handler
15384 .handle_control(
15385 ConnectionId::new(connection),
15386 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15387 )
15388 .unwrap();
15389 assert_eq!(replies[0].header.ty, FrameType::Error);
15390 assert_eq!(
15391 parse_error(&replies[0])["code"],
15392 "swap_token_invalid",
15393 "nonce {nonce:?}"
15394 );
15395 }
15396 assert!(registry.get_candidate("aft").unwrap().is_none());
15397 assert_eq!(
15398 registry.get_module("aft").unwrap().unwrap().connection_id,
15399 ConnectionId::new(1)
15400 );
15401
15402 let admitted = handler
15405 .handle_control(
15406 ConnectionId::new(4),
15407 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15408 )
15409 .unwrap();
15410 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15411 assert_eq!(
15412 registry
15413 .get_candidate("aft")
15414 .unwrap()
15415 .unwrap()
15416 .connection_id,
15417 ConnectionId::new(4)
15418 );
15419 assert_eq!(
15420 registry.get_module("aft").unwrap().unwrap().connection_id,
15421 ConnectionId::new(1),
15422 "the candidate must not take the active slot"
15423 );
15424 let replayed = handler
15425 .handle_control(
15426 ConnectionId::new(5),
15427 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15428 )
15429 .unwrap();
15430 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15431
15432 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15436 let squatter = handler
15437 .handle_control(
15438 ConnectionId::new(6),
15439 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15440 )
15441 .unwrap();
15442 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15443 assert!(
15444 registry.get_module("aft").unwrap().is_none(),
15445 "a squatter took the active slot of an id being swapped"
15446 );
15447 }
15448
15449 #[test]
15454 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15455 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15456
15457 let replies = handler
15458 .handle_control(
15459 ConnectionId::new(2),
15460 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15461 )
15462 .unwrap();
15463
15464 assert_eq!(
15465 replies[0].header.ty,
15466 FrameType::HelloAck,
15467 "reserved candidate refused: {:?}",
15468 serde_json::from_slice::<Value>(&replies[0].body).ok()
15469 );
15470 assert_eq!(
15471 registry
15472 .get_candidate("vault")
15473 .unwrap()
15474 .unwrap()
15475 .connection_id,
15476 ConnectionId::new(2)
15477 );
15478 }
15479
15480 #[test]
15483 fn without_an_open_swap_the_ordinary_gates_decide() {
15484 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15485 supervisor.close_swap("vault");
15486
15487 let candidate = handler
15488 .handle_control(
15489 ConnectionId::new(2),
15490 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15491 )
15492 .unwrap();
15493 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15494 let duplicate = handler
15495 .handle_control(
15496 ConnectionId::new(3),
15497 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15498 )
15499 .unwrap();
15500 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15501 assert!(registry.get_candidate("vault").unwrap().is_none());
15502 }
15503 }
15504
15505 mod scopes {
15509 mod idle_tests {
15510 use super::*;
15511 include!("scope_idle_tests.rs");
15512 }
15513 mod route_open_timings {
15514 use super::*;
15515 include!("route_open_timing_tests.rs");
15516 }
15517 mod activity {
15518 include!("activity_scope_tests.rs");
15519 }
15520 mod delta {
15521 include!("scope_handler_delta_tests.rs");
15522 }
15523 mod call_modes {
15524 include!("scope_call_mode_tests.rs");
15525 }
15526 #[tokio::test]
15527 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15528 let rig = rig().await;
15529 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15530 "s".into(),
15531 1,
15532 "tool".into(),
15533 ))
15534 .unwrap();
15535 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
15536 let (direct, _rx) = wide_ctx(999);
15537 for (ctx, expected) in [
15538 (&rig.owner, FrameType::Response),
15539 (&direct, FrameType::Error),
15540 ] {
15541 let frame =
15542 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
15543 .unwrap();
15544 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
15545 assert_eq!(replies[0].header.ty, expected);
15546 if expected == FrameType::Response {
15547 let reply: subc_control::ScopeServedByReply =
15548 serde_json::from_slice(&replies[0].body).unwrap();
15549 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
15550 } else {
15551 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
15552 }
15553 }
15554 let reply = call(
15555 &rig.handler,
15556 &rig.owner,
15557 &ModuleControlRequestFromModule::ScopeDescribe {
15558 owner: Principal::Reserved {
15559 module_id: OWNER.into(),
15560 },
15561 scope_ref: "s".into(),
15562 },
15563 )
15564 .await;
15565 assert!(matches!(
15566 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
15567 ModuleControlResponseToModule::ScopeDescribe { .. }
15568 ));
15569 }
15570
15571 use subc_protocol::scope::{
15572 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
15573 ScopeStatus,
15574 };
15575
15576 use super::*;
15577
15578 const OWNER: &str = "prefrontal-core";
15579
15580 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
15581 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
15582 }
15583
15584 async fn call(
15585 handler: &ControlHandler,
15586 ctx: &RouteCtx,
15587 request: &ModuleControlRequestFromModule,
15588 ) -> Frame {
15589 let body = serde_json::to_vec(request).unwrap();
15590 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
15591 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
15592 assert_eq!(replies.len(), 1, "{replies:?}");
15593 replies.pop().unwrap()
15594 }
15595
15596 async fn sync(
15597 handler: &ControlHandler,
15598 ctx: &RouteCtx,
15599 generation: u64,
15600 scopes: Vec<ScopeRecord>,
15601 ) -> Result<ModuleControlResponseToModule, String> {
15602 let reply = call(
15603 handler,
15604 ctx,
15605 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
15606 )
15607 .await;
15608 match reply.header.ty {
15609 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
15610 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
15611 }
15612 }
15613
15614 async fn describe(
15615 handler: &ControlHandler,
15616 ctx: &RouteCtx,
15617 owner: &str,
15618 scope_ref: &str,
15619 ) -> ModuleControlResponseToModule {
15620 let reply = call(
15621 handler,
15622 ctx,
15623 &ModuleControlRequestFromModule::ScopeDescribe {
15624 owner: Principal::Reserved {
15625 module_id: owner.to_string(),
15626 },
15627 scope_ref: scope_ref.to_string(),
15628 },
15629 )
15630 .await;
15631 assert_eq!(
15632 reply.header.ty,
15633 FrameType::Response,
15634 "{:?}",
15635 parse_error(&reply)
15636 );
15637 serde_json::from_slice(&reply.body).unwrap()
15638 }
15639
15640 #[tokio::test]
15641 async fn scope_describe_never_echoes_bind_proof_or_hash() {
15642 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15643 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15644 let mut rig = rig().await;
15645 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15646 .await;
15647 let _bound = rig
15648 .bound(
15649 Some(AFT),
15650 PLEXUS,
15651 Some(
15652 rig.selector("s", Some(1))
15653 .with_bind_proof(Some(PROOF.into())),
15654 ),
15655 )
15656 .await;
15657 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
15658 let output = serde_json::to_string(&reply).unwrap();
15659 assert!(output.contains("\"status\":\"live\""));
15660 assert!(!output.contains(PROOF));
15661 assert!(!output.contains(HASH));
15662 }
15663
15664 async fn module(
15666 handler: &ControlHandler,
15667 connection: u64,
15668 module_id: &str,
15669 nonce: Option<&str>,
15670 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15671 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
15672 hello_via_sink(
15673 handler,
15674 &ctx,
15675 &mut rx,
15676 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
15677 )
15678 .await;
15679 (ctx, rx)
15680 }
15681
15682 #[tokio::test]
15685 async fn a_client_connection_cannot_sync_or_describe() {
15686 let handler = ControlHandler::new(Arc::new(Registry::default()));
15687 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
15688 for request in [
15689 ModuleControlRequestFromModule::ScopeSync {
15690 generation: 1,
15691 scopes: vec![head("s", 1)],
15692 },
15693 ModuleControlRequestFromModule::ScopeDescribe {
15694 owner: Principal::Direct,
15695 scope_ref: "s".to_string(),
15696 },
15697 ] {
15698 let reply = call(&handler, &ctx, &request).await;
15699 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
15700 }
15701 assert!(
15702 !handler
15703 .scopes
15704 .read()
15705 .unwrap()
15706 .describe(
15707 &Principal::Reserved {
15708 module_id: OWNER.to_string()
15709 },
15710 "s"
15711 )
15712 .owner_synced
15713 );
15714 }
15715
15716 #[tokio::test]
15719 async fn a_module_without_a_supervised_launch_cannot_sync() {
15720 let handler = ControlHandler::new(Arc::new(Registry::default()));
15721 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
15722 assert_eq!(
15723 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
15724 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15725 );
15726 }
15727
15728 #[tokio::test]
15729 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
15730 let supervisor = SupervisorHandle::new();
15731 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15732 let handler = ControlHandler::new(Arc::new(Registry::default()))
15733 .with_supervisor(supervisor.clone());
15734 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15735 sync(&handler, &incumbent, 1, vec![head("s", 1)])
15736 .await
15737 .expect("the current launch syncs");
15738
15739 supervisor.open_swap(OWNER, "n2".to_string());
15742 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
15743 assert_eq!(
15744 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15745 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15746 );
15747 sync(&handler, &incumbent, 2, vec![head("s", 1)])
15748 .await
15749 .expect("the serving owner syncs during the swap");
15750
15751 supervisor.close_swap(OWNER);
15754 assert_eq!(
15755 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15756 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15757 );
15758 sync(&handler, &incumbent, 3, vec![head("s", 1)])
15759 .await
15760 .expect("the serving owner syncs after the rollback");
15761 handler.cleanup_connection(candidate.connection_id).unwrap();
15762
15763 supervisor.open_swap(OWNER, "n3".to_string());
15768 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
15769 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
15770 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
15771 .await
15772 .expect("the promoted launch takes authority");
15773 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15774 panic!("unexpected reply {reply:?}");
15775 };
15776 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
15777 assert_eq!(
15778 sync(&handler, &incumbent, 4, Vec::new()).await,
15779 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15780 );
15781 }
15782
15783 #[tokio::test]
15786 async fn closing_the_authority_connection_frees_sync_authority() {
15787 let supervisor = SupervisorHandle::new();
15788 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15789 let handler = ControlHandler::new(Arc::new(Registry::default()))
15790 .with_supervisor(supervisor.clone());
15791 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15792 sync(&handler, &first, 10, vec![head("s", 1)])
15793 .await
15794 .unwrap();
15795 handler.cleanup_connection(first.connection_id).unwrap();
15796
15797 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
15798 sync(&handler, &second, 1, vec![head("s", 1)])
15799 .await
15800 .expect("the next connection takes the released authority");
15801 }
15802
15803 #[tokio::test]
15804 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
15805 let supervisor = SupervisorHandle::new();
15806 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15807 let handler =
15808 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
15809 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
15810 sync(&handler, &first, 10, vec![head("s", 1)])
15811 .await
15812 .unwrap();
15813 handler
15814 .handle_control_frame(
15815 &first,
15816 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
15817 )
15818 .await
15819 .unwrap();
15820 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
15821 sync(&handler, &second, 1, vec![head("s", 1)])
15822 .await
15823 .expect("GOODBYE releases authority even if the old socket remains open");
15824 }
15825
15826 #[tokio::test]
15827 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
15828 let registry = Arc::new(Registry::default());
15829 let supervisor_handle = SupervisorHandle::new();
15830 let supervisor =
15831 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15832 .with_handle(supervisor_handle.clone())
15833 .with_daemon_incarnation("incarnation-7".to_string());
15834 supervisor
15837 .supervise_configured(
15838 ModuleSpec {
15839 module_id: OWNER.to_string(),
15840 program: PathBuf::from("/nonexistent/prefrontal-core"),
15841 args: Vec::new(),
15842 env: Vec::new(),
15843 reserved: false,
15844 reserved_prefixes: Vec::new(),
15845 protocol: ModuleProtocol::Subc,
15846 overlap: Default::default(),
15847 },
15848 false,
15849 )
15850 .unwrap();
15851 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
15852 let handler =
15853 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15854 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
15855
15856 let ModuleControlResponseToModule::ScopeDescribe {
15858 status,
15859 daemon_incarnation,
15860 owner_synced,
15861 owner_configured,
15862 scope,
15863 ..
15864 } = describe(&handler, &reader, OWNER, "s").await
15865 else {
15866 panic!("not a describe reply");
15867 };
15868 assert_eq!(status, ScopeStatus::NotLive);
15869 assert_eq!(daemon_incarnation, "incarnation-7");
15870 assert!(!owner_synced);
15871 assert!(owner_configured);
15872 assert!(scope.is_none());
15873
15874 let ModuleControlResponseToModule::ScopeDescribe {
15877 status,
15878 owner_configured,
15879 ..
15880 } = describe(&handler, &reader, "ghost", "s").await
15881 else {
15882 panic!("not a describe reply");
15883 };
15884 assert_eq!(status, ScopeStatus::NotLive);
15885 assert!(!owner_configured);
15886
15887 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15889 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15890 let ModuleControlResponseToModule::ScopeDescribe {
15891 status,
15892 scope_epoch,
15893 owner_synced,
15894 scope,
15895 ..
15896 } = describe(&handler, &reader, OWNER, "s").await
15897 else {
15898 panic!("not a describe reply");
15899 };
15900 assert_eq!(status, ScopeStatus::Live);
15901 assert_eq!(scope_epoch, Some(4));
15902 assert!(owner_synced);
15903 let stamp = scope.expect("a live scope carries its stamp");
15904 assert!(
15905 stamp.owner_authorized,
15906 "prefrontal-core is the default authority"
15907 );
15908 assert_eq!(stamp.kind, ScopeKind::Head);
15909 }
15910
15911 #[tokio::test]
15912 async fn scope_authority_owners_decides_owner_authorized() {
15913 let supervisor = SupervisorHandle::new();
15914 supervisor.set_spawn_nonce("broca", "b1".to_string());
15915 let handler = ControlHandler::new(Arc::new(Registry::default()))
15916 .with_supervisor(supervisor)
15917 .with_scope_authority_owners(vec!["broca".to_string()]);
15918 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15919 let mut gated = head("s", 1);
15920 gated.attributes.agent_id = Some("agent".to_string());
15921 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15922 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15923 describe(&handler, &broca, "broca", "s").await
15924 else {
15925 panic!("not a describe reply");
15926 };
15927 assert!(scope.unwrap().owner_authorized);
15928 }
15929
15930 #[tokio::test]
15934 async fn scope_ops_and_the_scopes_capability_are_advertised() {
15935 let handler = ControlHandler::new(Arc::new(Registry::default()));
15936 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15937 let ack = hello_via_sink(
15938 &handler,
15939 &ctx,
15940 &mut rx,
15941 hello_frame("m", PROTOCOL_VERSION, 1),
15942 )
15943 .await;
15944 let ack = parse_ack(&ack);
15945 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15946 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15947 }
15948 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
15949
15950 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
15951 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15952 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15953 let reply = handler
15954 .handle_control_frame(&client, frame)
15955 .await
15956 .unwrap()
15957 .pop()
15958 .unwrap();
15959 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15960 serde_json::from_slice(&reply.body).unwrap()
15961 else {
15962 panic!("not a server.describe reply");
15963 };
15964 assert!(
15965 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
15966 "{capabilities:?}"
15967 );
15968 }
15969
15970 const PLEXUS: &str = "plexus";
15973 const OTHER: &str = "other";
15974 const AFT: &str = "aft";
15975 const BROCA: &str = "broca";
15976 const MAGIC: &str = "magic-context";
15977
15978 fn nonce(module_id: &str) -> String {
15979 format!("nonce-{module_id}")
15980 }
15981
15982 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15983 let (tx, rx) = mpsc::channel(64);
15984 (
15985 RouteCtx {
15986 connection_id: ConnectionId::new(connection),
15987 egress: FrameSink::new(tx),
15988 },
15989 rx,
15990 )
15991 }
15992
15993 struct Rig {
15997 handler: ControlHandler,
15998 forwarding: Arc<ForwardingTable>,
15999 owner: RouteCtx,
16000 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16001 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16002 generation: u64,
16003 next_connection: u64,
16004 _supervisor: Supervisor,
16005 }
16006
16007 async fn rig() -> Rig {
16008 rig_with_flow_support(true).await
16009 }
16010
16011 async fn rig_with_flow_support(flow_support: bool) -> Rig {
16012 rig_with_scope_capabilities(flow_support, false).await
16013 }
16014
16015 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16016 let mut provides = Vec::new();
16017 if flow_support {
16018 provides.push("flow-scopes/v1");
16019 }
16020 if run_support {
16021 provides.push("agent-run-scopes/v1");
16022 }
16023 rig_with_target_capabilities(&provides).await
16024 }
16025
16026 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16027 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16028 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16029 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16032 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16033 Frame::build(
16034 FrameType::Hello,
16035 control_flags(),
16036 0,
16037 0,
16038 connection,
16039 serde_json::to_vec(&body).unwrap(),
16040 )
16041 .unwrap()
16042 }
16043
16044 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16045 let registry = Arc::new(Registry::default());
16046 let forwarding = Arc::new(ForwardingTable::default());
16047 let supervisor_handle = SupervisorHandle::new();
16048 let supervisor =
16049 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16050 .with_handle(supervisor_handle.clone());
16051 supervisor
16052 .supervise_configured(
16053 ModuleSpec {
16054 module_id: OWNER.to_string(),
16055 program: PathBuf::from("/nonexistent/prefrontal-core"),
16056 args: Vec::new(),
16057 env: Vec::new(),
16058 reserved: false,
16059 reserved_prefixes: Vec::new(),
16060 protocol: ModuleProtocol::Subc,
16061 overlap: Default::default(),
16062 },
16063 false,
16064 )
16065 .unwrap();
16066 for module_id in [OWNER, AFT, BROCA, MAGIC] {
16067 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16068 }
16069 let handler =
16070 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16071 .with_supervisor(supervisor_handle);
16072 let (owner, mut owner_rx) = wide_ctx(1);
16073 hello_via_sink(
16074 &handler,
16075 &owner,
16076 &mut owner_rx,
16077 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16078 )
16079 .await;
16080 let mut modules = BTreeMap::new();
16081 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16082 let (ctx, mut rx) = wide_ctx(connection);
16083 let hello = hello_with_capabilities(module_id, connection, provides);
16084 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16085 modules.insert(module_id.to_string(), (ctx, rx));
16086 }
16087 Rig {
16088 handler,
16089 forwarding,
16090 owner,
16091 _owner_rx: owner_rx,
16092 modules,
16093 generation: 0,
16094 next_connection: 100,
16095 _supervisor: supervisor,
16096 }
16097 }
16098
16099 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16100 ScopeCarrier::new(Principal::Reserved {
16101 module_id: module_id.to_string(),
16102 })
16103 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16104 }
16105
16106 fn session(scope_epoch: u64) -> ScopeRecord {
16109 let mut record = head("s", scope_epoch);
16110 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16111 record.attributes.agent_id = Some("agent-1".to_string());
16112 record.attributes.delegates = true;
16113 record
16114 }
16115
16116 impl Rig {
16117 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16118 self.generation += 1;
16119 sync(&self.handler, &self.owner, self.generation, scopes)
16120 .await
16121 .expect("the owner's sync is accepted");
16122 }
16123
16124 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16125 ScopeSelector::new(
16126 Principal::Reserved {
16127 module_id: OWNER.to_string(),
16128 },
16129 scope_ref.to_string(),
16130 scope_epoch,
16131 )
16132 }
16133
16134 fn open_frame(
16135 &mut self,
16136 opener: Option<&str>,
16137 target: &str,
16138 scope: Option<ScopeSelector>,
16139 ) -> (
16140 RouteCtx,
16141 mpsc::Receiver<crate::router::OutboundFrame>,
16142 Frame,
16143 ) {
16144 self.next_connection += 1;
16145 let (ctx, rx) = wide_ctx(self.next_connection);
16146 let root = unique_project_root("scoped-open");
16147 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16148 target: RouteTarget::ToolProvider {
16149 module_id: target.to_string(),
16150 },
16151 identity: BindIdentity::new(
16152 root.path().to_path_buf(),
16153 "unit".to_string(),
16154 "session".to_string(),
16155 ),
16156 consumer_identity: opener.map(|module_id| ConsumerIdentity {
16157 module_id: module_id.to_string(),
16158 launch_nonce: nonce(module_id),
16159 }),
16160 consumer_capabilities: None,
16161 role_versions: None,
16162 admission_facts: None,
16163 scope,
16164 })
16165 .unwrap();
16166 let frame = Frame::build(
16167 FrameType::Request,
16168 control_flags(),
16169 0,
16170 0,
16171 self.next_connection,
16172 body,
16173 )
16174 .unwrap();
16175 (ctx, rx, frame)
16176 }
16177
16178 async fn refused(
16180 &mut self,
16181 opener: Option<&str>,
16182 target: &str,
16183 scope: Option<ScopeSelector>,
16184 ) -> String {
16185 self.refusal_body(opener, target, scope).await["code"]
16186 .as_str()
16187 .unwrap()
16188 .to_string()
16189 }
16190
16191 async fn refusal_body(
16192 &mut self,
16193 opener: Option<&str>,
16194 target: &str,
16195 scope: Option<ScopeSelector>,
16196 ) -> Value {
16197 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16198 let replies = tokio::time::timeout(
16199 Duration::from_secs(2),
16200 self.handler.handle_control_frame(&ctx, frame),
16201 )
16202 .await
16203 .expect("the open must be refused before waiting for a bind ack")
16204 .unwrap();
16205 assert_eq!(replies.len(), 1, "{replies:?}");
16206 assert_eq!(replies[0].header.ty, FrameType::Error);
16207 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16208 assert!(
16209 module_rx.try_recv().is_err(),
16210 "a refused open relays nothing"
16211 );
16212 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16213 parse_error(&replies[0])
16214 }
16215
16216 async fn relayed(
16218 &mut self,
16219 opener: Option<&str>,
16220 target: &str,
16221 scope: Option<ScopeSelector>,
16222 ) -> Relayed {
16223 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16224 let handler = self.handler.clone();
16225 let task_ctx = ctx.clone();
16226 let task = tokio::spawn(async move {
16227 handler
16228 .handle_control_frame(&task_ctx, frame)
16229 .await
16230 .unwrap()
16231 });
16232 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16233 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16234 .await
16235 .expect("the target receives the relayed route.bind")
16236 .unwrap()
16237 .frame;
16238 Relayed {
16239 target: target.to_string(),
16240 client: ctx,
16241 client_rx: rx,
16242 task,
16243 bind,
16244 }
16245 }
16246
16247 async fn ack(&self, relayed: &Relayed) {
16248 let (module, _) = &self.modules[&relayed.target];
16249 self.handler
16250 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16251 .await
16252 .unwrap();
16253 }
16254
16255 async fn bound(
16257 &mut self,
16258 opener: Option<&str>,
16259 target: &str,
16260 scope: Option<ScopeSelector>,
16261 ) -> Bound {
16262 let relayed = self.relayed(opener, target, scope).await;
16263 self.ack(&relayed).await;
16264 let Relayed {
16265 target,
16266 client,
16267 mut client_rx,
16268 task,
16269 bind,
16270 } = relayed;
16271 assert!(
16272 task.await.unwrap().is_empty(),
16273 "the open is answered by commit"
16274 );
16275 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16276 Bound {
16277 target,
16278 client,
16279 client_rx,
16280 channel,
16281 epoch,
16282 bind,
16283 }
16284 }
16285
16286 fn live(&self, route: &Bound) -> bool {
16287 matches!(
16288 self.forwarding
16289 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16290 .unwrap(),
16291 DataRoute::Client(DataRouteState::Bound(_))
16292 )
16293 }
16294 }
16295
16296 struct Relayed {
16297 target: String,
16298 client: RouteCtx,
16299 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16300 task: tokio::task::JoinHandle<Vec<Frame>>,
16301 bind: Frame,
16302 }
16303
16304 struct Bound {
16305 target: String,
16306 client: RouteCtx,
16307 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16308 channel: u16,
16309 epoch: u32,
16310 bind: Frame,
16311 }
16312
16313 impl Bound {
16314 fn closed_reason(&mut self) -> RouteCloseReason {
16317 let mut reason = None;
16318 let mut goodbye = false;
16319 while let Ok(outbound) = self.client_rx.try_recv() {
16320 let frame = outbound.frame;
16321 match frame.header.ty {
16322 FrameType::Goodbye => {
16323 assert_eq!(
16324 (frame.header.channel, frame.header.epoch),
16325 (self.channel, self.epoch)
16326 );
16327 goodbye = true;
16328 }
16329 FrameType::Push => {
16330 let ClientControlPush::RouteClosed {
16331 reason: r,
16332 module_id,
16333 ..
16334 } = serde_json::from_slice(&frame.body).unwrap()
16335 else {
16336 panic!("unexpected push");
16337 };
16338 assert_eq!(module_id, self.target);
16339 reason = Some(r);
16340 }
16341 other => panic!("unexpected frame {other:?}"),
16342 }
16343 }
16344 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16345 reason.expect("the client is told why the route closed")
16346 }
16347
16348 fn untouched(&mut self) -> bool {
16349 self.client_rx.try_recv().is_err()
16350 }
16351
16352 fn stamp(&self) -> Option<ScopeStamp> {
16353 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16354 ModuleControlRequest::RouteBind { scope, .. } => scope,
16355 other => panic!("expected a route.bind, got {other:?}"),
16356 }
16357 }
16358 }
16359
16360 #[tokio::test]
16361 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16362 ) {
16363 let mut rig = rig().await;
16364 let mut record = session(1);
16365 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16366 record.child_owners = vec![Principal::Reserved {
16367 module_id: MAGIC.to_string(),
16368 }];
16369 rig.sync(vec![record]).await;
16370 let scope = || Some(rig_selector("s", Some(1)));
16371
16372 rig.bound(Some(OWNER), PLEXUS, scope()).await;
16375 rig.bound(Some(AFT), OTHER, scope()).await;
16376 rig.bound(Some(BROCA), PLEXUS, scope()).await;
16377
16378 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16382 assert_eq!(
16383 rig.refused(opener, target, scope()).await,
16384 error_codes::SCOPE_NOT_CARRIER,
16385 "{opener:?} -> {target}"
16386 );
16387 }
16388 }
16389
16390 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16391 ScopeSelector::new(
16392 Principal::Reserved {
16393 module_id: OWNER.to_string(),
16394 },
16395 scope_ref.to_string(),
16396 scope_epoch,
16397 )
16398 }
16399
16400 #[tokio::test]
16401 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16402 let mut rig = rig().await;
16403 rig.sync(vec![session(1)]).await;
16404 for opener in [OWNER, AFT] {
16405 assert_eq!(
16406 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16407 .await,
16408 error_codes::SCOPE_EPOCH_REQUIRED,
16409 "{opener}"
16410 );
16411 }
16412 }
16413
16414 #[tokio::test]
16415 async fn admission_separates_not_synced_not_live_and_ended() {
16416 let mut rig = rig().await;
16417 let code = rig
16419 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16420 .await;
16421 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16422 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16423
16424 let ghost = ScopeSelector::new(
16426 Principal::Reserved {
16427 module_id: "ghost".to_string(),
16428 },
16429 "s".to_string(),
16430 Some(1),
16431 );
16432 assert_eq!(
16433 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16434 error_codes::SCOPE_NOT_LIVE
16435 );
16436
16437 rig.sync(vec![session(2)]).await;
16438 assert_eq!(
16439 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16440 .await,
16441 error_codes::SCOPE_NOT_LIVE
16442 );
16443 for epoch in [1, 3] {
16444 assert_eq!(
16445 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16446 .await,
16447 error_codes::SCOPE_ENDED,
16448 "epoch {epoch}"
16449 );
16450 }
16451 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16453 .await;
16454 }
16455
16456 #[tokio::test]
16457 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16458 let mut rig = rig().await;
16459 rig.sync(vec![session(1)]).await;
16460 let route = rig
16461 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16462 .await;
16463 let stamp = route.stamp().expect("a scoped bind carries the stamp");
16464 assert_eq!(stamp.scope_ref, "s");
16465 assert_eq!(stamp.scope_epoch, 1);
16466 assert_eq!(stamp.kind, ScopeKind::Head);
16467 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16468 assert!(stamp.attributes.delegates);
16469 assert!(stamp.owner_authorized);
16470 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16471 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16472
16473 let (broca, mut broca_rx) = wide_ctx(50);
16476 hello_via_sink(
16477 &rig.handler,
16478 &broca,
16479 &mut broca_rx,
16480 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16481 )
16482 .await;
16483 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16484 .await
16485 .unwrap();
16486 let own = ScopeSelector::new(
16487 Principal::Reserved {
16488 module_id: BROCA.to_string(),
16489 },
16490 "b".to_string(),
16491 Some(1),
16492 );
16493 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16494 assert!(!route.stamp().unwrap().owner_authorized);
16495 }
16496
16497 #[tokio::test]
16498 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16499 let mut rig = rig().await;
16500 let mut record = head("s", 1);
16501 record.carriers = vec![carrier(AFT, None)];
16502 let flow_id = "Flow:run-7/step_2!~";
16503 record.attributes.flow_id = Some(flow_id.to_string());
16504 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16505 .await
16506 .unwrap();
16507 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16508 panic!("not a sync reply");
16509 };
16510 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16511 let route = rig
16512 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16513 .await;
16514 assert!(rig.live(&route), "the stamped bind committed");
16515 let stamp = route.stamp().expect("a flow scope carries a stamp");
16516 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16517 assert_eq!(stamp.attributes.agent_id, None);
16518 assert!(!stamp.attributes.delegates);
16519 assert!(stamp.owner_authorized);
16520 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16521 assert_eq!(unscoped.stamp(), None);
16522 }
16523
16524 #[tokio::test]
16525 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16526 let mut rig = rig_with_flow_support(false).await;
16527 let mut record = session(1);
16528 record.attributes.flow_id = Some("flow:7".to_string());
16529 rig.sync(vec![record]).await;
16530 for opener in [OWNER, AFT] {
16531 let body = rig
16532 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16533 .await;
16534 assert_eq!(body["code"], "target_flow_unsupported");
16535 let message = body["message"].as_str().unwrap();
16536 for required in [PLEXUS, "flow-scopes/v1"] {
16537 assert!(message.contains(required), "{message}");
16538 }
16539 }
16540 }
16541
16542 #[tokio::test]
16543 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
16544 let mut rig = rig_with_flow_support(true).await;
16545 let mut record = session(1);
16546 record.attributes.flow_id = Some("flow:7".to_string());
16547 rig.sync(vec![record]).await;
16548 for opener in [OWNER, AFT] {
16549 let route = rig
16550 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16551 .await;
16552 assert!(rig.live(&route));
16553 assert_eq!(
16554 route.stamp().unwrap().attributes.flow_id.as_deref(),
16555 Some("flow:7")
16556 );
16557 }
16558 }
16559
16560 #[tokio::test]
16561 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
16562 use std::future::Future;
16563
16564 let mut rig = rig_with_flow_support(true).await;
16565 let mut record = session(1);
16566 record.attributes.flow_id = Some("flow:7".to_string());
16567 rig.sync(vec![record]).await;
16568 let (client, mut client_rx, frame) =
16569 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
16570 for _ in 0..64 {
16573 client.egress.try_send(route_bind_ack(1)).unwrap();
16574 }
16575 let handler = rig.handler.clone();
16576 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16577 std::future::poll_fn(|cx| {
16578 assert!(open.as_mut().poll(cx).is_pending());
16579 std::task::Poll::Ready(())
16580 })
16581 .await;
16582 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16583
16584 let old_connection = rig.modules[PLEXUS].0.connection_id;
16585 rig.handler.cleanup_connection(old_connection).unwrap();
16586 let (replacement, mut replacement_rx) = wide_ctx(200);
16587 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
16588 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16589 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16590 let hello = Frame::build(
16591 FrameType::Hello,
16592 control_flags(),
16593 0,
16594 0,
16595 200,
16596 serde_json::to_vec(&body).unwrap(),
16597 )
16598 .unwrap();
16599 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
16600
16601 client_rx.try_recv().unwrap();
16602 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16603 .await
16604 .expect("the replacement is refused without waiting for a bind ack")
16605 .unwrap();
16606 assert_eq!(replies.len(), 1);
16607 let body = parse_error(&replies[0]);
16608 assert_eq!(body["code"], "target_flow_unsupported");
16609 for required in [PLEXUS, "flow-scopes/v1"] {
16610 assert!(body["message"].as_str().unwrap().contains(required));
16611 }
16612 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
16613 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16614 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16615 assert!(rig
16616 .handler
16617 .registry
16618 .get_module_by_connection(replacement.connection_id)
16619 .unwrap()
16620 .is_some());
16621 }
16622
16623 #[tokio::test]
16624 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
16625 use std::future::Future;
16626 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16627 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16628 const NEXT_HASH: &str =
16629 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16630 let mut rig = rig().await;
16631 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
16632 record.carriers.push(ScopeCarrier::new(Principal::Direct));
16633 rig.sync(vec![record.clone()]).await;
16634 let (client, mut rx, frame) = rig.open_frame(
16635 None,
16636 PLEXUS,
16637 Some(
16638 rig.selector("s", Some(1))
16639 .with_bind_proof(Some(PROOF.into())),
16640 ),
16641 );
16642 for _ in 0..64 {
16645 client.egress.try_send(route_bind_ack(1)).unwrap();
16646 }
16647 let handler = rig.handler.clone();
16648 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16649 std::future::poll_fn(|cx| {
16650 assert!(open.as_mut().poll(cx).is_pending());
16651 std::task::Poll::Ready(())
16652 })
16653 .await;
16654 record.bind_proof_sha256 = Some(NEXT_HASH.into());
16655 rig.sync(vec![record]).await;
16656 rx.try_recv().unwrap();
16657 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16658 .await
16659 .unwrap()
16660 .unwrap();
16661 assert_eq!(
16662 parse_error(&replies[0])["code"],
16663 error_codes::SCOPE_BIND_PROOF_MISMATCH
16664 );
16665 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
16666 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16667 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16668 }
16669
16670 #[tokio::test]
16671 async fn scope_bind_proof_reserved_openers_ignore_proof() {
16672 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16673 let mut rig = rig().await;
16674 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16675 .await;
16676 for opener in [OWNER, AFT] {
16677 for proof in [None, Some("malformed+proof=")] {
16678 let route = rig
16679 .bound(
16680 Some(opener),
16681 PLEXUS,
16682 Some(
16683 rig.selector("s", Some(1))
16684 .with_bind_proof(proof.map(str::to_string)),
16685 ),
16686 )
16687 .await;
16688 assert!(rig.live(&route));
16689 }
16690 }
16691 }
16692
16693 #[tokio::test]
16694 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
16695 use std::future::Future;
16696 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16697 const NEXT_HASH: &str =
16698 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16699 let mut rig = rig().await;
16700 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16701 .await;
16702 let (client, mut rx, frame) =
16703 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
16704 for _ in 0..64 {
16705 client.egress.try_send(route_bind_ack(1)).unwrap();
16706 }
16707 let handler = rig.handler.clone();
16708 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16709 std::future::poll_fn(|cx| {
16710 assert!(open.as_mut().poll(cx).is_pending());
16711 std::task::Poll::Ready(())
16712 })
16713 .await;
16714 rig.sync(vec![
16715 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
16716 ])
16717 .await;
16718 rx.try_recv().unwrap();
16719 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16720 let bind = std::future::poll_fn(|cx| {
16721 assert!(open.as_mut().poll(cx).is_pending());
16722 module_rx.poll_recv(cx)
16723 })
16724 .await
16725 .unwrap()
16726 .frame;
16727 handler
16728 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
16729 .await
16730 .unwrap();
16731 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16732 .await
16733 .unwrap()
16734 .unwrap();
16735 assert_eq!(replies.len(), 1);
16736 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
16739 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16740 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
16741 let route = rig
16742 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
16743 .await;
16744 assert!(rig.live(&route));
16745 }
16746
16747 #[tokio::test]
16748 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
16749 {
16750 let mut rig = rig_with_flow_support(false).await;
16751 rig.sync(vec![session(1)]).await;
16752 let route = rig
16753 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16754 .await;
16755 assert!(rig.live(&route));
16756 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
16757 let mut record = session(1);
16758 record.attributes.flow_id = Some("flow:7".to_string());
16759 rig.sync(vec![record]).await;
16760 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
16761 assert!(rig.live(&unscoped));
16762 assert_eq!(unscoped.stamp(), None);
16763 }
16764
16765 #[tokio::test]
16766 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
16767 let mut rig = rig().await;
16768 let mut record = session(1);
16769 record.attributes.flow_id = Some("flow:7".to_string());
16770 rig.sync(vec![record.clone()]).await;
16771 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16772 let mut owner_route = rig
16773 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16774 .await;
16775 let mut carrier_route = rig
16776 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
16777 .await;
16778 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16779 rig.sync(vec![record.clone()]).await;
16780 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16781 assert!(rig.live(&owner_route) && owner_route.untouched());
16782 assert!(rig.live(&carrier_route) && carrier_route.untouched());
16783
16784 record.attributes.flow_id = Some("flow:8".to_string());
16785 rig.sync(vec![record]).await;
16786 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
16787 let before = before.unwrap();
16788 assert_eq!(after.scope_epoch, before.scope_epoch);
16789 assert!(after.version > before.version);
16790 for route in [&mut owner_route, &mut carrier_route] {
16791 assert!(!rig.live(route));
16792 assert_eq!(
16793 route.closed_reason(),
16794 RouteCloseReason::ScopeDelegationChanged
16795 );
16796 }
16797 assert!(rig.live(&unscoped) && unscoped.untouched());
16798 for target in [PLEXUS, OTHER] {
16801 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
16802 let goodbye = module_rx
16803 .try_recv()
16804 .expect("the provider sees the drain")
16805 .frame;
16806 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16807 assert!(module_rx.try_recv().is_err());
16808 }
16809 let rebound = rig
16810 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16811 .await;
16812 assert_eq!(
16813 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
16814 Some("flow:8")
16815 );
16816 }
16817
16818 #[tokio::test]
16823 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
16824 let mut rig = rig().await;
16825 rig.sync(vec![session(1)]).await;
16826 let mut cotenant = rig
16827 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16828 .await;
16829
16830 let mut changed = session(1);
16831 changed.child_owners.push(Principal::Reserved {
16832 module_id: MAGIC.to_string(),
16833 });
16834 let mut ended = None;
16835 for (code, next) in [
16836 (error_codes::SCOPE_CHANGED, vec![changed]),
16837 (error_codes::SCOPE_ENDED, Vec::new()),
16838 ] {
16839 let relayed = rig
16840 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16841 .await;
16842 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
16843 ended = Some(next.is_empty());
16844 rig.sync(next).await;
16845 rig.ack(&relayed).await;
16846 let replies = relayed.task.await.unwrap();
16847 assert_eq!(replies.len(), 1, "{replies:?}");
16848 assert_eq!(parse_error(&replies[0])["code"], code);
16849 assert_eq!(
16850 subc_protocol::error_codes::is_retryable_route_open(code),
16851 code == error_codes::SCOPE_CHANGED
16852 );
16853 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16854 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16856 let mut goodbyes = Vec::new();
16859 while let Ok(outbound) = plexus_rx.try_recv() {
16860 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
16861 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
16862 }
16863 assert!(
16864 goodbyes.contains(&(bind_channel, bind_epoch)),
16865 "{goodbyes:?}"
16866 );
16867 assert!(rig
16868 .handler
16869 .registry
16870 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
16871 .unwrap()
16872 .is_some());
16873 }
16874 assert_eq!(ended, Some(true));
16875 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
16878 }
16879
16880 #[tokio::test]
16883 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
16884 struct Case {
16885 name: &'static str,
16886 change: fn(&mut ScopeRecord),
16887 closed: [Option<RouteCloseReason>; 4],
16890 }
16891 use RouteCloseReason::*;
16892 let cases = [
16893 Case {
16894 name: "a carrier entry removed",
16895 change: |r| {
16896 r.carriers.retain(|c| {
16897 c.principal
16898 != Principal::Reserved {
16899 module_id: AFT.to_string(),
16900 }
16901 })
16902 },
16903 closed: [None, Some(ScopeCarrierRemoved), None, None],
16904 },
16905 Case {
16906 name: "a target removed from a carrier",
16907 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16908 closed: [None, None, None, Some(ScopeCarrierRemoved)],
16909 },
16910 Case {
16911 name: "a bare carrier narrowed to targets",
16912 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16913 closed: [None, Some(ScopeCarrierRemoved), None, None],
16914 },
16915 Case {
16916 name: "delegates turned off",
16917 change: |r| r.attributes.delegates = false,
16918 closed: [Some(ScopeDelegationChanged); 4],
16919 },
16920 Case {
16921 name: "agent_id changed",
16922 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16923 closed: [Some(ScopeDelegationChanged); 4],
16924 },
16925 Case {
16926 name: "a carrier added, child owners changed, the record re-sent",
16927 change: |r| {
16928 r.carriers.push(carrier(MAGIC, None));
16929 r.child_owners.push(Principal::Reserved {
16930 module_id: MAGIC.to_string(),
16931 });
16932 },
16933 closed: [None; 4],
16934 },
16935 Case {
16936 name: "a target added",
16937 change: |r| {
16938 r.carriers[1]
16939 .targets
16940 .as_mut()
16941 .unwrap()
16942 .push("third".to_string())
16943 },
16944 closed: [None; 4],
16945 },
16946 Case {
16947 name: "delegates turned on",
16948 change: |r| r.attributes.delegates = true,
16949 closed: [None; 4],
16950 },
16951 ];
16952 for case in cases {
16953 let mut rig = rig().await;
16954 rig.sync(vec![session(1)]).await;
16955 let scope = || Some(rig_selector("s", Some(1)));
16956 let mut routes = [
16957 rig.bound(Some(OWNER), PLEXUS, scope()).await,
16958 rig.bound(Some(AFT), PLEXUS, scope()).await,
16959 rig.bound(Some(BROCA), PLEXUS, scope()).await,
16960 rig.bound(Some(BROCA), OTHER, scope()).await,
16961 ];
16962 let mut record = session(1);
16963 (case.change)(&mut record);
16964 rig.sync(vec![record]).await;
16965 for (index, expected) in case.closed.iter().enumerate() {
16966 let route = &mut routes[index];
16967 match expected {
16968 Some(reason) => {
16969 assert!(!rig.live(route), "{}: route {index} still live", case.name);
16970 assert_eq!(
16971 route.closed_reason(),
16972 *reason,
16973 "{}: route {index}",
16974 case.name
16975 );
16976 }
16977 None => {
16978 assert!(rig.live(route), "{}: route {index} closed", case.name);
16979 assert!(
16980 route.untouched(),
16981 "{}: route {index} was told something",
16982 case.name
16983 );
16984 }
16985 }
16986 }
16987 }
16988 }
16989
16990 #[tokio::test]
16991 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16992 for next in [Vec::new(), vec![session(2)]] {
16994 let mut rig = rig().await;
16995 rig.sync(vec![session(1)]).await;
16996 let mut route = rig
16997 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16998 .await;
16999 rig.sync(next).await;
17000 assert!(!rig.live(&route));
17001 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17002 }
17003
17004 let mut rig = rig().await;
17007 let mut child = session(1);
17008 child.scope_ref = "child".to_string();
17009 child.kind = ScopeKind::Worker;
17010 child.parent = Some(ScopeParent::new(
17011 Principal::Reserved {
17012 module_id: OWNER.to_string(),
17013 },
17014 "s".to_string(),
17015 1,
17016 ));
17017 rig.sync(vec![session(1), child.clone()]).await;
17018 let mut child_route = rig
17019 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17020 .await;
17021 assert_eq!(
17022 child_route.stamp().unwrap().parent_state,
17023 Some(ParentState::Linked)
17024 );
17025 rig.sync(vec![child]).await;
17026 assert!(!rig.live(&child_route));
17027 assert_eq!(
17028 child_route.closed_reason(),
17029 RouteCloseReason::ScopeParentEnded
17030 );
17031 }
17032
17033 #[tokio::test]
17034 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17035 ) {
17036 let mut rig = rig().await;
17037 rig.sync(vec![session(1)]).await;
17038 let mut route = rig
17039 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17040 .await;
17041 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17042 rig.sync(vec![session(1)]).await;
17043 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17044 assert!(rig.live(&route) && route.untouched());
17045
17046 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17050 .forwarding
17051 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17052 .unwrap()
17053 else {
17054 panic!("the route is bound");
17055 };
17056 binding.flow.acquire_tagged(9, false).await.unwrap();
17057 let mut widened = session(1);
17058 widened.carriers.push(carrier(MAGIC, None));
17059 rig.sync(vec![widened]).await;
17060 assert!(rig.live(&route) && route.untouched());
17061 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17062 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17063 assert_eq!(binding.flow.in_flight(), 1);
17066 binding
17067 .flow
17068 .acquire_tagged(10, false)
17069 .await
17070 .expect("the flow is still open");
17071 }
17072
17073 #[tokio::test]
17076 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17077 let mut rig = rig().await;
17078 rig.sync(vec![session(1)]).await;
17079 let mut on_incumbent = rig
17080 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17081 .await;
17082
17083 let (candidate, _candidate_rx) = wide_ctx(9);
17086 let registration = rig
17087 .handler
17088 .registry
17089 .register_candidate_with_control_ops(
17090 manifest(PLEXUS, PROTOCOL_VERSION),
17091 PROTOCOL_VERSION,
17092 candidate.connection_id,
17093 module_baseline_control_ops(),
17094 )
17095 .unwrap();
17096 rig.forwarding
17097 .register_candidate_module_connection(
17098 candidate.connection_id,
17099 PLEXUS.to_string(),
17100 PROTOCOL_VERSION,
17101 manifest_concurrency(®istration.manifest),
17102 candidate.egress.clone(),
17103 )
17104 .unwrap();
17105 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17106 rig.handler
17107 .registry
17108 .promote_candidate(PLEXUS)
17109 .unwrap()
17110 .unwrap();
17111 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17112
17113 rig.sync(Vec::new()).await;
17114 assert!(!rig.live(&on_incumbent));
17115 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17116 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17117 let goodbye = incumbent_rx
17118 .try_recv()
17119 .expect("the superseded endpoint is told")
17120 .frame;
17121 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17122 }
17123 }
17124}
17125
17126#[cfg(test)]
17127mod concurrency_default_exposure_tests {
17128 use super::*;
17129
17130 fn hello_body(role_json: &str) -> Vec<u8> {
17131 format!(
17132 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17133 )
17134 .into_bytes()
17135 }
17136
17137 fn manifest_from(body: &[u8]) -> ModuleManifest {
17138 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17139 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17140 .expect("manifest parses")
17141 }
17142
17143 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17144
17145 #[test]
17146 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17147 let body = hello_body(&format!(
17148 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17149 ));
17150 let manifest = manifest_from(&body);
17151 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17154 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17155 }
17156
17157 #[test]
17158 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17159 let body = hello_body(&format!(
17160 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17161 ));
17162 let manifest = manifest_from(&body);
17163 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17164 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17165 }
17166
17167 #[test]
17168 fn non_management_roles_are_never_reported() {
17169 let body = hello_body(
17170 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17171 );
17172 let manifest = manifest_from(&body);
17173 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17174 }
17175}