1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
26 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
27 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
28 },
29 scope::{
30 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32 SCOPE_SYNC_OP,
33 },
34 session::{
35 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38 ROLE_VERSIONS_FIELD,
39 },
40 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41 Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47 caller_identity::CallerIdentity,
48 capability_requirements::{
49 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51 RuntimeModule,
52 },
53 child_roster::DaemonShutdownFlag,
54 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55 forwarding::{
56 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59 },
60 observability::{
61 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62 },
63 privileged_prompt::{prompt_text, PrivilegedOperation},
64 provenance::{
65 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66 },
67 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68 router::{RouteCtx, RouterError},
69 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71 stderr_tail::{CaptureState, TailEntry},
72 supervise::{
73 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74 SupervisorHandle, SwapHelloAdmission,
75 },
76 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93 ops::ACTIVITY_RECORD,
94 ops::ACTIVITY_QUERY,
95 ops::ACTIVITY_SUBSCRIBE,
96 ops::SERVER_DESCRIBE,
97 ops::CATALOG_LIST,
98 ops::ROUTE_OPEN,
99 ops::ROUTE_POLL,
100 ops::ROUTE_CLOSING,
101 ops::ROUTE_CLOSED,
102 ops::SUPERVISOR_LIST,
103 ops::SUPERVISOR_RESTART,
104 ops::SUPERVISOR_SWAP,
105 ops::SUPERVISOR_RELOAD,
106 ops::SUPERVISOR_RESCAN,
107 ops::SUPERVISOR_RELEASE_RESERVED,
108 ops::SUPERVISOR_SET_ENABLED,
109 ops::SUPERVISOR_HEALTH_PROBE,
110 ops::SUPERVISOR_HEALTH,
111 ops::SUPERVISOR_STDERR_TAIL,
112 ops::SUPERVISOR_TERMINALS,
113 ops::SUPERVISOR_ROUTES,
114 ops::SUPERVISOR_PROVENANCE,
115 ops::SUPERVISOR_SPAWN_SNAPSHOT,
116 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132 "supervisor.live_roots",
133 SCOPE_SYNC_OP,
134 SCOPE_APPLY_OP,
135 SCOPE_DESCRIBE_OP,
136 "operator.confirm",
137];
138
139const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194 configured: &Path,
195 spawned_from: Option<&Path>,
196 image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198 let path = match spawned_from {
199 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200 ReloadPathAgreement::Match
201 }
202 Some(spawned_from) => ReloadPathAgreement::Mismatch {
203 configured: configured.to_path_buf(),
204 spawned_from: spawned_from.to_path_buf(),
205 },
206 None => ReloadPathAgreement::Unavailable {
207 reason: if matches!(
208 image,
209 subc_control::RunningImageAgreement::Unavailable {
210 reason: subc_control::RunningImageUnavailableReason::NotRunning
211 }
212 ) {
213 ReloadPathUnavailableReason::NotRunning
214 } else {
215 ReloadPathUnavailableReason::SpawnedPathUnavailable
216 },
217 },
218 };
219 PendingReloadVerdict {
220 path,
221 image,
222 spec: None,
223 }
224}
225
226fn launch_spec_agreement(
227 configured: &crate::supervise::ModuleSpec,
228 running: bool,
229 spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232 if !running {
233 return Agreement::Unavailable {
234 reason: Reason::NotRunning,
235 };
236 }
237 match spawned {
238 None => Agreement::Unavailable {
239 reason: Reason::SpawnedSpecUnavailable,
240 },
241 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242 Agreement::Match
243 }
244 Some(_) => Agreement::Mismatch,
245 }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249 if configured == spawned_from {
250 return true;
251 }
252 #[cfg(windows)]
253 {
254 match (
260 std::fs::canonicalize(configured),
261 std::fs::canonicalize(spawned_from),
262 ) {
263 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264 _ => false,
265 }
266 }
267 #[cfg(not(windows))]
268 {
269 false
270 }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275 build: DaemonBuildProvenance,
276 pid: Option<u32>,
277 started_at_ms: Option<u64>,
278 start_clock: Option<crate::clock::StartClock>,
279 executable_path: Option<PathBuf>,
280 executable_identity: Option<SpawnedFileIdentity>,
281 process_start_time: Option<u64>,
282 probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286 fn default() -> Self {
287 Self {
288 build: DaemonBuildProvenance {
289 build_git_sha: None,
290 build_lock_digest: None,
291 },
292 pid: None,
293 started_at_ms: None,
294 start_clock: None,
295 executable_path: None,
296 executable_identity: None,
297 process_start_time: None,
298 probe: ExecutableIdentityProbe::default(),
299 }
300 }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305 supervisor: Supervisor,
306 config_path: PathBuf,
307 configured_port: Option<u16>,
308 storage_config: Option<crate::daemon_config::StorageConfig>,
309 admission_facts_carrier_module_id: Option<String>,
310 admission_facts_targets: Option<Vec<String>>,
311 scope_authority_owners: Vec<String>,
312 privileged_operator_prompt: PrivilegedOperatorPrompt,
313 generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342 "reloading",
343 "supervisor_not_live",
344 "registration_not_active",
345 "no_forwarding_connection",
346 "relay_send_failed",
347];
348
349#[derive(Clone)]
351pub struct ControlHandler {
352 registry: Arc<Registry>,
353 activity: Option<crate::activity::ActivityLedger>,
354 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355 forwarding: Arc<ForwardingTable>,
356 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357 supervisor: SupervisorHandle,
358 subc_capabilities: Arc<[String]>,
359 route_bind_relay_timeout: Duration,
363 route_bind_relay_timeouts: BTreeMap<String, Duration>,
367 route_bind_breakers: RouteBindBreakers,
370 route_bind_concurrency: RouteBindConcurrency,
373 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378 route_bind_breaker_threshold: u32,
380 route_bind_breaker_cooldown: Duration,
382 health_probe_timeout: Duration,
383 storage_config: Option<crate::daemon_config::StorageConfig>,
386 machine_id: Option<crate::machine_id::MachineId>,
390 admission_facts_carrier_module_id: Option<String>,
391 admission_facts_targets: Option<Vec<String>>,
392 scopes: Arc<RwLock<ScopeTable>>,
396 scope_deadline_added: Arc<tokio::sync::Notify>,
400 scope_authority_owners: Vec<String>,
403 privileged_operator_prompt: PrivilegedOperatorPrompt,
405 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408 #[cfg(test)]
409 caller_lookup: Option<TestCallerLookup>,
410 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413 rescan: Option<SupervisorRescanContext>,
414 connected_clients: ConnectedClients,
415 counters: DaemonCounters,
416 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417 daemon_shutdown: DaemonShutdownFlag,
422 daemon_provenance: DaemonProvenanceFacts,
423 #[cfg(test)]
424 control_dispatch_delay: Option<Duration>,
425 #[cfg(test)]
426 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427 #[cfg(test)]
431 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432 #[cfg(test)]
438 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443 f.debug_struct("ControlHandler")
444 .field("registry", &self.registry)
445 .field("forwarding", &self.forwarding)
446 .field("process_liveness", &self.process_liveness.is_some())
447 .field("supervisor", &self.supervisor)
448 .field("subc_capabilities", &self.subc_capabilities)
449 .finish()
450 }
451}
452
453struct RouteOpenRequest {
454 target: RouteTarget,
455 identity: BindIdentity,
456 consumer_identity: Option<ConsumerIdentity>,
457 consumer_capabilities: Option<Vec<String>>,
458 role_versions: Option<BTreeMap<String, String>>,
459 admission_facts: Option<serde_json::Value>,
460 scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465 Admission,
466 ProjectRoot,
467 EgressReservation,
468 BindSend,
469 BindAckWait,
470}
471
472struct RouteOpenTimings {
476 module_id: String,
477 principal: String,
478 admission: Duration,
479 project_root: Duration,
480 egress_reservation: Duration,
481 bind_send: Duration,
482 bind_ack_wait: Duration,
483 phase: Option<RouteOpenPhase>,
484 phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488 fn new() -> Self {
489 Self {
490 module_id: String::new(),
491 principal: "unresolved".to_string(),
493 admission: Duration::ZERO,
494 project_root: Duration::ZERO,
495 egress_reservation: Duration::ZERO,
496 bind_send: Duration::ZERO,
497 bind_ack_wait: Duration::ZERO,
498 phase: Some(RouteOpenPhase::Admission),
499 phase_started_at: Instant::now(),
500 }
501 }
502
503 fn finish(&mut self) {
504 let elapsed = self.phase_started_at.elapsed();
505 match self.phase.take() {
506 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511 None => {}
512 }
513 }
514
515 fn enter(&mut self, phase: RouteOpenPhase) {
516 self.finish();
517 self.phase = Some(phase);
518 self.phase_started_at = Instant::now();
519 }
520}
521
522struct RouteBindReservationGuard {
523 forwarding: Arc<ForwardingTable>,
524 endpoint: ModuleEndpointId,
525 relay_corr: u64,
526 armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530 forwarding: Arc<ForwardingTable>,
531 endpoint: ModuleEndpointId,
532 corr: u64,
533 armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538 Self {
539 forwarding,
540 endpoint,
541 corr,
542 armed: true,
543 }
544 }
545
546 fn disarm(&mut self) {
547 self.armed = false;
548 }
549}
550
551impl Drop for ModuleControlRpcGuard {
552 fn drop(&mut self) {
553 if self.armed {
554 let _ = self
555 .forwarding
556 .cancel_module_control_rpc(self.endpoint, self.corr);
557 }
558 }
559}
560
561impl RouteBindReservationGuard {
562 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563 Self {
564 forwarding,
565 endpoint,
566 relay_corr,
567 armed: true,
568 }
569 }
570
571 fn release_and_disarm(&mut self) {
572 if !self.armed {
573 return;
574 }
575 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576 self.endpoint,
577 self.relay_corr,
578 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579 ) {
580 send_goodbye_target_best_effort(
581 &self.forwarding.counters(),
582 &target,
583 "canceled route.bind",
584 );
585 }
586 self.armed = false;
587 }
588
589 fn disarm(&mut self) {
590 self.armed = false;
591 }
592}
593
594impl Drop for RouteBindReservationGuard {
595 fn drop(&mut self) {
596 self.release_and_disarm();
597 }
598}
599
600#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636 modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640 concurrency: RouteBindConcurrency,
641 module_id: String,
642}
643
644impl RouteBindConcurrency {
645 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648 let mut modules = self
649 .modules
650 .lock()
651 .expect("route.bind concurrency mutex poisoned");
652 let in_flight = modules.entry(module_id.to_string()).or_default();
653 if *in_flight >= limit {
654 return Err(*in_flight);
655 }
656 *in_flight += 1;
657 Ok(RouteBindConcurrencyGuard {
658 concurrency: self.clone(),
659 module_id: module_id.to_string(),
660 })
661 }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665 fn drop(&mut self) {
666 let mut modules = self
667 .concurrency
668 .modules
669 .lock()
670 .expect("route.bind concurrency mutex poisoned");
671 let remove = {
672 let in_flight = modules
673 .get_mut(&self.module_id)
674 .expect("admitted route.bind has a concurrency entry");
675 *in_flight -= 1;
676 *in_flight == 0
677 };
678 if remove {
679 modules.remove(&self.module_id);
680 }
681 }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686 consecutive_timeouts: u32,
688 cooldown_until: Option<Instant>,
691 probe_in_flight: Option<Arc<()>>,
696}
697
698enum RouteBindAdmission<'a> {
700 Admitted {
701 guard: RouteBindBreakerGuard<'a>,
702 probe: bool,
705 },
706 Refused {
707 consecutive_timeouts: u32,
708 retry_in: Duration,
712 probe_in_flight: bool,
713 },
714}
715
716struct RouteBindBreakerGuard<'a> {
723 breakers: RouteBindBreakers,
724 module_id: &'a str,
725 probe_token: Option<Arc<()>>,
726 settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730 fn record_accepted(&mut self) -> bool {
734 self.settled = true;
735 self.breakers.record_accepted(self.module_id)
736 }
737
738 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741 self.settled = true;
742 self.breakers.record_timeout(
743 self.module_id,
744 self.probe_token.as_ref(),
745 threshold,
746 cooldown,
747 )
748 }
749
750 fn record_inconclusive(&mut self) {
759 self.settled = true;
760 self.breakers
761 .record_inconclusive(self.module_id, self.probe_token.as_ref());
762 }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766 fn drop(&mut self) {
767 if !self.settled {
768 self.breakers
769 .record_inconclusive(self.module_id, self.probe_token.as_ref());
770 }
771 }
772}
773
774struct BreakerOpened {
778 consecutive_timeouts: u32,
779 reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786 self.modules
787 .lock()
788 .expect("route.bind breaker mutex poisoned")
789 }
790
791 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795 probe: probe_token.is_some(),
796 guard: RouteBindBreakerGuard {
797 breakers: self.clone(),
798 module_id,
799 probe_token,
800 settled: false,
801 },
802 };
803
804 let mut modules = self.lock();
805 let Some(state) = modules.get_mut(module_id) else {
806 return admitted(None);
807 };
808 let Some(cooldown_until) = state.cooldown_until else {
809 return admitted(None);
810 };
811 if state.probe_in_flight.is_some() {
812 return RouteBindAdmission::Refused {
813 consecutive_timeouts: state.consecutive_timeouts,
814 retry_in: Duration::ZERO,
815 probe_in_flight: true,
816 };
817 }
818 let now = Instant::now();
819 if now < cooldown_until {
820 return RouteBindAdmission::Refused {
821 consecutive_timeouts: state.consecutive_timeouts,
822 retry_in: cooldown_until - now,
823 probe_in_flight: false,
824 };
825 }
826 let token = Arc::new(());
827 state.probe_in_flight = Some(Arc::clone(&token));
828 admitted(Some(token))
829 }
830
831 fn record_accepted(&self, module_id: &str) -> bool {
832 self.lock()
833 .remove(module_id)
834 .is_some_and(|state| state.cooldown_until.is_some())
835 }
836
837 fn record_timeout(
838 &self,
839 module_id: &str,
840 probe_token: Option<&Arc<()>>,
841 threshold: u32,
842 cooldown: Duration,
843 ) -> Option<BreakerOpened> {
844 let mut modules = self.lock();
845 let state = modules.entry(module_id.to_string()).or_default();
846 let was_open = state.cooldown_until.is_some();
847 let was_probe = Self::owns_probe(state, probe_token);
848 if was_probe {
849 state.probe_in_flight = None;
850 }
851 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852 if state.consecutive_timeouts < threshold {
853 return None;
854 }
855 state.cooldown_until = Some(Instant::now() + cooldown);
856 Some(BreakerOpened {
857 consecutive_timeouts: state.consecutive_timeouts,
858 reopened_after_probe: was_open && was_probe,
859 })
860 }
861
862 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863 state
867 .probe_in_flight
868 .as_ref()
869 .zip(token)
870 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871 }
872
873 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874 if let Some(state) = self.lock().get_mut(module_id) {
875 if Self::owns_probe(state, probe_token) {
876 state.probe_in_flight = None;
877 }
878 }
879 }
880
881 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896 self.lock()
897 .remove(module_id)
898 .map(|state| state.consecutive_timeouts)
899 .filter(|discarded| *discarded > 0)
900 }
901
902 fn open_snapshot(&self) -> Option<serde_json::Value> {
909 let now = Instant::now();
910 let modules = self.lock();
911 let open = modules
912 .iter()
913 .filter_map(|(module_id, state)| {
914 let cooldown_until = state.cooldown_until?;
915 Some((
916 module_id.clone(),
917 serde_json::json!({
918 "consecutive_timeouts": state.consecutive_timeouts,
919 "cooldown_remaining_ms":
920 cooldown_until.saturating_duration_since(now).as_millis() as u64,
921 "probe_in_flight": state.probe_in_flight.is_some(),
922 }),
923 ))
924 })
925 .collect::<serde_json::Map<String, serde_json::Value>>();
926 (!open.is_empty()).then_some(serde_json::Value::Object(open))
927 }
928}
929
930impl ControlHandler {
931 pub fn new(registry: Arc<Registry>) -> Self {
932 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933 }
934
935 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936 let counters = forwarding.counters();
937 let route_bind_breakers = forwarding.route_bind_breakers();
941 let route_bind_concurrency = forwarding.route_bind_concurrency();
942 let route_outages = forwarding.route_outages();
943 Self {
944 registry,
945 activity: None,
946 activity_streams: Arc::new(Mutex::new(HashMap::new())),
947 forwarding,
948 process_liveness: None,
949 supervisor: SupervisorHandle::new(),
950 subc_capabilities: Arc::from([
951 CAP_MANIFEST_REGISTRATION.to_string(),
952 CAP_CHANNEL_LIFECYCLE.to_string(),
953 CAP_PING_PONG.to_string(),
954 CAP_SESSION_ATTACH.to_string(),
955 CAP_ADMISSION_FACTS_RELAY.to_string(),
956 CAP_SCOPES_V1.to_string(),
957 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
964 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
965 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
966 ]),
967 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
968 route_bind_relay_timeouts: BTreeMap::new(),
969 route_bind_breakers,
970 route_bind_concurrency,
971 route_outages,
972 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
973 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
974 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
975 storage_config: None,
976 machine_id: None,
977 admission_facts_carrier_module_id: None,
978 admission_facts_targets: None,
979 scopes: Arc::new(RwLock::new(ScopeTable::new(
980 crate::daemon_config::default_scope_authority_owners(),
981 ))),
982 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
983 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
984 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
985 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
986 #[cfg(test)]
987 caller_lookup: None,
988 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
989 rescan: None,
990 connected_clients: ConnectedClients::new(),
991 counters,
992 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
993 daemon_shutdown: DaemonShutdownFlag::default(),
994 daemon_provenance: DaemonProvenanceFacts::default(),
995 #[cfg(test)]
996 control_dispatch_delay: None,
997 #[cfg(test)]
998 provenance_probe_override: None,
999 #[cfg(test)]
1000 route_bind_relay_sent_hold: None,
1001 #[cfg(test)]
1002 hello_registry_to_forwarding_hold: None,
1003 }
1004 }
1005
1006 pub fn with_storage_config(
1009 mut self,
1010 storage_config: Option<crate::daemon_config::StorageConfig>,
1011 ) -> Self {
1012 self.storage_config = storage_config;
1013 self
1014 }
1015
1016 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1019 self.activity = Some(crate::activity::ActivityLedger::open(
1020 run_dir.join("activity.jsonl"),
1021 incarnation,
1022 ));
1023 self.supervisor.set_activity(self.activity.clone());
1024 self
1025 }
1026
1027 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1030 self.machine_id = machine_id;
1031 self
1032 }
1033
1034 pub fn with_admission_facts_config(
1038 mut self,
1039 carrier_module_id: Option<String>,
1040 targets: Option<Vec<String>>,
1041 ) -> Self {
1042 self.admission_facts_carrier_module_id = carrier_module_id;
1043 self.admission_facts_targets = targets;
1044 self
1045 }
1046
1047 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1051 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1052 self.scope_authority_owners = owners;
1053 self
1054 }
1055
1056 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1058 self.privileged_operator_prompt = mode;
1059 self
1060 }
1061
1062 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1065 self.route_bind_relay_timeout = timeout;
1066 self
1067 }
1068
1069 pub fn with_route_bind_relay_timeouts(
1075 mut self,
1076 timeouts: impl IntoIterator<Item = (String, Duration)>,
1077 ) -> Self {
1078 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1079 self
1080 }
1081
1082 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1088 self.route_bind_relay_timeouts
1089 .get(module_id)
1090 .copied()
1091 .unwrap_or(self.route_bind_relay_timeout)
1092 }
1093
1094 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1102 self.route_bind_breaker_threshold = threshold.max(1);
1103 self.route_bind_breaker_cooldown = cooldown;
1104 self
1105 }
1106
1107 #[cfg(test)]
1108 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1109 self.health_probe_timeout = timeout;
1110 self
1111 }
1112
1113 #[cfg(test)]
1114 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1115 self.control_dispatch_delay = Some(delay);
1116 self
1117 }
1118
1119 #[cfg(test)]
1123 pub(crate) fn with_route_bind_relay_sent_hold(
1124 mut self,
1125 hold: Arc<tokio::sync::Barrier>,
1126 ) -> Self {
1127 self.route_bind_relay_sent_hold = Some(hold);
1128 self
1129 }
1130
1131 #[cfg(test)]
1136 pub(crate) fn with_hello_registry_to_forwarding_hold(
1137 mut self,
1138 hold: Arc<std::sync::Barrier>,
1139 ) -> Self {
1140 self.hello_registry_to_forwarding_hold = Some(hold);
1141 self
1142 }
1143
1144 #[cfg(test)]
1145 fn hold_between_registry_and_forwarding(&self) {
1146 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1147 hold.wait();
1148 hold.wait();
1149 }
1150 }
1151
1152 pub fn with_process_liveness(
1153 mut self,
1154 process_liveness: Arc<dyn ModuleProcessLiveness>,
1155 ) -> Self {
1156 self.process_liveness = Some(process_liveness);
1157 self
1158 }
1159
1160 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1161 self.supervisor = supervisor;
1162 self.supervisor.set_activity(self.activity.clone());
1163 self
1164 }
1165
1166 pub fn with_daemon_provenance(
1167 mut self,
1168 pid: u32,
1169 started_at_ms: u64,
1170 executable_path: Option<PathBuf>,
1171 build_git_sha: Option<String>,
1172 build_lock_digest: Option<String>,
1173 ) -> Self {
1174 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1175 let process_start_time = process_start_time(pid);
1176 self.daemon_provenance = DaemonProvenanceFacts {
1177 build: DaemonBuildProvenance {
1178 build_git_sha,
1179 build_lock_digest,
1180 },
1181 pid: Some(pid),
1182 started_at_ms: Some(started_at_ms),
1183 start_clock: None,
1184 executable_path,
1185 executable_identity,
1186 process_start_time,
1187 probe: ExecutableIdentityProbe::default(),
1188 };
1189 self
1190 }
1191
1192 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1193 self.daemon_provenance.start_clock = Some(clock);
1194 self
1195 }
1196
1197 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1199 self.scopes
1200 .write()
1201 .unwrap_or_else(|poisoned| poisoned.into_inner())
1202 .set_wall_clock(Arc::new(clock));
1203 self
1204 }
1205
1206 #[cfg(test)]
1207 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1208 self.provenance_probe_override = Some(result);
1209 self
1210 }
1211
1212 pub fn with_capability_config(
1216 self,
1217 modules: impl IntoIterator<Item = (String, bool)>,
1218 reserved_capabilities: BTreeMap<String, String>,
1219 ) -> Self {
1220 self.capability_evaluator
1221 .configure(modules, reserved_capabilities);
1222 self
1223 }
1224
1225 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1229 self.daemon_shutdown = flag;
1230 self
1231 }
1232
1233 pub fn with_supervisor_rescan(
1234 mut self,
1235 supervisor: Supervisor,
1236 config_path: impl Into<PathBuf>,
1237 configured_port: Option<u16>,
1238 ) -> Self {
1239 self.rescan = Some(SupervisorRescanContext {
1240 supervisor,
1241 config_path: config_path.into(),
1242 configured_port,
1243 storage_config: self.storage_config.clone(),
1244 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1245 admission_facts_targets: self.admission_facts_targets.clone(),
1246 scope_authority_owners: self.scope_authority_owners.clone(),
1247 privileged_operator_prompt: self.privileged_operator_prompt,
1248 generation: Arc::default(),
1249 });
1250 self
1251 }
1252
1253 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1254 self.connected_clients = connected_clients;
1255 self
1256 }
1257
1258 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1259 Arc::clone(&self.forwarding)
1260 }
1261
1262 pub(crate) fn counters(&self) -> DaemonCounters {
1263 self.counters.clone()
1264 }
1265
1266 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1269 tokio::spawn(async move {
1270 loop {
1271 self.capability_evaluator
1272 .wait_for_change_or_deadline()
1273 .await;
1274 self.refresh_capability_requirements();
1275 }
1276 });
1277 }
1278
1279 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1285 tokio::spawn(async move {
1286 loop {
1287 let has_deadlines = self
1288 .scopes
1289 .read()
1290 .unwrap_or_else(|poisoned| poisoned.into_inner())
1291 .has_deadlines();
1292 if !has_deadlines {
1293 self.scope_deadline_added.notified().await;
1294 continue;
1295 }
1296 tokio::time::sleep(Duration::from_secs(1)).await;
1297 if let Err(error) = self.sweep_expired_scopes() {
1298 warn!(%error, "scope expiry sweep failed");
1299 }
1300 }
1301 });
1302 }
1303
1304 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1305 let mut table = self
1306 .scopes
1307 .write()
1308 .unwrap_or_else(|poisoned| poisoned.into_inner());
1309 let swept = table.sweep_expired();
1310 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1311 drop(table);
1312 self.close_scope_drained_routes(drained);
1313 Ok(())
1314 }
1315
1316 fn publish_scope_changes(
1317 &self,
1318 changes: &[crate::scopes::ScopeTagChange],
1319 expired: &[crate::scopes::ScopeExpired],
1320 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1321 if changes.is_empty() {
1322 return Ok(Vec::new());
1323 }
1324 let drained = self
1325 .forwarding
1326 .publish_scope_changes(changes)
1327 .map_err(RouterError::Forwarding)?;
1328 let mut counts = HashMap::new();
1329 for route in &drained {
1330 *counts
1331 .entry((
1332 route.scope.owner.as_str(),
1333 route.scope.scope_ref.as_str(),
1334 route.scope.tag.scope_epoch,
1335 ))
1336 .or_insert(0usize) += 1;
1337 }
1338 for scope in expired {
1339 let routes_closed = counts
1340 .get(&(
1341 scope.owner.as_str(),
1342 scope.scope_ref.as_str(),
1343 scope.scope_epoch,
1344 ))
1345 .copied()
1346 .unwrap_or(0);
1347 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1348 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1349 routes_closed, "scope expired");
1350 }
1351 Ok(drained)
1352 }
1353
1354 fn runtime_capability_snapshot(
1355 &self,
1356 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1357 let runtime = self
1358 .supervisor
1359 .list()
1360 .into_iter()
1361 .map(|module| {
1362 let status = module.status().map_err(|err| {
1363 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1364 })?;
1365 Ok(RuntimeModule {
1366 module_id: status.module_id,
1367 state: status.state,
1368 enabled: status.enabled,
1369 })
1370 })
1371 .collect::<Result<Vec<_>, RouterError>>()?;
1372 let registrations = self.registry.list_registered_modules().map_err(|err| {
1378 RouterError::backend(
1379 0,
1380 0,
1381 format!("failed to list capability registrations: {err}"),
1382 )
1383 })?;
1384 let registrations = registrations
1385 .into_iter()
1386 .map(|registration| RegisteredModule {
1387 module_id: registration.manifest.module_id,
1388 module_version: registration.manifest.module_version,
1389 capabilities: registration.manifest.capabilities,
1390 })
1391 .collect();
1392 Ok((runtime, registrations))
1393 }
1394
1395 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1402 let cached_registration = RegisteredModule {
1403 module_id: registration.manifest.module_id.clone(),
1404 module_version: registration.manifest.module_version.clone(),
1405 capabilities: registration.manifest.capabilities.clone(),
1406 };
1407 if self.capability_evaluator.record_hello(&cached_registration) {
1408 warn!(
1409 module_id = %cached_registration.module_id,
1410 "capability claims drifted from the cached manifest"
1411 );
1412 }
1413 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1414 self.enforce_capability_denies();
1415 }
1416 self.refresh_capability_requirements();
1417 }
1418
1419 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1423 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1424 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1425 self.supervisor.set_swap_promotion_observer(observer);
1426 }
1427
1428 pub fn refresh_capability_requirements(&self) {
1429 match self.runtime_capability_snapshot() {
1430 Ok((runtime, registrations)) => {
1431 log_requirement_events(
1432 self.capability_evaluator
1433 .evaluate_now(&runtime, ®istrations),
1434 &self.daemon_shutdown,
1435 );
1436 }
1437 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1438 }
1439 }
1440
1441 fn enforce_capability_denies(&self) {
1445 let registrations = match self.registry.list_registered_modules() {
1446 Ok(snapshot) => snapshot,
1447 Err(err) => {
1448 warn!(error = %err, "failed to read registrations for capability deny census");
1449 return;
1450 }
1451 };
1452 let manifests = registrations
1453 .into_iter()
1454 .map(|registration| {
1455 (
1456 registration.manifest.module_id.clone(),
1457 registration.manifest,
1458 )
1459 })
1460 .collect::<BTreeMap<_, _>>();
1461 let census = match self.forwarding.route_census(None) {
1462 Ok(census) => census,
1463 Err(err) => {
1464 warn!(error = %err, "failed to read route census for capability deny enforcement");
1465 return;
1466 }
1467 };
1468
1469 for (target_module_id, routes) in census {
1470 let Some(target_manifest) = manifests.get(&target_module_id) else {
1471 continue;
1472 };
1473 let mut closed_routes = Vec::new();
1474 let mut module_goodbyes = Vec::new();
1475 for route in routes {
1476 let Principal::Reserved {
1477 module_id: opening_module_id,
1478 } = &route.principal
1479 else {
1480 continue;
1481 };
1482 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1483 continue;
1484 };
1485 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1486 continue;
1487 };
1488
1489 match self.forwarding.release_client_route(
1490 route.goodbye_target.connection_id,
1491 route.goodbye_target.channel,
1492 route.goodbye_target.epoch,
1493 ) {
1494 Ok(RouteRelease::Removed(module_goodbye)) => {
1495 warn!(
1496 opening_module_id,
1497 target_module_id,
1498 capability,
1499 "force-closing route because an attested capability deny edge now matches"
1500 );
1501 closed_routes.push(route);
1502 module_goodbyes.push(module_goodbye);
1503 }
1504 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1505 Err(err) => warn!(
1506 opening_module_id,
1507 target_module_id,
1508 capability,
1509 error = %err,
1510 "failed to force-close capability-denied route"
1511 ),
1512 }
1513 }
1514
1515 if closed_routes.is_empty() {
1516 continue;
1517 }
1518 send_route_control_pushes(
1519 &self.forwarding,
1520 closed_routes,
1521 ClientControlPush::RouteClosed {
1522 module_id: target_module_id,
1523 channels: Vec::new(),
1524 reason: RouteCloseReason::CapabilityDenied,
1525 drained: false,
1526 abandoned: 0,
1527 excluded_subscriptions: 0,
1528 terminal: Some(false),
1529 },
1530 );
1531 self.emit_route_goodbyes(module_goodbyes);
1532 }
1533 }
1534
1535 fn not_ready_reason(
1541 &self,
1542 registration: &crate::registry::ModuleRegistration,
1543 ) -> Option<NotReadyReason> {
1544 if !registration.ready {
1545 return Some(NotReadyReason {
1546 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1547 capability: None,
1548 });
1549 }
1550 self.first_unprovided_required_capability(registration)
1551 .map(|capability| NotReadyReason {
1552 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1553 capability: Some(capability),
1554 })
1555 }
1556
1557 fn first_unprovided_required_capability(
1575 &self,
1576 registration: &crate::registry::ModuleRegistration,
1577 ) -> Option<String> {
1578 let required = registration
1579 .manifest
1580 .capabilities
1581 .iter()
1582 .flat_map(|declarations| declarations.requires.iter())
1583 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1584 .map(|requirement| requirement.capability.as_str())
1585 .collect::<BTreeSet<_>>();
1586 if required.is_empty() {
1587 return None;
1588 }
1589 let module_id = registration.manifest.module_id.as_str();
1590 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1591 if required
1592 .iter()
1593 .any(|capability| verdict(capability).is_none())
1594 {
1595 self.refresh_capability_requirements();
1596 }
1597 required
1598 .into_iter()
1599 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1600 .map(str::to_string)
1601 }
1602
1603 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1604 self.capability_evaluator
1605 .statuses()
1606 .into_iter()
1607 .map(capability_requirement_status)
1608 .collect()
1609 }
1610
1611 fn deregister_connection(
1618 &self,
1619 connection_id: ConnectionId,
1620 reason: RegistrationEndReason,
1621 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1622 self.registry
1623 .deregister_connection_with_reason(connection_id, reason)
1624 }
1625
1626 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1627 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1628 return None;
1629 }
1630 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1631 parse_client_control_request(&frame.body)
1632 else {
1633 return None;
1634 };
1635 Some(target_module_id(&target).to_string())
1636 }
1637
1638 pub(crate) fn route_open_capacity_refusal(
1639 &self,
1640 ctx: &RouteCtx,
1641 frame: &Frame,
1642 target_module_id: &str,
1643 in_flight: usize,
1644 limit: usize,
1645 ) -> Result<Frame, RouterError> {
1646 let caller = self.route_open_caller_from_frame(frame);
1649 self.route_open_admission_refusal_frame(
1650 ctx,
1651 &caller,
1652 frame,
1653 target_module_id,
1654 "open_admission_full",
1655 (in_flight, limit),
1656 format!(
1657 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1658 ),
1659 )
1660 }
1661
1662 fn route_open_target_capacity_refusal(
1663 &self,
1664 ctx: &RouteCtx,
1665 caller: &RouteOpenCaller,
1666 frame: &Frame,
1667 target_module_id: &str,
1668 in_flight: usize,
1669 ) -> Result<Frame, RouterError> {
1670 self.route_open_admission_refusal_frame(
1671 ctx,
1672 caller,
1673 frame,
1674 target_module_id,
1675 "target_binds_full",
1676 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1677 format!(
1678 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1679 ),
1680 )
1681 }
1682
1683 #[allow(clippy::too_many_arguments)]
1691 fn route_open_admission_refusal_frame(
1692 &self,
1693 ctx: &RouteCtx,
1694 caller: &RouteOpenCaller,
1695 frame: &Frame,
1696 target_module_id: &str,
1697 reason: &'static str,
1698 (in_flight, limit): (usize, usize),
1699 message: impl Into<String>,
1700 ) -> Result<Frame, RouterError> {
1701 let code = error_codes::TARGET_UNAVAILABLE;
1702 self.counters.increment_route_open_refused(code);
1703 info!(
1704 target: "control",
1705 code,
1706 reason,
1707 module_id = ?target_module_id,
1708 connection_id = ctx.connection_id.get(),
1709 principal = %caller.principal,
1710 scope = %caller.scope,
1711 harness = %caller.harness,
1712 in_flight,
1713 limit,
1714 "route.open refused"
1715 );
1716 control_error_frame(frame, code, message.into())
1717 }
1718
1719 #[cfg(test)]
1725 pub fn handle_control(
1726 &self,
1727 connection_id: ConnectionId,
1728 frame: Frame,
1729 ) -> Result<Vec<Frame>, RouterError> {
1730 match frame.header.ty {
1731 FrameType::Ping => Ok(vec![pong(&frame)?]),
1732 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1733 FrameType::Goodbye => self.handle_goodbye(connection_id),
1734 ty => Ok(vec![control_error_frame(
1735 &frame,
1736 "unsupported_control_frame",
1737 format!("unsupported channel-0 frame {ty:?}"),
1738 )?]),
1739 }
1740 }
1741
1742 pub async fn handle_control_frame(
1743 &self,
1744 ctx: &RouteCtx,
1745 frame: Frame,
1746 ) -> Result<Vec<Frame>, RouterError> {
1747 self.handle_control_frame_timed(ctx, frame, None).await
1748 }
1749
1750 pub(crate) fn record_caller_endpoints(
1751 &self,
1752 connection: ConnectionId,
1753 endpoints: Option<(SocketAddr, SocketAddr)>,
1754 ) {
1755 let mut callers = self
1756 .caller_endpoints
1757 .lock()
1758 .unwrap_or_else(|p| p.into_inner());
1759 callers.remove(&connection);
1760 if let Some(endpoints) = endpoints {
1761 callers.insert(connection, endpoints);
1762 }
1763 }
1764
1765 #[cfg(test)]
1766 pub(crate) fn with_caller_lookup_for_test(
1767 mut self,
1768 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1769 ) -> Self {
1770 self.caller_lookup = Some(Arc::new(lookup));
1771 self
1772 }
1773
1774 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1775 let endpoints = self
1776 .caller_endpoints
1777 .lock()
1778 .unwrap_or_else(|p| p.into_inner())
1779 .get(&connection)
1780 .copied();
1781 #[cfg(test)]
1782 let lookup = self.caller_lookup.clone();
1783 tokio::task::spawn_blocking(move || {
1784 #[cfg(test)]
1785 if let Some(lookup) = lookup {
1786 return lookup(endpoints);
1787 }
1788 endpoints
1789 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1790 .unwrap_or_default()
1791 })
1792 .await
1793 .unwrap_or_default()
1794 }
1795
1796 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1797 frame.header.channel == 0
1798 && frame.header.ty == FrameType::Request
1799 && self
1800 .forwarding
1801 .module_endpoint_for_connection(ctx.connection_id)
1802 .is_ok_and(|endpoint| endpoint.is_none())
1803 && parse_client_control_request(&frame.body)
1804 .ok()
1805 .is_some_and(|request| privileged_operation(&request).is_some())
1806 }
1807
1808 pub(crate) async fn handle_control_frame_timed(
1809 &self,
1810 ctx: &RouteCtx,
1811 frame: Frame,
1812 dispatch_started_at: Option<StdInstant>,
1813 ) -> Result<Vec<Frame>, RouterError> {
1814 match frame.header.ty {
1815 FrameType::Ping => Ok(vec![pong(&frame)?]),
1816 FrameType::Hello => {
1817 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1818 }
1819 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1820 FrameType::Cancel => {
1821 if self
1824 .forwarding
1825 .operator_confirms()
1826 .cancel(ctx.connection_id, frame.header.corr)
1827 || self
1828 .supervisor
1829 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1830 || self.cancel_activity_subscription(ctx, &frame)?
1831 {
1832 Ok(Vec::new())
1833 } else {
1834 Ok(vec![control_error_frame(
1835 &frame,
1836 "unknown_subscription",
1837 "no subscription has this correlation id",
1838 )?])
1839 }
1840 }
1841 FrameType::Request => {
1842 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1845 if op
1846 .as_ref()
1847 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1848 {
1849 return self.handle_activity_record(ctx, frame).await;
1850 }
1851 if op.as_ref().is_some_and(|probe| {
1852 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1853 }) {
1854 return self.handle_activity_read(ctx, frame).await;
1855 }
1856 if op
1857 .as_ref()
1858 .is_some_and(|probe| probe.op == "operator.confirm")
1859 {
1860 return self.handle_operator_confirm(ctx, frame);
1861 }
1862 if op
1863 .as_ref()
1864 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1865 {
1866 return self.handle_scope_served_by(ctx.connection_id, frame);
1867 }
1868 if self
1869 .forwarding
1870 .module_endpoint_for_connection(ctx.connection_id)
1871 .map_err(RouterError::Forwarding)?
1872 .is_some()
1873 {
1874 if !is_known_module_request_op(&frame.body) {
1875 return Ok(vec![control_error_frame(
1876 &frame,
1877 "unsupported_control_frame",
1878 "module-originated channel-0 REQUEST is not supported",
1879 )?]);
1880 }
1881 let request = match parse_module_control_request_from_module(&frame.body) {
1882 Ok(request) => request,
1883 Err((err, ControlRequestBodyError::UnknownOp)) => {
1884 return Ok(vec![control_error_frame(
1885 &frame,
1886 "unsupported_control_frame",
1887 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1888 )?])
1889 }
1890 Err((err, ControlRequestBodyError::InvalidBody)) => {
1891 return Ok(vec![control_error_frame(
1892 &frame,
1893 "invalid_control_body",
1894 format!("malformed module control body: {err}"),
1895 )?])
1896 }
1897 };
1898 let op = module_control_request_op(&request);
1899 let corr = frame.header.corr;
1900 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1901 let result =
1902 self.handle_module_control_request(ctx.connection_id, frame, request);
1903 log_slow_control_dispatch(
1904 dispatch_started_at,
1905 op,
1906 ctx.connection_id,
1907 corr,
1908 None,
1909 );
1910 return result;
1911 }
1912
1913 if is_known_module_request_op(&frame.body) {
1914 return Ok(vec![control_error_frame(
1915 &frame,
1916 "not_registered",
1917 "catalog.update requires an active module registration owned by this connection",
1918 )?]);
1919 }
1920
1921 let request = match parse_client_control_request(&frame.body) {
1922 Ok(request) => request,
1923 Err((err, ControlRequestBodyError::UnknownOp)) => {
1924 return Ok(vec![control_error_frame(
1925 &frame,
1926 "unknown_control_op",
1927 format!("unknown client control op: {err}"),
1928 )?])
1929 }
1930 Err((err, ControlRequestBodyError::InvalidBody)) => {
1931 return Ok(vec![control_error_frame(
1932 &frame,
1933 "invalid_control_body",
1934 format!("malformed client control body: {err}"),
1935 )?])
1936 }
1937 };
1938 let op = client_control_request_op(&request);
1939 let corr = frame.header.corr;
1940 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1941 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1942 #[cfg(test)]
1943 if let Some(delay) = self.control_dispatch_delay {
1944 tokio::time::sleep(delay).await;
1945 }
1946 let result = self
1947 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1948 .await;
1949 if let Some(timings) = route_open_timings.as_mut() {
1950 timings.finish();
1951 }
1952 log_slow_control_dispatch(
1953 dispatch_started_at,
1954 op,
1955 ctx.connection_id,
1956 corr,
1957 route_open_timings.as_ref(),
1958 );
1959 result
1960 }
1961 FrameType::Push => {
1962 let Some(endpoint) = self
1963 .forwarding
1964 .module_endpoint_for_connection(ctx.connection_id)
1965 .map_err(RouterError::Forwarding)?
1966 else {
1967 return Ok(vec![control_error_frame(
1968 &frame,
1969 "unsupported_control_frame",
1970 "client-originated channel-0 PUSH is not supported",
1971 )?]);
1972 };
1973 self.handle_status_update(endpoint, frame)
1974 }
1975 FrameType::Response | FrameType::Error
1976 if self
1977 .forwarding
1978 .module_endpoint_for_connection(ctx.connection_id)
1979 .map_err(RouterError::Forwarding)?
1980 .is_some() =>
1981 {
1982 self.handle_module_relay_response(ctx.connection_id, frame)
1983 }
1984 ty => Ok(vec![control_error_frame(
1985 &frame,
1986 "unsupported_control_frame",
1987 format!("unsupported channel-0 frame {ty:?}"),
1988 )?]),
1989 }
1990 }
1991
1992 pub fn cleanup_connection(
1993 &self,
1994 connection_id: ConnectionId,
1995 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1996 self.caller_endpoints
1997 .lock()
1998 .unwrap_or_else(|p| p.into_inner())
1999 .remove(&connection_id);
2000 self.forwarding
2001 .operator_confirms()
2002 .connection_closed(connection_id);
2003 self.cleanup_connection_with_end_reason(
2004 connection_id,
2005 RegistrationEndReason::ConnectionClosed,
2006 )
2007 }
2008
2009 fn cleanup_connection_with_end_reason(
2010 &self,
2011 connection_id: ConnectionId,
2012 requested_reason: RegistrationEndReason,
2013 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2014 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2015 self.registry
2016 .get_module_by_connection(connection_id)?
2017 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2018 .and_then(|module| module.registration_end_reason().ok().flatten())
2019 .unwrap_or(requested_reason)
2020 } else {
2021 requested_reason
2022 };
2023 let crash_closed = self
2024 .registry
2025 .get_module_by_connection(connection_id)?
2026 .and_then(|registration| {
2027 self.forwarding
2028 .module_endpoint_for_connection(connection_id)
2029 .ok()
2030 .flatten()
2031 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2032 .map(|routes| (registration.manifest.module_id, routes))
2033 });
2034 let crash_closed = crash_closed.map(|(module_id, routes)| {
2035 let terminal = match self.supervisor.get(&module_id) {
2036 None => false,
2037 Some(module) => match module.will_recover_after_connection_loss() {
2038 Ok(will_recover) => !will_recover,
2039 Err(err) => {
2040 warn!(
2041 %module_id,
2042 error = %err,
2043 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2044 );
2045 false
2046 }
2047 },
2048 };
2049 let (reason, drained) = match self
2053 .forwarding
2054 .daemon_drain_result_for_connection(connection_id)
2055 {
2056 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2057 Ok(None) => (RouteCloseReason::Crash, false),
2058 Err(err) => {
2059 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2060 (RouteCloseReason::Crash, false)
2061 }
2062 };
2063 (module_id, routes, reason, drained, terminal)
2064 });
2065 let registrations = self.deregister_connection(connection_id, end_reason);
2066 let cleanup = if crash_closed.is_some() {
2067 self.forwarding.cleanup_connection_counted(connection_id)
2068 } else {
2069 self.forwarding
2075 .cleanup_connection(connection_id)
2076 .map(|released| crate::forwarding::ConnectionCleanup {
2077 released,
2078 abandoned_relays: 0,
2079 })
2080 };
2081 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2086 let abandoned = cleanup
2087 .as_ref()
2088 .map_or(0, |cleanup| cleanup.abandoned_relays);
2089 send_route_control_pushes(
2090 &self.forwarding,
2091 routes,
2092 ClientControlPush::RouteClosed {
2093 module_id,
2094 channels: Vec::new(),
2095 reason,
2096 drained,
2097 abandoned,
2098 excluded_subscriptions: 0,
2099 terminal: Some(terminal),
2100 },
2101 );
2102 }
2103 if let Ok(cleanup) = cleanup {
2104 self.emit_route_goodbyes(cleanup.released);
2105 }
2106 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2110 crate::supervise::notify_registration_release();
2111 self.capability_evaluator.wake_deadline_loop();
2112 self.refresh_capability_requirements();
2113 }
2114 self.supervisor.remove_spawn_subscribers(connection_id);
2115 self.activity_streams
2116 .lock()
2117 .unwrap_or_else(|p| p.into_inner())
2118 .retain(|(id, _), task| {
2119 if *id == connection_id {
2120 task.abort();
2121 false
2122 } else {
2123 true
2124 }
2125 });
2126 self.hello_launch_nonces
2129 .lock()
2130 .unwrap_or_else(|poisoned| poisoned.into_inner())
2131 .forget(connection_id);
2132 self.scopes
2133 .write()
2134 .unwrap_or_else(|poisoned| poisoned.into_inner())
2135 .release_connection(connection_id);
2136 registrations
2137 }
2138
2139 pub(crate) fn handle_route_goodbye(
2140 &self,
2141 connection_id: ConnectionId,
2142 route_channel: u16,
2143 route_epoch: u32,
2144 ) -> Result<bool, RouterError> {
2145 debug!(
2146 connection_id = connection_id.get(),
2147 route_channel, route_epoch, "handling route GOODBYE"
2148 );
2149 let RouteRelease::Removed(released_route) = self
2150 .forwarding
2151 .release_client_route(connection_id, route_channel, route_epoch)
2152 .map_err(RouterError::Forwarding)?
2153 else {
2154 return Ok(false);
2155 };
2156 self.emit_route_goodbyes(vec![released_route]);
2157 Ok(true)
2158 }
2159
2160 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2161 for released in released_routes {
2162 let frame = match Frame::build_with_version(
2163 released.negotiated_ver,
2164 FrameType::Goodbye,
2165 control_flags(),
2166 released.channel,
2167 released.epoch,
2168 0,
2169 Vec::new(),
2170 ) {
2171 Ok(frame) => frame,
2172 Err(err) => {
2173 warn!(
2174 route_channel = released.channel,
2175 error = %err,
2176 "failed to build route GOODBYE frame"
2177 );
2178 continue;
2179 }
2180 };
2181 if !released.close_on_delivery_failure() {
2182 crate::forwarding::send_module_route_goodbye(
2183 &self.counters,
2184 &released.sink,
2185 frame,
2186 released.module_id.as_deref(),
2187 "client route released",
2188 );
2189 continue;
2190 }
2191 if let Err(err) = released.sink.try_send(frame) {
2192 warn!(
2193 target_connection_id = released.connection_id.get(),
2194 route_channel = released.channel,
2195 error = %err,
2196 "route GOODBYE was not delivered to client; closing target connection"
2197 );
2198 if self
2199 .forwarding
2200 .escalate_client_delivery_failure(
2201 released.connection_id,
2202 released.channel,
2203 released.epoch,
2204 CloseReason::new(
2205 "route_goodbye_delivery_failed",
2206 format!(
2207 "failed to enqueue route GOODBYE for channel {}: {err}",
2208 released.channel
2209 ),
2210 ),
2211 crate::forwarding::UndeliveredFrame {
2212 module_id: released.module_id.as_deref(),
2213 sink: &released.sink,
2214 },
2215 )
2216 .unwrap_or(false)
2217 {
2218 self.counters.increment_goodbye_relay_client_failed();
2219 }
2220 }
2221 }
2222 }
2223
2224 fn send_abandoned_route_bind_goodbye(
2236 &self,
2237 module_sink: &crate::FrameSink,
2238 negotiated_ver: u8,
2239 module_channel: u16,
2240 module_epoch: u32,
2241 ) {
2242 let frame = match Frame::build_with_version(
2243 negotiated_ver,
2244 FrameType::Goodbye,
2245 control_flags(),
2246 module_channel,
2247 module_epoch,
2248 0,
2249 Vec::new(),
2250 ) {
2251 Ok(frame) => frame,
2252 Err(err) => {
2253 warn!(
2254 route_channel = module_channel,
2255 error = %err,
2256 "failed to build GOODBYE for abandoned route.bind"
2257 );
2258 return;
2259 }
2260 };
2261 crate::forwarding::send_module_route_goodbye(
2262 &self.counters,
2263 module_sink,
2264 frame,
2265 None,
2266 "abandoned route.bind",
2267 );
2268 }
2269
2270 fn handle_hello(
2271 &self,
2272 connection_id: ConnectionId,
2273 sink: Option<crate::FrameSink>,
2274 frame: Frame,
2275 ) -> Result<Vec<Frame>, RouterError> {
2276 debug!(
2277 connection_id = connection_id.get(),
2278 corr = frame.header.corr,
2279 "handling HELLO"
2280 );
2281 if self
2285 .registry
2286 .get_module_by_connection(connection_id)
2287 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2288 .is_some()
2289 {
2290 return Ok(vec![control_error_frame(
2291 &frame,
2292 "invalid_hello",
2293 "connection is already registered as a module",
2294 )?]);
2295 }
2296 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2297 Ok(value) => value,
2298 Err(err) => {
2299 return Ok(vec![control_error_frame(
2300 &frame,
2301 "invalid_hello",
2302 format!("malformed HELLO body: {err}"),
2303 )?])
2304 }
2305 };
2306 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2307 return Ok(vec![control_error_frame(
2308 &frame,
2309 "invalid_capability_grammar",
2310 err.to_string(),
2311 )?]);
2312 }
2313 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2314 return Ok(vec![control_error_frame(
2315 &frame,
2316 "invalid_manifest",
2317 err.to_string(),
2318 )?]);
2319 }
2320 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2321 return Ok(vec![control_error_frame(
2322 &frame,
2323 "invalid_event_declaration",
2324 err.to_string(),
2325 )?]);
2326 }
2327 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2328 return Ok(vec![control_error_frame(
2329 &frame,
2330 "invalid_manifest",
2331 err.to_string(),
2332 )?]);
2333 }
2334 if let Some(provenance) = hello_value
2335 .get("manifest")
2336 .and_then(|manifest| manifest.get("provenance"))
2337 {
2338 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2339 return Ok(vec![control_error_frame(
2340 &frame,
2341 "invalid_manifest",
2342 format!("malformed manifest provenance: {err}"),
2343 )?]);
2344 }
2345 }
2346 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2347 let module_id = hello_value["manifest"]["module_id"]
2348 .as_str()
2349 .unwrap_or("<unknown>");
2350 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2351 }
2352 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2353 Ok(hello) => hello,
2354 Err(err) => {
2355 return Ok(vec![control_error_frame(
2356 &frame,
2357 "invalid_hello",
2358 format!("malformed HELLO body: {err}"),
2359 )?])
2360 }
2361 };
2362
2363 if hello.protocol_ver != hello.manifest.protocol_ver {
2364 return Ok(vec![control_error_frame(
2365 &frame,
2366 "invalid_manifest",
2367 format!(
2368 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2369 hello.protocol_ver, hello.manifest.protocol_ver
2370 ),
2371 )?]);
2372 }
2373
2374 if hello.manifest.module_id.trim().is_empty() {
2375 return Ok(vec![control_error_frame(
2376 &frame,
2377 "invalid_manifest",
2378 "manifest module_id must not be empty",
2379 )?]);
2380 }
2381
2382 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2383 Ok(negotiated_ver) => negotiated_ver,
2384 Err(message) => {
2385 return Ok(vec![control_error_frame(
2386 &frame,
2387 "version_unsupported",
2388 message,
2389 )?])
2390 }
2391 };
2392
2393 let swap_admission = self
2401 .supervisor
2402 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2403 if swap_admission == SwapHelloAdmission::Refused {
2404 warn!(
2405 module_id = %hello.manifest.module_id,
2406 connection_id = connection_id.get(),
2407 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2408 );
2409 return Ok(vec![control_error_frame(
2410 &frame,
2411 "swap_token_invalid",
2412 format!(
2413 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2414 hello.manifest.module_id
2415 ),
2416 )?]);
2417 }
2418 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2419
2420 let nonce_verified = if swap_candidate {
2425 true
2426 } else {
2427 match self.supervisor.reserved_hello_verification(
2428 &hello.manifest.module_id,
2429 hello.launch_nonce.as_deref(),
2430 ) {
2431 Ok(verified) => verified,
2432 Err(rejection) => {
2433 let message = match rejection {
2434 ReservedHelloRejection::Exact { module_id } => format!(
2435 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2436 ),
2437 ReservedHelloRejection::Prefix {
2438 prefix,
2439 owner_module_id,
2440 } => format!(
2441 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2442 hello.manifest.module_id
2443 ),
2444 };
2445 return Ok(vec![control_error_frame(
2446 &frame,
2447 "reserved_module",
2448 message,
2449 )?]);
2450 }
2451 }
2452 };
2453
2454 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2455 &hello.manifest.module_id,
2456 hello.manifest.capabilities.as_ref(),
2457 );
2458 if let Some(refusal) = reserved_capability_refusals.first() {
2459 let capability = refusal.capability.clone();
2460 let bound_module = refusal.claimants[0].clone();
2461 log_duplicate_claim_events(reserved_capability_refusals);
2462 return Ok(vec![control_error_frame(
2463 &frame,
2464 "reserved_capability",
2465 format!(
2466 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2467 capability, bound_module, hello.manifest.module_id
2468 ),
2469 )?]);
2470 }
2471
2472 if self
2475 .forwarding
2476 .connection_has_client_routes(connection_id)
2477 .map_err(RouterError::Forwarding)?
2478 {
2479 return Ok(vec![control_error_frame(
2480 &frame,
2481 "invalid_hello",
2482 "connection has open client routes and cannot also register as a module",
2483 )?]);
2484 }
2485
2486 self.hello_launch_nonces
2491 .lock()
2492 .unwrap_or_else(|poisoned| poisoned.into_inner())
2493 .record(connection_id, hello.launch_nonce.as_deref());
2494 let control_ops = effective_module_control_ops(hello.control_ops);
2495 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2498 if swap_candidate {
2499 return self.register_swap_candidate(
2500 connection_id,
2501 sink,
2502 &frame,
2503 hello.manifest,
2504 negotiated_ver,
2505 control_ops,
2506 hello_ack,
2507 );
2508 }
2509 let registration = match self
2516 .registry
2517 .register_unpublished_verified_with_control_ops(
2518 hello.manifest,
2519 negotiated_ver,
2520 connection_id,
2521 control_ops,
2522 nonce_verified,
2523 ) {
2524 Ok(registration) => registration,
2525 Err(RegistryError::DuplicateModuleId { module_id }) => {
2526 return Ok(vec![control_error_frame(
2527 &frame,
2528 "duplicate_module_id",
2529 format!(
2530 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2531 ),
2532 )?])
2533 }
2534 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2535 return Ok(vec![control_error_frame(
2536 &frame,
2537 "invalid_module_id",
2538 err.to_string(),
2539 )?])
2540 }
2541 Err(err) => {
2542 return Ok(vec![control_error_frame(
2543 &frame,
2544 "registry_error",
2545 err.to_string(),
2546 )?])
2547 }
2548 };
2549 #[cfg(test)]
2550 self.hold_between_registry_and_forwarding();
2551
2552 let reply = if let Some(sink) = sink {
2553 let concurrency = manifest_concurrency(®istration.manifest);
2569 if let Err(err) = self.forwarding.register_module_connection_acked(
2570 connection_id,
2571 registration.manifest.module_id.clone(),
2572 negotiated_ver,
2573 concurrency,
2574 sink,
2575 hello_ack,
2576 ) {
2577 if matches!(
2582 self.deregister_connection(
2583 connection_id,
2584 RegistrationEndReason::RegistrationFailed,
2585 ),
2586 Ok(r) if !r.is_empty()
2587 ) {
2588 crate::supervise::notify_registration_release();
2589 }
2590 return Ok(vec![control_error_frame(
2591 &frame,
2592 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2593 "invalid_hello"
2594 } else {
2595 forwarding_error_code(&err)
2596 },
2597 err.to_string(),
2598 )?]);
2599 }
2600 Vec::new()
2601 } else {
2602 vec![hello_ack]
2605 };
2606
2607 if !self
2612 .registry
2613 .publish_registration(connection_id)
2614 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2615 {
2616 debug!(
2617 module_id = %registration.manifest.module_id,
2618 connection_id = connection_id.get(),
2619 "registration was removed before it could be published"
2620 );
2621 }
2622
2623 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2632 info!(
2633 module_id = %registration.manifest.module_id,
2634 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2635 );
2636 }
2637
2638 self.apply_registration_capabilities(®istration);
2639
2640 self.supervisor
2641 .record_activity_registration(®istration.manifest.module_id);
2642
2643 info!(
2644 module_id = %registration.manifest.module_id,
2645 module_version = %registration.manifest.module_version,
2646 negotiated_ver,
2647 routable_provider = manifest_provides_routable_role(®istration.manifest),
2648 connection_id = connection_id.get(),
2649 "module registered"
2650 );
2651
2652 Ok(reply)
2653 }
2654
2655 #[allow(clippy::too_many_arguments)]
2672 fn register_swap_candidate(
2673 &self,
2674 connection_id: ConnectionId,
2675 sink: Option<crate::FrameSink>,
2676 frame: &Frame,
2677 manifest: ModuleManifest,
2678 negotiated_ver: u8,
2679 control_ops: Vec<String>,
2680 hello_ack: Frame,
2681 ) -> Result<Vec<Frame>, RouterError> {
2682 let module_id = manifest.module_id.clone();
2683 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2684 manifest,
2685 negotiated_ver,
2686 connection_id,
2687 control_ops,
2688 ) {
2689 Ok(registration) => registration,
2690 Err(RegistryError::DuplicateModuleId { module_id }) => {
2691 return Ok(vec![control_error_frame(
2692 frame,
2693 "duplicate_module_id",
2694 format!(
2695 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2696 ),
2697 )?])
2698 }
2699 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2700 return Ok(vec![control_error_frame(
2701 frame,
2702 "invalid_module_id",
2703 err.to_string(),
2704 )?])
2705 }
2706 Err(err) => {
2707 return Ok(vec![control_error_frame(
2708 frame,
2709 "registry_error",
2710 err.to_string(),
2711 )?])
2712 }
2713 };
2714 #[cfg(test)]
2715 self.hold_between_registry_and_forwarding();
2716 let reply = if let Some(sink) = sink {
2717 let concurrency = manifest_concurrency(®istration.manifest);
2721 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2722 connection_id,
2723 module_id.clone(),
2724 negotiated_ver,
2725 concurrency,
2726 sink,
2727 hello_ack,
2728 ) {
2729 if matches!(
2730 self.deregister_connection(
2731 connection_id,
2732 RegistrationEndReason::RegistrationFailed,
2733 ),
2734 Ok(r) if !r.is_empty()
2735 ) {
2736 crate::supervise::notify_registration_release();
2737 }
2738 return Ok(vec![control_error_frame(
2739 frame,
2740 forwarding_error_code(&err),
2741 err.to_string(),
2742 )?]);
2743 }
2744 Vec::new()
2745 } else {
2746 vec![hello_ack]
2747 };
2748 if !self
2749 .registry
2750 .publish_registration(connection_id)
2751 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2752 {
2753 debug!(
2754 module_id = %module_id,
2755 connection_id = connection_id.get(),
2756 "swap candidate registration was removed before it could be published"
2757 );
2758 }
2759 self.supervisor.mark_swap_candidate_admitted(&module_id);
2760 info!(
2761 module_id = %module_id,
2762 module_version = %registration.manifest.module_version,
2763 negotiated_ver,
2764 ready = registration.ready,
2765 connection_id = connection_id.get(),
2766 "swap candidate registered; not routable until cutover"
2767 );
2768 Ok(reply)
2769 }
2770
2771 fn build_hello_ack(
2772 &self,
2773 frame: &Frame,
2774 negotiated_ver: u8,
2775 module_id: &str,
2776 ) -> Result<Frame, RouterError> {
2777 let ack = ModuleHelloAckBody {
2778 negotiated_ver,
2779 subc_ops: module_subc_ops(),
2780 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2781 storage: self
2782 .storage_config
2783 .as_ref()
2784 .map(|cfg| cfg.descriptor_for(module_id)),
2785 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2786 };
2787 let body = serde_json::to_vec(&ack).map_err(|err| {
2788 RouterError::backend(
2789 0,
2790 frame.header.corr,
2791 format!("failed to encode HELLO_ACK: {err}"),
2792 )
2793 })?;
2794
2795 Frame::build_with_version(
2796 negotiated_ver,
2797 FrameType::HelloAck,
2798 control_flags(),
2799 0,
2800 0,
2801 frame.header.corr,
2802 body,
2803 )
2804 .map_err(RouterError::FrameBuild)
2805 }
2806
2807 async fn handle_client_control_request(
2808 &self,
2809 ctx: &RouteCtx,
2810 frame: Frame,
2811 request: ClientControlRequest,
2812 route_open_timings: &mut Option<RouteOpenTimings>,
2813 ) -> Result<Vec<Frame>, RouterError> {
2814 if let Some(operation) = privileged_operation(&request) {
2815 let caller = self.resolve_caller(ctx.connection_id).await;
2816 let op = client_control_request_op(&request);
2817 let text = prompt_text(operation, &caller);
2818 match self.privileged_operator_prompt {
2819 PrivilegedOperatorPrompt::Log => {
2820 info!(target: "subc_daemon::privileged_prompt", operation = op,
2821 target = ?operation, caller = %caller.description(), summary = %text,
2822 "privileged request would prompt the operator");
2823 }
2824 PrivilegedOperatorPrompt::Enforce => {
2825 let key =
2826 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2827 let outcome = self
2828 .forwarding
2829 .operator_confirms()
2830 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2831 .await;
2832 if outcome.code != "confirmed" {
2833 return Ok(vec![outcome.frame(&frame)]);
2834 }
2835 }
2836 }
2837 }
2838 match request {
2839 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2840 ClientControlRequest::CatalogList { module_id } => {
2841 self.handle_catalog_list(frame, module_id)
2842 }
2843 ClientControlRequest::RouteOpen {
2844 target,
2845 identity,
2846 consumer_identity,
2847 consumer_capabilities,
2848 role_versions,
2849 admission_facts,
2850 scope,
2851 } => {
2852 self.handle_route_open(
2853 ctx,
2854 frame,
2855 RouteOpenRequest {
2856 target,
2857 identity,
2858 consumer_identity,
2859 consumer_capabilities,
2860 role_versions,
2861 admission_facts,
2862 scope,
2863 },
2864 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2865 )
2866 .await
2867 }
2868 ClientControlRequest::RoutePoll {
2869 route_channel,
2870 route_epoch,
2871 kind,
2872 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2873 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2874 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2875 self.handle_supervisor_spawn_snapshot(frame)
2876 }
2877 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2878 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2879 }
2880 ClientControlRequest::SupervisorRestart {
2881 module_id,
2882 drain_timeout_ms,
2883 } => {
2884 self.log_supervisor_request_received(
2885 ctx,
2886 frame.header.corr,
2887 ops::SUPERVISOR_RESTART,
2888 Some(&module_id),
2889 None,
2890 )?;
2891 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2892 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2893 .await
2894 }
2895 ClientControlRequest::SupervisorSwap {
2896 module_id,
2897 ready_timeout_ms,
2898 } => {
2899 self.log_supervisor_request_received(
2900 ctx,
2901 frame.header.corr,
2902 ops::SUPERVISOR_SWAP,
2903 Some(&module_id),
2904 None,
2905 )?;
2906 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2907 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2908 .await
2909 }
2910 ClientControlRequest::SupervisorReload { module_id } => {
2911 self.log_supervisor_request_received(
2912 ctx,
2913 frame.header.corr,
2914 ops::SUPERVISOR_RELOAD,
2915 Some(&module_id),
2916 None,
2917 )?;
2918 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2919 self.handle_supervisor_reload(frame, module_id, requester)
2920 .await
2921 }
2922 ClientControlRequest::SupervisorRescan { preview } => {
2923 if !preview {
2924 self.log_supervisor_request_received(
2925 ctx,
2926 frame.header.corr,
2927 ops::SUPERVISOR_RESCAN,
2928 None,
2929 None,
2930 )?;
2931 }
2932 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2933 self.handle_supervisor_rescan(
2934 frame,
2935 if preview {
2936 RescanMode::Preview
2937 } else {
2938 RescanMode::Execute
2939 },
2940 requester,
2941 )
2942 .await
2943 }
2944 ClientControlRequest::SupervisorRescanChecked {
2945 preview,
2946 candidate,
2947 expect,
2948 } => {
2949 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2950 .await
2951 }
2952 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2953 self.log_supervisor_request_received(
2954 ctx,
2955 frame.header.corr,
2956 ops::SUPERVISOR_RELEASE_RESERVED,
2957 Some(&module_id),
2958 None,
2959 )?;
2960 self.handle_supervisor_release_reserved(frame, module_id)
2961 .await
2962 }
2963 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2964 self.log_supervisor_request_received(
2965 ctx,
2966 frame.header.corr,
2967 ops::SUPERVISOR_SET_ENABLED,
2968 Some(&module_id),
2969 Some(enabled),
2970 )?;
2971 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2972 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2973 .await
2974 }
2975 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2976 self.handle_supervisor_health_probe(frame, module_id).await
2977 }
2978 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2979 ClientControlRequest::SupervisorRoutes { module_id } => {
2980 self.handle_supervisor_routes(frame, module_id)
2981 }
2982 ClientControlRequest::SupervisorProvenance { module_id } => {
2983 self.handle_supervisor_provenance(frame, module_id).await
2984 }
2985 ClientControlRequest::SupervisorStderrTail {
2986 module_id,
2987 max_lines,
2988 max_bytes,
2989 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2990 ClientControlRequest::SupervisorTerminals { module_id } => {
2991 self.handle_supervisor_terminals(frame, module_id).await
2992 }
2993 }
2994 }
2995
2996 fn log_supervisor_request_received(
2997 &self,
2998 ctx: &RouteCtx,
2999 corr: u64,
3000 op: &'static str,
3001 module_id: Option<&str>,
3002 enabled: Option<bool>,
3003 ) -> Result<(), RouterError> {
3004 let caller = self.supervisor_requester(ctx, corr)?;
3005 let caller = principal_label(&caller);
3006 info!(
3009 target: "control",
3010 op,
3011 module_id,
3012 enabled,
3013 connection_id = ctx.connection_id.get(),
3014 caller = %caller,
3015 "supervisor request received"
3016 );
3017 Ok(())
3018 }
3019
3020 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3021 Ok(self
3022 .registry
3023 .get_module_by_connection(ctx.connection_id)
3024 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3025 .map(|registration| Principal::Reserved {
3026 module_id: registration.manifest.module_id,
3027 })
3028 .unwrap_or(Principal::Direct))
3029 }
3030
3031 fn handle_module_control_request(
3032 &self,
3033 connection_id: ConnectionId,
3034 frame: Frame,
3035 request: ModuleControlRequestFromModule,
3036 ) -> Result<Vec<Frame>, RouterError> {
3037 match request {
3038 ModuleControlRequestFromModule::CatalogUpdate {
3039 provides,
3040 capabilities,
3041 ready,
3042 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3043 ModuleControlRequestFromModule::LiveRoots {} => {
3044 let registered = self
3045 .registry
3046 .get_module_by_connection(connection_id)
3047 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3048 let Some(registration) = registered else {
3049 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3050 };
3051 let response = self
3052 .forwarding
3053 .live_roots(®istration.manifest.module_id)
3054 .map_err(RouterError::Forwarding)?;
3055 Ok(vec![control_response_body_frame(
3056 &frame,
3057 &response,
3058 "ModuleControlResponseToModule::LiveRoots",
3059 )?])
3060 }
3061 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3062 self.handle_scope_sync(connection_id, frame, generation, scopes)
3063 }
3064 ModuleControlRequestFromModule::ScopeApply {
3065 generation,
3066 upsert,
3067 end,
3068 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3069 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3070 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3071 }
3072 }
3073 }
3074
3075 fn handle_scope_served_by(
3076 &self,
3077 connection_id: ConnectionId,
3078 frame: Frame,
3079 ) -> Result<Vec<Frame>, RouterError> {
3080 let registered = self
3081 .registry
3082 .get_module_by_connection(connection_id)
3083 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3084 let Some(registered) = registered else {
3085 return Ok(vec![control_error_frame(
3086 &frame,
3087 "not_registered",
3088 "scope.served_by requires an active module registration owned by this connection",
3089 )?]);
3090 };
3091 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3092 {
3093 Ok(request) => request,
3094 Err(err) => {
3095 return Ok(vec![control_error_frame(
3096 &frame,
3097 "invalid_control_body",
3098 format!("malformed scope.served_by body: {err}"),
3099 )?])
3100 }
3101 };
3102 let answer = self
3103 .scopes
3104 .read()
3105 .unwrap_or_else(|poisoned| poisoned.into_inner())
3106 .served_by(
3107 ®istered.manifest.module_id,
3108 &request.scope_ref,
3109 request.scope_epoch,
3110 &request.module_id,
3111 );
3112 let reply = subc_control::ScopeServedByReply::new(
3113 answer,
3114 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3115 );
3116 Ok(vec![control_response_body_frame(
3117 &frame,
3118 &reply,
3119 "ScopeServedByReply",
3120 )?])
3121 }
3122
3123 fn handle_operator_confirm(
3124 &self,
3125 ctx: &RouteCtx,
3126 frame: Frame,
3127 ) -> Result<Vec<Frame>, RouterError> {
3128 use crate::operator_confirm::{audit, Outcome};
3129 let registration = self
3130 .registry
3131 .get_module_by_connection(ctx.connection_id)
3132 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3133 let Some(registration) = registration else {
3134 let outcome = Outcome::refusal("not_registered");
3135 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3136 return Ok(vec![outcome.frame(&frame)]);
3137 };
3138 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3139 Ok(request) => request,
3140 Err(_) => {
3141 let outcome = Outcome::refusal("invalid_control_body");
3142 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3143 return Ok(vec![outcome.frame(&frame)]);
3144 }
3145 };
3146 let module_id = registration.manifest.module_id;
3147 let nonce = self
3151 .hello_launch_nonces
3152 .lock()
3153 .unwrap_or_else(|p| p.into_inner())
3154 .nonce(ctx.connection_id)
3155 .map(str::to_owned);
3156 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3157 self.supervisor
3158 .spawned_consumer_authorized(&module_id, nonce)
3159 });
3160 let confirms = self.forwarding.operator_confirms();
3161 self.forwarding
3162 .with_operator_route(
3163 ctx.connection_id,
3164 request.route_channel,
3165 request.route_epoch,
3166 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3167 )
3168 .map_err(RouterError::Forwarding)
3169 }
3170
3171 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3174 let mut scopes = ScopeRecords::from(scopes);
3175 for (index, record) in scopes.records.iter_mut().enumerate() {
3176 if let Some(refusal) = Self::canonicalize_workdir(record) {
3177 scopes.refusals.insert(index, refusal);
3178 }
3179 }
3180 scopes
3181 }
3182
3183 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3184 let workdir = record.workdir.as_mut()?;
3185 let path = std::path::Path::new(workdir);
3186 if !path.is_absolute() {
3187 return Some((
3188 error_codes::INVALID_WORKDIR,
3189 "workdir must be absolute".to_string(),
3190 ));
3191 }
3192 match ProjectRootId::from_path_allowing_missing(path) {
3193 Ok(root) => match root.as_path().to_str() {
3194 Some(canonical) => {
3195 *workdir = canonical.to_string();
3196 None
3197 }
3198 None => Some((
3199 error_codes::INVALID_WORKDIR,
3200 "canonical workdir is not UTF-8".to_string(),
3201 )),
3202 },
3203 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3204 }
3205 }
3206
3207 fn handle_scope_sync(
3211 &self,
3212 connection_id: ConnectionId,
3213 frame: Frame,
3214 generation: u64,
3215 scopes: Vec<ScopeRecord>,
3216 ) -> Result<Vec<Frame>, RouterError> {
3217 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3218 }
3219
3220 fn handle_scope_change(
3221 &self,
3222 connection_id: ConnectionId,
3223 frame: Frame,
3224 generation: u64,
3225 scopes: Vec<ScopeRecord>,
3226 end: Option<Vec<ScopeEnd>>,
3227 ) -> Result<Vec<Frame>, RouterError> {
3228 let is_apply = end.is_some();
3229 let Some(registration) = self
3230 .registry
3231 .get_module_by_connection(connection_id)
3232 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3233 else {
3234 return Ok(vec![control_error_frame(
3235 &frame,
3236 "not_registered",
3237 "scope.sync and scope.apply require an active module registration owned by this connection",
3238 )?]);
3239 };
3240 let owner = registration.manifest.module_id;
3241 let scopes = Self::prepare_scope_records(scopes);
3242 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3243 let is_current_launch = |connection: ConnectionId| {
3244 self.hello_launch_nonces
3245 .lock()
3246 .unwrap_or_else(|poisoned| poisoned.into_inner())
3247 .presented(connection, current_nonce.as_deref())
3248 };
3249 let mut table = self
3254 .scopes
3255 .write()
3256 .unwrap_or_else(|poisoned| poisoned.into_inner());
3257 let outcome = match end {
3258 Some(end) => table.apply(
3259 &owner,
3260 connection_id,
3261 is_current_launch,
3262 generation,
3263 scopes,
3264 end,
3265 ),
3266 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3267 };
3268 let drained = match &outcome {
3269 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3270 Err(_) => Vec::new(),
3271 };
3272 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3273 drop(table);
3274 if has_deadlines {
3275 self.scope_deadline_added.notify_one();
3276 }
3277 match outcome {
3278 Ok(applied) => {
3279 let counts = ScopeOutcomeCounts::of(&applied.results);
3280 info!(
3281 owner = %owner,
3282 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3283 generation,
3284 records = applied.results.len(),
3285 created = counts.created,
3286 replaced = counts.replaced,
3287 updated = counts.updated,
3288 unchanged = counts.unchanged,
3289 refused = counts.refused,
3290 ended = applied.ended.len(),
3291 tag_changes = applied.tag_changes.len(),
3292 routes_closed = drained.len(),
3293 "scope change accepted"
3294 );
3295 for refused in applied
3301 .results
3302 .iter()
3303 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3304 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3305 {
3306 warn!(
3307 owner = %owner,
3308 generation,
3309 scope_ref = %refused.scope_ref,
3310 scope_epoch = refused.scope_epoch,
3311 code = refused.code.as_deref().unwrap_or(""),
3312 "scope record refused"
3313 );
3314 }
3315 self.close_scope_drained_routes(drained);
3316 let response = if is_apply {
3317 ModuleControlResponseToModule::ScopeApply {
3318 generation,
3319 results: applied.results,
3320 end_results: applied.end_results,
3321 ended: applied.ended,
3322 }
3323 } else {
3324 ModuleControlResponseToModule::ScopeSync {
3325 generation,
3326 results: applied.results,
3327 ended: applied.ended,
3328 }
3329 };
3330 Ok(vec![control_response_body_frame(
3331 &frame,
3332 &response,
3333 "scope change response",
3334 )?])
3335 }
3336 Err(refusal) => {
3337 info!(
3338 owner = %owner,
3339 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3340 generation,
3341 code = refusal.code,
3342 "scope change refused"
3343 );
3344 Ok(vec![control_error_frame(
3345 &frame,
3346 refusal.code,
3347 refusal.message,
3348 )?])
3349 }
3350 }
3351 }
3352
3353 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3359 if drained.is_empty() {
3360 return;
3361 }
3362 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3363 BTreeMap::new();
3364 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3365 for route in drained {
3366 warn!(
3367 module_id = %route.module_id,
3368 reason = ?route.reason,
3369 client_connection_id = route.client.connection_id.get(),
3370 route_channel = route.client.channel,
3371 "closing route because its scope changed"
3372 );
3373 pushes
3374 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3375 .or_insert_with(|| (route.reason, Vec::new()))
3376 .1
3377 .push(EndpointRoute {
3378 goodbye_target: route.client.clone(),
3379 principal: Principal::Unverified,
3380 bound_at: Instant::now(),
3381 draining: false,
3382 drain_reason: None,
3383 });
3384 goodbyes.push(route.module);
3385 goodbyes.push(route.client);
3386 }
3387 for ((module_id, _), (reason, routes)) in pushes {
3388 send_route_control_pushes(
3389 &self.forwarding,
3390 routes,
3391 ClientControlPush::RouteClosed {
3392 module_id,
3393 channels: Vec::new(),
3394 reason,
3395 drained: false,
3396 abandoned: 0,
3397 excluded_subscriptions: 0,
3398 terminal: Some(false),
3399 },
3400 );
3401 }
3402 self.emit_route_goodbyes(goodbyes);
3403 }
3404
3405 fn handle_scope_describe(
3408 &self,
3409 connection_id: ConnectionId,
3410 frame: Frame,
3411 owner: Principal,
3412 scope_ref: String,
3413 ) -> Result<Vec<Frame>, RouterError> {
3414 let registered = self
3415 .registry
3416 .get_module_by_connection(connection_id)
3417 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3418 let Some(registered) = registered else {
3419 return Ok(vec![control_error_frame(
3420 &frame,
3421 "not_registered",
3422 "scope.describe requires an active module registration owned by this connection",
3423 )?]);
3424 };
3425 let mut description = self
3426 .scopes
3427 .read()
3428 .unwrap_or_else(|poisoned| poisoned.into_inner())
3429 .describe(&owner, &scope_ref);
3430 if !registered
3433 .manifest
3434 .capabilities
3435 .as_ref()
3436 .is_some_and(|caps| {
3437 caps.provides
3438 .iter()
3439 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3440 })
3441 {
3442 if let Some(stamp) = description.stamp.as_mut() {
3443 stamp.attributes.on_behalf_of_agent = None;
3444 }
3445 }
3446 let owner_configured = match &owner {
3447 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3452 _ => false,
3453 };
3454 let response = ModuleControlResponseToModule::ScopeDescribe {
3455 status: description.status,
3456 scope_epoch: description.scope_epoch,
3457 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3458 owner_synced: description.owner_synced,
3459 owner_configured,
3460 scope: description.stamp,
3461 };
3462 Ok(vec![control_response_body_frame(
3463 &frame,
3464 &response,
3465 "ModuleControlResponseToModule::ScopeDescribe",
3466 )?])
3467 }
3468
3469 fn handle_catalog_update(
3470 &self,
3471 connection_id: ConnectionId,
3472 frame: Frame,
3473 provides: Vec<ProviderRole>,
3474 capabilities: Option<CapabilityDeclarations>,
3475 ready: Option<bool>,
3476 ) -> Result<Vec<Frame>, RouterError> {
3477 self.refresh_capability_requirements();
3478 let Some(registration) = self
3479 .registry
3480 .get_module_by_connection(connection_id)
3481 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3482 else {
3483 return Ok(vec![control_error_frame(
3484 &frame,
3485 "not_registered",
3486 "catalog.update requires an active module registration owned by this connection",
3487 )?]);
3488 };
3489
3490 if let Some(message) =
3491 catalog_update_frozen_field_message(®istration.manifest, &provides)
3492 {
3493 return Ok(vec![control_error_frame(
3494 &frame,
3495 "catalog_update_frozen_field",
3496 message,
3497 )?]);
3498 }
3499
3500 let mut candidate = registration.manifest.clone();
3501 candidate.provides = provides.clone();
3502 candidate.capabilities = capabilities
3503 .clone()
3504 .or_else(|| registration.manifest.capabilities.clone());
3505 if let Some(rich_ui) = &candidate.rich_ui {
3506 if let Err(err) = rich_ui.validate() {
3507 return Ok(vec![control_error_frame(
3508 &frame,
3509 "invalid_manifest",
3510 err.to_string(),
3511 )?]);
3512 }
3513 }
3514 if let Err(err) = candidate.validate_capability_grammar() {
3515 return Ok(vec![control_error_frame(
3516 &frame,
3517 "invalid_capability_grammar",
3518 err.to_string(),
3519 )?]);
3520 }
3521
3522 let mut conflicts = self
3525 .capability_evaluator
3526 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3527 if let Some(conflict) = conflicts.first() {
3528 let message = format!(
3529 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3530 conflict.capability, conflict.claimants[0], candidate.module_id
3531 );
3532 for conflict in &mut conflicts {
3533 conflict.source = DuplicateClaimSource::CatalogUpdate;
3534 }
3535 log_duplicate_claim_events(conflicts);
3536 return Ok(vec![control_error_frame(
3537 &frame,
3538 "reserved_capability",
3539 message,
3540 )?]);
3541 }
3542
3543 let updated = self
3544 .registry
3545 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3546 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3547 if updated.is_none() {
3548 return Ok(vec![control_error_frame(
3549 &frame,
3550 "not_registered",
3551 "catalog.update requires an active module registration owned by this connection",
3552 )?]);
3553 }
3554 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3555 log_duplicate_claim_events(
3556 self.capability_evaluator
3557 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3558 );
3559 }
3560 if capability_census_trigger(
3561 registration.manifest.capabilities.as_ref(),
3562 updated
3563 .as_ref()
3564 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3565 ) {
3566 self.enforce_capability_denies();
3567 }
3568 self.refresh_capability_requirements();
3569
3570 let response = ModuleControlResponseToModule::CatalogUpdate {};
3571 control_response_body_frame(
3572 &frame,
3573 &response,
3574 "ModuleControlResponseToModule::CatalogUpdate",
3575 )
3576 .map(|frame| vec![frame])
3577 }
3578
3579 pub(crate) fn with_activity_ledger_config(
3580 mut self,
3581 run_dir: &Path,
3582 incarnation: String,
3583 config: crate::activity::ActivityConfig,
3584 ) -> Self {
3585 self.activity = Some(crate::activity::ActivityLedger::open_config(
3586 run_dir.join("activity.jsonl"),
3587 incarnation,
3588 config,
3589 ));
3590 self.supervisor.set_activity(self.activity.clone());
3591 self
3592 }
3593
3594 pub(crate) async fn configure_activity(
3595 &self,
3596 config: crate::activity::ActivityConfig,
3597 ) -> Result<(), &'static str> {
3598 match &self.activity {
3599 Some(ledger) => ledger.configure(config).await,
3600 None => Ok(()),
3601 }
3602 }
3603
3604 fn cancel_activity_subscription(
3605 &self,
3606 ctx: &RouteCtx,
3607 frame: &Frame,
3608 ) -> Result<bool, RouterError> {
3609 let task = self
3610 .activity_streams
3611 .lock()
3612 .unwrap_or_else(|p| p.into_inner())
3613 .remove(&(ctx.connection_id, frame.header.corr));
3614 let Some(task) = task else {
3615 return Ok(false);
3616 };
3617 task.abort();
3618 let end = Frame::build_with_version(
3619 response_version(frame),
3620 FrameType::StreamEnd,
3621 control_flags(),
3622 0,
3623 0,
3624 frame.header.corr,
3625 Vec::new(),
3626 )
3627 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3628 let sink = ctx.egress.clone();
3629 tokio::spawn(async move {
3630 let _ = sink.send(end).await;
3631 });
3632 Ok(true)
3633 }
3634
3635 async fn handle_activity_read(
3636 &self,
3637 ctx: &RouteCtx,
3638 frame: Frame,
3639 ) -> Result<Vec<Frame>, RouterError> {
3640 use subc_control::{
3641 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3642 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3643 };
3644 #[derive(Deserialize)]
3645 struct Identity {
3646 #[serde(default)]
3647 consumer_identity: Option<ConsumerIdentity>,
3648 }
3649 let principal = match self
3653 .forwarding
3654 .module_id_for_connection(ctx.connection_id)
3655 .map_err(RouterError::Forwarding)?
3656 {
3657 Some(module_id) => Ok(Principal::Reserved { module_id }),
3658 None => {
3659 let identity: Identity = match serde_json::from_slice(&frame.body) {
3660 Ok(identity) => identity,
3661 Err(_) => {
3662 return Ok(vec![control_error_frame(
3663 &frame,
3664 ACTIVITY_MALFORMED,
3665 "malformed activity read",
3666 )?])
3667 }
3668 };
3669 self.route_open_principal(&frame, identity.consumer_identity)?
3670 }
3671 };
3672 let admitted = matches!(principal, Ok(Principal::Direct))
3673 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3674 if !admitted {
3675 return Ok(vec![control_error_frame(
3676 &frame,
3677 ACTIVITY_READ_FORBIDDEN,
3678 "activity reads require an operator or trusted mirror principal",
3679 )?]);
3680 }
3681 let Some(ledger) = &self.activity else {
3682 return Ok(vec![control_error_frame(
3683 &frame,
3684 ACTIVITY_UNAVAILABLE,
3685 "activity ledger unavailable",
3686 )?]);
3687 };
3688 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3689 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3690 {
3691 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3692 Ok(request) => ledger.query(request).await,
3693 Err(_) => Err(ACTIVITY_MALFORMED),
3694 };
3695 return match result {
3696 Ok(reply) => Ok(vec![control_response_body_frame(
3697 &frame,
3698 &reply,
3699 ops::ACTIVITY_QUERY,
3700 )?]),
3701 Err(code) => Ok(vec![control_error_frame(
3702 &frame,
3703 code,
3704 "activity query refused",
3705 )?]),
3706 };
3707 }
3708 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3709 Ok(request) => request,
3710 Err(_) => {
3711 return Ok(vec![control_error_frame(
3712 &frame,
3713 ACTIVITY_MALFORMED,
3714 "malformed activity subscription",
3715 )?])
3716 }
3717 };
3718 let mut subscription = match ledger.subscribe(request.since).await {
3719 Ok(subscription) => subscription,
3720 Err(code) => {
3721 return Ok(vec![control_error_frame(
3722 &frame,
3723 code,
3724 "activity subscription refused",
3725 )?])
3726 }
3727 };
3728 let key = (ctx.connection_id, frame.header.corr);
3729 let sink = ctx.egress.clone();
3730 let mut streams = self
3731 .activity_streams
3732 .lock()
3733 .unwrap_or_else(|p| p.into_inner());
3734 streams.retain(|_, task| !task.is_finished());
3735 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3736 return Ok(vec![control_error_frame(
3737 &frame,
3738 "duplicate_subscription",
3739 "activity subscription correlation id already in use",
3740 )?]);
3741 }
3742 let task = tokio::spawn(async move {
3743 loop {
3744 let entry = if let Some(entry) = subscription.replay.pop_front() {
3745 Some(entry)
3746 } else {
3747 subscription.receiver.recv().await
3748 };
3749 let Some(entry) = entry else {
3750 break;
3751 };
3752 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3753 return;
3754 };
3755 let Ok(next) = Frame::build_with_version(
3756 response_version(&frame),
3757 FrameType::StreamData,
3758 control_flags(),
3759 0,
3760 0,
3761 frame.header.corr,
3762 body,
3763 ) else {
3764 return;
3765 };
3766 if sink.send(next).await.is_err() {
3767 return;
3768 }
3769 }
3770 if let Some(code) = subscription.terminal_code() {
3773 if let Ok(end) =
3774 control_error_frame(&frame, code, "activity subscriber fell behind")
3775 {
3776 let _ = sink.send(end).await;
3777 }
3778 }
3779 });
3780 streams.insert(key, task.abort_handle());
3781 Ok(Vec::new())
3782 }
3783
3784 async fn handle_activity_record(
3785 &self,
3786 ctx: &RouteCtx,
3787 frame: Frame,
3788 ) -> Result<Vec<Frame>, RouterError> {
3789 use subc_control::{
3790 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3791 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3792 };
3793 let result = async {
3794 let module = self
3795 .forwarding
3796 .module_id_for_connection(ctx.connection_id)
3797 .ok()
3798 .flatten()
3799 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3800 let request: ActivityRecordRequest =
3801 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3802 crate::activity::validate(&module, &request)?;
3803 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3804 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3805 let at_ms = std::time::SystemTime::now()
3806 .duration_since(std::time::UNIX_EPOCH)
3807 .unwrap_or_default()
3808 .as_millis() as u64;
3809 ledger.record(module, request, actor, at_ms).await
3810 }
3811 .await;
3812 match result {
3813 Ok(entry) => Ok(vec![control_response_body_frame(
3814 &frame,
3815 &ActivityRecordResponse { seq: entry.seq },
3816 "activity.record",
3817 )?]),
3818 Err(code) => Ok(vec![control_error_frame(
3819 &frame,
3820 code,
3821 "activity recording refused",
3822 )?]),
3823 }
3824 }
3825
3826 fn activity_actor(
3827 &self,
3828 connection: ConnectionId,
3829 module: &str,
3830 on: Option<&subc_control::ActivityOn>,
3831 ) -> Result<subc_control::ActivityActor, &'static str> {
3832 use crate::forwarding::{DataRoute, DataRouteState};
3833 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3834 let principal = |id: &str| Principal::Reserved {
3835 module_id: id.to_owned(),
3836 };
3837 let mut actor = ActivityActor {
3838 principal: principal(module),
3839 scope_owner: None,
3840 agent_id: None,
3841 run_id: None,
3842 flow_id: None,
3843 project_root: None,
3844 workdir: None,
3845 project_id: None,
3846 };
3847 let scope = match on {
3848 None => return Ok(actor),
3849 Some(ActivityOn::Scope(scope)) => {
3850 let selector = subc_protocol::scope::ScopeSelector::new(
3851 principal(module),
3852 scope.scope_ref.clone(),
3853 Some(scope.epoch),
3854 );
3855 self.scopes
3856 .read()
3857 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3858 .attribute(&principal(module), module, &selector, true)
3859 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3860 }
3861 Some(ActivityOn::Route(route)) => {
3862 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3863 .forwarding
3864 .lookup_data_route(connection, route.channel, route.epoch)
3865 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3866 else {
3867 return Err(ACTIVITY_ACTOR_UNKNOWN);
3868 };
3869 if binding.module_id != module {
3870 return Err(ACTIVITY_ACTOR_UNKNOWN);
3871 }
3872 actor.principal = binding.principal.clone();
3873 actor.project_root = binding
3874 .project_root
3875 .as_ref()
3876 .map(|root| root.as_path().to_path_buf());
3877 let Some(bound) = &binding.scope else {
3878 return Ok(actor);
3879 };
3880 let owner = principal(&bound.owner);
3881 let selector = subc_protocol::scope::ScopeSelector::new(
3882 owner.clone(),
3883 bound.scope_ref.clone(),
3884 Some(bound.tag.scope_epoch),
3885 );
3886 let admitted = self
3887 .scopes
3888 .read()
3889 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3890 .attribute(&owner, module, &selector, true)
3891 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3892 if admitted.tag != bound.tag {
3893 return Err(ACTIVITY_ACTOR_UNKNOWN);
3894 }
3895 admitted
3896 }
3897 };
3898 actor.scope_owner = Some(scope.stamp.owner);
3899 actor.agent_id = scope.stamp.attributes.agent_id;
3900 actor.run_id = scope.stamp.attributes.run_id;
3901 actor.flow_id = scope.stamp.attributes.flow_id;
3902 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3903 actor.project_id = scope.stamp.project_id;
3904 Ok(actor)
3905 }
3906
3907 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3908 self.refresh_capability_requirements();
3909 let mut counters = self.counters.snapshot();
3915 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3916 obj.insert(
3917 "activity_unparseable_lines".into(),
3918 ledger.read_errors().into(),
3919 );
3920 obj.insert(
3921 "activity_lifecycle_append_failures".into(),
3922 ledger.lifecycle_failures().into(),
3923 );
3924 obj.insert(
3925 "activity_started_digest_skips".into(),
3926 ledger.lifecycle_digest_skips().into(),
3927 );
3928 }
3929 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3930 self.forwarding.client_route_concentration(),
3931 counters.as_object_mut(),
3932 ) {
3933 obj.insert(
3934 "client_connections_with_routes".into(),
3935 connections_with_routes.into(),
3936 );
3937 obj.insert("max_routes_on_one_connection".into(), max.into());
3938 }
3939 if let (Some(open_breakers), Some(obj)) = (
3945 self.route_bind_breakers.open_snapshot(),
3946 counters.as_object_mut(),
3947 ) {
3948 obj.insert("route_bind_breakers_open".into(), open_breakers);
3949 }
3950 let response = ClientControlResponse::ServerDescribe {
3951 protocol_ver: PROTOCOL_VERSION,
3952 subc_ops: subc_ops(),
3953 capabilities: self.describe_capabilities(),
3954 connected_clients: self.connected_clients.count(),
3955 counters: Some(counters),
3956 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3957 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3958 capability_requirements: self.capability_requirement_statuses(),
3959 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3960 config_path: self
3961 .rescan
3962 .as_ref()
3963 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3964 };
3965 Ok(vec![control_response_body_frame(
3966 &frame,
3967 &response,
3968 "ClientControlResponse::ServerDescribe",
3969 )?])
3970 }
3971
3972 fn handle_catalog_list(
3973 &self,
3974 frame: Frame,
3975 module_id: Option<String>,
3976 ) -> Result<Vec<Frame>, RouterError> {
3977 let (generation, modules) = self
3978 .registry
3979 .list_modules_with_nonce_verification()
3980 .map_err(|err| {
3981 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3982 })?;
3983 let entries = modules
3984 .into_iter()
3985 .filter(|(registration, _)| {
3986 module_id
3987 .as_deref()
3988 .map(|wanted| registration.manifest.module_id == wanted)
3989 .unwrap_or(true)
3990 })
3991 .map(|(registration, nonce_verified)| {
3992 let not_ready = self.not_ready_reason(®istration);
3993 let roles = registration.manifest.provides;
3994 CatalogEntry::new(
3995 registration.manifest.module_id,
3996 roles,
3997 registration.control_ops,
3998 )
3999 .with_ready(not_ready.is_none())
4000 .with_not_ready(not_ready)
4001 .with_module_version(Some(registration.manifest.module_version))
4002 .with_capabilities(registration.manifest.capabilities)
4003 .with_self_signals(registration.manifest.self_signals)
4004 .with_events(registration.manifest.events)
4005 .with_operation_delivery(registration.manifest.operation_delivery)
4006 .with_operation_payload(registration.manifest.operation_payload)
4007 .with_rich_ui(registration.manifest.rich_ui)
4008 .with_role_declarations(registration.manifest.role_declarations)
4009 .with_nonce_verified(nonce_verified)
4010 })
4011 .collect();
4012 let response = ClientControlResponse::CatalogList {
4013 generation,
4014 modules: entries,
4015 subc_ops: subc_ops(),
4016 };
4017 Ok(vec![control_response_body_frame(
4018 &frame,
4019 &response,
4020 "ClientControlResponse::CatalogList",
4021 )?])
4022 }
4023
4024 fn attest_route_open_consumer(
4029 &self,
4030 consumer_identity: Option<&ConsumerIdentity>,
4031 ) -> Option<Principal> {
4032 let Some(consumer_identity) = consumer_identity else {
4033 return Some(Principal::Direct);
4034 };
4035 self.supervisor
4036 .spawned_consumer_authorized(
4037 &consumer_identity.module_id,
4038 &consumer_identity.launch_nonce,
4039 )
4040 .then(|| Principal::Reserved {
4041 module_id: consumer_identity.module_id.clone(),
4042 })
4043 }
4044
4045 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4049 let Ok(ClientControlRequest::RouteOpen {
4050 identity,
4051 consumer_identity,
4052 scope,
4053 ..
4054 }) = parse_client_control_request(&frame.body)
4055 else {
4056 return RouteOpenCaller::unknown();
4057 };
4058 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4059 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4060 }
4061
4062 fn route_open_principal(
4063 &self,
4064 frame: &Frame,
4065 consumer_identity: Option<ConsumerIdentity>,
4066 ) -> Result<Result<Principal, Frame>, RouterError> {
4067 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4068 self.route_open_principal_attested(frame, consumer_identity, attested)
4069 }
4070
4071 fn route_open_principal_attested(
4075 &self,
4076 frame: &Frame,
4077 consumer_identity: Option<ConsumerIdentity>,
4078 attested: Option<Principal>,
4079 ) -> Result<Result<Principal, Frame>, RouterError> {
4080 if let Some(principal) = attested {
4081 return Ok(Ok(principal));
4082 }
4083 let Some(consumer_identity) = consumer_identity else {
4084 return Ok(Ok(Principal::Direct));
4085 };
4086
4087 Ok(Err(control_error_frame(
4088 frame,
4089 "bad_consumer_identity",
4090 format!(
4091 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4092 consumer_identity.module_id
4093 ),
4094 )?))
4095 }
4096
4097 #[allow(clippy::too_many_arguments)]
4104 fn route_open_refusal_frame(
4105 &self,
4106 ctx: &RouteCtx,
4107 caller: &RouteOpenCaller,
4108 frame: &Frame,
4109 module_id: &str,
4110 reason: &'static str,
4111 code: &'static str,
4112 message: impl Into<String>,
4113 ) -> Result<Frame, RouterError> {
4114 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4115 control_error_frame(frame, code, message.into())
4116 }
4117
4118 #[allow(clippy::too_many_arguments)]
4136 fn route_open_breaker_refusal_frame(
4137 &self,
4138 ctx: &RouteCtx,
4139 caller: &RouteOpenCaller,
4140 frame: &Frame,
4141 module_id: &str,
4142 consecutive_timeouts: u32,
4143 retry_in: Duration,
4144 probe_in_flight: bool,
4145 ) -> Result<Frame, RouterError> {
4146 self.counters
4147 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4148 debug!(
4149 target: "control",
4150 code = "module_timeout",
4151 module_id = ?module_id,
4152 connection_id = ctx.connection_id.get(),
4153 principal = %caller.principal,
4154 scope = %caller.scope,
4155 harness = %caller.harness,
4156 consecutive_timeouts,
4157 retry_in_ms = retry_in.as_millis() as u64,
4158 probe_in_flight,
4159 "route.open refused by open bind-relay breaker"
4160 );
4161 let detail = if probe_in_flight {
4168 "one test route is already being tried; retry once it settles".to_string()
4169 } else {
4170 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4171 };
4172 control_error_frame(
4173 frame,
4174 "module_timeout",
4175 format!(
4176 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4177 timed out in a row); {detail}; its established routes are unaffected"
4178 ),
4179 )
4180 }
4181
4182 fn observe_route_open_refusal(
4195 &self,
4196 ctx: &RouteCtx,
4197 caller: &RouteOpenCaller,
4198 module_id: &str,
4199 reason: &'static str,
4200 code: &'static str,
4201 ) {
4202 self.counters.increment_route_open_refused(code);
4203 info!(
4204 target: "control",
4205 code,
4206 reason,
4207 module_id = ?module_id,
4208 connection_id = ctx.connection_id.get(),
4209 principal = %caller.principal,
4210 scope = %caller.scope,
4211 harness = %caller.harness,
4212 "route.open refused"
4213 );
4214 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4215 self.route_outages.record_not_serving(module_id, reason);
4216 }
4217 }
4218
4219 fn observe_route_open_accept(
4264 &self,
4265 ctx: &RouteCtx,
4266 module_id: &str,
4267 principal: &str,
4268 scope: &str,
4269 ) {
4270 self.route_outages.record_accepted(module_id);
4271 self.counters.increment_route_open_accepted(principal);
4272 info!(
4273 target: "control",
4274 principal,
4275 module_id,
4276 connection_id = ctx.connection_id.get(),
4277 scope = %scope,
4278 "route.open accepted"
4279 );
4280 }
4281
4282 fn supervised_absent_route_open_refusal_frame(
4283 &self,
4284 ctx: &RouteCtx,
4285 caller: &RouteOpenCaller,
4286 frame: &Frame,
4287 module_id: &str,
4288 code: &'static str,
4289 status: &crate::supervise::ModuleStatus,
4290 ) -> Result<Frame, RouterError> {
4291 self.counters.increment_route_open_refused(code);
4292 info!(
4293 target: "control",
4294 code,
4295 reason = "supervised_not_registered",
4296 module_id = ?module_id,
4297 connection_id = ctx.connection_id.get(),
4298 principal = %caller.principal,
4299 scope = %caller.scope,
4300 harness = %caller.harness,
4301 state = %status.state,
4302 enabled = status.enabled,
4303 live = status.live,
4304 "route.open refused"
4305 );
4306 self.route_outages
4310 .record_not_serving(module_id, "supervised_not_registered");
4311 let message = if let Some(pid) = status.waiting_for_previous_pid {
4312 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4313 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4314 })
4315 } else {
4316 format!(
4317 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4318 status.state, status.enabled, status.live
4319 )
4320 };
4321 control_error_frame(frame, code, message)
4322 }
4323
4324 async fn handle_route_open(
4325 &self,
4326 ctx: &RouteCtx,
4327 frame: Frame,
4328 request: RouteOpenRequest,
4329 timings: &mut RouteOpenTimings,
4330 ) -> Result<Vec<Frame>, RouterError> {
4331 let RouteOpenRequest {
4332 target,
4333 mut identity,
4334 consumer_identity,
4335 consumer_capabilities,
4336 role_versions,
4337 admission_facts,
4338 scope,
4339 } = request;
4340 let target_module_id = target_module_id(&target).to_string();
4341 timings.module_id.clone_from(&target_module_id);
4342 if self
4343 .registry
4344 .get_module_by_connection(ctx.connection_id)
4345 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4346 .is_some()
4347 {
4348 return Ok(vec![control_error_frame(
4349 &frame,
4350 "invalid_request",
4351 "module connections cannot open client routes",
4352 )?]);
4353 }
4354 debug!(
4355 connection_id = ctx.connection_id.get(),
4356 corr = frame.header.corr,
4357 module_id = %target_module_id,
4358 "handling route.open"
4359 );
4360
4361 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4366 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4367
4368 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4374 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4375 self.observe_route_open_refusal(
4376 ctx,
4377 &caller,
4378 &target_module_id,
4379 "invalid_role_versions",
4380 error_codes::INVALID_REQUEST,
4381 );
4382 return Ok(vec![control_error_body_frame(
4383 &frame,
4384 ErrorBody {
4385 code: error_codes::INVALID_REQUEST.to_string(),
4386 message: error.to_string(),
4387 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4388 },
4389 )?]);
4390 }
4391
4392 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4393 if status.waiting_for_previous_pid.is_some() {
4394 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4395 ctx,
4396 &caller,
4397 &frame,
4398 &target_module_id,
4399 "module_warming",
4400 &status,
4401 )?]);
4402 }
4403 }
4404
4405 let Some(registration) = self
4424 .registry
4425 .get_module(&target_module_id)
4426 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4427 else {
4428 if let Some((status, warming)) =
4429 self.supervisor_status(&target_module_id, frame.header.corr)?
4430 {
4431 if status.protocol == ModuleProtocol::None {
4438 return Ok(vec![self.route_open_refusal_frame(
4439 ctx,
4440 &caller,
4441 &frame,
4442 &target_module_id,
4443 "protocol_none",
4444 error_codes::MODULE_NO_PROTOCOL,
4445 format!(
4446 "module_id '{target_module_id}' is declared protocol: none; \
4447 it speaks no subc wire and serves no routes"
4448 ),
4449 )?]);
4450 }
4451 let code = if warming {
4452 "module_warming"
4453 } else {
4454 "target_unavailable"
4455 };
4456 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4457 ctx,
4458 &caller,
4459 &frame,
4460 &target_module_id,
4461 code,
4462 &status,
4463 )?]);
4464 }
4465 if let Some(removed_ago_ms) =
4466 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4467 {
4468 return Ok(vec![self.route_open_refusal_frame(
4469 ctx,
4470 &caller,
4471 &frame,
4472 &target_module_id,
4473 "removed",
4474 error_codes::MODULE_REMOVED,
4475 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4476 )?]);
4477 }
4478 return Ok(vec![self.route_open_refusal_frame(
4479 ctx,
4480 &caller,
4481 &frame,
4482 &target_module_id,
4483 "not_registered",
4484 error_codes::UNKNOWN_MODULE,
4485 format!("module_id '{target_module_id}' is not registered"),
4486 )?]);
4487 };
4488
4489 if !registration.ready {
4493 self.counters
4494 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4495 info!(
4496 target: "control",
4497 code = error_codes::MODULE_WARMING,
4498 module_id = ?target_module_id,
4499 connection_id = ctx.connection_id.get(),
4500 principal = %caller.principal,
4501 scope = %caller.scope,
4502 harness = %caller.harness,
4503 reason = "declared_not_ready",
4504 "route.open refused"
4505 );
4506 self.route_outages
4509 .record_not_serving(&target_module_id, "declared_not_ready");
4510 return Ok(vec![control_error_body_frame(
4511 &frame,
4512 ErrorBody {
4513 code: error_codes::MODULE_WARMING.to_string(),
4514 message: format!(
4515 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4516 ),
4517 detail: Some(serde_json::json!({
4518 "reason": "declared_not_ready"
4519 })),
4520 },
4521 )?]);
4522 }
4523
4524 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4543 self.counters
4544 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4545 info!(
4546 target: "control",
4547 code = error_codes::MODULE_WARMING,
4548 module_id = ?target_module_id,
4549 connection_id = ctx.connection_id.get(),
4550 principal = %caller.principal,
4551 scope = %caller.scope,
4552 harness = %caller.harness,
4553 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4554 capability = %capability,
4555 "route.open refused"
4556 );
4557 return Ok(vec![control_error_body_frame(
4558 &frame,
4559 ErrorBody {
4560 code: error_codes::MODULE_WARMING.to_string(),
4561 message: format!(
4562 "module_id '{target_module_id}' requires capability '{capability}', \
4563 which no registered module provides; retry"
4564 ),
4565 detail: Some(serde_json::json!({
4566 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4567 "capability": capability,
4568 })),
4569 },
4570 )?]);
4571 }
4572
4573 if !target_has_required_role(&target, ®istration.manifest.provides) {
4574 return Ok(vec![self.route_open_refusal_frame(
4575 ctx,
4576 &caller,
4577 &frame,
4578 &target_module_id,
4579 "role_not_provided",
4580 "target_unavailable",
4581 format!("module_id '{target_module_id}' does not provide the requested target"),
4582 )?]);
4583 }
4584
4585 if registration.state != ChannelState::Active {
4586 return Ok(vec![self.route_open_refusal_frame(
4587 ctx,
4588 &caller,
4589 &frame,
4590 &target_module_id,
4591 "registration_not_active",
4592 "target_unavailable",
4593 format!("module_id '{target_module_id}' is not active"),
4594 )?]);
4595 }
4596
4597 if self
4598 .forwarding
4599 .module_is_draining(&target_module_id)
4600 .map_err(RouterError::Forwarding)?
4601 {
4602 return Ok(vec![self.route_open_refusal_frame(
4603 ctx,
4604 &caller,
4605 &frame,
4606 &target_module_id,
4607 "reloading",
4608 "module_reloading",
4609 format!("module_id '{target_module_id}' is reloading"),
4610 )?]);
4611 }
4612
4613 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4614 process_liveness.process_live(&target_module_id) == Some(false)
4615 }) {
4616 if process_liveness.process_replacing(&target_module_id) {
4623 return Ok(vec![self.route_open_refusal_frame(
4624 ctx,
4625 &caller,
4626 &frame,
4627 &target_module_id,
4628 "reloading",
4629 "module_reloading",
4630 format!("module_id '{target_module_id}' is reloading"),
4631 )?]);
4632 }
4633 return Ok(vec![self.route_open_refusal_frame(
4634 ctx,
4635 &caller,
4636 &frame,
4637 &target_module_id,
4638 "supervisor_not_live",
4639 "target_unavailable",
4640 format!("module_id '{target_module_id}' is not live"),
4641 )?]);
4642 }
4643
4644 if !self
4645 .forwarding
4646 .has_live_module_connection(&target_module_id)
4647 .map_err(RouterError::Forwarding)?
4648 {
4649 return Ok(vec![self.route_open_refusal_frame(
4650 ctx,
4651 &caller,
4652 &frame,
4653 &target_module_id,
4654 "no_forwarding_connection",
4655 "target_unavailable",
4656 format!("module_id '{target_module_id}' has no live forwarding connection"),
4657 )?]);
4658 }
4659
4660 if let Some(error) =
4661 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4662 {
4663 self.observe_route_open_refusal(
4664 ctx,
4665 &caller,
4666 &target_module_id,
4667 "op_not_allowed",
4668 "op_not_allowed",
4669 );
4670 return Ok(vec![error]);
4671 }
4672
4673 let principal =
4674 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4675 Ok(principal) => principal,
4676 Err(error) => {
4677 self.observe_route_open_refusal(
4678 ctx,
4679 &caller,
4680 &target_module_id,
4681 "bad_consumer_identity",
4682 "bad_consumer_identity",
4683 );
4684 return Ok(vec![error]);
4685 }
4686 };
4687 timings.principal = principal_label(&principal);
4688
4689 if let Principal::Reserved {
4693 module_id: opening_module_id,
4694 } = &principal
4695 {
4696 if let Some(opening_registration) = self
4697 .registry
4698 .get_module(opening_module_id)
4699 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4700 {
4701 if let Some(capability) =
4702 denied_capability(&opening_registration.manifest, ®istration.manifest)
4703 {
4704 warn!(
4705 opening_module_id,
4706 target_module_id,
4707 capability,
4708 "refusing route.open because an attested capability deny edge matches"
4709 );
4710 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4711 &frame,
4712 &target_module_id,
4713 "capability_deny_edge",
4714 "capability_forbidden",
4715 format!(
4716 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4717 ),
4718 )?]);
4719 }
4720 }
4721 }
4722
4723 if admission_facts.is_some() {
4724 let carrier_matches = matches!(
4725 &principal,
4726 Principal::Reserved { module_id }
4727 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4728 );
4729 if !carrier_matches {
4730 return Ok(vec![self.route_open_refusal_frame(
4731 ctx,
4732 &caller,
4733 &frame,
4734 &target_module_id,
4735 "admission_facts_carrier_not_permitted",
4736 "admission_facts_not_permitted",
4737 "admission facts may only be carried by the configured reserved module",
4738 )?]);
4739 }
4740
4741 let target_allowed = self
4742 .admission_facts_targets
4743 .as_ref()
4744 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4745 if !target_allowed {
4746 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4747 &frame,
4748 &target_module_id,
4749 "admission_facts_target_not_listed",
4750 "admission_facts_target_not_allowed",
4751 format!(
4752 "admission facts are not permitted for target module_id '{target_module_id}'"
4753 ),
4754 )?]);
4755 }
4756
4757 }
4761
4762 let (bound_scope, mut scope_stamp) = match &scope {
4768 None => (None, None),
4769 Some(selector) => {
4770 let owner_configured = match &selector.owner {
4771 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4777 _ => false,
4778 };
4779 let mut scopes = self
4780 .scopes
4781 .write()
4782 .unwrap_or_else(|poisoned| poisoned.into_inner());
4783 let admitted = scopes
4784 .admit(&principal, &target_module_id, selector, owner_configured)
4785 .and_then(|admission| {
4786 crate::scopes::check_target_flow_support(
4787 &admission.stamp,
4788 &target_module_id,
4789 registration.manifest.capabilities.as_ref(),
4790 )?;
4791 crate::scopes::check_target_agent_run_support(
4792 &admission.stamp,
4793 &target_module_id,
4794 registration.manifest.capabilities.as_ref(),
4795 )?;
4796 let bound = scopes.reserve_bind(
4797 &admission,
4798 &principal,
4799 ctx.connection_id,
4800 selector.hold_presence,
4801 );
4802 Ok((admission, bound))
4803 });
4804 match admitted {
4805 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4806 Err(refusal) => {
4807 return Ok(vec![self.route_open_refusal_frame(
4808 ctx,
4809 &caller,
4810 &frame,
4811 &target_module_id,
4812 refusal.code,
4813 refusal.code,
4814 refusal.message,
4815 )?]);
4816 }
4817 }
4818 }
4819 };
4820 if let Some(stamp) = &scope_stamp {
4823 caller.scope = route_open_scope_label(stamp);
4824 }
4825
4826 timings.enter(RouteOpenPhase::ProjectRoot);
4848 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4849 Ok(project_root) => project_root,
4850 Err(err) => {
4851 return Ok(vec![control_error_frame(
4852 &frame,
4853 "invalid_project_root",
4854 err.to_string(),
4855 )?])
4856 }
4857 };
4858 timings.enter(RouteOpenPhase::Admission);
4859 identity.project_root = project_root.as_path().to_path_buf();
4860
4861 let _concurrency_guard = match self
4872 .route_bind_concurrency
4873 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4874 {
4875 Ok(guard) => guard,
4876 Err(in_flight) => {
4877 return Ok(vec![self.route_open_target_capacity_refusal(
4878 ctx,
4879 &caller,
4880 &frame,
4881 &target_module_id,
4882 in_flight,
4883 )?]);
4884 }
4885 };
4886
4887 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4890 RouteBindAdmission::Admitted { guard, probe } => {
4891 if probe {
4892 info!(
4893 module_id = %target_module_id,
4894 connection_id = ctx.connection_id.get(),
4895 "route.bind breaker half-open: admitting one probe"
4896 );
4897 }
4898 guard
4899 }
4900 RouteBindAdmission::Refused {
4901 consecutive_timeouts,
4902 retry_in,
4903 probe_in_flight,
4904 } => {
4905 return Ok(vec![self.route_open_breaker_refusal_frame(
4906 ctx,
4907 &caller,
4908 &frame,
4909 &target_module_id,
4910 consecutive_timeouts,
4911 retry_in,
4912 probe_in_flight,
4913 )?]);
4914 }
4915 };
4916
4917 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4922 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4923 timings.enter(RouteOpenPhase::EgressReservation);
4924 let pending = match self
4925 .forwarding
4926 .begin_route_bind_relay_for(
4927 ctx.connection_id,
4928 ctx.egress.clone(),
4929 response_version(&frame),
4930 frame.header.corr,
4931 &target_module_id,
4932 principal.clone(),
4933 bound_scope,
4934 Some(project_root),
4935 relay_deadline,
4936 )
4937 .await
4938 {
4939 Ok(pending) => pending,
4940 Err(err) => {
4941 return Ok(vec![self.route_open_refusal_frame(
4942 ctx,
4943 &caller,
4944 &frame,
4945 &target_module_id,
4946 "relay_reservation_failed",
4947 forwarding_error_code(&err),
4948 err.to_string(),
4949 )?])
4950 }
4951 };
4952 timings.enter(RouteOpenPhase::Admission);
4953 let crate::forwarding::PendingRouteBindRelay {
4954 endpoint,
4955 module_sink,
4956 negotiated_ver,
4957 client_channel,
4958 client_epoch,
4959 module_channel,
4960 module_epoch,
4961 corr: relay_corr,
4962 receiver,
4963 } = pending;
4964 let mut reservation =
4965 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4966
4967 let relay_registration = self
4972 .registry
4973 .get_module_by_connection(endpoint.connection_id)
4974 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4975 let relay_capabilities = relay_registration
4976 .as_ref()
4977 .and_then(|registration| registration.manifest.capabilities.as_ref());
4978 if let Some(selector) = &scope {
4979 if let Err(refusal) = self
4982 .scopes
4983 .read()
4984 .unwrap_or_else(|poisoned| poisoned.into_inner())
4985 .check_bind_proof(&principal, selector)
4986 {
4987 reservation.release_and_disarm();
4988 return Ok(vec![self.route_open_refusal_frame(
4989 ctx,
4990 &caller,
4991 &frame,
4992 &target_module_id,
4993 refusal.code,
4994 refusal.code,
4995 refusal.message,
4996 )?]);
4997 }
4998 }
4999 if let Some(stamp) = scope_stamp
5000 .as_ref()
5001 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5002 {
5003 if let Err(refusal) = crate::scopes::check_target_flow_support(
5004 stamp,
5005 &target_module_id,
5006 relay_capabilities,
5007 )
5008 .and_then(|()| {
5009 crate::scopes::check_target_agent_run_support(
5010 stamp,
5011 &target_module_id,
5012 relay_capabilities,
5013 )
5014 }) {
5015 reservation.release_and_disarm();
5016 return Ok(vec![self.route_open_refusal_frame(
5017 ctx,
5018 &caller,
5019 &frame,
5020 &target_module_id,
5021 refusal.code,
5022 refusal.code,
5023 refusal.message,
5024 )?]);
5025 }
5026 }
5027
5028 if !relay_capabilities.is_some_and(|caps| {
5031 caps.provides
5032 .iter()
5033 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5034 }) {
5035 if let Some(stamp) = scope_stamp.as_mut() {
5036 stamp.workdir = None;
5037 stamp.project_id = None;
5038 }
5039 }
5040 if !relay_capabilities.is_some_and(|caps| {
5041 caps.provides
5042 .iter()
5043 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5044 }) {
5045 if let Some(stamp) = scope_stamp.as_mut() {
5046 stamp.attributes.on_behalf_of_agent = None;
5047 }
5048 }
5049 let call_mode = if relay_capabilities
5050 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5051 {
5052 Some(derive_call_mode(scope_stamp.as_ref()))
5053 } else {
5054 if let Some(stamp) = scope_stamp.as_mut() {
5055 stamp.expires_at_ms = None;
5056 }
5057 None
5058 };
5059
5060 debug!(
5061 connection_id = ctx.connection_id.get(),
5062 client_channel,
5063 client_epoch,
5064 module_channel,
5065 module_epoch,
5066 "reserved route handle pair"
5067 );
5068 let route_principal_label = principal_label(&principal);
5071 let relay = ModuleControlRequest::RouteBind {
5072 route_channel: module_channel,
5073 epoch: module_epoch,
5074 target,
5075 identity,
5076 principal: Some(principal),
5077 consumer_capabilities,
5078 role_versions,
5079 admission_facts,
5080 scope: scope_stamp,
5081 call_mode,
5082 };
5083 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5084 RouterError::backend(
5085 0,
5086 frame.header.corr,
5087 format!("failed to encode route.bind request: {err}"),
5088 )
5089 })?;
5090 let relay_frame = Frame::build_with_version(
5091 negotiated_ver,
5092 FrameType::Request,
5093 control_flags(),
5094 0,
5095 0,
5096 relay_corr,
5097 relay_body,
5098 )
5099 .map_err(RouterError::FrameBuild)?;
5100
5101 timings.enter(RouteOpenPhase::BindSend);
5102 let sent = module_sink.send(relay_frame).await;
5103 timings.enter(RouteOpenPhase::Admission);
5104 if let Err(err) = sent {
5105 reservation.release_and_disarm();
5106 return Ok(vec![self.route_open_refusal_frame(
5107 ctx,
5108 &caller,
5109 &frame,
5110 &target_module_id,
5111 "relay_send_failed",
5112 "target_unavailable",
5113 err.to_string(),
5114 )?]);
5115 }
5116
5117 #[cfg(test)]
5118 if let Some(hold) = &self.route_bind_relay_sent_hold {
5119 hold.wait().await;
5123 hold.wait().await;
5124 }
5125
5126 if self
5131 .forwarding
5132 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5133 .map_err(RouterError::Forwarding)?
5134 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5135 {
5136 self.send_abandoned_route_bind_goodbye(
5137 &module_sink,
5138 negotiated_ver,
5139 module_channel,
5140 module_epoch,
5141 );
5142 }
5143
5144 timings.enter(RouteOpenPhase::BindAckWait);
5145 let outcome = timeout_at(relay_deadline, receiver).await;
5146 timings.enter(RouteOpenPhase::Admission);
5147 match outcome {
5148 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5149 reservation.disarm();
5150 if breaker.record_accepted() {
5151 info!(
5152 module_id = %target_module_id,
5153 "route.bind breaker closed: the probe was accepted"
5154 );
5155 }
5156 self.observe_route_open_accept(
5157 ctx,
5158 &target_module_id,
5159 &route_principal_label,
5160 &caller.scope,
5161 );
5162 Ok(Vec::new())
5163 }
5164 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5165 reservation.release_and_disarm();
5166 breaker.record_inconclusive();
5170 let scope_code = match body.code.as_str() {
5174 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5175 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5176 _ => None,
5177 };
5178 if let Some(code) = scope_code {
5179 self.observe_route_open_refusal(
5180 ctx,
5181 &caller,
5182 &target_module_id,
5183 "scope_changed_before_commit",
5184 code,
5185 );
5186 return Ok(vec![control_error_body_frame(&frame, body)?]);
5187 }
5188 self.counters
5189 .increment_route_open_refused("module_rejected");
5190 info!(
5191 target: "control",
5192 code = "module_rejected",
5193 module_code = ?body.code,
5194 module_id = ?target_module_id,
5195 connection_id = ctx.connection_id.get(),
5196 principal = %caller.principal,
5197 scope = %caller.scope,
5198 harness = %caller.harness,
5199 "route.open refused"
5200 );
5201 Ok(vec![control_error_body_frame(&frame, body)?])
5202 }
5203 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5204 reservation.release_and_disarm();
5205 breaker.record_inconclusive();
5206 tracing::warn!(
5213 module_id = %target_module_id,
5214 "route.bind relay abandoned: {message}"
5215 );
5216 Ok(vec![self.route_open_refusal_frame(
5217 ctx,
5218 &caller,
5219 &frame,
5220 &target_module_id,
5221 "relay_abandoned",
5222 "target_unavailable",
5223 message,
5224 )?])
5225 }
5226 Ok(Err(_)) => {
5227 reservation.release_and_disarm();
5228 breaker.record_inconclusive();
5229 Ok(vec![self.route_open_refusal_frame(
5230 ctx,
5231 &caller,
5232 &frame,
5233 &target_module_id,
5234 "relay_waiter_canceled",
5235 "target_unavailable",
5236 "route.bind relay waiter was canceled before the module responded",
5237 )?])
5238 }
5239 Err(_) => {
5240 reservation.release_and_disarm();
5241 if let Some(opened) = breaker.record_timeout(
5245 self.route_bind_breaker_threshold,
5246 self.route_bind_breaker_cooldown,
5247 ) {
5248 warn!(
5249 module_id = %target_module_id,
5250 consecutive_timeouts = opened.consecutive_timeouts,
5251 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5252 reopened_after_probe = opened.reopened_after_probe,
5253 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5254 );
5255 }
5256 tracing::warn!(
5264 module_id = %target_module_id,
5265 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5266 "route.bind relay timed out: module did not ack within budget"
5267 );
5268 Ok(vec![self.route_open_refusal_frame(
5269 ctx,
5270 &caller,
5271 &frame,
5272 &target_module_id,
5273 "relay_timed_out",
5274 "module_timeout",
5275 format!(
5276 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5277 route_bind_relay_timeout
5278 ),
5279 )?])
5280 }
5281 }
5282 }
5283
5284 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5285 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5286 snapshot: self.supervisor.spawn_snapshot(),
5287 };
5288 Ok(vec![control_response_body_frame(
5289 &frame,
5290 &response,
5291 "ClientControlResponse::SupervisorSpawnSnapshot",
5292 )?])
5293 }
5294
5295 fn handle_supervisor_spawn_subscribe(
5296 &self,
5297 ctx: &RouteCtx,
5298 frame: Frame,
5299 since: Option<SpawnCursor>,
5300 ) -> Result<Vec<Frame>, RouterError> {
5301 match self.supervisor.subscribe_spawns(
5302 ctx.connection_id,
5303 frame.header.corr,
5304 response_version(&frame),
5305 since,
5306 ctx.egress.clone(),
5307 ) {
5308 Ok(()) => Ok(Vec::new()),
5309 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5310 Ok(vec![control_error_body_frame(
5311 &frame,
5312 ErrorBody {
5313 code: "spawn_cursor_incarnation_mismatch".to_string(),
5314 message: "spawn cursor belongs to a different daemon incarnation"
5315 .to_string(),
5316 detail: Some(serde_json::json!({
5317 "current_daemon_incarnation": current
5318 })),
5319 },
5320 )?])
5321 }
5322 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5323 &frame,
5324 ErrorBody {
5325 code: "spawn_cursor_too_old".to_string(),
5326 message: "spawn cursor predates the retained event ring".to_string(),
5327 detail: Some(serde_json::json!({
5328 "oldest_retained_cursor": oldest
5329 })),
5330 },
5331 )?]),
5332 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5333 0,
5334 frame.header.corr,
5335 format!("failed to open supervisor spawn subscription: {error}"),
5336 )),
5337 }
5338 }
5339
5340 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5341 let generation = self
5342 .registry
5343 .generation()
5344 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5345 let mut modules = Vec::new();
5346 for module in self.supervisor.list() {
5347 let status = module.status_for_control("list").map_err(|err| {
5348 RouterError::backend(
5349 0,
5350 frame.header.corr,
5351 format!("failed to read supervisor status: {err}"),
5352 )
5353 })?;
5354 let (configured, _) = module.configuration().map_err(|err| {
5355 RouterError::backend(
5356 0,
5357 frame.header.corr,
5358 format!("failed to read module configuration: {err}"),
5359 )
5360 })?;
5361 let image = module.running_image_agreement().await;
5363 let resources = Some(module.child_resource_usage());
5366 let mut verdict =
5367 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5368 verdict.spec = Some(launch_spec_agreement(
5369 &configured,
5370 status.process_alive,
5371 status.launch_spec_digest.as_ref(),
5372 ));
5373 let pending_reload = Some(verdict);
5374 let configured_enabled = self
5375 .capability_evaluator
5376 .configured_enabled(&status.module_id);
5377 modules.push(
5380 SupervisorEntry::new(
5381 status.module_id,
5382 status.state.to_string(),
5383 status.enabled,
5384 status.live,
5385 status.health.status,
5386 )
5387 .with_configured_enabled(configured_enabled)
5388 .with_launch_nonce_env(Some(!cfg!(unix)))
5389 .with_launch_nonce_source(status.launch_nonce_source)
5390 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5391 .with_protocol(status.protocol)
5392 .with_pending_reload(pending_reload)
5393 .with_last_probe_ms(status.health.last_probe_ms)
5394 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5395 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5396 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5397 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5398 .with_restart_count(Some(status.restart_count))
5399 .with_max_restarts(Some(status.max_restarts))
5400 .with_lifetime_restarts(Some(status.lifetime_restarts))
5401 .with_spawn_generation(Some(status.spawn_generation))
5402 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5403 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5404 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5405 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5406 .with_resources(resources),
5407 );
5408 }
5409 let response = ClientControlResponse::SupervisorList {
5410 generation,
5411 modules,
5412 };
5413 Ok(vec![control_response_body_frame(
5414 &frame,
5415 &response,
5416 "ClientControlResponse::SupervisorList",
5417 )?])
5418 }
5419
5420 fn handle_supervisor_stderr_tail(
5421 &self,
5422 frame: Frame,
5423 module_id: String,
5424 max_lines: Option<u32>,
5425 max_bytes: Option<u32>,
5426 ) -> Result<Vec<Frame>, RouterError> {
5427 let Some(module) = self.supervisor.get(&module_id) else {
5428 return Ok(vec![control_error_frame(
5429 &frame,
5430 "unknown_module",
5431 format!("module_id '{module_id}' is not supervised"),
5432 )?]);
5433 };
5434
5435 let snapshot = module.stderr_tail(
5436 max_lines.map(|value| value as usize),
5437 max_bytes.map(|value| value as usize),
5438 );
5439
5440 let response = ClientControlResponse::SupervisorStderrTail {
5441 module_id,
5442 tail: StderrTail {
5443 capture: match snapshot.capture {
5444 CaptureState::Captured => StderrCaptureState::Captured,
5445 CaptureState::Incomplete { reason } => {
5446 StderrCaptureState::Incomplete { reason }
5447 }
5448 CaptureState::NotCaptured { reason } => {
5449 StderrCaptureState::NotCaptured { reason }
5450 }
5451 },
5452 entries: snapshot
5453 .entries
5454 .into_iter()
5455 .map(|entry| match entry {
5456 TailEntry::Line {
5457 text,
5458 truncated,
5459 at_ms,
5460 } => StderrTailEntry::Line {
5461 text,
5462 truncated,
5463 at_ms,
5464 },
5465 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5466 })
5467 .collect(),
5468 dropped_lines: snapshot.dropped_lines,
5469 },
5470 };
5471 Ok(vec![control_response_body_frame(
5472 &frame,
5473 &response,
5474 "ClientControlResponse::SupervisorStderrTail",
5475 )?])
5476 }
5477
5478 async fn handle_supervisor_terminals(
5479 &self,
5480 frame: Frame,
5481 module_id: String,
5482 ) -> Result<Vec<Frame>, RouterError> {
5483 let Some(module) = self.supervisor.get(&module_id) else {
5484 return Ok(vec![control_error_frame(
5485 &frame,
5486 "unknown_module",
5487 format!("module_id '{module_id}' is not supervised"),
5488 )?]);
5489 };
5490
5491 let terminals = module
5494 .read_durable_terminal_history()
5495 .await
5496 .map_err(|error| {
5497 RouterError::backend(
5498 0,
5499 frame.header.corr,
5500 format!("failed to read terminal history: {error}"),
5501 )
5502 })?;
5503 let response = ClientControlResponse::SupervisorTerminals {
5504 module_id,
5505 terminals,
5506 };
5507 Ok(vec![control_response_body_frame(
5508 &frame,
5509 &response,
5510 "ClientControlResponse::SupervisorTerminals",
5511 )?])
5512 }
5513
5514 fn handle_supervisor_routes(
5515 &self,
5516 frame: Frame,
5517 module_id: Option<String>,
5518 ) -> Result<Vec<Frame>, RouterError> {
5519 let modules = self
5520 .forwarding
5521 .route_census(module_id.as_deref())
5522 .map_err(RouterError::Forwarding)?
5523 .into_iter()
5524 .map(|(module_id, routes)| SupervisorRouteModule {
5525 module_id,
5526 routes: routes
5527 .into_iter()
5528 .map(|route| SupervisorRoute {
5529 consumer: match route.principal {
5530 Principal::Reserved { module_id } => {
5531 SupervisorRouteConsumer::Reserved { module_id }
5532 }
5533 Principal::Direct | Principal::Unverified => {
5534 SupervisorRouteConsumer::Direct {
5535 connection_id: route.goodbye_target.connection_id.get(),
5536 }
5537 }
5538 },
5539 age_ms: Instant::now()
5540 .saturating_duration_since(route.bound_at)
5541 .as_millis()
5542 .try_into()
5543 .unwrap_or(u64::MAX),
5544 draining: route.draining,
5545 drain_reason: route.drain_reason,
5546 })
5547 .collect(),
5548 })
5549 .collect();
5550 let response = ClientControlResponse::SupervisorRoutes { modules };
5551 Ok(vec![control_response_body_frame(
5552 &frame,
5553 &response,
5554 "ClientControlResponse::SupervisorRoutes",
5555 )?])
5556 }
5557
5558 async fn handle_supervisor_provenance(
5559 &self,
5560 frame: Frame,
5561 module_id: Option<String>,
5562 ) -> Result<Vec<Frame>, RouterError> {
5563 let mut selected = if let Some(module_id) = module_id {
5564 let Some(module) = self.supervisor.get(&module_id) else {
5565 return Ok(vec![control_error_frame(
5566 &frame,
5567 "unknown_module",
5568 format!("module_id '{module_id}' is not supervised"),
5569 )?]);
5570 };
5571 vec![module]
5572 } else {
5573 self.supervisor.list()
5574 };
5575
5576 let mut modules = Vec::with_capacity(selected.len());
5577 for module in selected.drain(..) {
5578 let (status, observed_image) = module
5579 .status_and_running_image_agreement()
5580 .await
5581 .map_err(|err| {
5582 RouterError::backend(
5583 0,
5584 frame.header.corr,
5585 format!("failed to read supervisor status: {err}"),
5586 )
5587 })?;
5588 let module_declared = self
5589 .registry
5590 .get_module(&status.module_id)
5591 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5592 .and_then(|registration| registration.manifest.provenance)
5593 .map(|build| ModuleDeclaredProvenance::Reported { build })
5594 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5595 #[cfg(test)]
5596 let running_image = match &self.provenance_probe_override {
5597 Some(result) => result.clone(),
5598 None => observed_image,
5599 };
5600 #[cfg(not(test))]
5601 let running_image = observed_image;
5602 modules.push(SupervisorModuleProvenance {
5603 module_id: status.module_id,
5604 module_declared,
5605 daemon_observed: SupervisorObservedProcess {
5606 launch_nonce_source: status.launch_nonce_source,
5607 pid: status.pid,
5608 spawned_at_ms: status.spawned_at_ms,
5609 spawned_from: status.spawned_from,
5610 running_image,
5611 },
5612 });
5613 }
5614 let daemon = SupervisorDaemonProvenance {
5615 daemon_build: self.daemon_provenance.build.clone(),
5616 daemon_observed: DaemonObservedProcess {
5617 pid: self.daemon_provenance.pid,
5618 started_at_ms: self
5619 .daemon_provenance
5620 .start_clock
5621 .map(|clock| clock.started_at_ms())
5622 .or(self.daemon_provenance.started_at_ms),
5623 running_image: self
5624 .daemon_provenance
5625 .probe
5626 .observe(
5627 self.daemon_provenance.pid,
5628 self.daemon_provenance.executable_path.as_deref(),
5629 self.daemon_provenance.executable_identity.clone(),
5630 self.daemon_provenance.process_start_time,
5631 )
5632 .await,
5633 },
5634 };
5635 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5636 Ok(vec![control_response_body_frame(
5637 &frame,
5638 &response,
5639 "ClientControlResponse::SupervisorProvenance",
5640 )?])
5641 }
5642
5643 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5644 self.refresh_capability_requirements();
5645 let generation = self
5646 .registry
5647 .generation()
5648 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5649 let modules = self
5650 .supervisor
5651 .list()
5652 .into_iter()
5653 .map(|module| {
5654 let status = module.status_for_control("health").map_err(|err| {
5655 RouterError::backend(
5656 0,
5657 frame.header.corr,
5658 format!("failed to read supervisor health: {err}"),
5659 )
5660 })?;
5661 let module_id = status.module_id;
5662 let capability_detail = self
5663 .capability_evaluator
5664 .required_problem_detail(&module_id);
5665 Ok(SupervisorHealthEntry {
5666 module_id,
5667 status: status.health.status,
5668 detail: append_capability_problem_detail(
5669 status.health.detail,
5670 capability_detail,
5671 ),
5672 metrics: status.health.metrics,
5673 consecutive_failures: status.health.consecutive_failures,
5674 late_answer_count: status.health.late_answer_count,
5675 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5676 last_action: status.health.last_action,
5677 last_action_ms: status.health.last_action_ms,
5678 last_probe_ms: status.health.last_probe_ms,
5679 })
5680 })
5681 .collect::<Result<Vec<_>, RouterError>>()?;
5682 let response = ClientControlResponse::SupervisorHealth {
5683 generation,
5684 modules,
5685 };
5686 Ok(vec![control_response_body_frame(
5687 &frame,
5688 &response,
5689 "ClientControlResponse::SupervisorHealth",
5690 )?])
5691 }
5692
5693 async fn handle_supervisor_restart(
5694 &self,
5695 frame: Frame,
5696 module_id: String,
5697 drain_timeout_ms: Option<u64>,
5698 requester: Principal,
5699 ) -> Result<Vec<Frame>, RouterError> {
5700 let operation_lock = self.supervisor.operation_lock();
5701 let _operation_guard = operation_lock.lock().await;
5702 let Some(module) = self.supervisor.get(&module_id) else {
5703 return Ok(vec![control_error_frame(
5704 &frame,
5705 "unknown_module",
5706 format!("module_id '{module_id}' is not supervised"),
5707 )?]);
5708 };
5709
5710 self.route_outages.mark_operator_action(&module_id);
5711 if let Err(err) = module
5712 .restart_requested(drain_timeout_ms, Some(requester))
5713 .await
5714 {
5715 self.route_outages
5716 .operator_action_ended_unrefused(&module_id);
5717 let (code, message) = match err {
5718 crate::supervise::SuperviseError::Disabled { .. } => {
5719 ("module_disabled", err.to_string())
5720 }
5721 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5722 ("swap_in_progress", err.to_string())
5723 }
5724 _ => (
5725 "target_unavailable",
5726 format!("failed to restart module_id '{module_id}': {err}"),
5727 ),
5728 };
5729 return Ok(vec![control_error_frame(&frame, code, message)?]);
5730 }
5731
5732 let response = ClientControlResponse::SupervisorAck {
5733 module_id,
5734 applied: true,
5735 };
5736 Ok(vec![control_response_body_frame(
5737 &frame,
5738 &response,
5739 "ClientControlResponse::SupervisorAck",
5740 )?])
5741 }
5742
5743 async fn handle_supervisor_swap(
5747 &self,
5748 frame: Frame,
5749 module_id: String,
5750 ready_timeout_ms: Option<u64>,
5751 requester: Principal,
5752 ) -> Result<Vec<Frame>, RouterError> {
5753 let module = {
5760 let operation_lock = self.supervisor.operation_lock();
5761 let _operation_guard = operation_lock.lock().await;
5762 self.supervisor.get(&module_id)
5763 };
5764 let Some(module) = module else {
5765 return Ok(vec![control_error_frame(
5766 &frame,
5767 "unknown_module",
5768 format!("module_id '{module_id}' is not supervised"),
5769 )?]);
5770 };
5771
5772 self.route_outages.mark_operator_action(&module_id);
5773 if let Err(err) = module
5774 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5775 .await
5776 {
5777 self.route_outages
5778 .operator_action_ended_unrefused(&module_id);
5779 use crate::supervise::SuperviseError;
5780 let message = err.to_string();
5781 let error = match err {
5782 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5783 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5784 code: "swap_refused".to_string(),
5785 message,
5786 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5787 },
5788 SuperviseError::SwapFailed {
5789 arm,
5790 candidate_exit,
5791 ..
5792 } => ErrorBody {
5793 code: "swap_failed".to_string(),
5794 message,
5795 detail: Some(serde_json::json!({
5796 "arm": arm.as_str(),
5797 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5798 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5799 })),
5800 },
5801 _ => ErrorBody::new(
5802 "target_unavailable",
5803 format!("failed to swap module_id '{module_id}': {message}"),
5804 ),
5805 };
5806 return Ok(vec![control_error_body_frame(&frame, error)?]);
5807 }
5808 self.route_outages
5812 .operator_action_ended_unrefused(&module_id);
5813
5814 let response = ClientControlResponse::SupervisorAck {
5815 module_id,
5816 applied: true,
5817 };
5818 Ok(vec![control_response_body_frame(
5819 &frame,
5820 &response,
5821 "ClientControlResponse::SupervisorAck",
5822 )?])
5823 }
5824
5825 async fn handle_supervisor_reload(
5826 &self,
5827 frame: Frame,
5828 module_id: String,
5829 requester: Principal,
5830 ) -> Result<Vec<Frame>, RouterError> {
5831 let operation_lock = self.supervisor.operation_lock();
5832 let _operation_guard = operation_lock.lock().await;
5833 let Some(module) = self.supervisor.get(&module_id) else {
5834 return Ok(vec![control_error_frame(
5835 &frame,
5836 "unknown_module",
5837 format!("module_id '{module_id}' is not supervised"),
5838 )?]);
5839 };
5840
5841 self.route_outages.mark_operator_action(&module_id);
5842 if let Err(err) = module.reload_requested(Some(requester)).await {
5843 self.route_outages
5844 .operator_action_ended_unrefused(&module_id);
5845 let (code, message) = match err {
5846 crate::supervise::SuperviseError::Disabled { .. } => {
5847 ("module_disabled", err.to_string())
5848 }
5849 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5850 ("swap_in_progress", err.to_string())
5851 }
5852 _ => (
5853 "reload_failed",
5854 format!("failed to reload module_id '{module_id}': {err}"),
5855 ),
5856 };
5857 return Ok(vec![control_error_frame(&frame, code, message)?]);
5858 }
5859
5860 let response = ClientControlResponse::SupervisorAck {
5861 module_id,
5862 applied: true,
5863 };
5864 Ok(vec![control_response_body_frame(
5865 &frame,
5866 &response,
5867 "ClientControlResponse::SupervisorAck",
5868 )?])
5869 }
5870
5871 async fn handle_supervisor_rescan(
5875 &self,
5876 frame: Frame,
5877 mode: RescanMode,
5878 requester: Principal,
5879 ) -> Result<Vec<Frame>, RouterError> {
5880 let preview = mode.previews();
5881 let Some(context) = self.rescan.clone() else {
5882 return Ok(vec![control_error_frame(
5883 &frame,
5884 "rescan_unavailable",
5885 "the daemon was not started with a reloadable config path".to_string(),
5886 )?]);
5887 };
5888
5889 let operation_lock = self.supervisor.operation_lock();
5890 let _operation_guard = operation_lock.lock().await;
5891 let loaded = match self.load_rescan_config(&mode, &context) {
5895 Ok(config) => config,
5896 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5897 };
5898 let Some((config, activity_config)) = loaded else {
5906 return Ok(vec![control_error_frame(
5907 &frame,
5908 "invalid_daemon_config",
5909 format!(
5910 "daemon config not found at {}; refusing to rescan (an absent config would \
5911 retire every supervised module)",
5912 context.config_path.display()
5913 ),
5914 )?]);
5915 };
5916 let (
5917 configured_port,
5918 storage_config,
5919 admission_facts_carrier_module_id,
5920 admission_facts_targets,
5921 scope_authority_owners,
5922 privileged_operator_prompt,
5923 modules,
5924 reserved_capabilities,
5925 ) = (
5926 config.port,
5927 config.storage,
5928 config.admission_facts_carrier_module_id,
5929 config.admission_facts_targets,
5930 config.scope_authority_owners,
5931 config.privileged_operator_prompt,
5932 config.modules,
5933 config.reserved_capabilities,
5934 );
5935
5936 let mut restart_required = Vec::new();
5945 for section in RestartRequiredSection::ALL {
5946 let changed = match section {
5947 RestartRequiredSection::Port => configured_port != context.configured_port,
5948 RestartRequiredSection::Storage => storage_config != context.storage_config,
5949 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5950 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5951 }
5952 RestartRequiredSection::AdmissionFactsTargets => {
5953 admission_facts_targets != context.admission_facts_targets
5954 }
5955 RestartRequiredSection::ScopeAuthorityOwners => {
5956 scope_authority_owners != context.scope_authority_owners
5957 }
5958 RestartRequiredSection::PrivilegedOperatorPrompt => {
5959 privileged_operator_prompt != context.privileged_operator_prompt
5960 }
5961 };
5962 if changed {
5963 restart_required.push(section.label().to_string());
5964 }
5965 }
5966 if !restart_required.is_empty() {
5967 warn!(
5968 config_path = %context.config_path.display(),
5969 sections = %restart_required.join(", "),
5970 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5971 );
5972 }
5973
5974 for configured in &modules {
5975 if let Err(err) = validate_spec(&configured.module_spec()) {
5976 return Ok(vec![control_error_frame(
5977 &frame,
5978 "invalid_daemon_config",
5979 format!("supervisor rescan rejected daemon config: {err}"),
5980 )?]);
5981 }
5982 }
5983
5984 let configured_capabilities = modules
5985 .iter()
5986 .map(|module| (module.module_id.clone(), module.enabled))
5987 .collect::<Vec<_>>();
5988 let preview_capability_warnings = if preview {
5989 let (_, registrations) = self.runtime_capability_snapshot()?;
5990 let current_modules = self
5991 .supervisor
5992 .list()
5993 .into_iter()
5994 .map(|module| module.module_id().to_string())
5995 .collect::<BTreeSet<_>>();
5996 let resulting_modules = configured_capabilities.clone();
5997 let removed = current_modules
5998 .into_iter()
5999 .filter(|module_id| {
6000 !resulting_modules
6001 .iter()
6002 .any(|(configured_id, _)| configured_id == module_id)
6003 })
6004 .collect::<Vec<_>>();
6005 let disabled = match &mode {
6008 RescanMode::Candidate(_) => resulting_modules
6009 .iter()
6010 .filter(|(_, enabled)| !enabled)
6011 .map(|(module_id, _)| module_id.clone())
6012 .collect::<Vec<_>>(),
6013 _ => Vec::new(),
6014 };
6015 self.capability_evaluator.preview_warnings(
6016 resulting_modules,
6017 &removed,
6018 &disabled,
6019 ®istrations,
6020 )
6021 } else {
6022 Vec::new()
6023 };
6024 let reconciled = match &mode {
6025 RescanMode::Candidate(candidate) => self
6026 .preview_candidate(&context, modules, candidate)
6027 .map(|result| (result, Vec::new())),
6028 _ => {
6029 self.reconcile_supervised_modules(
6030 &context.supervisor,
6031 modules,
6032 preview,
6033 requester.clone(),
6034 )
6035 .await
6036 }
6037 };
6038 let (result, changed) = match reconciled {
6039 Ok(result) => result,
6040 Err(message) => {
6041 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6042 }
6043 };
6044 if !preview {
6045 if let Some(ledger) = &self.activity {
6046 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6047 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6048 }
6049 if let Err(code) = self.configure_activity(activity_config).await {
6050 warn!(
6051 code,
6052 "activity retention could not be applied during rescan"
6053 );
6054 }
6055 self.capability_evaluator
6056 .configure(configured_capabilities, reserved_capabilities);
6057 self.capability_evaluator.wake_deadline_loop();
6058 self.refresh_capability_requirements();
6059 }
6060 let mut result = result;
6061 result.restart_required = restart_required;
6062 result.capability_warnings = preview_capability_warnings;
6063 let response = ClientControlResponse::SupervisorRescan { result };
6064 Ok(vec![control_response_body_frame(
6065 &frame,
6066 &response,
6067 "ClientControlResponse::SupervisorRescan",
6068 )?])
6069 }
6070
6071 async fn handle_supervisor_release_reserved(
6072 &self,
6073 frame: Frame,
6074 module_id: String,
6075 ) -> Result<Vec<Frame>, RouterError> {
6076 let Some(context) = self.rescan.clone() else {
6077 return Ok(vec![control_error_frame(
6078 &frame,
6079 "release_unavailable",
6080 "reserved-id release requires a daemon started with a reloadable config path",
6081 )?]);
6082 };
6083 let operation_lock = self.supervisor.operation_lock();
6084 let _operation_guard = operation_lock.lock().await;
6085 let loaded = match crate::daemon_config::load(&context.config_path) {
6086 Ok(Some(config)) => config,
6087 Ok(None) => {
6088 return Ok(vec![control_error_frame(
6089 &frame,
6090 "invalid_daemon_config",
6091 format!(
6092 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6093 context.config_path.display()
6094 ),
6095 )?])
6096 }
6097 Err(err) => {
6098 return Ok(vec![control_error_frame(
6099 &frame,
6100 "invalid_daemon_config",
6101 format!("unable to verify reserved-id release against daemon config: {err}"),
6102 )?])
6103 }
6104 };
6105 if loaded
6106 .modules
6107 .iter()
6108 .any(|configured| configured.module_id == module_id)
6109 {
6110 return Ok(vec![control_error_frame(
6111 &frame,
6112 "reserved_module_configured",
6113 format!(
6114 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6115 ),
6116 )?]);
6117 }
6118 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6119 return Ok(vec![control_error_frame(
6120 &frame,
6121 "reserved_gate_not_retained",
6122 format!(
6123 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6124 ),
6125 )?]);
6126 }
6127
6128 let response = ClientControlResponse::SupervisorAck {
6129 module_id,
6130 applied: true,
6131 };
6132 Ok(vec![control_response_body_frame(
6133 &frame,
6134 &response,
6135 "ClientControlResponse::SupervisorAck",
6136 )?])
6137 }
6138
6139 fn plan_rescan(
6143 &self,
6144 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6145 ) -> Result<RescanPlan, String> {
6146 let mut current = BTreeMap::new();
6147 for module in self.supervisor.list() {
6148 let (spec, health) = module.configuration().map_err(|err| {
6149 format!(
6150 "failed to read configuration for module_id '{}': {err}",
6151 module.module_id()
6152 )
6153 })?;
6154 let enabled = module
6155 .status()
6156 .map_err(|err| {
6157 format!(
6158 "failed to read status for module_id '{}': {err}",
6159 module.module_id()
6160 )
6161 })?
6162 .enabled;
6163 current.insert(
6164 module.module_id().to_string(),
6165 (module, spec, health, enabled),
6166 );
6167 }
6168 let configured = configured_modules
6169 .into_iter()
6170 .map(|module| (module.module_id.clone(), module))
6171 .collect::<BTreeMap<_, _>>();
6172
6173 let added = configured
6174 .keys()
6175 .filter(|module_id| !current.contains_key(*module_id))
6176 .cloned()
6177 .collect::<Vec<_>>();
6178 let removed = current
6179 .keys()
6180 .filter(|module_id| !configured.contains_key(*module_id))
6181 .cloned()
6182 .collect::<Vec<_>>();
6183 let mut changed_pending_reload = Vec::new();
6184 let mut configuration_changes = BTreeSet::new();
6185 let mut enabled_changes = BTreeSet::new();
6186 let mut unchanged = 0_u32;
6187
6188 for (module_id, configured_module) in &configured {
6189 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6190 else {
6191 continue;
6192 };
6193 let launch_changed = *current_spec != configured_module.module_spec();
6197 let configuration_changed =
6198 launch_changed || *current_health != configured_module.health;
6199 let enabled_changed = *current_enabled != configured_module.enabled;
6200 if configuration_changed {
6201 configuration_changes.insert(module_id.clone());
6202 }
6203 if launch_changed {
6204 changed_pending_reload.push(module_id.clone());
6205 }
6206 if enabled_changed {
6207 enabled_changes.insert(module_id.clone());
6208 }
6209 if !configuration_changed && !enabled_changed {
6210 unchanged = unchanged.saturating_add(1);
6211 }
6212 }
6213 Ok(RescanPlan {
6214 current,
6215 configured,
6216 added,
6217 removed,
6218 changed_pending_reload,
6219 configuration_changes,
6220 enabled_changes,
6221 unchanged,
6222 })
6223 }
6224
6225 async fn reconcile_supervised_modules(
6234 &self,
6235 supervisor: &Supervisor,
6236 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6237 preview: bool,
6238 requester: Principal,
6239 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6240 let plan = self.plan_rescan(configured_modules)?;
6241
6242 if preview {
6248 return Ok((plan.preview_result(), Vec::new()));
6249 }
6250 let changed = plan
6251 .configuration_changes
6252 .union(&plan.enabled_changes)
6253 .cloned()
6254 .collect::<Vec<_>>();
6255 let RescanPlan {
6256 current,
6257 configured,
6258 added,
6259 removed,
6260 changed_pending_reload,
6261 configuration_changes,
6262 enabled_changes,
6263 unchanged,
6264 } = plan;
6265 if !(added.is_empty()
6270 && removed.is_empty()
6271 && configuration_changes.is_empty()
6272 && enabled_changes.is_empty())
6273 {
6274 if let Some(context) = &self.rescan {
6275 context
6276 .generation
6277 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6278 }
6279 }
6280
6281 for module_id in &removed {
6282 let module = ¤t
6283 .get(module_id)
6284 .expect("removed module came from current supervisor state")
6285 .0;
6286 module
6287 .retire_requested(Some(requester.clone()))
6288 .await
6289 .map_err(|err| {
6290 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6291 })?;
6292 self.supervisor.record_rescan_removal(module_id);
6326 self.supervisor.retire(module_id);
6327 self.route_outages.forget(module_id);
6328 }
6329
6330 for module_id in configured.keys() {
6331 let Some((module, _, _, _)) = current.get(module_id) else {
6332 continue;
6333 };
6334 let configured_module = configured
6335 .get(module_id)
6336 .expect("configured module id came from configured map");
6337 if configuration_changes.contains(module_id) {
6338 module
6339 .update_configuration(
6340 configured_module.module_spec(),
6341 configured_module.health.clone(),
6342 configured_module.drain_timeout_ms,
6343 )
6344 .await
6345 .map_err(|err| {
6346 format!(
6347 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6348 )
6349 })?;
6350 }
6351 if enabled_changes.contains(module_id) {
6352 self.route_outages.mark_operator_action(module_id);
6355 module
6356 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6357 .await
6358 .map_err(|err| {
6359 self.route_outages.operator_action_ended_unrefused(module_id);
6360 format!(
6361 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6362 configured_module.enabled
6363 )
6364 })?;
6365 }
6366 }
6367
6368 for module_id in &added {
6369 let configured_module = configured
6370 .get(module_id)
6371 .expect("added module id came from configured map");
6372 supervisor
6373 .supervise_configured_requested(
6374 configured_module.module_spec(),
6375 configured_module.enabled,
6376 configured_module.health.clone(),
6377 configured_module.drain_timeout_ms,
6378 configured_module.restart,
6379 Some(requester.clone()),
6380 )
6381 .map_err(|err| {
6382 format!("failed to add module_id '{module_id}' during rescan: {err}")
6383 })?;
6384 }
6385
6386 Ok((
6387 SupervisorRescanResult {
6388 added,
6389 removed,
6390 changed_pending_reload,
6391 enabled_changes: enabled_changes.iter().cloned().collect(),
6392 unchanged,
6393 preview: false,
6394 restart_required: Vec::new(),
6397 capability_warnings: Vec::new(),
6398 module_actions: Vec::new(),
6399 earlier_pending: Vec::new(),
6400 expect: None,
6401 },
6402 changed,
6403 ))
6404 }
6405
6406 async fn handle_supervisor_set_enabled(
6407 &self,
6408 frame: Frame,
6409 module_id: String,
6410 enabled: bool,
6411 requester: Principal,
6412 ) -> Result<Vec<Frame>, RouterError> {
6413 let operation_lock = self.supervisor.operation_lock();
6414 let _operation_guard = operation_lock.lock().await;
6415 let Some(module) = self.supervisor.get(&module_id) else {
6416 return Ok(vec![control_error_frame(
6417 &frame,
6418 "unknown_module",
6419 format!("module_id '{module_id}' is not supervised"),
6420 )?]);
6421 };
6422
6423 self.route_outages.mark_operator_action(&module_id);
6426 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6427 Ok(applied) => applied,
6428 Err(err) => {
6429 self.route_outages
6430 .operator_action_ended_unrefused(&module_id);
6431 return Ok(vec![control_error_frame(
6432 &frame,
6433 "target_unavailable",
6434 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6435 )?]);
6436 }
6437 };
6438 if !applied {
6439 self.route_outages
6442 .operator_action_ended_unrefused(&module_id);
6443 }
6444
6445 self.capability_evaluator.wake_deadline_loop();
6446 self.refresh_capability_requirements();
6447 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6448 Ok(vec![control_response_body_frame(
6449 &frame,
6450 &response,
6451 "ClientControlResponse::SupervisorAck",
6452 )?])
6453 }
6454
6455 async fn handle_supervisor_health_probe(
6456 &self,
6457 frame: Frame,
6458 module_id: String,
6459 ) -> Result<Vec<Frame>, RouterError> {
6460 self.refresh_capability_requirements();
6461 let Some(registration) = self
6462 .registry
6463 .get_module(&module_id)
6464 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6465 else {
6466 return Ok(vec![control_error_frame(
6467 &frame,
6468 "unknown_module",
6469 format!("module_id '{module_id}' is not registered"),
6470 )?]);
6471 };
6472
6473 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6486 {
6487 return Ok(vec![control_error_frame(
6488 &frame,
6489 "health_not_advertised",
6490 format!("module_id '{module_id}' did not advertise health.check"),
6491 )?]);
6492 }
6493
6494 let deadline = Instant::now() + self.health_probe_timeout;
6495 let pending = match self.forwarding.begin_module_control_rpc_for(
6496 &module_id,
6497 MODULE_CONTROL_OP_HEALTH_CHECK,
6498 deadline,
6499 ) {
6500 Ok(pending) => pending,
6501 Err(err) => {
6502 return Ok(vec![control_error_frame(
6503 &frame,
6504 forwarding_error_code(&err),
6505 err.to_string(),
6506 )?])
6507 }
6508 };
6509
6510 let PendingModuleControlRpc {
6511 endpoint,
6512 module_sink,
6513 negotiated_ver,
6514 corr: probe_corr,
6515 receiver,
6516 } = pending;
6517 let mut guard =
6518 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6519 let probe_body =
6520 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6521 RouterError::backend(
6522 0,
6523 frame.header.corr,
6524 format!("failed to encode health.check request: {err}"),
6525 )
6526 })?;
6527 let probe_frame = Frame::build_with_version(
6528 negotiated_ver,
6529 FrameType::Request,
6530 control_flags(),
6531 0,
6532 0,
6533 probe_corr,
6534 probe_body,
6535 )
6536 .map_err(RouterError::FrameBuild)?;
6537
6538 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6541 Ok(Ok(())) => {}
6542 Ok(Err(err)) => {
6543 return Ok(vec![control_error_frame(
6544 &frame,
6545 "target_unavailable",
6546 err.to_string(),
6547 )?]);
6548 }
6549 Err(_) => {
6550 return Ok(vec![control_error_frame(
6551 &frame,
6552 "module_timeout",
6553 format!(
6554 "module_id '{module_id}' did not answer health.check within {:?}",
6555 self.health_probe_timeout
6556 ),
6557 )?]);
6558 }
6559 }
6560
6561 match timeout_at(deadline, receiver).await {
6562 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6563 guard.disarm();
6564 let Some(report) = response.health_report() else {
6565 return Ok(vec![control_error_frame(
6566 &frame,
6567 "invalid_control_body",
6568 "health.check RPC returned a non-health response",
6569 )?]);
6570 };
6571 let HealthReport {
6576 status,
6577 detail,
6578 metrics,
6579 } = report;
6580 let capability_detail = self
6581 .capability_evaluator
6582 .required_problem_detail(&module_id);
6583 let response = ClientControlResponse::SupervisorHealthProbe {
6584 module_id,
6585 status,
6586 detail: append_capability_problem_detail(detail, capability_detail),
6587 metrics,
6588 };
6589 Ok(vec![control_response_body_frame(
6590 &frame,
6591 &response,
6592 "ClientControlResponse::SupervisorHealthProbe",
6593 )?])
6594 }
6595 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6596 guard.disarm();
6597 Ok(vec![control_error_body_frame(&frame, body)?])
6598 }
6599 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6600 guard.disarm();
6601 Ok(vec![control_error_frame(
6602 &frame,
6603 "target_unavailable",
6604 message,
6605 )?])
6606 }
6607 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6608 guard.disarm();
6609 Ok(vec![control_error_frame(
6610 &frame,
6611 "invalid_control_body",
6612 message,
6613 )?])
6614 }
6615 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6616 guard.disarm();
6617 Ok(vec![control_error_frame(
6618 &frame,
6619 "invalid_control_body",
6620 format!("expected module-control op '{expected}', got '{actual}'"),
6621 )?])
6622 }
6623 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6624 guard.disarm();
6625 Ok(vec![control_error_frame(
6626 &frame,
6627 "module_timeout",
6628 format!(
6629 "module_id '{module_id}' answered health.check after {:?}",
6630 self.health_probe_timeout
6631 ),
6632 )?])
6633 }
6634 Ok(Err(_)) => Ok(vec![control_error_frame(
6635 &frame,
6636 "target_unavailable",
6637 "health.check waiter was canceled before the module responded",
6638 )?]),
6639 Err(_) => Ok(vec![control_error_frame(
6640 &frame,
6641 "module_timeout",
6642 format!(
6643 "module_id '{module_id}' did not answer health.check within {:?}",
6644 self.health_probe_timeout
6645 ),
6646 )?]),
6647 }
6648 }
6649
6650 fn supervisor_status(
6651 &self,
6652 module_id: &str,
6653 corr: u64,
6654 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6655 self.supervisor
6656 .get(module_id)
6657 .map(|module| {
6658 let warming = module.is_warming_for_control("status").map_err(|err| {
6659 RouterError::backend(
6660 0,
6661 corr,
6662 format!(
6663 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6664 ),
6665 )
6666 })?;
6667 module.status_for_control("status").map_err(|err| {
6668 RouterError::backend(
6669 0,
6670 corr,
6671 format!(
6672 "failed to read supervisor status for module_id '{module_id}': {err}"
6673 ),
6674 )
6675 }).map(|status| (status, warming))
6676 })
6677 .transpose()
6678 }
6679
6680 fn guard_module_control_op(
6681 &self,
6682 frame: &Frame,
6683 module_id: &str,
6684 op: &str,
6685 ) -> Result<Option<Frame>, RouterError> {
6686 if self.module_grants_op(module_id, op, frame.header.corr)? {
6687 return Ok(None);
6688 }
6689
6690 Ok(Some(control_error_frame(
6691 frame,
6692 "op_not_allowed",
6693 format!("module_id '{module_id}' did not grant control op '{op}'"),
6694 )?))
6695 }
6696
6697 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6698 let Some(registration) = self
6699 .registry
6700 .get_module(module_id)
6701 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6702 else {
6703 return Ok(false);
6704 };
6705 Ok(module_registration_grants_op(®istration.control_ops, op))
6706 }
6707
6708 fn handle_status_update(
6709 &self,
6710 endpoint: ModuleEndpointId,
6711 frame: Frame,
6712 ) -> Result<Vec<Frame>, RouterError> {
6713 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6714 Ok(update) => update,
6715 Err(err) => {
6716 if is_known_module_push_op(&frame.body) {
6721 return Ok(vec![control_error_frame(
6722 &frame,
6723 "invalid_control_body",
6724 format!("malformed module control push body: {err}"),
6725 )?]);
6726 }
6727 return Ok(Vec::new());
6728 }
6729 };
6730
6731 match update {
6732 ModuleControlPush::RouteStatus {
6733 route_channel,
6734 route_epoch,
6735 status,
6736 } => {
6737 self.forwarding
6738 .cache_status(endpoint, route_channel, route_epoch, status)
6739 .map_err(RouterError::Forwarding)?;
6740 }
6741 }
6742 Ok(Vec::new())
6743 }
6744
6745 fn handle_route_poll(
6746 &self,
6747 ctx: &RouteCtx,
6748 frame: Frame,
6749 route_channel: u16,
6750 route_epoch: u32,
6751 kind: PollKind,
6752 ) -> Result<Vec<Frame>, RouterError> {
6753 let snapshot = self
6754 .forwarding
6755 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6756 .map_err(RouterError::Forwarding)?;
6757 let response = match (kind, snapshot) {
6758 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6759 ClientControlResponse::RoutePoll {
6760 route_channel,
6761 route_epoch,
6762 status,
6763 live: None,
6764 }
6765 }
6766 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6767 route_channel,
6768 route_epoch,
6769 status: None,
6770 live: None,
6771 },
6772 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6773 let live = self
6799 .process_liveness
6800 .as_ref()
6801 .and_then(|source| source.process_live(&module_id))
6802 .unwrap_or(true);
6803 ClientControlResponse::RoutePoll {
6804 route_channel,
6805 route_epoch,
6806 status: None,
6807 live: Some(live),
6808 }
6809 }
6810 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6811 route_channel,
6812 route_epoch,
6813 status: None,
6814 live: Some(false),
6815 },
6816 };
6817
6818 Ok(vec![control_response_body_frame(
6819 &frame,
6820 &response,
6821 "ClientControlResponse::RoutePoll",
6822 )?])
6823 }
6824
6825 pub(crate) fn observe_module_control_completion(
6826 &self,
6827 completion: ModuleControlRpcCompletion,
6828 ) -> bool {
6829 match completion {
6830 ModuleControlRpcCompletion::Unknown => false,
6831 ModuleControlRpcCompletion::Settled => true,
6832 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6833 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6834 info!(
6835 module_id = %module_id,
6836 latency_ms,
6837 "late health.check answer proves the module is alive"
6838 );
6839 match self
6840 .supervisor
6841 .record_late_health_answer(&module_id, latency_ms)
6842 {
6843 Ok(true) => {}
6844 Ok(false) => debug!(
6845 module_id = %module_id,
6846 latency_ms,
6847 "late health.check answer has no active supervisor snapshot"
6848 ),
6849 Err(err) => warn!(
6850 module_id = %module_id,
6851 latency_ms,
6852 error = %err,
6853 "failed to record late health.check answer"
6854 ),
6855 }
6856 true
6857 }
6858 }
6859 }
6860
6861 fn refuse_to_end_module_connection_for_a_client(
6882 &self,
6883 module_connection_id: ConnectionId,
6884 corr: u64,
6885 err: ForwardingError,
6886 ) -> Result<(), RouterError> {
6887 if let ForwardingError::ConnectionClosing { connection_id } = err {
6888 if connection_id != module_connection_id {
6889 warn!(
6890 module_connection_id = module_connection_id.get(),
6891 client_connection_id = connection_id.get(),
6892 corr,
6893 "dropping a route.bind response for a closing client; the module connection keeps serving"
6894 );
6895 return Ok(());
6896 }
6897 }
6898 Err(RouterError::Forwarding(err))
6899 }
6900
6901 fn handle_module_relay_response(
6902 &self,
6903 connection_id: ConnectionId,
6904 frame: Frame,
6905 ) -> Result<Vec<Frame>, RouterError> {
6906 let mut secondary_error = None;
6907 let outcome = match frame.header.ty {
6908 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6909 Ok(probe) if probe.op == "route.bind" => {
6910 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6911 Ok(ModuleControlResponse::RouteBindAck {}) => {
6912 RouteBindRelayOutcome::Accepted
6913 }
6914 Ok(other) => {
6915 let message =
6916 format!("route.bind response carried unexpected body: {other:?}");
6917 secondary_error = Some(control_error_frame(
6918 &frame,
6919 "invalid_control_body",
6920 message.clone(),
6921 )?);
6922 RouteBindRelayOutcome::ModuleGone(message)
6923 }
6924 Err(err) => {
6925 let message = format!("malformed route.bind response body: {err}");
6926 secondary_error = Some(control_error_frame(
6927 &frame,
6928 "invalid_control_body",
6929 message.clone(),
6930 )?);
6931 RouteBindRelayOutcome::ModuleGone(message)
6932 }
6933 }
6934 }
6935 Ok(probe) => {
6936 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6937 {
6938 Ok(response) => ModuleControlRpcOutcome::Response(response),
6939 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6940 "malformed {} response body: {err}",
6941 probe.op
6942 )),
6943 };
6944 let completion = self
6945 .forwarding
6946 .complete_module_control_rpc(
6947 connection_id,
6948 frame.header.corr,
6949 Some(&probe.op),
6950 outcome,
6951 )
6952 .map_err(RouterError::Forwarding)?;
6953 if !self.observe_module_control_completion(completion) {
6954 debug!(
6955 connection_id = connection_id.get(),
6956 corr = frame.header.corr,
6957 op = %probe.op,
6958 "dropping late or unknown module-control RPC response"
6959 );
6960 }
6961 return Ok(Vec::new());
6962 }
6963 Err(err) => {
6964 if let Some(expected_op) = self
6965 .forwarding
6966 .pending_module_control_op(connection_id, frame.header.corr)
6967 .map_err(RouterError::Forwarding)?
6968 {
6969 let completion = self
6970 .forwarding
6971 .complete_module_control_rpc(
6972 connection_id,
6973 frame.header.corr,
6974 None,
6975 ModuleControlRpcOutcome::MalformedResponse(format!(
6976 "malformed {expected_op} response body: {err}"
6977 )),
6978 )
6979 .map_err(RouterError::Forwarding)?;
6980 if !self.observe_module_control_completion(completion) {
6981 debug!(
6982 connection_id = connection_id.get(),
6983 corr = frame.header.corr,
6984 "dropping late malformed module-control RPC response"
6985 );
6986 }
6987 return Ok(Vec::new());
6988 }
6989 let message = format!("malformed route.bind response body: {err}");
6990 secondary_error = Some(control_error_frame(
6991 &frame,
6992 "invalid_control_body",
6993 message.clone(),
6994 )?);
6995 RouteBindRelayOutcome::ModuleGone(message)
6996 }
6997 },
6998 FrameType::Error => {
6999 if self
7000 .forwarding
7001 .pending_module_control_op(connection_id, frame.header.corr)
7002 .map_err(RouterError::Forwarding)?
7003 .is_some()
7004 {
7005 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7006 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7007 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7008 "malformed module-control ERROR body: {err}"
7009 )),
7010 };
7011 let completion = self
7012 .forwarding
7013 .complete_module_control_rpc(
7014 connection_id,
7015 frame.header.corr,
7016 None,
7017 outcome,
7018 )
7019 .map_err(RouterError::Forwarding)?;
7020 if !self.observe_module_control_completion(completion) {
7021 debug!(
7022 connection_id = connection_id.get(),
7023 corr = frame.header.corr,
7024 "dropping late or unknown module-control RPC error"
7025 );
7026 }
7027 return Ok(Vec::new());
7028 }
7029 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7030 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7031 Err(err) => {
7032 let message = format!("malformed route.bind ERROR body: {err}");
7033 secondary_error = Some(control_error_frame(
7034 &frame,
7035 "invalid_control_body",
7036 message.clone(),
7037 )?);
7038 RouteBindRelayOutcome::ModuleGone(message)
7039 }
7040 }
7041 }
7042 ty => {
7043 return Ok(vec![control_error_frame(
7044 &frame,
7045 "unsupported_control_frame",
7046 format!("unsupported module channel-0 frame {ty:?}"),
7047 )?])
7048 }
7049 };
7050
7051 let settled =
7052 self.forwarding
7053 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7054 let completion = match settled {
7055 Ok(completion) => completion,
7056 Err(err) => {
7057 self.refuse_to_end_module_connection_for_a_client(
7058 connection_id,
7059 frame.header.corr,
7060 err,
7061 )?;
7062 return Ok(secondary_error.into_iter().collect());
7063 }
7064 };
7065 if let Some(target) = completion.abandoned.as_ref() {
7066 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7067 }
7068 if !completion.settled {
7069 debug!(
7070 connection_id = connection_id.get(),
7071 corr = frame.header.corr,
7072 frame_type = ?frame.header.ty,
7073 "dropping late or unknown route.bind relay response"
7074 );
7075 }
7076 Ok(secondary_error.into_iter().collect())
7077 }
7078
7079 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7080 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7081 self.cleanup_connection_with_end_reason(
7085 connection_id,
7086 RegistrationEndReason::ExplicitGoodbye,
7087 )
7088 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7089 Ok(Vec::new())
7090 }
7091}
7092
7093impl Default for ControlHandler {
7094 fn default() -> Self {
7095 Self::new(Arc::new(Registry::default()))
7096 }
7097}
7098
7099impl crate::supervise::SwapPromotionObserver for ControlHandler {
7100 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7101 self.apply_registration_capabilities(registration);
7102 }
7103}
7104
7105fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7106 CapabilityRequirementStatus {
7107 consumer: status.consumer,
7108 capability: status.capability,
7109 need: match status.need {
7110 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7111 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7112 },
7113 verdict: status.verdict.as_str().to_string(),
7114 episode_seq: status.episode_seq,
7115 config_satisfiable: status.config_satisfiable,
7116 runtime_available: status.runtime_available,
7117 detail: status.detail,
7118 }
7119}
7120
7121fn append_capability_problem_detail(
7122 detail: Option<String>,
7123 capability_detail: Option<String>,
7124) -> Option<String> {
7125 match (detail, capability_detail) {
7126 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7127 (Some(detail), None) => Some(detail),
7128 (None, Some(capability_detail)) => Some(capability_detail),
7129 (None, None) => None,
7130 }
7131}
7132
7133fn subc_ops() -> Vec<String> {
7134 SUBC_CONTROL_OPS
7135 .iter()
7136 .map(|op| (*op).to_string())
7137 .collect()
7138}
7139
7140fn module_subc_ops() -> Vec<String> {
7141 SUBC_CONTROL_OPS
7142 .iter()
7143 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7144 .map(|op| (*op).to_string())
7145 .collect()
7146}
7147
7148#[cfg(test)]
7149fn module_baseline_control_ops() -> Vec<String> {
7150 MODULE_BASELINE_CONTROL_OPS
7151 .iter()
7152 .map(|op| (*op).to_string())
7153 .collect()
7154}
7155
7156fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7157 let mut seen = HashSet::new();
7158 let mut effective = Vec::new();
7159 for op in MODULE_BASELINE_CONTROL_OPS {
7160 if seen.insert((*op).to_string()) {
7161 effective.push((*op).to_string());
7162 }
7163 }
7164 for op in declared.unwrap_or_default() {
7165 if seen.insert(op.clone()) {
7166 effective.push(op);
7167 }
7168 }
7169 effective
7170}
7171
7172fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7173 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7174}
7175
7176fn target_module_id(target: &RouteTarget) -> &str {
7177 match target {
7178 RouteTarget::ToolProvider { module_id }
7179 | RouteTarget::ManagementSurface { module_id }
7180 | RouteTarget::InternalService { module_id, .. } => module_id,
7181 }
7182}
7183
7184fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7185 roles.iter().any(|role| match (target, role) {
7186 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7187 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7188 (
7189 RouteTarget::InternalService { service_id, .. },
7190 ProviderRole::InternalService {
7191 service_id: provided,
7192 ..
7193 },
7194 ) => service_id == provided,
7195 _ => false,
7196 })
7197}
7198
7199fn is_routable_role(role: &ProviderRole) -> bool {
7200 matches!(
7201 role,
7202 ProviderRole::ToolProvider { .. }
7203 | ProviderRole::ManagementSurface { .. }
7204 | ProviderRole::InternalService { .. }
7205 )
7206}
7207
7208#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7209enum ControlRequestBodyError {
7210 UnknownOp,
7211 InvalidBody,
7212}
7213
7214#[derive(Debug, Deserialize)]
7215struct ControlOpProbe {
7216 op: String,
7217}
7218
7219const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7222
7223fn is_known_module_push_op(body: &[u8]) -> bool {
7224 serde_json::from_slice::<ControlOpProbe>(body)
7225 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7226 .unwrap_or(false)
7227}
7228
7229fn is_known_module_request_op(body: &[u8]) -> bool {
7230 serde_json::from_slice::<ControlOpProbe>(body)
7231 .map(|probe| is_module_to_subc_op(&probe.op))
7232 .unwrap_or(false)
7233}
7234
7235fn is_module_to_subc_op(op: &str) -> bool {
7236 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7237}
7238
7239fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7240 debug!(
7241 op = %op,
7242 connection_id = connection_id.get(),
7243 corr,
7244 "control dispatch"
7245 );
7246}
7247
7248fn log_slow_control_dispatch(
7249 dispatch_started_at: Option<StdInstant>,
7250 op: &'static str,
7251 connection_id: ConnectionId,
7252 corr: u64,
7253 route_open_timings: Option<&RouteOpenTimings>,
7254) {
7255 let Some(dispatch_started_at) = dispatch_started_at else {
7256 return;
7257 };
7258 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7260 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7261 if let Some(timings) = route_open_timings {
7262 warn!(
7263 op = %op,
7264 connection_id = connection_id.get(),
7265 corr,
7266 elapsed_ms = elapsed.as_millis() as u64,
7267 module_id = ?timings.module_id,
7268 principal = %timings.principal,
7269 admission_ms = timings.admission.as_millis() as u64,
7270 project_root_ms = timings.project_root.as_millis() as u64,
7271 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7272 bind_send_ms = timings.bind_send.as_millis() as u64,
7273 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7274 "slow control dispatch"
7275 );
7276 return;
7277 }
7278 warn!(
7279 op = %op,
7280 connection_id = connection_id.get(),
7281 corr,
7282 elapsed_ms = elapsed.as_millis() as u64,
7283 "slow control dispatch"
7284 );
7285 }
7286}
7287
7288#[cfg(test)]
7290type TestCallerLookup =
7291 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7292
7293fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7295 match request {
7296 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7297 Some(PrivilegedOperation::Restart { module_id })
7298 }
7299 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7300 Some(PrivilegedOperation::Swap { module_id })
7301 }
7302 ClientControlRequest::SupervisorReload { module_id } => {
7303 Some(PrivilegedOperation::Reload { module_id })
7304 }
7305 ClientControlRequest::SupervisorRescan { preview } => {
7306 if *preview {
7307 None
7308 } else {
7309 Some(PrivilegedOperation::Rescan)
7310 }
7311 }
7312 ClientControlRequest::SupervisorRescanChecked {
7317 preview,
7318 candidate,
7319 expect,
7320 } => (!*preview && candidate.is_none() && expect.is_some())
7321 .then_some(PrivilegedOperation::Rescan),
7322 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7323 Some(PrivilegedOperation::SetEnabled {
7324 module_id,
7325 enabled: *enabled,
7326 })
7327 }
7328 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7329 Some(PrivilegedOperation::ReleaseReserved { module_id })
7330 }
7331 ClientControlRequest::ServerDescribe {}
7332 | ClientControlRequest::CatalogList { .. }
7333 | ClientControlRequest::RouteOpen { .. }
7334 | ClientControlRequest::RoutePoll { .. }
7335 | ClientControlRequest::SupervisorList {}
7336 | ClientControlRequest::SupervisorHealth {}
7337 | ClientControlRequest::SupervisorHealthProbe { .. }
7338 | ClientControlRequest::SupervisorRoutes { .. }
7339 | ClientControlRequest::SupervisorProvenance { .. }
7340 | ClientControlRequest::SupervisorStderrTail { .. }
7341 | ClientControlRequest::SupervisorTerminals { .. }
7342 | ClientControlRequest::SupervisorSpawnSnapshot {}
7343 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7344 }
7345}
7346
7347fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7348 match request {
7349 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7350 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7351 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7352 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7353 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7354 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7355 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7356 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7357 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7358 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7359 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7360 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7361 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7362 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7363 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7364 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7365 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7366 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7367 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7368 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7369 }
7370}
7371
7372struct RouteOpenCaller {
7380 principal: String,
7384 scope: String,
7387 harness: String,
7390}
7391
7392impl RouteOpenCaller {
7393 fn new(
7394 principal: Option<&Principal>,
7395 scope: Option<&ScopeSelector>,
7396 identity: &BindIdentity,
7397 ) -> Self {
7398 Self {
7399 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7400 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7401 harness: bounded_log_text(&identity.harness),
7402 }
7403 }
7404
7405 fn unknown() -> Self {
7408 Self {
7409 principal: "unknown".to_string(),
7410 scope: "unknown".to_string(),
7411 harness: "unknown".to_string(),
7412 }
7413 }
7414}
7415
7416const MAX_LOGGED_TEXT_CHARS: usize = 64;
7419
7420fn bounded_log_text(value: &str) -> String {
7424 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7425 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7426 bounded.push('…');
7427 }
7428 format!("{bounded:?}")
7429}
7430
7431fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7437 use subc_protocol::scope::ScopeKind;
7438 let kind = match stamp.kind {
7439 ScopeKind::Head => "head",
7440 ScopeKind::Worker => "worker",
7441 ScopeKind::Ephemeral => "ephemeral",
7442 };
7443 format!(
7444 "{kind}/{}/{}@{}",
7445 principal_label(&stamp.owner),
7446 bounded_log_text(&stamp.scope_ref),
7447 stamp.scope_epoch
7448 )
7449}
7450
7451fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7458 let epoch = selector
7459 .scope_epoch
7460 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7461 format!(
7462 "requested/{}/{}@{epoch}",
7463 bounded_log_text(&principal_label(&selector.owner)),
7464 bounded_log_text(&selector.scope_ref),
7465 )
7466}
7467
7468fn principal_label(principal: &Principal) -> String {
7469 match principal {
7470 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7471 Principal::Direct => "direct".to_string(),
7472 other => format!("{other:?}"),
7473 }
7474}
7475
7476fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7477 match request {
7478 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7479 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7480 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7481 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7482 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7483 }
7484}
7485
7486fn parse_client_control_request(
7487 body: &[u8],
7488) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7489 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7490 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7491 Ok(probe)
7492 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7493 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7494 {
7495 ControlRequestBodyError::InvalidBody
7496 }
7497 Ok(_) => ControlRequestBodyError::UnknownOp,
7498 Err(_) => ControlRequestBodyError::InvalidBody,
7499 };
7500 (err, classification)
7501 })
7502}
7503
7504fn parse_module_control_request_from_module(
7505 body: &[u8],
7506) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7507 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7508 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7509 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7510 Ok(_) => ControlRequestBodyError::UnknownOp,
7511 Err(_) => ControlRequestBodyError::InvalidBody,
7512 };
7513 (err, classification)
7514 })
7515}
7516
7517#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7518enum ProviderRoleKind {
7519 ToolProvider,
7520 PipelineStage,
7521 ManagementSurface,
7522 InternalService,
7523}
7524
7525fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7526 match role {
7527 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7528 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7529 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7530 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7531 }
7532}
7533
7534fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7535 roles.iter().map(provider_role_kind).collect()
7536}
7537
7538const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7541
7542#[derive(Debug, Default, PartialEq, Eq)]
7544struct ScopeOutcomeCounts {
7545 created: usize,
7546 replaced: usize,
7547 updated: usize,
7548 unchanged: usize,
7549 refused: usize,
7550}
7551
7552impl ScopeOutcomeCounts {
7553 fn of(results: &[ScopeRecordResult]) -> Self {
7554 let mut counts = Self::default();
7555 for result in results {
7556 let slot = match result.outcome {
7557 ScopeRecordOutcome::Created => &mut counts.created,
7558 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7559 ScopeRecordOutcome::Updated => &mut counts.updated,
7560 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7561 ScopeRecordOutcome::Refused => &mut counts.refused,
7562 };
7563 *slot += 1;
7564 }
7565 counts
7566 }
7567}
7568
7569#[cfg(test)]
7570mod scope_outcome_count_tests {
7571 use super::*;
7572
7573 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7574 ScopeRecordResult {
7575 scope_ref: "r".to_string(),
7576 scope_epoch: 1,
7577 outcome,
7578 code: None,
7579 message: None,
7580 version: None,
7581 parent_state: None,
7582 }
7583 }
7584
7585 #[test]
7588 fn every_outcome_is_counted_in_its_own_field() {
7589 let results = [
7590 result(ScopeRecordOutcome::Created),
7591 result(ScopeRecordOutcome::Created),
7592 result(ScopeRecordOutcome::Replaced),
7593 result(ScopeRecordOutcome::Updated),
7594 result(ScopeRecordOutcome::Unchanged),
7595 result(ScopeRecordOutcome::Refused),
7596 result(ScopeRecordOutcome::Refused),
7597 result(ScopeRecordOutcome::Refused),
7598 ];
7599 assert_eq!(
7600 ScopeOutcomeCounts::of(&results),
7601 ScopeOutcomeCounts {
7602 created: 2,
7603 replaced: 1,
7604 updated: 1,
7605 unchanged: 1,
7606 refused: 3,
7607 }
7608 );
7609 }
7610}
7611
7612fn capability_census_trigger(
7616 old: Option<&CapabilityDeclarations>,
7617 new: Option<&CapabilityDeclarations>,
7618) -> bool {
7619 let old_provides = old
7620 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7621 .unwrap_or_default();
7622 let old_denies = old
7623 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7624 .unwrap_or_default();
7625 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7626 provides: Vec::new(),
7627 requires: Vec::new(),
7628 must_never_reach: Vec::new(),
7629 });
7630
7631 new.provides
7632 .iter()
7633 .any(|capability| !old_provides.contains(capability))
7634 || new
7635 .must_never_reach
7636 .iter()
7637 .any(|capability| !old_denies.contains(capability))
7638}
7639
7640fn denied_capability<'a>(
7643 opening_manifest: &'a ModuleManifest,
7644 target_manifest: &ModuleManifest,
7645) -> Option<&'a str> {
7646 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7647 let target_capabilities = target_manifest.capabilities.as_ref()?;
7648 opening_capabilities
7649 .must_never_reach
7650 .iter()
7651 .find(|denied| {
7652 target_capabilities
7653 .provides
7654 .iter()
7655 .any(|provided| provided == *denied)
7656 })
7657 .map(String::as_str)
7658}
7659
7660fn catalog_update_frozen_field_message(
7661 registered: &ModuleManifest,
7662 provides: &[ProviderRole],
7663) -> Option<String> {
7664 let old_has_provides = !registered.provides.is_empty();
7665 let new_has_provides = !provides.is_empty();
7666 if old_has_provides != new_has_provides {
7667 return Some(format!(
7668 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7669 registered.module_id
7670 ));
7671 }
7672
7673 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7674 return Some(format!(
7675 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7676 registered.module_id
7677 ));
7678 }
7679
7680 let registered_concurrency = manifest_concurrency(registered);
7681 let mut candidate = registered.clone();
7682 candidate.provides = provides.to_vec();
7683 let candidate_concurrency = manifest_concurrency(&candidate);
7684 if candidate_concurrency != registered_concurrency {
7685 return Some(format!(
7686 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7687 registered.module_id, registered_concurrency, candidate_concurrency
7688 ));
7689 }
7690
7691 None
7694}
7695
7696fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7697 manifest.provides.iter().any(is_routable_role)
7698}
7699
7700fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7706 manifest
7707 .provides
7708 .iter()
7709 .find_map(|provider| match provider {
7710 ProviderRole::ToolProvider { concurrency, .. }
7711 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7712 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7713 })
7714 .unwrap_or(Concurrency::ModuleManaged)
7715}
7716
7717fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7724 let has_management_surface = manifest
7725 .provides
7726 .iter()
7727 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7728 if !has_management_surface {
7729 return false;
7730 }
7731 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7732 return false;
7733 };
7734 let Some(provides) = raw
7735 .get("manifest")
7736 .and_then(|manifest| manifest.get("provides"))
7737 .and_then(serde_json::Value::as_array)
7738 else {
7739 return false;
7740 };
7741 provides.iter().any(|role| {
7746 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7747 && role.get("concurrency").is_none()
7748 })
7749}
7750
7751fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7752 if peer_version != PROTOCOL_VERSION {
7753 return Err(format!(
7754 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7755 ));
7756 }
7757 Ok(PROTOCOL_VERSION)
7758}
7759
7760fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7761 Frame::build_with_version(
7762 response_version(frame),
7763 FrameType::Pong,
7764 frame.header.flags,
7765 0,
7766 0,
7767 frame.header.corr,
7768 Vec::new(),
7769 )
7770 .map_err(RouterError::FrameBuild)
7771}
7772
7773pub(crate) fn control_error_frame(
7774 frame: &Frame,
7775 code: &'static str,
7776 message: impl Into<String>,
7777) -> Result<Frame, RouterError> {
7778 control_error_body_frame(
7779 frame,
7780 ErrorBody {
7781 code: code.to_string(),
7782 message: message.into(),
7783 detail: None,
7784 },
7785 )
7786}
7787
7788fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7789 let body = serde_json::to_vec(&error).map_err(|err| {
7790 RouterError::backend(
7791 0,
7792 frame.header.corr,
7793 format!("failed to encode control ERROR: {err}"),
7794 )
7795 })?;
7796
7797 Frame::build_with_version(
7798 response_version(frame),
7799 FrameType::Error,
7800 control_flags(),
7801 0,
7802 0,
7803 frame.header.corr,
7804 body,
7805 )
7806 .map_err(RouterError::FrameBuild)
7807}
7808
7809fn control_response_body_frame<T: Serialize>(
7810 frame: &Frame,
7811 reply: &T,
7812 label: &'static str,
7813) -> Result<Frame, RouterError> {
7814 let body = serde_json::to_vec(reply).map_err(|err| {
7815 RouterError::backend(
7816 0,
7817 frame.header.corr,
7818 format!("failed to encode {label}: {err}"),
7819 )
7820 })?;
7821
7822 Frame::build_with_version(
7823 response_version(frame),
7824 FrameType::Response,
7825 control_flags(),
7826 0,
7827 0,
7828 frame.header.corr,
7829 body,
7830 )
7831 .map_err(RouterError::FrameBuild)
7832}
7833
7834fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7864 match err {
7865 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7866 ForwardingError::NoModuleConnection => "target_unavailable",
7867 ForwardingError::ModuleReloading { .. } => "module_reloading",
7868 ForwardingError::ClientRouteChannelExhausted { .. }
7869 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7870 ForwardingError::StaleModuleEndpoint
7871 | ForwardingError::UnknownReservation { .. }
7872 | ForwardingError::ConnectionClosing { .. }
7873 | ForwardingError::ClientEgressClosed { .. }
7874 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7875 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7878 ForwardingError::RelayCorrelationExhausted
7879 | ForwardingError::RouteOpenBuild(_)
7880 | ForwardingError::Poisoned => "forwarding_error",
7881 }
7882}
7883
7884fn response_version(frame: &Frame) -> u8 {
7885 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7886 frame.header.ver
7887 } else {
7888 PROTOCOL_VERSION
7889 }
7890}
7891
7892fn control_flags() -> Flags {
7893 Flags::new(false, Priority::Passive, false)
7894}
7895
7896fn send_goodbye_target_best_effort(
7901 counters: &DaemonCounters,
7902 target: &GoodbyeTarget,
7903 context: &'static str,
7904) {
7905 let Ok(frame) = Frame::build_with_version(
7906 target.negotiated_ver,
7907 FrameType::Goodbye,
7908 control_flags(),
7909 target.channel,
7910 target.epoch,
7911 0,
7912 Vec::new(),
7913 ) else {
7914 return;
7915 };
7916 crate::forwarding::send_module_route_goodbye(
7917 counters,
7918 &target.sink,
7919 frame,
7920 target.module_id.as_deref(),
7921 context,
7922 );
7923}
7924
7925pub(crate) fn send_route_control_pushes(
7926 forwarding: &ForwardingTable,
7927 routes: Vec<EndpointRoute>,
7928 push: ClientControlPush,
7929) {
7930 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7931 for route in routes {
7932 let target = route.goodbye_target;
7933 if let Some((existing, channels)) = targets
7934 .iter_mut()
7935 .find(|(existing, _)| existing.connection_id == target.connection_id)
7936 {
7937 debug_assert_eq!(
7938 existing.negotiated_ver, target.negotiated_ver,
7939 "one connection cannot negotiate multiple frame versions"
7940 );
7941 if !channels.contains(&target.channel) {
7942 channels.push(target.channel);
7943 }
7944 continue;
7945 }
7946 let channel = target.channel;
7947 targets.push((target, vec![channel]));
7948 }
7949 for (target, mut channels) in targets {
7950 channels.sort_unstable();
7951 let mut push = push.clone();
7952 match &mut push {
7953 ClientControlPush::RouteClosing {
7954 channels: covered, ..
7955 }
7956 | ClientControlPush::RouteClosed {
7957 channels: covered, ..
7958 } => *covered = channels,
7959 }
7960 let body = match serde_json::to_vec(&push) {
7961 Ok(body) => body,
7962 Err(err) => {
7963 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7964 continue;
7965 }
7966 };
7967 let frame = match Frame::build_with_version(
7968 target.negotiated_ver,
7969 FrameType::Push,
7970 control_flags(),
7971 0,
7972 0,
7973 0,
7974 body.clone(),
7975 ) {
7976 Ok(frame) => frame,
7977 Err(err) => {
7978 warn!(
7979 route_channel = target.channel,
7980 error = %err,
7981 "failed to build route lifecycle control PUSH frame"
7982 );
7983 continue;
7984 }
7985 };
7986 if let Err(err) = target.sink.try_send(frame) {
7987 if target.close_on_delivery_failure() {
7988 warn!(
7989 target_connection_id = target.connection_id.get(),
7990 route_channel = target.channel,
7991 error = %err,
7992 "route lifecycle control PUSH was not delivered to client; closing target connection"
7993 );
7994 let _ = forwarding.escalate_client_delivery_failure(
7995 target.connection_id,
7996 target.channel,
7997 target.epoch,
7998 CloseReason::new(
7999 "route_lifecycle_push_delivery_failed",
8000 format!(
8001 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8002 target.channel
8003 ),
8004 ),
8005 crate::forwarding::UndeliveredFrame {
8006 module_id: target.module_id.as_deref(),
8007 sink: &target.sink,
8008 },
8009 );
8010 }
8011 }
8012 }
8013}
8014
8015#[cfg(test)]
8016mod tests {
8017 mod activity_read {
8018 include!("activity_read_tests.rs");
8019 }
8020
8021 mod lifecycle_activity {
8022 include!("control_lifecycle_tests.rs");
8023 }
8024 #[cfg(unix)]
8025 #[tokio::test]
8026 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8027 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8028 let path = dir.join("subc.jsonc");
8029 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8030 "program":"/bin/sleep","args":["60"],"protocol":"none",
8031 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8032 }}}).to_string()).unwrap();
8033 let mut configured = crate::daemon_config::load(&path)
8034 .unwrap()
8035 .unwrap()
8036 .modules
8037 .pop()
8038 .unwrap();
8039 let registry = std::sync::Arc::new(crate::Registry::default());
8040 let handle = crate::SupervisorHandle::new();
8041 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8042 .with_handle(handle.clone());
8043 let module = supervisor
8044 .supervise_configured_with_health(
8045 configured.module_spec(),
8046 true,
8047 configured.health.clone(),
8048 None,
8049 configured.restart,
8050 )
8051 .unwrap();
8052 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8053 let before = module.status().unwrap().pid;
8054 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8055 configured.health.cadence = std::time::Duration::from_secs(3600);
8056 let health_only = handler
8057 .reconcile_supervised_modules(
8058 &supervisor,
8059 vec![configured.clone()],
8060 false,
8061 Principal::Direct,
8062 )
8063 .await
8064 .unwrap();
8065 assert!(
8066 health_only.0.changed_pending_reload.is_empty(),
8067 "health policy is already applied live"
8068 );
8069 assert_eq!(module.status().unwrap().pid, before);
8070 assert_eq!(
8071 module.configuration().unwrap().1.http,
8072 configured.health.http
8073 );
8074 configured.args = vec!["61".into()];
8075 let launch = handler
8076 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8077 .await
8078 .unwrap();
8079 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8080 assert_eq!(
8081 module.status().unwrap().pid,
8082 before,
8083 "a launch edit is stored until reload"
8084 );
8085 module.drain().await.unwrap();
8086 }
8087 use cortexkit_test_support::ScratchDir;
8088 use std::{
8089 collections::BTreeMap,
8090 fmt,
8091 path::PathBuf,
8092 sync::{Arc, Mutex},
8093 time::Duration,
8094 };
8095
8096 use serde_json::{json, Value};
8097 use subc_protocol::{
8098 manifest::{
8099 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8100 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8101 },
8102 session::HealthStatus,
8103 FrameType,
8104 };
8105
8106 use super::*;
8107 use crate::{
8108 forwarding::{DataRoute, DataRouteState},
8109 registry::ChannelState,
8110 router::FrameSink,
8111 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8112 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8113 RouteCtx, Router,
8114 };
8115 use tokio::{
8116 sync::mpsc,
8117 time::{sleep, Instant},
8118 };
8119 use tracing::{
8120 field::{Field, Visit},
8121 Event, Subscriber,
8122 };
8123 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8124
8125 fn fake_aft_stub_path() -> PathBuf {
8144 let mut path = std::env::current_exe().expect("current_exe available in tests");
8145 path.pop(); path.pop(); path.push(if cfg!(windows) {
8148 "fake-aft-stub.exe"
8149 } else {
8150 "fake-aft-stub"
8151 });
8152 assert!(
8153 path.exists(),
8154 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8155 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8156 path.display()
8157 );
8158 path
8159 }
8160
8161 fn client_retries(code: &str) -> bool {
8166 subc_protocol::error_codes::is_retryable_route_open(code)
8167 }
8168
8169 #[test]
8180 fn retryability_of_forwarding_codes_matches_the_failure() {
8181 let transient = [
8184 ForwardingError::NoModuleConnection,
8185 ForwardingError::ModuleReloading {
8186 module_id: "m".into(),
8187 },
8188 ForwardingError::StaleModuleEndpoint,
8189 ForwardingError::UnknownReservation {
8190 client_channel: 1,
8191 module_channel: 1,
8192 },
8193 ForwardingError::ConnectionClosing {
8194 connection_id: ConnectionId::new(1),
8195 },
8196 ForwardingError::ClientEgressClosed {
8197 connection_id: ConnectionId::new(1),
8198 },
8199 ForwardingError::ModuleEgressUnavailable {
8200 connection_id: ConnectionId::new(1),
8201 },
8202 ];
8203 for err in transient {
8204 let code = forwarding_error_code(&err);
8205 assert!(
8206 client_retries(code),
8207 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8208 );
8209 }
8210
8211 let permanent = [
8216 ForwardingError::ConnectionRoleConflict {
8217 connection_id: ConnectionId::new(1),
8218 },
8219 ForwardingError::ClientRouteChannelExhausted {
8220 connection_id: ConnectionId::new(1),
8221 },
8222 ForwardingError::ModuleRouteChannelExhausted {
8223 endpoint: ModuleEndpointId {
8224 connection_id: ConnectionId::new(1),
8225 generation: 1,
8226 },
8227 },
8228 ForwardingError::RelayCorrelationExhausted,
8229 ForwardingError::RouteOpenBuild("x".into()),
8230 ForwardingError::Poisoned,
8231 ];
8232 for err in permanent {
8233 let code = forwarding_error_code(&err);
8234 assert!(
8235 !client_retries(code),
8236 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8237 );
8238 }
8239 }
8240
8241 #[tokio::test]
8269 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8270 let handler = ControlHandler::default();
8271 let frame =
8272 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8273
8274 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8278 assert_eq!(
8279 stamped,
8280 Principal::Direct,
8281 "a caller that proved nothing must not be stamped as a supervised module"
8282 );
8283
8284 let forged = handler
8289 .route_open_principal(
8290 &frame,
8291 Some(ConsumerIdentity {
8292 module_id: "aft".to_string(),
8293 launch_nonce: "not-a-real-nonce".to_string(),
8294 }),
8295 )
8296 .unwrap();
8297 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8298 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8299 }
8300
8301 #[test]
8318 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8319 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8320 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8321 let ClientControlRequest::RouteOpen {
8322 consumer_identity, ..
8323 } = parsed
8324 else {
8325 panic!("route.open body must parse as RouteOpen");
8326 };
8327 assert_eq!(
8328 consumer_identity,
8329 Some(ConsumerIdentity {
8330 module_id: "aft".to_string(),
8331 launch_nonce: "n".to_string(),
8332 }),
8333 "the wire field name must reach the value route_open_principal reads"
8334 );
8335 }
8336
8337 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8338 ModuleManifest::builder(module_id, "0.1.0")
8339 .protocol_ver(protocol_ver)
8340 .provides(vec![ProviderRole::ToolProvider {
8341 tools: vec![Tool {
8342 name: "read".to_string(),
8343 description: None,
8344 execution_mode: ExecutionMode::Pure,
8345 schema: json!({"type": "object"}),
8346 }],
8347 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8348 concurrency: Concurrency::ModuleManaged,
8349 emits_push: true,
8350 sub_supervises: true,
8351 }])
8352 .build()
8353 }
8354
8355 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8356 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8357 }
8358
8359 fn hello_frame_with_control_ops(
8360 module_id: &str,
8361 protocol_ver: u8,
8362 corr: u64,
8363 control_ops: Option<Vec<String>>,
8364 ) -> Frame {
8365 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8366 }
8367
8368 fn hello_frame_with_nonce(
8369 module_id: &str,
8370 protocol_ver: u8,
8371 corr: u64,
8372 launch_nonce: Option<&str>,
8373 ) -> Frame {
8374 hello_frame_full(
8375 module_id,
8376 protocol_ver,
8377 corr,
8378 None,
8379 launch_nonce.map(ToOwned::to_owned),
8380 )
8381 }
8382
8383 fn hello_frame_full(
8384 module_id: &str,
8385 protocol_ver: u8,
8386 corr: u64,
8387 control_ops: Option<Vec<String>>,
8388 launch_nonce: Option<String>,
8389 ) -> Frame {
8390 let body = serde_json::to_vec(&ModuleHelloBody {
8391 manifest: manifest(module_id, protocol_ver),
8392 protocol_ver,
8393 control_ops,
8394 launch_nonce,
8395 })
8396 .unwrap();
8397 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8398 }
8399
8400 fn hello_frame_with_rich_ui(
8401 module_id: &str,
8402 protocol_ver: u8,
8403 corr: u64,
8404 launch_nonce: Option<&str>,
8405 rich_ui: Value,
8406 ) -> Frame {
8407 let mut body = serde_json::to_value(ModuleHelloBody {
8408 manifest: manifest(module_id, protocol_ver),
8409 protocol_ver,
8410 control_ops: None,
8411 launch_nonce: launch_nonce.map(str::to_string),
8412 })
8413 .unwrap();
8414 body["manifest"]["rich_ui"] = rich_ui;
8415 Frame::build(
8416 FrameType::Hello,
8417 control_flags(),
8418 0,
8419 0,
8420 corr,
8421 serde_json::to_vec(&body).unwrap(),
8422 )
8423 .unwrap()
8424 }
8425
8426 fn non_routable_hello_frame_with_control_ops(
8427 module_id: &str,
8428 corr: u64,
8429 control_ops: Option<Vec<String>>,
8430 ) -> Frame {
8431 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8432 manifest.provides.clear();
8433 let body = serde_json::to_vec(&ModuleHelloBody {
8434 manifest,
8435 protocol_ver: PROTOCOL_VERSION,
8436 control_ops,
8437 launch_nonce: None,
8438 })
8439 .unwrap();
8440 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8441 }
8442
8443 fn capability_grammar_hello_frame(
8444 capabilities: Value,
8445 runtime_computed: Option<Value>,
8446 corr: u64,
8447 ) -> Frame {
8448 let mut body = serde_json::to_value(ModuleHelloBody {
8449 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8450 protocol_ver: PROTOCOL_VERSION,
8451 control_ops: None,
8452 launch_nonce: None,
8453 })
8454 .expect("HELLO body serializes");
8455 body["manifest"]["capabilities"] = capabilities;
8456 if let Some(runtime_computed) = runtime_computed {
8457 body["runtime_computed"] = runtime_computed;
8458 }
8459 Frame::build(
8460 FrameType::Hello,
8461 control_flags(),
8462 0,
8463 0,
8464 corr,
8465 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8466 )
8467 .expect("HELLO frame builds")
8468 }
8469
8470 fn channel_request(channel: u16, corr: u64) -> Frame {
8471 Frame::build(
8472 FrameType::Request,
8473 Flags::new(true, Priority::Interactive, false),
8474 channel,
8475 0,
8476 corr,
8477 b"opaque".to_vec(),
8478 )
8479 .unwrap()
8480 }
8481
8482 fn route_ctx(
8483 connection_id: ConnectionId,
8484 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8485 let (tx, rx) = mpsc::channel(8);
8486 (
8487 RouteCtx {
8488 connection_id,
8489 egress: FrameSink::new(tx),
8490 },
8491 rx,
8492 )
8493 }
8494
8495 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8496 serde_json::from_slice(&frame.body).unwrap()
8497 }
8498
8499 async fn hello_via_sink(
8504 handler: &ControlHandler,
8505 ctx: &RouteCtx,
8506 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8507 hello: Frame,
8508 ) -> Frame {
8509 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8510 assert!(
8511 replies.is_empty(),
8512 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8513 );
8514 let ack = rx
8515 .try_recv()
8516 .expect("HELLO_ACK is queued on the module sink")
8517 .frame;
8518 assert_eq!(ack.header.ty, FrameType::HelloAck);
8519 ack
8520 }
8521
8522 fn parse_error(frame: &Frame) -> Value {
8523 serde_json::from_slice(&frame.body).unwrap()
8524 }
8525
8526 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8527 serde_json::from_slice(&frame.body).unwrap()
8528 }
8529
8530 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8531 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8532 route_channel,
8533 route_epoch: 0,
8534 kind,
8535 })
8536 .unwrap();
8537 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8538 }
8539
8540 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8541 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8542 module_id: module_id.to_string(),
8543 })
8544 .unwrap();
8545 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8546 }
8547
8548 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8549 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8550 }
8551
8552 fn route_open_frame_with_consumer_capabilities(
8553 corr: u64,
8554 module_id: &str,
8555 project_root: ScratchDir,
8556 consumer_capabilities: Option<Vec<String>>,
8557 ) -> Frame {
8558 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8559 target: RouteTarget::ToolProvider {
8560 module_id: module_id.to_string(),
8561 },
8562 identity: BindIdentity::new(
8563 project_root.path().to_path_buf(),
8564 "unit".to_string(),
8565 "session".to_string(),
8566 ),
8567 consumer_identity: None,
8568 consumer_capabilities,
8569 role_versions: None,
8570 admission_facts: None,
8571 scope: None,
8572 })
8573 .unwrap();
8574 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8575 }
8576
8577 fn route_open_frame_with_role_versions(
8578 corr: u64,
8579 module_id: &str,
8580 project_root: ScratchDir,
8581 role_versions: Option<BTreeMap<String, String>>,
8582 ) -> Frame {
8583 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8584 target: RouteTarget::ToolProvider {
8585 module_id: module_id.to_string(),
8586 },
8587 identity: BindIdentity::new(
8588 project_root.path().to_path_buf(),
8589 "unit".to_string(),
8590 format!("session-{corr}"),
8591 ),
8592 consumer_identity: None,
8593 consumer_capabilities: None,
8594 role_versions,
8595 admission_facts: None,
8596 scope: None,
8597 })
8598 .unwrap();
8599 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8600 }
8601
8602 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8603 entries
8604 .iter()
8605 .map(|(role, version)| (role.to_string(), version.to_string()))
8606 .collect()
8607 }
8608
8609 fn route_open_frame_with_admission_facts(
8610 corr: u64,
8611 module_id: &str,
8612 project_root: ScratchDir,
8613 consumer_identity: Option<subc_control::ConsumerIdentity>,
8614 facts: Option<Value>,
8615 ) -> Frame {
8616 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8617 target: RouteTarget::ToolProvider {
8618 module_id: module_id.to_string(),
8619 },
8620 identity: BindIdentity::new(
8621 project_root.path().to_path_buf(),
8622 "unit".to_string(),
8623 format!("session-{corr}"),
8624 ),
8625 consumer_identity,
8626 consumer_capabilities: None,
8627 role_versions: None,
8628 admission_facts: facts,
8629 scope: None,
8630 })
8631 .unwrap();
8632 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8633 }
8634
8635 #[derive(Clone, Default)]
8636 struct EventCapture {
8637 events: Arc<Mutex<Vec<CapturedEvent>>>,
8638 }
8639
8640 #[derive(Clone, Debug)]
8641 struct CapturedEvent {
8642 target: String,
8643 level: tracing::Level,
8644 fields: BTreeMap<String, String>,
8645 }
8646
8647 impl EventCapture {
8648 fn events(&self) -> Vec<CapturedEvent> {
8649 self.events.lock().unwrap().clone()
8650 }
8651 }
8652
8653 impl<S> Layer<S> for EventCapture
8654 where
8655 S: Subscriber,
8656 {
8657 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8658 let mut visitor = EventFieldVisitor::default();
8659 event.record(&mut visitor);
8660 self.events.lock().unwrap().push(CapturedEvent {
8661 target: event.metadata().target().to_string(),
8662 level: *event.metadata().level(),
8663 fields: visitor.fields,
8664 });
8665 }
8666 }
8667
8668 #[derive(Default)]
8669 struct EventFieldVisitor {
8670 fields: BTreeMap<String, String>,
8671 }
8672
8673 impl Visit for EventFieldVisitor {
8674 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8675 self.fields
8676 .insert(field.name().to_string(), format!("{value:?}"));
8677 }
8678 }
8679
8680 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8681 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8682 status,
8683 detail: Some("warming".to_string()),
8684 metrics: Some(json!({"queue_depth": 3})),
8685 })
8686 .unwrap();
8687 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8688 }
8689
8690 fn route_bind_ack(corr: u64) -> Frame {
8691 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8692 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8693 }
8694
8695 fn unique_project_root(label: &str) -> ScratchDir {
8696 ScratchDir::new(label)
8697 }
8698
8699 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8700 match parse_route_poll(frame) {
8701 ClientControlResponse::RoutePoll {
8702 status: None,
8703 live: Some(live),
8704 ..
8705 } => assert_eq!(live, expected_live),
8706 other => panic!("unexpected route.poll response: {other:?}"),
8707 }
8708 }
8709
8710 fn bind_liveness_route(
8711 registry: &Registry,
8712 forwarding: &ForwardingTable,
8713 module_id: &str,
8714 ) -> (RouteCtx, u16, u32) {
8715 let module_connection = ConnectionId::new(101);
8716 let client_connection = ConnectionId::new(202);
8717 let registration = registry
8718 .register_with_control_ops(
8719 manifest(module_id, PROTOCOL_VERSION),
8720 PROTOCOL_VERSION,
8721 module_connection,
8722 module_baseline_control_ops(),
8723 )
8724 .unwrap();
8725 let (module_tx, _module_rx) = mpsc::channel(8);
8726 let endpoint = forwarding
8727 .register_module_connection(
8728 module_connection,
8729 module_id.to_string(),
8730 PROTOCOL_VERSION,
8731 manifest_concurrency(®istration.manifest),
8732 FrameSink::new(module_tx),
8733 )
8734 .unwrap();
8735 let (client_ctx, _client_rx) = route_ctx(client_connection);
8736 let pending = forwarding
8737 .begin_route_bind_relay_for_test(
8738 client_connection,
8739 client_ctx.egress.clone(),
8740 1,
8741 module_id,
8742 )
8743 .unwrap();
8744 assert_eq!(pending.endpoint, endpoint);
8745 let route_channel = pending.client_channel;
8746 let route_epoch = pending.client_epoch;
8747 forwarding
8748 .complete_pending_relay(
8749 module_connection,
8750 pending.corr,
8751 RouteBindRelayOutcome::Accepted,
8752 )
8753 .unwrap();
8754 (client_ctx, route_channel, route_epoch)
8755 }
8756
8757 struct FakeProcessLiveness {
8758 live: Option<bool>,
8759 }
8760
8761 impl ModuleProcessLiveness for FakeProcessLiveness {
8762 fn process_live(&self, _module_id: &str) -> Option<bool> {
8763 self.live
8764 }
8765 }
8766
8767 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8768 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8769 {
8770 let registry = Arc::new(Registry::default());
8771 let supervisor_handle = SupervisorHandle::new();
8772 let supervisor = Supervisor::new_for_test(
8773 Arc::clone(®istry),
8774 RestartPolicy::new(1, Duration::from_millis(10)),
8775 )
8776 .with_handle(supervisor_handle.clone());
8777 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8778 let module = supervisor
8779 .spawn(ModuleSpec {
8780 module_id: "stderr-tail-wire".to_string(),
8781 program: fake_aft_stub_path(),
8782 args: Vec::new(),
8783 env: vec![
8784 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8785 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8786 ],
8787 reserved: false,
8788 reserved_prefixes: Vec::new(),
8789 protocol: ModuleProtocol::Subc,
8790 overlap: Default::default(),
8791 })
8792 .unwrap();
8793
8794 let deadline = Instant::now() + Duration::from_secs(5);
8795 loop {
8796 let tail = module.stderr_tail(None, None);
8797 if tail
8798 .entries
8799 .iter()
8800 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8801 && tail.entries.iter().any(|entry| {
8802 matches!(
8803 entry,
8804 TailEntry::Line {
8805 truncated: true,
8806 ..
8807 }
8808 )
8809 })
8810 {
8811 break;
8812 }
8813 assert!(
8814 Instant::now() < deadline,
8815 "module did not produce a truncated line and restart boundary: {tail:?}"
8816 );
8817 sleep(Duration::from_millis(10)).await;
8818 }
8819
8820 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8821 let request = ClientControlRequest::SupervisorStderrTail {
8822 module_id: "stderr-tail-wire".to_string(),
8823 max_lines: None,
8824 max_bytes: None,
8825 };
8826 let frame = Frame::build(
8827 FrameType::Request,
8828 control_flags(),
8829 0,
8830 0,
8831 1,
8832 serde_json::to_vec(&request).unwrap(),
8833 )
8834 .unwrap();
8835 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8836 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8837 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8838 serde_json::from_slice(&responses[0].body).unwrap()
8839 else {
8840 panic!("expected supervisor.stderr_tail response");
8841 };
8842
8843 assert!(
8844 tail.entries
8845 .iter()
8846 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8847 "the control response lost the restart boundary"
8848 );
8849 let Some(StderrTailEntry::Line {
8850 text,
8851 truncated,
8852 at_ms,
8853 }) = tail.entries.iter().find(|entry| {
8854 matches!(
8855 entry,
8856 StderrTailEntry::Line {
8857 truncated: true,
8858 ..
8859 }
8860 )
8861 })
8862 else {
8863 panic!("the control response lost the truncated line");
8864 };
8865 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8866 assert!(*truncated);
8867 assert!(
8868 at_ms.is_some(),
8869 "the control response lost the line's capture time"
8870 );
8871 }
8872
8873 #[tokio::test(flavor = "current_thread")]
8878 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8879 let dir = ScratchDir::new("terminals-off-worker");
8880 let journal_path = dir.join("terminals.jsonl");
8881 let registry = Arc::new(Registry::default());
8882 let supervisor_handle = SupervisorHandle::new();
8883 let supervisor =
8884 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8885 .with_handle(supervisor_handle.clone())
8886 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8887 let module = supervisor
8888 .spawn(ModuleSpec {
8889 module_id: "terminal-off-worker".to_string(),
8890 program: fake_aft_stub_path(),
8891 args: Vec::new(),
8892 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8893 reserved: false,
8894 reserved_prefixes: Vec::new(),
8895 protocol: ModuleProtocol::Subc,
8896 overlap: Default::default(),
8897 })
8898 .unwrap();
8899 let deadline = Instant::now() + Duration::from_secs(5);
8900 while module.terminal_history().entries.len() != 2 {
8901 assert!(Instant::now() < deadline, "module did not record two exits");
8902 sleep(Duration::from_millis(10)).await;
8903 }
8904
8905 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8906 let handler =
8907 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8908 let frame = Frame::build(
8909 FrameType::Request,
8910 control_flags(),
8911 0,
8912 0,
8913 1,
8914 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8915 module_id: "terminal-off-worker".to_string(),
8916 })
8917 .unwrap(),
8918 )
8919 .unwrap();
8920 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8921 let spawned_at = std::time::Instant::now();
8922 let read = tokio::spawn({
8923 let handler = Arc::clone(&handler);
8924 async move { handler.handle_control_frame(&ctx, frame).await }
8925 });
8926 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8929 .await
8930 .unwrap()
8931 .expect("the history read reached its pause");
8932 let elapsed = spawned_at.elapsed();
8933 assert!(
8934 elapsed < Duration::from_secs(2) && !read.is_finished(),
8935 "this task could not run while the history read was paused \
8936 (resumed after {elapsed:?}, read finished: {})",
8937 read.is_finished()
8938 );
8939
8940 drop(release);
8941 let responses = read.await.unwrap().unwrap();
8942 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8943 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8944 panic!("expected supervisor.terminals response");
8945 };
8946 assert_eq!(terminals.entries.len(), 2);
8947 assert_eq!(terminals.journal_skipped_lines, 0);
8948 assert_eq!(terminals.journal_read_errors, 0);
8949 }
8950
8951 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8952 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8953 let registry = Arc::new(Registry::default());
8954 let supervisor_handle = SupervisorHandle::new();
8955 let supervisor =
8956 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8957 .with_handle(supervisor_handle.clone());
8958 let module = supervisor
8959 .spawn(ModuleSpec {
8960 module_id: "terminal-golden".to_string(),
8961 program: fake_aft_stub_path(),
8962 args: Vec::new(),
8963 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8964 reserved: false,
8965 reserved_prefixes: Vec::new(),
8966 protocol: ModuleProtocol::Subc,
8967 overlap: Default::default(),
8968 })
8969 .unwrap();
8970
8971 let deadline = Instant::now() + Duration::from_secs(5);
8972 while module.terminal_history().entries.len() != 2 {
8973 assert!(
8974 Instant::now() < deadline,
8975 "module did not retain two terminal exits: {:?}",
8976 module.terminal_history()
8977 );
8978 sleep(Duration::from_millis(10)).await;
8979 }
8980
8981 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8982 let request = ClientControlRequest::SupervisorTerminals {
8983 module_id: "terminal-golden".to_string(),
8984 };
8985 let frame = Frame::build(
8986 FrameType::Request,
8987 control_flags(),
8988 0,
8989 0,
8990 1,
8991 serde_json::to_vec(&request).unwrap(),
8992 )
8993 .unwrap();
8994 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8995 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8996 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8997 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8998 panic!("expected supervisor.terminals response");
8999 };
9000 assert_eq!(terminals.entries.len(), 2);
9001 assert_eq!(terminals.dropped, 0);
9002
9003 let mut rendered = serde_json::to_value(response).unwrap();
9004 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9007 for (index, entry) in rendered["entries"]
9008 .as_array_mut()
9009 .expect("terminal response entries array")
9010 .iter_mut()
9011 .enumerate()
9012 {
9013 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9014 }
9015
9016 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9017 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9018 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9019 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9020 std::fs::write(&golden_path, &serialized).unwrap();
9021 }
9022 let expected: Value =
9023 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9024 assert_eq!(rendered, expected);
9025 }
9026
9027 #[test]
9028 fn hello_registers_manifest_and_returns_ack() {
9029 let registry = Arc::new(Registry::default());
9030 let handler = ControlHandler::new(Arc::clone(®istry));
9031 let conn = ConnectionId::new(1);
9032
9033 let responses = handler
9034 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9035 .unwrap();
9036
9037 assert_eq!(responses.len(), 1);
9038 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9039 assert_eq!(responses[0].header.channel, 0);
9040 assert_eq!(responses[0].header.corr, 7);
9041 let ack = parse_ack(&responses[0]);
9042 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9043 assert!(ack
9044 .subc_capabilities
9045 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9046 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9047 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9048 assert!(ack
9049 .subc_ops
9050 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9051 assert!(ack
9052 .subc_ops
9053 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9054
9055 let registration = registry.get_module("aft").unwrap().unwrap();
9056 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9057 assert_eq!(registration.state, ChannelState::Active);
9058 assert_eq!(registration.connection_id, conn);
9059 assert_eq!(registration.control_ops, module_baseline_control_ops());
9060 }
9061
9062 #[test]
9063 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9064 let invalid_identifiers = [
9065 ("case_change", "credentials-Provider/v1"),
9066 ("leading_zero", "credentials-provider/v01"),
9067 ("trailing_hyphen", "credentials-provider-/v1"),
9068 ("consecutive_hyphens", "credentials--provider/v1"),
9069 ("uppercase", "Credentials-provider/v1"),
9070 ("missing_v", "credentials-provider/1"),
9071 ("whitespace", "credentials provider/v1"),
9072 ("zero_version", "credentials-provider/v0"),
9073 ("out_of_range_version", "credentials-provider/v4294967296"),
9074 (
9075 "overlength_name",
9076 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9077 ),
9078 ];
9079 let mut cases = invalid_identifiers
9080 .into_iter()
9081 .map(|(name, identifier)| {
9082 (
9083 format!("identifier_{name}"),
9084 "capabilities.provides[0]".to_string(),
9085 identifier.to_string(),
9086 json!({ "provides": [identifier] }),
9087 None,
9088 )
9089 })
9090 .collect::<Vec<_>>();
9091 cases.extend([
9092 (
9093 "unknown_need".to_string(),
9094 "capabilities.requires[0].need".to_string(),
9095 "deferred".to_string(),
9096 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9097 None,
9098 ),
9099 (
9100 "duplicate_provides".to_string(),
9101 "capabilities.provides[1]".to_string(),
9102 "credentials-provider/v1".to_string(),
9103 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9104 None,
9105 ),
9106 (
9107 "duplicate_must_never_reach".to_string(),
9108 "capabilities.must_never_reach[1]".to_string(),
9109 "credentials-provider/v1".to_string(),
9110 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9111 None,
9112 ),
9113 (
9114 "duplicate_requires_same_need".to_string(),
9115 "capabilities.requires[1]".to_string(),
9116 "credentials-provider/v1".to_string(),
9117 json!({ "requires": [
9118 { "capability": "credentials-provider/v1", "need": "required" },
9119 { "capability": "credentials-provider/v1", "need": "required" }
9120 ] }),
9121 None,
9122 ),
9123 (
9124 "duplicate_requires_conflicting_need".to_string(),
9125 "capabilities.requires[1]".to_string(),
9126 "credentials-provider/v1".to_string(),
9127 json!({ "requires": [
9128 { "capability": "credentials-provider/v1", "need": "required" },
9129 { "capability": "credentials-provider/v1", "need": "optional" }
9130 ] }),
9131 None,
9132 ),
9133 (
9134 "capabilities_root_pointer".to_string(),
9135 "runtime_computed[0]".to_string(),
9136 "/capabilities".to_string(),
9137 json!({}),
9138 Some(json!(["/capabilities"])),
9139 ),
9140 (
9141 "capabilities_descendant_pointer".to_string(),
9142 "runtime_computed[0]".to_string(),
9143 "/capabilities/provides".to_string(),
9144 json!({}),
9145 Some(json!(["/capabilities/provides"])),
9146 ),
9147 (
9148 "malformed_pointer_without_leading_slash".to_string(),
9149 "runtime_computed[0]".to_string(),
9150 "capabilities".to_string(),
9151 json!({}),
9152 Some(json!(["capabilities"])),
9153 ),
9154 (
9155 "malformed_pointer_escape".to_string(),
9156 "runtime_computed[0]".to_string(),
9157 "/roles/~2/tools".to_string(),
9158 json!({}),
9159 Some(json!(["/roles/~2/tools"])),
9160 ),
9161 (
9162 "unknown_capabilities_field".to_string(),
9163 "capabilities.future".to_string(),
9164 "<array>".to_string(),
9165 json!({ "future": [] }),
9166 None,
9167 ),
9168 ]);
9169
9170 for (index, (name, field, value, capabilities, runtime_computed)) in
9171 cases.into_iter().enumerate()
9172 {
9173 let registry = Arc::new(Registry::default());
9174 let handler = ControlHandler::new(Arc::clone(®istry));
9175 let response = handler
9176 .handle_control(
9177 ConnectionId::new((index + 1) as u64),
9178 capability_grammar_hello_frame(
9179 capabilities,
9180 runtime_computed,
9181 index as u64 + 1,
9182 ),
9183 )
9184 .expect("invalid HELLO returns a refusal");
9185
9186 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9187 let error = parse_error(&response[0]);
9188 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9189 let message = error["message"]
9190 .as_str()
9191 .expect("error message is a string");
9192 assert!(
9193 message.contains(&field),
9194 "{name}: field missing from {message}"
9195 );
9196 assert!(
9197 message.contains(&value),
9198 "{name}: value missing from {message}"
9199 );
9200 assert_eq!(
9201 registry
9202 .active_registration_count()
9203 .expect("registry reads"),
9204 0,
9205 "{name}: refused HELLO must not create a catalog entry"
9206 );
9207 }
9208 }
9209
9210 #[test]
9211 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9212 let registry = Arc::new(Registry::default());
9213 let handler = ControlHandler::new(Arc::clone(®istry));
9214 let capabilities = json!({
9215 "provides": ["credentials-provider/v1"],
9216 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9217 "must_never_reach": ["federation-transport/v1"]
9218 });
9219 let response = handler
9220 .handle_control(
9221 ConnectionId::new(99),
9222 capability_grammar_hello_frame(
9223 capabilities.clone(),
9224 Some(json!(["/roles/0/tools"])),
9225 99,
9226 ),
9227 )
9228 .expect("valid HELLO registers");
9229 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9230
9231 let request = Frame::build(
9232 FrameType::Request,
9233 control_flags(),
9234 0,
9235 0,
9236 100,
9237 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9238 .expect("catalog request serializes"),
9239 )
9240 .expect("catalog request frame builds");
9241 let response = handler
9242 .handle_catalog_list(request, None)
9243 .expect("catalog list succeeds");
9244 let ClientControlResponse::CatalogList { modules, .. } =
9245 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9246 else {
9247 panic!("catalog request must return catalog.list");
9248 };
9249 assert_eq!(modules.len(), 1);
9250 assert_eq!(
9251 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9252 capabilities
9253 );
9254 }
9255
9256 #[test]
9257 fn catalog_list_mirrors_management_operation_description() {
9258 let registry = Arc::new(Registry::default());
9259 let handler = ControlHandler::new(Arc::clone(®istry));
9260 let description = "List managed records and return their identifiers and metadata.";
9261 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9262 manifest.provides = vec![ProviderRole::ManagementSurface {
9263 operations: vec![ManagementOperation {
9264 name: "records.list".to_string(),
9265 kind: ManagementOperationKind::Query,
9266 description: Some(description.to_string()),
9267 }],
9268 config_schema: json!({"type": "object"}),
9269 observability: vec![ObservabilitySurface {
9270 name: "records.stats".to_string(),
9271 kind: ObservabilityKind::Snapshot,
9272 }],
9273 identity_scope: vec![IdentityScope::Project],
9274 concurrency: Concurrency::ModuleManaged,
9275 }];
9276 registry
9277 .register_with_control_ops(
9278 manifest,
9279 PROTOCOL_VERSION,
9280 ConnectionId::new(99),
9281 Vec::new(),
9282 )
9283 .expect("described management manifest registers");
9284
9285 let request = Frame::build(
9286 FrameType::Request,
9287 control_flags(),
9288 0,
9289 0,
9290 100,
9291 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9292 .expect("catalog request serializes"),
9293 )
9294 .expect("catalog request frame builds");
9295 let response = handler
9296 .handle_catalog_list(request, None)
9297 .expect("catalog list succeeds");
9298 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9299 assert_eq!(
9300 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9301 "catalog.list must preserve the declared operation description verbatim"
9302 );
9303 }
9304
9305 #[test]
9306 fn catalog_list_mirrors_registered_operation_delivery() {
9307 let registry = Arc::new(Registry::default());
9308 let handler = ControlHandler::new(Arc::clone(®istry));
9309 let delivery =
9310 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9311 let mut body = serde_json::to_value(ModuleHelloBody {
9312 manifest: manifest("watch-management", PROTOCOL_VERSION),
9313 protocol_ver: PROTOCOL_VERSION,
9314 control_ops: None,
9315 launch_nonce: None,
9316 })
9317 .unwrap();
9318 body["manifest"]["provides"] = json!([{
9319 "role": "management_surface",
9320 "operations": [
9321 {"name": "status", "kind": "query"},
9322 {"name": "future", "kind": "query"},
9323 {"name": "reset", "kind": "mutate"}
9324 ],
9325 "config_schema": {}, "observability": [], "identity_scope": []
9326 }]);
9327 body["manifest"]["operation_delivery"] = delivery.clone();
9328 let hello = Frame::build(
9329 FrameType::Hello,
9330 control_flags(),
9331 0,
9332 0,
9333 99,
9334 serde_json::to_vec(&body).unwrap(),
9335 )
9336 .unwrap();
9337 let response = handler
9338 .handle_control(ConnectionId::new(99), hello)
9339 .unwrap();
9340 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9341
9342 let (_, registrations) = registry.list_modules().unwrap();
9343 let registered = ®istrations[0].manifest;
9344 assert!(registered.is_watch_operation("status"));
9345 assert!(!registered.is_watch_operation("future"));
9346 assert!(!registered.is_watch_operation("reset"));
9347 assert!(!registered.is_watch_operation("missing"));
9348
9349 let request = Frame::build(
9350 FrameType::Request,
9351 control_flags(),
9352 0,
9353 0,
9354 100,
9355 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9356 )
9357 .unwrap();
9358 let response = handler.handle_catalog_list(request, None).unwrap();
9359 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9360 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9361 let ClientControlResponse::CatalogList { modules, .. } =
9362 serde_json::from_slice(&response[0].body).unwrap()
9363 else {
9364 panic!("expected catalog.list");
9365 };
9366 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9367 }
9368
9369 #[test]
9370 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9371 let registry = Arc::new(Registry::default());
9372 let supervisor = SupervisorHandle::new();
9373 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9374 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9375 let descriptor = json!({
9376 "schema": "ckios.richui.manifest/1",
9377 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9378 "bytes": 4096,
9379 "chunks": 4
9380 });
9381 let connection = ConnectionId::new(201);
9382 let hello = handler
9383 .handle_control(
9384 connection,
9385 hello_frame_with_rich_ui(
9386 "vault",
9387 PROTOCOL_VERSION,
9388 201,
9389 Some("the-real-nonce"),
9390 descriptor.clone(),
9391 ),
9392 )
9393 .unwrap();
9394 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9395
9396 let request =
9397 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9398 let provides = registry
9399 .get_module_by_connection(connection)
9400 .unwrap()
9401 .unwrap()
9402 .manifest
9403 .provides;
9404 handler
9405 .handle_catalog_update(connection, request, provides, None, Some(false))
9406 .unwrap();
9407
9408 let request = Frame::build(
9409 FrameType::Request,
9410 control_flags(),
9411 0,
9412 0,
9413 203,
9414 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9415 )
9416 .unwrap();
9417 let response = handler.handle_catalog_list(request, None).unwrap();
9418 let ClientControlResponse::CatalogList { modules, .. } =
9419 serde_json::from_slice(&response[0].body).unwrap()
9420 else {
9421 panic!("expected catalog.list");
9422 };
9423 assert_eq!(
9424 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9425 descriptor
9426 );
9427 assert!(modules[0].nonce_verified);
9428 }
9429
9430 fn hello_with_role_declarations(declarations: Value) -> Frame {
9431 let mut body = serde_json::to_value(ModuleHelloBody {
9432 manifest: manifest("role-test", PROTOCOL_VERSION),
9433 protocol_ver: PROTOCOL_VERSION,
9434 control_ops: None,
9435 launch_nonce: None,
9436 })
9437 .unwrap();
9438 body["manifest"]["role_declarations"] = declarations;
9439 Frame::build(
9440 FrameType::Hello,
9441 control_flags(),
9442 0,
9443 0,
9444 301,
9445 serde_json::to_vec(&body).unwrap(),
9446 )
9447 .unwrap()
9448 }
9449
9450 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9451 let request = Frame::build(
9452 FrameType::Request,
9453 control_flags(),
9454 0,
9455 0,
9456 302,
9457 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9458 )
9459 .unwrap();
9460 let response = handler.handle_catalog_list(request, None).unwrap();
9461 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9462 let ClientControlResponse::CatalogList { mut modules, .. } =
9463 serde_json::from_slice(&response[0].body).unwrap()
9464 else {
9465 panic!("expected catalog.list");
9466 };
9467 (raw["modules"][0].clone(), modules.remove(0))
9468 }
9469
9470 #[test]
9471 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9472 let registry = Arc::new(Registry::default());
9473 let handler = ControlHandler::new(Arc::clone(®istry));
9474 let declarations = json!({
9475 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9476 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9477 });
9478 let connection = ConnectionId::new(301);
9479 let response = handler
9480 .handle_control(
9481 connection,
9482 hello_with_role_declarations(declarations.clone()),
9483 )
9484 .unwrap();
9485 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9486 let registered = registry
9487 .get_module_by_connection(connection)
9488 .unwrap()
9489 .unwrap()
9490 .manifest;
9491 assert_eq!(
9495 serde_json::to_value(®istered.role_declarations).unwrap(),
9496 declarations
9497 );
9498 let (raw, entry) = role_declarations_catalog(&handler);
9499 assert_eq!(raw["role_declarations"], declarations);
9500 assert_eq!(entry.role_declarations, registered.role_declarations);
9501
9502 let request =
9503 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9504 handler
9505 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9506 .unwrap();
9507 let (raw, entry) = role_declarations_catalog(&handler);
9508 assert_eq!(raw["role_declarations"], declarations);
9509 assert_eq!(entry.role_declarations, registered.role_declarations);
9510 }
9511
9512 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9513 let registry = Arc::new(Registry::default());
9514 let handler = ControlHandler::new(Arc::clone(®istry));
9515 let capture = EventCapture::default();
9516 let _subscriber =
9517 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9518 let response = handler
9519 .handle_control(
9520 ConnectionId::new(301),
9521 hello_with_role_declarations(declarations),
9522 )
9523 .unwrap();
9524 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9525 assert!(registry
9526 .get_module_by_connection(ConnectionId::new(301))
9527 .unwrap()
9528 .unwrap()
9529 .manifest
9530 .role_declarations
9531 .is_empty());
9532 let (raw, entry) = role_declarations_catalog(&handler);
9533 assert!(raw.get("role_declarations").is_none());
9534 assert!(entry.role_declarations.is_empty());
9535 let warnings: Vec<_> = capture
9536 .events()
9537 .into_iter()
9538 .filter(|event| {
9539 event.fields.get("message").map(String::as_str)
9540 == Some("dropping whole role_declarations map from HELLO")
9541 })
9542 .collect();
9543 assert_eq!(warnings.len(), 1);
9544 assert_eq!(warnings[0].level, tracing::Level::WARN);
9545 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9546 assert_eq!(warnings[0].fields["reason"], reason);
9547 }
9548
9549 #[test]
9550 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9551 for value in [
9552 json!(null),
9553 json!(false),
9554 json!(0),
9555 json!("opaque"),
9556 json!([]),
9557 ] {
9558 assert_hello_drops_role_declarations_with_warning(
9559 json!({"routing/v1": {}, "broken/v1": value}),
9560 "role_declarations values must be objects",
9561 );
9562 }
9563 assert_hello_drops_role_declarations_with_warning(
9564 json!([]),
9565 "role_declarations must be an object",
9566 );
9567 }
9568
9569 #[test]
9570 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9571 assert_hello_drops_role_declarations_with_warning(
9572 json!({"routing/v1": {}, "bad key": {}}),
9573 "role_declarations key must use capability-name grammar",
9574 );
9575 }
9576
9577 #[test]
9578 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9579 assert_hello_drops_role_declarations_with_warning(
9580 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9581 "role_declarations exceeds 16384 compact JSON bytes",
9582 );
9583 }
9584
9585 #[test]
9586 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9587 let registry = Arc::new(Registry::default());
9588 let handler = ControlHandler::new(registry.clone());
9589 let descriptor = json!({
9590 "schema": "ckios.richui.manifest/1",
9591 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9592 "bytes": 128,
9593 "chunks": 2
9594 });
9595 let malformed = [
9596 ("schema", json!("")),
9597 ("schema", json!("a".repeat(65))),
9598 ("schema", json!("Bad schema")),
9599 (
9600 "manifest_digest",
9601 json!(format!("sha256:{}", "A".repeat(64))),
9602 ),
9603 ("manifest_digest", json!("sha256:abc")),
9604 ("bytes", json!(0)),
9605 ("bytes", json!(16_777_217)),
9606 ("bytes", json!("128")),
9607 ("chunks", json!(0)),
9608 ("chunks", json!(1025)),
9609 ("chunks", json!("2")),
9610 ];
9611 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9612 let mut invalid = descriptor.clone();
9613 invalid[field] = invalid_value;
9614 let response = handler
9615 .handle_control(
9616 ConnectionId::new(210 + index as u64),
9617 hello_frame_with_rich_ui(
9618 &format!("bad-rich-ui-{index}"),
9619 PROTOCOL_VERSION,
9620 210 + index as u64,
9621 None,
9622 invalid,
9623 ),
9624 )
9625 .unwrap();
9626 assert_eq!(response[0].header.ty, FrameType::Error);
9627 let error = parse_error(&response[0]);
9628 assert_eq!(error["code"], "invalid_manifest");
9629 assert!(error["message"]
9630 .as_str()
9631 .unwrap()
9632 .contains(&format!("rich_ui.{field}")));
9633 assert!(registry
9634 .get_module(&format!("bad-rich-ui-{index}"))
9635 .unwrap()
9636 .is_none());
9637 }
9638 }
9639
9640 #[test]
9641 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9642 let registry = Arc::new(Registry::default());
9643 let handler = ControlHandler::new(Arc::clone(®istry));
9644 let connection = ConnectionId::new(219);
9645 registry
9646 .register_with_control_ops(
9647 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9648 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9649 "bad schema",
9650 format!("sha256:{}", "a".repeat(64)),
9651 128,
9652 2,
9653 )))
9654 .build(),
9655 PROTOCOL_VERSION,
9656 connection,
9657 Vec::new(),
9658 )
9659 .unwrap();
9660 let request =
9661 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9662 let response = handler
9663 .handle_catalog_update(connection, request, Vec::new(), None, None)
9664 .unwrap();
9665 assert_eq!(response[0].header.ty, FrameType::Error);
9666 let error = parse_error(&response[0]);
9667 assert_eq!(error["code"], "invalid_manifest");
9668 assert!(error["message"]
9669 .as_str()
9670 .unwrap()
9671 .contains("rich_ui.schema"));
9672 }
9673
9674 #[test]
9675 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9676 let registry = Arc::new(Registry::default());
9677 let supervisor = SupervisorHandle::new();
9678 supervisor.open_swap("provider", "candidate-nonce".to_string());
9679 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9680 let connection = ConnectionId::new(218);
9681 let hello = handler
9682 .handle_control(
9683 connection,
9684 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9685 )
9686 .unwrap();
9687 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9688 assert!(registry.list_modules().unwrap().1.is_empty());
9689
9690 registry.promote_candidate("provider").unwrap().unwrap();
9691 let request = Frame::build(
9692 FrameType::Request,
9693 control_flags(),
9694 0,
9695 0,
9696 219,
9697 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9698 )
9699 .unwrap();
9700 let response = handler.handle_catalog_list(request, None).unwrap();
9701 let ClientControlResponse::CatalogList { modules, .. } =
9702 serde_json::from_slice(&response[0].body).unwrap()
9703 else {
9704 panic!("expected catalog.list");
9705 };
9706 assert_eq!(modules.len(), 1);
9707 assert!(modules[0].nonce_verified);
9708 }
9709
9710 #[test]
9711 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9712 let registry = Arc::new(Registry::default());
9713 let handler = ControlHandler::new(Arc::clone(®istry));
9714 let hello = Frame::build(
9715 FrameType::Hello,
9716 control_flags(),
9717 0,
9718 0,
9719 220,
9720 serde_json::to_vec(&ModuleHelloBody {
9721 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9722 protocol_ver: PROTOCOL_VERSION,
9723 control_ops: None,
9724 launch_nonce: Some("unverified".to_string()),
9725 })
9726 .unwrap(),
9727 )
9728 .unwrap();
9729 assert_eq!(
9730 handler
9731 .handle_control(ConnectionId::new(220), hello)
9732 .unwrap()[0]
9733 .header
9734 .ty,
9735 FrameType::HelloAck
9736 );
9737 let request = Frame::build(
9738 FrameType::Request,
9739 control_flags(),
9740 0,
9741 0,
9742 221,
9743 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9744 )
9745 .unwrap();
9746 let response = handler.handle_catalog_list(request, None).unwrap();
9747 let ClientControlResponse::CatalogList { modules, .. } =
9748 serde_json::from_slice(&response[0].body).unwrap()
9749 else {
9750 panic!("expected catalog.list");
9751 };
9752 assert_eq!(
9753 serde_json::to_vec(&modules[0]).unwrap(),
9754 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9755 );
9756 assert!(!modules[0].nonce_verified);
9757 }
9758
9759 #[test]
9760 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9761 let registry = Arc::new(Registry::default());
9762 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9763 [("vault".to_string(), true), ("squatter".to_string(), true)],
9764 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9765 );
9766 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9767 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9768 provides: vec!["credentials-provider/v1".to_string()],
9769 requires: Vec::new(),
9770 must_never_reach: Vec::new(),
9771 });
9772 let frame = Frame::build(
9773 FrameType::Hello,
9774 control_flags(),
9775 0,
9776 0,
9777 77,
9778 serde_json::to_vec(&ModuleHelloBody {
9779 manifest: squatter,
9780 protocol_ver: PROTOCOL_VERSION,
9781 control_ops: None,
9782 launch_nonce: None,
9783 })
9784 .expect("HELLO serializes"),
9785 )
9786 .expect("HELLO frame builds");
9787 let response = handler
9788 .handle_control(ConnectionId::new(77), frame)
9789 .expect("reserved claim receives a typed refusal");
9790 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9791 assert_eq!(
9792 registry
9793 .active_registration_count()
9794 .expect("registry reads"),
9795 0,
9796 "a reserved capability refusal must not leave a catalog entry"
9797 );
9798 }
9799
9800 #[test]
9801 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9802 for settlement in ["timeout", "inconclusive", "drop"] {
9803 let breakers = RouteBindBreakers::default();
9804 let RouteBindAdmission::Admitted {
9805 guard: mut old,
9806 probe: false,
9807 } = breakers.admit("prov")
9808 else {
9809 panic!("ordinary relay admitted")
9810 };
9811 let RouteBindAdmission::Admitted {
9812 guard: mut opener, ..
9813 } = breakers.admit("prov")
9814 else {
9815 panic!("second relay admitted")
9816 };
9817 assert!(
9818 !opener
9819 .record_timeout(1, Duration::ZERO)
9820 .unwrap()
9821 .reopened_after_probe
9822 );
9823 let RouteBindAdmission::Admitted {
9824 guard: mut probe,
9825 probe: true,
9826 } = breakers.admit("prov")
9827 else {
9828 panic!("one cooldown probe admitted")
9829 };
9830 match settlement {
9831 "timeout" => assert!(
9832 !old.record_timeout(1, Duration::ZERO)
9833 .unwrap()
9834 .reopened_after_probe
9835 ),
9836 "inconclusive" => old.record_inconclusive(),
9837 "drop" => drop(old),
9838 _ => unreachable!(),
9839 }
9840 assert!(
9841 matches!(
9842 breakers.admit("prov"),
9843 RouteBindAdmission::Refused {
9844 probe_in_flight: true,
9845 ..
9846 }
9847 ),
9848 "{settlement} of a pre-open relay cannot release the real probe"
9849 );
9850 assert!(
9851 probe
9852 .record_timeout(1, Duration::ZERO)
9853 .unwrap()
9854 .reopened_after_probe
9855 );
9856 assert!(matches!(
9857 breakers.admit("prov"),
9858 RouteBindAdmission::Admitted { probe: true, .. }
9859 ));
9860 }
9861 let breakers = RouteBindBreakers::default();
9862 let admit = || match breakers.admit("prov") {
9863 RouteBindAdmission::Admitted { guard, .. } => guard,
9864 _ => panic!("relay admitted"),
9865 };
9866 admit().record_timeout(1, Duration::ZERO);
9867 let mut old_probe = admit();
9868 breakers.reset_for_new_module_connection("prov");
9869 admit().record_timeout(1, Duration::ZERO);
9870 let _new_probe = admit();
9871 old_probe.record_inconclusive();
9872 assert!(matches!(
9873 breakers.admit("prov"),
9874 RouteBindAdmission::Refused {
9875 probe_in_flight: true,
9876 ..
9877 }
9878 ));
9879 }
9880
9881 #[tokio::test]
9882 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9883 for candidate in [false, true] {
9884 let registry = Arc::new(Registry::default());
9885 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9886 [("vault".to_string(), true), ("squatter".to_string(), true)],
9887 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9888 );
9889 let conn = ConnectionId::new(77);
9890 let (ctx, mut rx) = route_ctx(conn);
9891 let initial = capability_manifest("squatter", &[], &[]);
9892 if candidate {
9893 registry
9894 .register_candidate_with_control_ops(
9895 initial.clone(),
9896 PROTOCOL_VERSION,
9897 conn,
9898 module_baseline_control_ops(),
9899 )
9900 .unwrap();
9901 handler
9902 .forwarding
9903 .register_candidate_module_connection(
9904 conn,
9905 "squatter".to_string(),
9906 PROTOCOL_VERSION,
9907 manifest_concurrency(&initial),
9908 ctx.egress.clone(),
9909 )
9910 .unwrap();
9911 } else {
9912 hello_via_sink(
9913 &handler,
9914 &ctx,
9915 &mut rx,
9916 hello_frame_with_manifest(initial.clone(), 1),
9917 )
9918 .await;
9919 }
9920 let response = handler
9921 .handle_control_frame(
9922 &ctx,
9923 catalog_update_with_capabilities_frame(
9924 2,
9925 capability_manifest("squatter", &["credentials-provider/v1"], &[])
9926 .capabilities
9927 .unwrap(),
9928 ),
9929 )
9930 .await
9931 .unwrap();
9932 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9933 assert_eq!(
9934 registry
9935 .get_module_by_connection(conn)
9936 .unwrap()
9937 .unwrap()
9938 .manifest,
9939 initial
9940 );
9941 }
9942 }
9943
9944 #[test]
9945 fn server_describe_surfaces_required_capability_verdict_fields() {
9946 let registry = Arc::new(Registry::default());
9947 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9948 [
9949 ("consumer".to_string(), true),
9950 ("provider".to_string(), false),
9951 ],
9952 BTreeMap::new(),
9953 );
9954 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9955 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9956 provides: Vec::new(),
9957 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9958 capability: "credentials-provider/v1".to_string(),
9959 need: subc_protocol::manifest::CapabilityNeed::Required,
9960 }],
9961 must_never_reach: Vec::new(),
9962 });
9963 let hello = Frame::build(
9964 FrameType::Hello,
9965 control_flags(),
9966 0,
9967 0,
9968 78,
9969 serde_json::to_vec(&ModuleHelloBody {
9970 manifest: consumer,
9971 protocol_ver: PROTOCOL_VERSION,
9972 control_ops: None,
9973 launch_nonce: None,
9974 })
9975 .expect("HELLO serializes"),
9976 )
9977 .expect("HELLO frame builds");
9978 handler
9979 .handle_control(ConnectionId::new(78), hello)
9980 .expect("consumer registers");
9981 let describe = Frame::build(
9982 FrameType::Request,
9983 control_flags(),
9984 0,
9985 0,
9986 79,
9987 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9988 .expect("request serializes"),
9989 )
9990 .expect("describe frame builds");
9991 let response = handler
9992 .handle_server_describe(describe)
9993 .expect("server.describe succeeds");
9994 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9995 let requirement = &rendered["capability_requirements"][0];
9996 assert_eq!(requirement["consumer"], "consumer");
9997 assert_eq!(requirement["verdict"], "never_provided");
9998 assert_eq!(requirement["episode_seq"], 1);
9999 assert_eq!(requirement["config_satisfiable"], false);
10000 assert_eq!(requirement["runtime_available"], false);
10001 assert!(requirement["detail"]
10002 .as_str()
10003 .expect("detail string")
10004 .contains("credentials-provider/v1"));
10005 }
10006
10007 fn deregister_required_provider(
10013 daemon_shutdown_begun: bool,
10014 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10015 const CAPABILITY: &str = "project-identity/v1";
10016 let registry = Arc::new(Registry::default());
10017 let daemon_shutdown = DaemonShutdownFlag::default();
10018 let handler = ControlHandler::new(Arc::clone(®istry))
10019 .with_capability_config(
10020 [
10021 ("consumer".to_string(), true),
10022 ("provider".to_string(), true),
10023 ],
10024 BTreeMap::new(),
10025 )
10026 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10027 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10028 consumer.capabilities = Some(CapabilityDeclarations {
10029 provides: Vec::new(),
10030 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10031 capability: CAPABILITY.to_string(),
10032 need: subc_protocol::manifest::CapabilityNeed::Required,
10033 }],
10034 must_never_reach: Vec::new(),
10035 });
10036 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10037 let provider_conn = ConnectionId::new(182);
10038 for (conn, manifest, corr) in [
10039 (ConnectionId::new(181), consumer, 181),
10040 (provider_conn, provider, 182),
10041 ] {
10042 let hello = handler
10043 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10044 .expect("module registers");
10045 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10046 }
10047 assert_eq!(
10048 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10049 Some(CapabilityVerdict::Provided),
10050 "the registered provider must satisfy the requirement first"
10051 );
10052 if daemon_shutdown_begun {
10053 daemon_shutdown.set_for_test();
10054 }
10055
10056 let capture = EventCapture::default();
10057 let _subscriber =
10058 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10059 handler
10060 .cleanup_connection(provider_conn)
10061 .expect("provider deregisters");
10062 let events = capture
10063 .events()
10064 .into_iter()
10065 .filter(|event| {
10066 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10067 })
10068 .collect();
10069 (
10070 events,
10071 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10072 )
10073 }
10074
10075 #[test]
10076 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10077 let (events, verdict) = deregister_required_provider(true);
10078 assert_eq!(
10079 verdict,
10080 Some(CapabilityVerdict::NeverProvided),
10081 "shutdown must not change the verdict route.open refusals read"
10082 );
10083 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10084 let event = &events[0];
10085 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10086 assert_eq!(
10087 event.fields.get("cause").map(String::as_str),
10088 Some("\"daemon_shutdown\""),
10089 "{event:?}"
10090 );
10091 assert_eq!(
10092 event.fields.get("verdict").map(String::as_str),
10093 Some("\"never_provided\""),
10094 "{event:?}"
10095 );
10096 }
10097
10098 #[test]
10099 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10100 let (events, verdict) = deregister_required_provider(false);
10101 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10102 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10103 let event = &events[0];
10104 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10105 assert_eq!(event.fields.get("cause"), None, "{event:?}");
10106 assert_eq!(
10107 event.fields.get("verdict").map(String::as_str),
10108 Some("\"never_provided\""),
10109 "{event:?}"
10110 );
10111 }
10112
10113 #[test]
10114 fn catalog_list_omits_capabilities_for_legacy_manifest() {
10115 let registry = Arc::new(Registry::default());
10116 let handler = ControlHandler::new(Arc::clone(®istry));
10117 let hello = handler
10118 .handle_control(
10119 ConnectionId::new(101),
10120 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10121 )
10122 .expect("legacy HELLO registers");
10123 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10124
10125 let request = Frame::build(
10126 FrameType::Request,
10127 control_flags(),
10128 0,
10129 0,
10130 102,
10131 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10132 .expect("catalog request serializes"),
10133 )
10134 .expect("catalog request frame builds");
10135 let response = handler
10136 .handle_catalog_list(request, None)
10137 .expect("catalog list succeeds");
10138 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10139 assert!(
10140 body["modules"][0].get("capabilities").is_none(),
10141 "legacy manifest must retain an absent capabilities field on catalog.list"
10142 );
10143 }
10144
10145 #[test]
10146 fn hello_ack_omits_storage_when_no_storage_config() {
10147 let registry = Arc::new(Registry::default());
10148 let handler = ControlHandler::new(Arc::clone(®istry));
10149 let responses = handler
10150 .handle_control(
10151 ConnectionId::new(1),
10152 hello_frame("aft", PROTOCOL_VERSION, 7),
10153 )
10154 .unwrap();
10155 let ack = parse_ack(&responses[0]);
10156 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10157 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10158 }
10159
10160 #[tokio::test]
10161 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10162 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10163 let registry = Arc::new(Registry::default());
10164 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
10165 let responses = handler
10166 .handle_control(
10167 ConnectionId::new(1),
10168 hello_frame("aft", PROTOCOL_VERSION, 7),
10169 )
10170 .unwrap();
10171 let ack = parse_ack(&responses[0]);
10172 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10173
10174 let described = handler
10175 .handle_control_frame(
10176 &route_ctx(ConnectionId::new(2)).0,
10177 Frame::build(
10178 FrameType::Request,
10179 control_flags(),
10180 0,
10181 0,
10182 9,
10183 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10184 )
10185 .unwrap(),
10186 )
10187 .await
10188 .unwrap();
10189 let ClientControlResponse::ServerDescribe { machine_id, .. } =
10190 serde_json::from_slice(&described[0].body).unwrap()
10191 else {
10192 panic!("server.describe answered with another shape");
10193 };
10194 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10195 }
10196
10197 #[test]
10198 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10199 let registry = Arc::new(Registry::default());
10202 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
10203 crate::daemon_config::StorageConfig::Sqlite {
10204 data_home: std::path::PathBuf::from("/data"),
10205 },
10206 ));
10207
10208 let responses = handler
10209 .handle_control(
10210 ConnectionId::new(1),
10211 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10212 )
10213 .unwrap();
10214 let ack = parse_ack(&responses[0]);
10215 assert_eq!(
10216 ack.storage,
10217 Some(serde_json::json!({
10218 "module_id": "alfonso-routing",
10219 "storage_namespace": "default",
10220 "isolation": { "kind": "module" },
10221 "backend": {
10222 "backend": "sqlite",
10223 "path": "/data/cortexkit/alfonso-routing/store.db"
10224 }
10225 })),
10226 "the delivered descriptor is the module's own sqlite store path"
10227 );
10228 }
10229
10230 #[test]
10231 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10232 let registry = Arc::new(Registry::default());
10233 let handler = ControlHandler::new(Arc::clone(®istry));
10234 let conn = ConnectionId::new(1);
10235 let responses = handler
10236 .handle_control(
10237 conn,
10238 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10239 )
10240 .unwrap();
10241 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10242 let registration = registry.get_module("aft").unwrap().unwrap();
10243 assert_eq!(registration.control_ops, module_baseline_control_ops());
10244
10245 let frame =
10246 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10247 assert!(handler
10248 .guard_module_control_op(&frame, "aft", "route.bind")
10249 .unwrap()
10250 .is_none());
10251 let error = handler
10252 .guard_module_control_op(&frame, "aft", "test.synthetic")
10253 .unwrap()
10254 .expect("synthetic ungranted op should be rejected");
10255 assert_eq!(error.header.ty, FrameType::Error);
10256 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10257 }
10258
10259 #[test]
10260 fn hello_control_ops_some_adds_optional_grants() {
10261 let registry = Arc::new(Registry::default());
10262 let handler = ControlHandler::new(Arc::clone(®istry));
10263 handler
10264 .handle_control(
10265 ConnectionId::new(1),
10266 hello_frame_with_control_ops(
10267 "aft",
10268 PROTOCOL_VERSION,
10269 7,
10270 Some(vec![
10271 "future.synthetic".to_string(),
10272 "route.bind".to_string(),
10273 ]),
10274 ),
10275 )
10276 .unwrap();
10277 let registration = registry.get_module("aft").unwrap().unwrap();
10278 assert_eq!(
10279 registration.control_ops,
10280 vec![
10281 "route.bind".to_string(),
10282 "route.status".to_string(),
10283 "future.synthetic".to_string(),
10284 ]
10285 );
10286 let frame =
10287 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10288 assert!(handler
10289 .guard_module_control_op(&frame, "aft", "future.synthetic")
10290 .unwrap()
10291 .is_none());
10292 }
10293
10294 #[tokio::test]
10295 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10296 let registry = Arc::new(Registry::default());
10297 let forwarding = Arc::new(ForwardingTable::default());
10298 let handler =
10299 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10300 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10301 hello_via_sink(
10302 &handler,
10303 &module_ctx,
10304 &mut module_rx,
10305 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10306 )
10307 .await;
10308
10309 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10310 let responses = handler
10311 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10312 .await
10313 .unwrap();
10314 assert_eq!(responses.len(), 1);
10315 assert_eq!(responses[0].header.ty, FrameType::Error);
10316 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10317 assert!(module_rx.try_recv().is_err());
10318 }
10319
10320 #[tokio::test]
10321 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10322 let registry = Arc::new(Registry::default());
10323 let forwarding = Arc::new(ForwardingTable::default());
10324 let handler =
10325 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10326 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10327 hello_via_sink(
10328 &handler,
10329 &module_ctx,
10330 &mut module_rx,
10331 hello_frame_with_control_ops(
10332 "aft",
10333 PROTOCOL_VERSION,
10334 7,
10335 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10336 ),
10337 )
10338 .await;
10339
10340 let project_root = unique_project_root("demux");
10341 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10342 let route_handler = handler.clone();
10343 let route_task = tokio::spawn(async move {
10344 route_handler
10345 .handle_control_frame(
10346 &route_client_ctx,
10347 route_open_frame(100, "aft", project_root),
10348 )
10349 .await
10350 .unwrap()
10351 });
10352 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10353 .await
10354 .unwrap()
10355 .unwrap();
10356 assert!(matches!(
10357 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10358 ModuleControlRequest::RouteBind { .. }
10359 ));
10360
10361 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10362 let health_handler = handler.clone();
10363 let health_task = tokio::spawn(async move {
10364 health_handler
10365 .handle_control_frame(
10366 &health_client_ctx,
10367 supervisor_health_probe_frame(101, "aft"),
10368 )
10369 .await
10370 .unwrap()
10371 });
10372 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10373 .await
10374 .unwrap()
10375 .unwrap();
10376 assert_eq!(
10377 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10378 ModuleControlRequest::HealthCheck {}
10379 );
10380
10381 handler
10382 .handle_control_frame(
10383 &module_ctx,
10384 health_response(health_frame.header.corr, HealthStatus::Degraded),
10385 )
10386 .await
10387 .unwrap();
10388 let health_response = health_task.await.unwrap();
10389 assert_eq!(health_response.len(), 1);
10390 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10391 ClientControlResponse::SupervisorHealthProbe {
10392 module_id,
10393 status,
10394 detail,
10395 metrics,
10396 } => {
10397 assert_eq!(module_id, "aft");
10398 assert_eq!(status, HealthStatus::Degraded);
10399 assert_eq!(detail.as_deref(), Some("warming"));
10400 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10401 }
10402 other => panic!("unexpected health response: {other:?}"),
10403 }
10404
10405 handler
10406 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10407 .await
10408 .unwrap();
10409 let route_response = route_task.await.unwrap();
10410 assert!(route_response.is_empty());
10411 let published = route_client_rx.recv().await.unwrap();
10412 assert!(matches!(
10413 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10414 ClientControlResponse::RouteOpen { .. }
10415 ));
10416 }
10417
10418 async fn relay_route_open(
10423 handler: &ControlHandler,
10424 client_connection: ConnectionId,
10425 client_egress: &FrameSink,
10426 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10427 corr: u64,
10428 module_id: &str,
10429 project_root_label: &str,
10430 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10431 let ctx = RouteCtx {
10432 connection_id: client_connection,
10433 egress: client_egress.clone(),
10434 };
10435 let handler = handler.clone();
10436 let project_root = unique_project_root(project_root_label);
10437 let module_id = module_id.to_string();
10438 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10439 let task = tokio::spawn(async move {
10440 let _guard = tracing::dispatcher::set_default(&dispatch);
10441 handler
10442 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10443 .await
10444 .unwrap()
10445 });
10446 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10447 .await
10448 .expect("module receives the relayed route.bind")
10449 .expect("module egress is open");
10450 (task, bind.frame)
10451 }
10452
10453 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10454 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10455 ModuleControlRequest::RouteBind {
10456 route_channel,
10457 epoch,
10458 ..
10459 } => (route_channel, epoch),
10460 other => panic!("expected a route.bind request, got {other:?}"),
10461 }
10462 }
10463
10464 fn published_route(frame: &Frame) -> (u16, u32) {
10465 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10466 ClientControlResponse::RouteOpen {
10467 route_channel,
10468 route_epoch,
10469 } => (route_channel, route_epoch),
10470 other => panic!("expected a route.open response, got {other:?}"),
10471 }
10472 }
10473
10474 #[tokio::test]
10494 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10495 let registry = Arc::new(Registry::default());
10496 let forwarding = Arc::new(ForwardingTable::default());
10497 let handler =
10498 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10499
10500 let module_connection = ConnectionId::new(30);
10501 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10502 hello_via_sink(
10503 &handler,
10504 &module_ctx,
10505 &mut module_rx,
10506 hello_frame("aft", PROTOCOL_VERSION, 7),
10507 )
10508 .await;
10509
10510 let dying_client = ConnectionId::new(31);
10511 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10512
10513 let (first_task, first_bind) = relay_route_open(
10516 &handler,
10517 dying_client,
10518 &dying_ctx.egress,
10519 &mut module_rx,
10520 100,
10521 "aft",
10522 "closing-first",
10523 )
10524 .await;
10525 handler
10526 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10527 .await
10528 .unwrap();
10529 assert!(first_task.await.unwrap().is_empty());
10530 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10531
10532 let (second_task, second_bind) = relay_route_open(
10534 &handler,
10535 dying_client,
10536 &dying_ctx.egress,
10537 &mut module_rx,
10538 101,
10539 "aft",
10540 "closing-second",
10541 )
10542 .await;
10543 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10544
10545 assert!(forwarding
10548 .escalate_client_delivery_failure(
10549 dying_client,
10550 first_channel,
10551 first_epoch,
10552 CloseReason::new(
10553 "module_to_client_delivery_failed",
10554 "client egress refused a module frame",
10555 ),
10556 crate::forwarding::UndeliveredFrame {
10557 module_id: None,
10558 sink: &dying_ctx.egress,
10559 },
10560 )
10561 .unwrap());
10562 assert!(!dying_ctx.egress.is_closed());
10563
10564 let ack = handler
10566 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10567 .await;
10568 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10569 if module_loop_error.is_some() {
10570 handler.cleanup_connection(module_connection).unwrap();
10574 }
10575 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10579 .await
10580 .ok()
10581 .flatten();
10582
10583 assert!(
10585 registry
10586 .get_module_by_connection(module_connection)
10587 .unwrap()
10588 .is_some(),
10589 "one client's closing connection ended the shared module connection: \
10590 {module_loop_error:?}"
10591 );
10592 let cotenant = ConnectionId::new(32);
10594 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10595 let (cotenant_task, cotenant_bind) = relay_route_open(
10596 &handler,
10597 cotenant,
10598 &cotenant_ctx.egress,
10599 &mut module_rx,
10600 102,
10601 "aft",
10602 "closing-cotenant",
10603 )
10604 .await;
10605 handler
10606 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10607 .await
10608 .unwrap();
10609 assert!(cotenant_task.await.unwrap().is_empty());
10610 let (cotenant_channel, cotenant_epoch) =
10611 published_route(&cotenant_rx.recv().await.unwrap());
10612 assert!(matches!(
10613 forwarding
10614 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10615 .unwrap(),
10616 DataRoute::Client(DataRouteState::Bound(_))
10617 ));
10618
10619 let goodbye = post_ack_module_frame
10622 .expect("module receives a GOODBYE for the abandoned route channel");
10623 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10624 assert_eq!(goodbye.header.channel, abandoned_channel);
10625 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10626
10627 assert!(dying_rx.try_recv().is_err());
10631 let second_response = second_task.await.unwrap();
10632 assert_eq!(second_response.len(), 1);
10633 assert_eq!(
10634 parse_error(&second_response[0])["code"],
10635 "target_unavailable"
10636 );
10637 }
10638
10639 async fn catalog_snapshot(
10642 handler: &ControlHandler,
10643 client_ctx: &RouteCtx,
10644 corr: u64,
10645 ) -> (u64, Vec<(String, bool)>) {
10646 let body =
10647 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10648 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10649 let reply = handler
10650 .handle_control_frame(client_ctx, request)
10651 .await
10652 .unwrap()
10653 .pop()
10654 .unwrap();
10655 let ClientControlResponse::CatalogList {
10656 generation,
10657 modules,
10658 ..
10659 } = serde_json::from_slice(&reply.body).unwrap()
10660 else {
10661 panic!("not a catalog.list reply");
10662 };
10663 (
10664 generation,
10665 modules
10666 .into_iter()
10667 .map(|entry| (entry.module_id, entry.ready))
10668 .collect(),
10669 )
10670 }
10671
10672 fn hello_held_before_forwarding() -> (
10678 ControlHandler,
10679 Arc<std::sync::Barrier>,
10680 std::thread::JoinHandle<Vec<Frame>>,
10681 RouteCtx,
10682 mpsc::Receiver<crate::router::OutboundFrame>,
10683 ) {
10684 let hold = Arc::new(std::sync::Barrier::new(2));
10685 let handler = ControlHandler::with_forwarding(
10686 Arc::new(Registry::default()),
10687 Arc::new(ForwardingTable::default()),
10688 )
10689 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10690 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10691 let hello = {
10692 let handler = handler.clone();
10693 let module_ctx = module_ctx.clone();
10694 std::thread::spawn(move || {
10695 handler
10696 .handle_hello(
10697 module_ctx.connection_id,
10698 Some(module_ctx.egress.clone()),
10699 hello_frame("aft", PROTOCOL_VERSION, 7),
10700 )
10701 .unwrap()
10702 })
10703 };
10704 (handler, hold, hello, module_ctx, module_rx)
10705 }
10706
10707 #[tokio::test]
10716 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10717 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10718 let client = ConnectionId::new(41);
10719 let (client_ctx, mut client_rx) = route_ctx(client);
10720
10721 hold.wait();
10722 assert!(handler.registry.get_module("aft").unwrap().is_some());
10724 assert!(!handler
10725 .forwarding
10726 .has_live_module_connection("aft")
10727 .unwrap());
10728 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10729 assert!(
10730 listed.is_empty(),
10731 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10732 );
10733 let refused = handler
10734 .handle_control_frame(
10735 &client_ctx,
10736 route_open_frame(301, "aft", unique_project_root("listed-window")),
10737 )
10738 .await
10739 .unwrap();
10740 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10741 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10742 let duplicate = handler
10743 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10744 .await
10745 .unwrap();
10746 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10747
10748 hold.wait();
10749 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10750 let ack = module_rx.try_recv().unwrap().frame;
10751 assert_eq!(ack.header.ty, FrameType::HelloAck);
10752 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10753 assert_eq!(listed, vec![("aft".to_string(), true)]);
10754
10755 let (open_task, bind) = relay_route_open(
10756 &handler,
10757 client,
10758 &client_ctx.egress,
10759 &mut module_rx,
10760 303,
10761 "aft",
10762 "listed-after",
10763 )
10764 .await;
10765 assert!(handler
10766 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10767 .await
10768 .unwrap()
10769 .is_empty());
10770 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10771 published_route(&client_rx.try_recv().unwrap().frame);
10772 }
10773
10774 #[tokio::test]
10781 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10782 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10783 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10784 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10785
10786 hold.wait();
10787 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10788 assert!(listed.is_empty());
10789 assert_eq!(
10790 during, before,
10791 "the generation moved while the module was not listed"
10792 );
10793
10794 hold.wait();
10795 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10796 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10797 assert_eq!(listed, vec![("aft".to_string(), true)]);
10798 assert_eq!(
10799 after,
10800 before + 1,
10801 "the module appeared without exactly one generation bump"
10802 );
10803 }
10804
10805 #[tokio::test]
10810 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10811 let registry = Arc::new(Registry::default());
10812 let handler = ControlHandler::with_forwarding(
10813 Arc::clone(®istry),
10814 Arc::new(ForwardingTable::default()),
10815 );
10816 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10817 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10818 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10819 drop(module_rx);
10820
10821 let replies = handler
10822 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10823 .await
10824 .unwrap();
10825 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10826 assert!(registry.get_module("aft").unwrap().is_none());
10827 assert!(registry
10828 .get_module_by_connection(module_ctx.connection_id)
10829 .unwrap()
10830 .is_none());
10831 assert_eq!(
10832 catalog_snapshot(&handler, &client_ctx, 301).await,
10833 (before, Vec::new())
10834 );
10835 }
10836
10837 async fn route_open_held_after_bind_send() -> (
10841 ControlHandler,
10842 Arc<tokio::sync::Barrier>,
10843 RouteCtx,
10844 mpsc::Receiver<crate::router::OutboundFrame>,
10845 ) {
10846 let hold = Arc::new(tokio::sync::Barrier::new(2));
10847 let handler = ControlHandler::with_forwarding(
10848 Arc::new(Registry::default()),
10849 Arc::new(ForwardingTable::default()),
10850 )
10851 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10852 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10853 hello_via_sink(
10854 &handler,
10855 &module_ctx,
10856 &mut module_rx,
10857 hello_frame("aft", PROTOCOL_VERSION, 7),
10858 )
10859 .await;
10860 (handler, hold, module_ctx, module_rx)
10861 }
10862
10863 #[tokio::test]
10870 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10871 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10872 let client = ConnectionId::new(41);
10873 let (client_ctx, mut client_rx) = route_ctx(client);
10874
10875 let (open_task, bind) = relay_route_open(
10876 &handler,
10877 client,
10878 &client_ctx.egress,
10879 &mut module_rx,
10880 200,
10881 "aft",
10882 "acked-before-mark",
10883 )
10884 .await;
10885 let (module_channel, module_epoch) = route_bind_channel(&bind);
10886 hold.wait().await;
10888
10889 assert!(handler
10891 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10892 .await
10893 .unwrap()
10894 .is_empty());
10895 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10896
10897 hold.wait().await;
10898 assert!(
10899 open_task.await.unwrap().is_empty(),
10900 "route.open answers accepted"
10901 );
10902
10903 match module_rx.try_recv() {
10906 Err(mpsc::error::TryRecvError::Empty) => {}
10907 Ok(outbound) => panic!(
10908 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10909 outbound.frame.header.ty,
10910 outbound.frame.header.channel,
10911 outbound.frame.header.epoch
10912 ),
10913 Err(err) => panic!("module egress closed: {err}"),
10914 }
10915
10916 let router = Router::with_control_handler(Arc::new(handler.clone()));
10919 let request = Frame::build(
10920 FrameType::Request,
10921 Flags::new(true, Priority::Interactive, false),
10922 client_channel,
10923 client_epoch,
10924 201,
10925 b"first request".to_vec(),
10926 )
10927 .unwrap();
10928 router
10929 .route_for_connection(&client_ctx, request)
10930 .await
10931 .unwrap();
10932 let delivered = module_rx.try_recv().unwrap().frame;
10933 assert_eq!(delivered.header.ty, FrameType::Request);
10934 assert_eq!(
10935 (delivered.header.channel, delivered.header.epoch),
10936 (module_channel, module_epoch)
10937 );
10938 assert_eq!(delivered.body, b"first request");
10939 }
10940
10941 #[tokio::test]
10947 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10948 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10949 let client = ConnectionId::new(42);
10950 let (client_ctx, _client_rx) = route_ctx(client);
10951
10952 let (open_task, bind) = relay_route_open(
10953 &handler,
10954 client,
10955 &client_ctx.egress,
10956 &mut module_rx,
10957 300,
10958 "aft",
10959 "released-before-mark",
10960 )
10961 .await;
10962 let (module_channel, module_epoch) = route_bind_channel(&bind);
10963 hold.wait().await;
10964
10965 handler.cleanup_connection(client).unwrap();
10966 assert!(
10967 module_rx.try_recv().is_err(),
10968 "client teardown must not GOODBYE a bind route.open is still queueing"
10969 );
10970
10971 hold.wait().await;
10972 let response = open_task.await.unwrap();
10973 assert_eq!(response.len(), 1);
10974 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10975
10976 let goodbye = module_rx
10977 .try_recv()
10978 .expect("module is told to drop the binding the daemon abandoned")
10979 .frame;
10980 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10981 assert_eq!(
10982 (goodbye.header.channel, goodbye.header.epoch),
10983 (module_channel, module_epoch)
10984 );
10985 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10986 }
10987
10988 #[tokio::test]
10993 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10994 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10995 let client = ConnectionId::new(43);
10996 let (client_ctx, client_rx) = route_ctx(client);
10997
10998 let (open_task, bind) = relay_route_open(
10999 &handler,
11000 client,
11001 &client_ctx.egress,
11002 &mut module_rx,
11003 400,
11004 "aft",
11005 "gone-client-before-mark",
11006 )
11007 .await;
11008 let (module_channel, module_epoch) = route_bind_channel(&bind);
11009 hold.wait().await;
11010
11011 drop(client_rx);
11012 assert!(client_ctx.egress.is_closed());
11013 handler
11014 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11015 .await
11016 .unwrap();
11017
11018 hold.wait().await;
11019 let response = open_task.await.unwrap();
11020 assert_eq!(response.len(), 1);
11021 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11022
11023 let goodbye = module_rx
11024 .try_recv()
11025 .expect("module is told to drop the binding it acked")
11026 .frame;
11027 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11028 assert_eq!(
11029 (goodbye.header.channel, goodbye.header.epoch),
11030 (module_channel, module_epoch)
11031 );
11032 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11033 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11034 }
11035
11036 #[test]
11043 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11044 let handler = ControlHandler::default();
11045 let module_connection = ConnectionId::new(30);
11046 let client_connection = ConnectionId::new(31);
11047
11048 handler
11049 .refuse_to_end_module_connection_for_a_client(
11050 module_connection,
11051 77,
11052 ForwardingError::ConnectionClosing {
11053 connection_id: client_connection,
11054 },
11055 )
11056 .expect("a closing client must never end the module connection");
11057
11058 assert!(matches!(
11059 handler.refuse_to_end_module_connection_for_a_client(
11060 module_connection,
11061 78,
11062 ForwardingError::ConnectionClosing {
11063 connection_id: module_connection,
11064 },
11065 ),
11066 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11067 connection_id
11068 })) if connection_id == module_connection
11069 ));
11070 assert!(matches!(
11071 handler.refuse_to_end_module_connection_for_a_client(
11072 module_connection,
11073 79,
11074 ForwardingError::Poisoned,
11075 ),
11076 Err(RouterError::Forwarding(ForwardingError::Poisoned))
11077 ));
11078 assert!(matches!(
11079 handler.refuse_to_end_module_connection_for_a_client(
11080 module_connection,
11081 80,
11082 ForwardingError::StaleModuleEndpoint,
11083 ),
11084 Err(RouterError::Forwarding(
11085 ForwardingError::StaleModuleEndpoint
11086 ))
11087 ));
11088 }
11089
11090 #[tokio::test]
11103 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11104 let registry = Arc::new(Registry::default());
11105 let forwarding = Arc::new(ForwardingTable::default());
11106 let supervisor = SupervisorHandle::new();
11107 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11108 let handler =
11109 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11110 .with_supervisor(supervisor);
11111
11112 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11113 hello_via_sink(
11114 &handler,
11115 &target_ctx,
11116 &mut target_rx,
11117 hello_frame("target", PROTOCOL_VERSION, 1),
11118 )
11119 .await;
11120
11121 let wrong_nonce = handler
11125 .handle_control_frame(
11126 &route_ctx(ConnectionId::new(91)).0,
11127 route_open_frame_with_admission_facts(
11128 20,
11129 "target",
11130 unique_project_root("admission-facts"),
11131 Some(subc_control::ConsumerIdentity {
11132 module_id: "fed".to_string(),
11133 launch_nonce: "not-the-real-nonce".to_string(),
11134 }),
11135 None,
11136 ),
11137 )
11138 .await
11139 .unwrap();
11140 assert_eq!(
11141 parse_error(&wrong_nonce[0])["code"],
11142 "bad_consumer_identity",
11143 "a mismatched launch nonce must be refused, not stamped Reserved"
11144 );
11145
11146 let never_spawned = handler
11150 .handle_control_frame(
11151 &route_ctx(ConnectionId::new(92)).0,
11152 route_open_frame_with_admission_facts(
11153 21,
11154 "target",
11155 unique_project_root("admission-facts"),
11156 Some(subc_control::ConsumerIdentity {
11157 module_id: "never-spawned".to_string(),
11158 launch_nonce: "any-nonce".to_string(),
11159 }),
11160 None,
11161 ),
11162 )
11163 .await
11164 .unwrap();
11165 assert_eq!(
11166 parse_error(&never_spawned[0])["code"],
11167 "bad_consumer_identity",
11168 "an unspawned module_id must be refused rather than accepted for lack of a record"
11169 );
11170 }
11171
11172 #[tokio::test]
11195 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11196 let registry = Arc::new(Registry::default());
11197 let forwarding = Arc::new(ForwardingTable::default());
11198 let supervisor = SupervisorHandle::new();
11199 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11200 let handler =
11201 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11202 .with_supervisor(supervisor);
11203
11204 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11205 hello_via_sink(
11206 &handler,
11207 &target_ctx,
11208 &mut target_rx,
11209 hello_frame("target", PROTOCOL_VERSION, 1),
11210 )
11211 .await;
11212
11213 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11214 let route_handler = handler.clone();
11215 let route_task = tokio::spawn(async move {
11216 route_handler
11217 .handle_control_frame(
11218 &client_ctx,
11219 route_open_frame_with_admission_facts(
11220 30,
11221 "target",
11222 unique_project_root("admission-facts"),
11223 Some(subc_control::ConsumerIdentity {
11224 module_id: "fed".to_string(),
11225 launch_nonce: "fed-nonce".to_string(),
11226 }),
11227 None,
11228 ),
11229 )
11230 .await
11231 .unwrap()
11232 });
11233
11234 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11241 .await
11242 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11243 .expect("module control channel closed before route.bind");
11244 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11245 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11246 panic!("expected route.bind")
11247 };
11248 assert_eq!(
11249 principal,
11250 Some(Principal::Reserved {
11251 module_id: "fed".to_string()
11252 }),
11253 "a correctly attested consumer must be stamped Reserved for its own id"
11254 );
11255
11256 handler
11257 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11258 .await
11259 .unwrap();
11260 assert!(route_task.await.unwrap().is_empty());
11261 assert!(
11262 matches!(
11263 serde_json::from_slice::<ClientControlResponse>(
11264 &client_rx.recv().await.unwrap().body
11265 )
11266 .unwrap(),
11267 ClientControlResponse::RouteOpen { .. }
11268 ),
11269 "the route must actually open, not merely avoid an error"
11270 );
11271 }
11272
11273 #[tokio::test(start_paused = true)]
11274 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11275 let registry = Arc::new(Registry::default());
11276 let forwarding = Arc::new(ForwardingTable::default());
11277 let supervisor = SupervisorHandle::new();
11278 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11279 let handler =
11280 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11281 .with_supervisor(supervisor);
11282
11283 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11284 hello_via_sink(
11285 &handler,
11286 &target_ctx,
11287 &mut target_rx,
11288 hello_frame("target", PROTOCOL_VERSION, 1),
11289 )
11290 .await;
11291
11292 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11293 let direct_handler = handler.clone();
11294 let direct_open = tokio::spawn(async move {
11295 direct_handler
11296 .handle_control_frame(
11297 &direct_ctx,
11298 route_open_frame(2, "target", unique_project_root("route-census-direct")),
11299 )
11300 .await
11301 .unwrap()
11302 });
11303 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11304 .await
11305 .expect("no direct route.bind within 5s")
11306 .expect("target control channel closed before direct route.bind");
11307 handler
11308 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11309 .await
11310 .unwrap();
11311 assert!(direct_open.await.unwrap().is_empty());
11312 let _ = direct_rx.recv().await.unwrap();
11313
11314 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11315 let reserved_handler = handler.clone();
11316 let reserved_open = tokio::spawn(async move {
11317 reserved_handler
11318 .handle_control_frame(
11319 &reserved_ctx,
11320 route_open_frame_with_admission_facts(
11321 3,
11322 "target",
11323 unique_project_root("admission-facts"),
11324 Some(ConsumerIdentity {
11325 module_id: "fed".to_string(),
11326 launch_nonce: "fed-nonce".to_string(),
11327 }),
11328 None,
11329 ),
11330 )
11331 .await
11332 .unwrap()
11333 });
11334 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11335 .await
11336 .expect("no reserved route.bind within 5s")
11337 .expect("target control channel closed before reserved route.bind");
11338 handler
11339 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11340 .await
11341 .unwrap();
11342 assert!(reserved_open.await.unwrap().is_empty());
11343 let _ = reserved_rx.recv().await.unwrap();
11344
11345 forwarding
11346 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11347 .unwrap();
11348 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11349 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11350 module_id: Some("target".to_string()),
11351 })
11352 .unwrap();
11353 let census_frame =
11354 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11355 let response = handler
11356 .handle_control_frame(&census_ctx, census_frame)
11357 .await
11358 .unwrap()
11359 .pop()
11360 .unwrap();
11361 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11362 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11363 assert!(matches!(
11364 decoded,
11365 ClientControlResponse::SupervisorRoutes { .. }
11366 ));
11367 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11368 assert_eq!(routes.len(), 2);
11369 assert!(routes.iter().all(|route| route["draining"] == true));
11370 assert!(
11373 routes.iter().all(|route| route["drain_reason"] == "reload"),
11374 "draining routes must name the drain's reason: {routes:?}"
11375 );
11376 assert!(routes.iter().any(|route| {
11377 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11378 }));
11379 assert!(routes.iter().any(|route| {
11380 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11381 }));
11382
11383 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11384 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11385 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11386 std::fs::write(
11387 &golden_path,
11388 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11389 )
11390 .unwrap();
11391 }
11392 let expected: Value =
11393 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11394 assert_eq!(actual, expected);
11395 }
11396
11397 async fn query_live_roots(
11398 handler: &ControlHandler,
11399 module_ctx: &RouteCtx,
11400 ) -> ModuleControlResponseToModule {
11401 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11402 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11403 let response = handler
11404 .handle_control_frame(module_ctx, frame)
11405 .await
11406 .unwrap()
11407 .pop()
11408 .unwrap();
11409 serde_json::from_slice(&response.body).unwrap()
11410 }
11411
11412 #[tokio::test(start_paused = true)]
11413 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11414 let registry = Arc::new(Registry::default());
11415 let forwarding = Arc::new(ForwardingTable::default());
11416 let handler = ControlHandler::with_forwarding(registry, forwarding);
11417 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11418 hello_via_sink(
11419 &handler,
11420 &target_ctx,
11421 &mut target_rx,
11422 hello_frame("target", PROTOCOL_VERSION, 1),
11423 )
11424 .await;
11425 let root = unique_project_root("live-roots-known");
11426 let path = ProjectRootId::from_path_allowing_missing(root.path())
11427 .unwrap()
11428 .as_path()
11429 .to_path_buf();
11430 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11431 let open_handler = handler.clone();
11432 let opened = tokio::spawn(async move {
11433 open_handler
11434 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11435 .await
11436 .unwrap()
11437 });
11438 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11439 .await
11440 .unwrap()
11441 .unwrap();
11442 handler
11443 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11444 .await
11445 .unwrap();
11446 assert!(opened.await.unwrap().is_empty());
11447 let _ = client_rx.recv().await.unwrap();
11448
11449 let root = unique_project_root("live-roots-pending");
11450 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11451 .unwrap()
11452 .as_path()
11453 .to_path_buf();
11454 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11455 let open_handler = handler.clone();
11456 let pending = tokio::spawn(async move {
11457 open_handler
11458 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11459 .await
11460 .unwrap()
11461 });
11462 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11463 .await
11464 .unwrap()
11465 .unwrap();
11466 let actual = query_live_roots(&handler, &target_ctx).await;
11467 let ModuleControlResponseToModule::LiveRoots {
11468 roots,
11469 unknown_root_bindings,
11470 total_bindings,
11471 } = actual
11472 else {
11473 panic!("expected live roots")
11474 };
11475 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11476 assert_eq!(unknown_root_bindings, 0);
11477 assert_eq!(
11478 roots.len(),
11479 2,
11480 "root-known arm must retain each canonical root"
11481 );
11482 assert_eq!(
11483 total_bindings,
11484 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11485 );
11486 let counts = roots
11487 .iter()
11488 .map(|root| (root.project_root.clone(), root.bound, root.pending))
11489 .collect::<Vec<_>>();
11490 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11491 expected.sort_by(|a, b| a.0.cmp(&b.0));
11492 assert_eq!(
11493 counts, expected,
11494 "roots must sort by path and count pending separately"
11495 );
11496 handler
11497 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11498 .await
11499 .unwrap();
11500 assert!(pending.await.unwrap().is_empty());
11501 }
11502
11503 #[tokio::test(start_paused = true)]
11504 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11505 let forwarding = Arc::new(ForwardingTable::default());
11506 let handler =
11507 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11508 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11509 hello_via_sink(
11510 &handler,
11511 &target_ctx,
11512 &mut target_rx,
11513 hello_frame("target", PROTOCOL_VERSION, 1),
11514 )
11515 .await;
11516 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11517 let pending = forwarding
11518 .begin_route_bind_relay_for_test(
11519 client_ctx.connection_id,
11520 client_ctx.egress.clone(),
11521 2,
11522 "target",
11523 )
11524 .unwrap();
11525 forwarding
11526 .complete_pending_relay(
11527 target_ctx.connection_id,
11528 pending.corr,
11529 RouteBindRelayOutcome::Accepted,
11530 )
11531 .unwrap();
11532 let actual = query_live_roots(&handler, &target_ctx).await;
11533 let ModuleControlResponseToModule::LiveRoots {
11534 roots,
11535 unknown_root_bindings,
11536 total_bindings,
11537 } = actual
11538 else {
11539 panic!("expected live roots")
11540 };
11541 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11542 assert_eq!(
11543 unknown_root_bindings, 1,
11544 "unknown-root arm must not read as no bindings"
11545 );
11546 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11547 assert_eq!(
11548 total_bindings,
11549 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11550 );
11551 }
11552
11553 #[tokio::test(start_paused = true)]
11561 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11562 let forwarding = Arc::new(ForwardingTable::default());
11563 let handler =
11564 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11565 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11566 let replies = handler
11567 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11568 .await
11569 .unwrap();
11570 let queued_by_hello = module_rx.len();
11571
11572 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11573 let open_handler = handler.clone();
11574 let open = tokio::spawn(async move {
11575 open_handler
11576 .handle_control_frame(
11577 &client_ctx,
11578 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11579 )
11580 .await
11581 .unwrap()
11582 });
11583 let mut spins = 0;
11585 while module_rx.len() == queued_by_hello {
11586 spins += 1;
11587 assert!(spins < 10_000, "route.open never queued a route.bind");
11588 tokio::task::yield_now().await;
11589 }
11590
11591 for reply in replies {
11593 module_ctx.egress.send(reply).await.unwrap();
11594 }
11595
11596 let first = module_rx.recv().await.unwrap().frame;
11597 assert_eq!(
11598 first.header.ty,
11599 FrameType::HelloAck,
11600 "the first frame a registering module reads must be its HELLO_ACK"
11601 );
11602 assert_eq!(first.header.corr, 7);
11603 let second = module_rx.recv().await.unwrap().frame;
11604 assert_eq!(second.header.ty, FrameType::Request);
11605 assert!(
11606 matches!(
11607 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11608 ModuleControlRequest::RouteBind { .. }
11609 ),
11610 "the route.bind follows the ack"
11611 );
11612 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11613
11614 handler
11615 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11616 .await
11617 .unwrap();
11618 assert!(open.await.unwrap().is_empty());
11619 let _ = client_rx.recv().await.unwrap();
11620 }
11621
11622 #[tokio::test(start_paused = true)]
11623 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11624 let handler = ControlHandler::with_forwarding(
11625 Arc::new(Registry::default()),
11626 Arc::new(ForwardingTable::default()),
11627 );
11628 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11629 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11630 hello_via_sink(
11631 &handler,
11632 &first_ctx,
11633 &mut first_rx,
11634 hello_frame("first", PROTOCOL_VERSION, 1),
11635 )
11636 .await;
11637 hello_via_sink(
11638 &handler,
11639 &second_ctx,
11640 &mut second_rx,
11641 hello_frame("second", PROTOCOL_VERSION, 2),
11642 )
11643 .await;
11644 let root = unique_project_root("second-only");
11645 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11646 let cloned = handler.clone();
11647 let open = tokio::spawn(async move {
11648 cloned
11649 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11650 .await
11651 .unwrap()
11652 });
11653 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11654 .await
11655 .unwrap()
11656 .unwrap();
11657 let first = query_live_roots(&handler, &first_ctx).await;
11658 let second = query_live_roots(&handler, &second_ctx).await;
11659 assert!(
11660 matches!(
11661 first,
11662 ModuleControlResponseToModule::LiveRoots {
11663 total_bindings: 0,
11664 ..
11665 }
11666 ),
11667 "cross-module scope must not expose another module's roots"
11668 );
11669 assert!(
11670 matches!(
11671 second,
11672 ModuleControlResponseToModule::LiveRoots {
11673 total_bindings: 1,
11674 ..
11675 }
11676 ),
11677 "second module must see its pending route"
11678 );
11679 handler
11680 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11681 .await
11682 .unwrap();
11683 assert!(open.await.unwrap().is_empty());
11684 }
11685
11686 #[tokio::test(start_paused = true)]
11687 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11688 let handler = ControlHandler::with_forwarding(
11689 Arc::new(Registry::default()),
11690 Arc::new(ForwardingTable::default()),
11691 );
11692 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11693 hello_via_sink(
11694 &handler,
11695 &target_ctx,
11696 &mut target_rx,
11697 hello_frame("target", PROTOCOL_VERSION, 1),
11698 )
11699 .await;
11700 let actual = query_live_roots(&handler, &target_ctx).await;
11701 let ModuleControlResponseToModule::LiveRoots {
11702 roots,
11703 unknown_root_bindings,
11704 total_bindings,
11705 } = actual
11706 else {
11707 panic!("expected live roots")
11708 };
11709 assert!(roots.is_empty());
11710 assert_eq!(unknown_root_bindings, 0);
11711 assert_eq!(total_bindings, 0);
11712 assert_eq!(
11713 total_bindings,
11714 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11715 );
11716 }
11717
11718 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11725 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11726 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11727 let text = std::fs::read_to_string(&path)
11728 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11729 let vectors: Vec<(String, Value)> = text
11730 .lines()
11731 .filter(|line| !line.trim().is_empty())
11732 .map(|line| {
11733 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11734 let id = entry["corpus_id"]
11735 .as_str()
11736 .expect("every vector carries a corpus_id")
11737 .to_string();
11738 (id, entry["package"].clone())
11739 })
11740 .collect();
11741 assert_eq!(
11745 vectors.len(),
11746 3,
11747 "vendored fed corpus changed size; re-sync from subc-federation"
11748 );
11749
11750 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11765 let richest = vectors
11766 .iter()
11767 .filter_map(|(_, package)| package.as_object())
11768 .map(|object| {
11769 object
11770 .keys()
11771 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11772 .count()
11773 })
11774 .max()
11775 .unwrap_or(0);
11776 assert!(
11777 richest >= 2,
11778 "vendored corpus no longer carries a package with unmodelled fields, \
11779 so the relay test can no longer distinguish a verbatim relay from a lossy one"
11780 );
11781
11782 vectors
11783 }
11784
11785 #[tokio::test]
11793 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11794 for (corpus_id, package) in fed_admission_facts_vectors() {
11795 let registry = Arc::new(Registry::default());
11796 let forwarding = Arc::new(ForwardingTable::default());
11797 let supervisor = SupervisorHandle::new();
11798 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11799 let handler =
11800 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11801 .with_supervisor(supervisor)
11802 .with_admission_facts_config(
11803 Some("fed".to_string()),
11804 Some(vec!["target".to_string()]),
11805 );
11806
11807 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11808 hello_via_sink(
11809 &handler,
11810 &target_ctx,
11811 &mut target_rx,
11812 hello_frame("target", PROTOCOL_VERSION, 1),
11813 )
11814 .await;
11815
11816 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11817 let route_handler = handler.clone();
11818 let expected = package.clone();
11819 let route_task = tokio::spawn(async move {
11820 route_handler
11821 .handle_control_frame(
11822 &client_ctx,
11823 route_open_frame_with_admission_facts(
11824 20,
11825 "target",
11826 unique_project_root("admission-facts"),
11827 Some(subc_control::ConsumerIdentity {
11828 module_id: "fed".to_string(),
11829 launch_nonce: "fed-nonce".to_string(),
11830 }),
11831 Some(package),
11832 ),
11833 )
11834 .await
11835 .unwrap()
11836 });
11837
11838 let bind_frame = target_rx.recv().await.unwrap();
11839 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11840 let ModuleControlRequest::RouteBind {
11841 admission_facts, ..
11842 } = bind
11843 else {
11844 panic!("{corpus_id}: expected route.bind")
11845 };
11846 assert_eq!(
11847 admission_facts,
11848 Some(expected),
11849 "{corpus_id}: relay must not add, drop or reshape any field"
11850 );
11851
11852 handler
11853 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11854 .await
11855 .unwrap();
11856 route_task.await.unwrap();
11857 }
11858 }
11859
11860 #[tokio::test]
11861 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11862 let registry = Arc::new(Registry::default());
11863 let forwarding = Arc::new(ForwardingTable::default());
11864 let supervisor = SupervisorHandle::new();
11865 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11866 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11867 let handler =
11868 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11869 .with_supervisor(supervisor)
11870 .with_admission_facts_config(
11871 Some("fed".to_string()),
11872 Some(vec!["target".to_string()]),
11873 );
11874
11875 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11876 hello_via_sink(
11877 &handler,
11878 &target_ctx,
11879 &mut target_rx,
11880 hello_frame("target", PROTOCOL_VERSION, 1),
11881 )
11882 .await;
11883 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11884 hello_via_sink(
11885 &handler,
11886 &other_ctx,
11887 &mut other_rx,
11888 hello_frame("other", PROTOCOL_VERSION, 2),
11889 )
11890 .await;
11891
11892 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11893 let expected_facts = facts.clone();
11894 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11895 let route_handler = handler.clone();
11896 let route_task = tokio::spawn(async move {
11897 route_handler
11898 .handle_control_frame(
11899 &client_ctx,
11900 route_open_frame_with_admission_facts(
11901 10,
11902 "target",
11903 unique_project_root("admission-facts"),
11904 Some(subc_control::ConsumerIdentity {
11905 module_id: "fed".to_string(),
11906 launch_nonce: "fed-nonce".to_string(),
11907 }),
11908 Some(facts.clone()),
11909 ),
11910 )
11911 .await
11912 .unwrap()
11913 });
11914 let bind_frame = target_rx.recv().await.unwrap();
11915 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11916 let ModuleControlRequest::RouteBind {
11917 admission_facts, ..
11918 } = bind
11919 else {
11920 panic!("expected route.bind")
11921 };
11922 assert_eq!(admission_facts, Some(expected_facts));
11923 handler
11924 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11925 .await
11926 .unwrap();
11927 assert!(route_task.await.unwrap().is_empty());
11928 assert!(matches!(
11929 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11930 .unwrap(),
11931 ClientControlResponse::RouteOpen { .. }
11932 ));
11933
11934 let direct = handler
11935 .handle_control_frame(
11936 &route_ctx(ConnectionId::new(73)).0,
11937 route_open_frame_with_admission_facts(
11938 11,
11939 "target",
11940 unique_project_root("admission-facts"),
11941 None,
11942 Some(json!({"x": 1})),
11943 ),
11944 )
11945 .await
11946 .unwrap();
11947 assert_eq!(
11948 parse_error(&direct[0])["code"],
11949 "admission_facts_not_permitted"
11950 );
11951
11952 let different_reserved = handler
11953 .handle_control_frame(
11954 &route_ctx(ConnectionId::new(77)).0,
11955 route_open_frame_with_admission_facts(
11956 15,
11957 "target",
11958 unique_project_root("admission-facts"),
11959 Some(subc_control::ConsumerIdentity {
11960 module_id: "other".to_string(),
11961 launch_nonce: "other-nonce".to_string(),
11962 }),
11963 Some(json!({"x": 1})),
11964 ),
11965 )
11966 .await
11967 .unwrap();
11968 assert_eq!(
11969 parse_error(&different_reserved[0])["code"],
11970 "admission_facts_not_permitted"
11971 );
11972
11973 let other_target = handler
11974 .handle_control_frame(
11975 &route_ctx(ConnectionId::new(74)).0,
11976 route_open_frame_with_admission_facts(
11977 12,
11978 "other",
11979 unique_project_root("admission-facts"),
11980 Some(subc_control::ConsumerIdentity {
11981 module_id: "fed".to_string(),
11982 launch_nonce: "fed-nonce".to_string(),
11983 }),
11984 Some(json!({"x": 1})),
11985 ),
11986 )
11987 .await
11988 .unwrap();
11989 assert_eq!(
11990 parse_error(&other_target[0])["code"],
11991 "admission_facts_target_not_allowed"
11992 );
11993
11994 let nonexistent = handler
11995 .handle_control_frame(
11996 &route_ctx(ConnectionId::new(75)).0,
11997 route_open_frame_with_admission_facts(
11998 13,
11999 "missing",
12000 unique_project_root("admission-facts"),
12001 None,
12002 Some(json!({"x": 1})),
12003 ),
12004 )
12005 .await
12006 .unwrap();
12007 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12008
12009 let described = handler
12010 .handle_control_frame(
12011 &route_ctx(ConnectionId::new(76)).0,
12012 Frame::build(
12013 FrameType::Request,
12014 control_flags(),
12015 0,
12016 0,
12017 14,
12018 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12019 )
12020 .unwrap(),
12021 )
12022 .await
12023 .unwrap();
12024 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12025 serde_json::from_slice(&described[0].body).unwrap()
12026 else {
12027 panic!("expected server.describe response")
12028 };
12029 assert!(capabilities
12030 .iter()
12031 .any(|cap| cap == "admission_facts_relay_v1"));
12032 }
12033
12034 #[tokio::test]
12035 async fn admission_facts_without_configured_carrier_are_rejected() {
12036 let registry = Arc::new(Registry::default());
12037 let forwarding = Arc::new(ForwardingTable::default());
12038 let handler = ControlHandler::with_forwarding(registry, forwarding);
12039 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12040 hello_via_sink(
12041 &handler,
12042 &target_ctx,
12043 &mut target_rx,
12044 hello_frame("target", PROTOCOL_VERSION, 1),
12045 )
12046 .await;
12047
12048 let responses = handler
12049 .handle_control_frame(
12050 &route_ctx(ConnectionId::new(79)).0,
12051 route_open_frame_with_admission_facts(
12052 16,
12053 "target",
12054 unique_project_root("admission-facts"),
12055 None,
12056 Some(json!({"x": 1})),
12057 ),
12058 )
12059 .await
12060 .unwrap();
12061 assert_eq!(
12062 parse_error(&responses[0])["code"],
12063 "admission_facts_not_permitted"
12064 );
12065 }
12066
12067 #[tokio::test]
12068 async fn route_open_relays_consumer_capabilities_verbatim() {
12069 let registry = Arc::new(Registry::default());
12070 let forwarding = Arc::new(ForwardingTable::default());
12071 let handler =
12072 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12073 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12074 hello_via_sink(
12075 &handler,
12076 &module_ctx,
12077 &mut module_rx,
12078 hello_frame("aft", PROTOCOL_VERSION, 7),
12079 )
12080 .await;
12081
12082 let expected = vec!["elicitation".to_string(), "roots".to_string()];
12083 let expected_for_request = expected.clone();
12084 let project_root = unique_project_root("consumer-capabilities-present");
12085 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12086 let route_handler = handler.clone();
12087 let route_task = tokio::spawn(async move {
12088 route_handler
12089 .handle_control_frame(
12090 &client_ctx,
12091 route_open_frame_with_consumer_capabilities(
12092 401,
12093 "aft",
12094 project_root,
12095 Some(expected_for_request),
12096 ),
12097 )
12098 .await
12099 .unwrap()
12100 });
12101 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12102 .await
12103 .unwrap()
12104 .unwrap();
12105 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12106 let ModuleControlRequest::RouteBind {
12107 consumer_capabilities,
12108 ..
12109 } = bind
12110 else {
12111 panic!("expected route.bind request, got {bind:?}");
12112 };
12113 assert_eq!(consumer_capabilities, Some(expected.clone()));
12114
12115 handler
12116 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12117 .await
12118 .unwrap();
12119 let route_response = route_task.await.unwrap();
12120 assert!(route_response.is_empty());
12121 let published = client_rx.recv().await.unwrap();
12122 assert!(matches!(
12123 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12124 ClientControlResponse::RouteOpen { .. }
12125 ));
12126 }
12127
12128 #[tokio::test]
12129 async fn route_open_without_consumer_capabilities_relays_none() {
12130 let registry = Arc::new(Registry::default());
12131 let forwarding = Arc::new(ForwardingTable::default());
12132 let handler =
12133 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12134 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12135 hello_via_sink(
12136 &handler,
12137 &module_ctx,
12138 &mut module_rx,
12139 hello_frame("aft", PROTOCOL_VERSION, 7),
12140 )
12141 .await;
12142
12143 let project_root = unique_project_root("consumer-capabilities-absent");
12144 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12145 let route_handler = handler.clone();
12146 let route_task = tokio::spawn(async move {
12147 route_handler
12148 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12149 .await
12150 .unwrap()
12151 });
12152 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12153 .await
12154 .unwrap()
12155 .unwrap();
12156 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12157 let ModuleControlRequest::RouteBind {
12158 consumer_capabilities,
12159 ..
12160 } = bind
12161 else {
12162 panic!("expected route.bind request, got {bind:?}");
12163 };
12164 assert_eq!(consumer_capabilities, None);
12165
12166 handler
12167 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12168 .await
12169 .unwrap();
12170 let route_response = route_task.await.unwrap();
12171 assert!(route_response.is_empty());
12172 let published = client_rx.recv().await.unwrap();
12173 assert!(matches!(
12174 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12175 ClientControlResponse::RouteOpen { .. }
12176 ));
12177 }
12178
12179 async fn bind_role_versions_for(
12183 sent: Option<BTreeMap<String, String>>,
12184 connection: u64,
12185 ) -> Option<BTreeMap<String, String>> {
12186 let registry = Arc::new(Registry::default());
12187 let forwarding = Arc::new(ForwardingTable::default());
12188 let handler =
12189 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12190 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12191 hello_via_sink(
12192 &handler,
12193 &module_ctx,
12194 &mut module_rx,
12195 hello_frame("aft", PROTOCOL_VERSION, 7),
12196 )
12197 .await;
12198 let project_root = unique_project_root("role-versions");
12199 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12200 let route_handler = handler.clone();
12201 let route_task = tokio::spawn(async move {
12202 route_handler
12203 .handle_control_frame(
12204 &client_ctx,
12205 route_open_frame_with_role_versions(403, "aft", project_root, sent),
12206 )
12207 .await
12208 .unwrap()
12209 });
12210 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12211 .await
12212 .expect("a well-formed route.open reaches the module as a bind")
12213 .unwrap();
12214 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12215 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12216 panic!("expected route.bind request, got {bind:?}");
12217 };
12218 handler
12219 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12220 .await
12221 .unwrap();
12222 assert!(route_task.await.unwrap().is_empty());
12223 let published = client_rx.recv().await.unwrap();
12224 assert!(matches!(
12225 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12226 ClientControlResponse::RouteOpen { .. }
12227 ));
12228 role_versions
12229 }
12230
12231 #[tokio::test]
12232 async fn route_open_relays_role_versions_verbatim() {
12233 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12234 assert_eq!(
12235 bind_role_versions_for(Some(sent.clone()), 141).await,
12236 Some(sent)
12237 );
12238 }
12239
12240 #[tokio::test]
12243 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12244 assert_eq!(bind_role_versions_for(None, 143).await, None);
12245 assert_eq!(
12246 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12247 None
12248 );
12249 }
12250
12251 #[tokio::test]
12252 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12253 let registry = Arc::new(Registry::default());
12254 let forwarding = Arc::new(ForwardingTable::default());
12255 let handler =
12256 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12257 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12258 hello_via_sink(
12259 &handler,
12260 &module_ctx,
12261 &mut module_rx,
12262 hello_frame("aft", PROTOCOL_VERSION, 7),
12263 )
12264 .await;
12265
12266 let nine: BTreeMap<String, String> = (0..9)
12267 .map(|index| (format!("role-{index}"), "v1".to_string()))
12268 .collect();
12269 for (label, malformed) in [
12270 (
12271 "invalid role name",
12272 role_versions(&[("Tool_Provider", "v1")]),
12273 ),
12274 ("invalid version", role_versions(&[("tool-provider", "v0")])),
12275 ("nine entries", nine),
12276 ] {
12277 let responses = tokio::time::timeout(
12280 Duration::from_secs(1),
12281 handler.handle_control_frame(
12282 &route_ctx(ConnectionId::new(148)).0,
12283 route_open_frame_with_role_versions(
12284 404,
12285 "aft",
12286 unique_project_root("role-versions-malformed"),
12287 Some(malformed),
12288 ),
12289 ),
12290 )
12291 .await
12292 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12293 .unwrap();
12294 assert_eq!(responses.len(), 1, "{label}");
12295 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12296 let error = parse_error(&responses[0]);
12297 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12298 assert_eq!(
12299 error["detail"]["field"], "role_versions",
12300 "{label}: {error}"
12301 );
12302 assert!(
12303 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12304 "{label}: a malformed declaration is terminal"
12305 );
12306 assert!(
12307 module_rx.try_recv().is_err(),
12308 "{label}: the module must never see a bind"
12309 );
12310 }
12311 }
12312
12313 #[tokio::test]
12317 async fn route_role_versions_capability_is_advertised() {
12318 let handler = ControlHandler::new(Arc::new(Registry::default()));
12319 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12320 let ack = hello_via_sink(
12321 &handler,
12322 &ctx,
12323 &mut rx,
12324 hello_frame("m", PROTOCOL_VERSION, 1),
12325 )
12326 .await;
12327 let ack = parse_ack(&ack);
12328 assert!(
12329 ack.subc_capabilities
12330 .iter()
12331 .any(|c| c == "route-role-versions/v1"),
12332 "{:?}",
12333 ack.subc_capabilities
12334 );
12335
12336 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12337 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12338 let reply = handler
12339 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12340 .await
12341 .unwrap()
12342 .pop()
12343 .unwrap();
12344 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12345 serde_json::from_slice(&reply.body).unwrap()
12346 else {
12347 panic!("not a server.describe reply");
12348 };
12349 assert!(
12350 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12351 "{capabilities:?}"
12352 );
12353 }
12354
12355 #[tokio::test]
12359 async fn catalog_events_capability_is_advertised() {
12360 let handler = ControlHandler::new(Arc::new(Registry::default()));
12361 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12362 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12363 let reply = handler
12364 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12365 .await
12366 .unwrap()
12367 .pop()
12368 .unwrap();
12369 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12370 serde_json::from_slice(&reply.body).unwrap()
12371 else {
12372 panic!("not a server.describe reply");
12373 };
12374 assert!(
12375 capabilities
12376 .iter()
12377 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12378 "{capabilities:?}"
12379 );
12380 }
12381
12382 #[tokio::test]
12383 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12384 let handler = ControlHandler::new(Arc::new(Registry::default()));
12385 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12386 let ack = hello_via_sink(
12387 &handler,
12388 &module,
12389 &mut module_rx,
12390 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12391 )
12392 .await;
12393 let ack = parse_ack(&ack);
12394 assert!(
12395 ack.subc_capabilities
12396 .iter()
12397 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12398 "{:?}",
12399 ack.subc_capabilities
12400 );
12401 assert!(ack
12402 .subc_capabilities
12403 .iter()
12404 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12405
12406 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12407 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12408 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12409 let reply = handler
12410 .handle_control_frame(&client, frame)
12411 .await
12412 .unwrap()
12413 .pop()
12414 .unwrap();
12415 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12416 serde_json::from_slice(&reply.body).unwrap()
12417 else {
12418 panic!("not a server.describe reply");
12419 };
12420 assert!(
12421 capabilities
12422 .iter()
12423 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12424 "{capabilities:?}"
12425 );
12426 assert!(capabilities
12427 .iter()
12428 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12429 }
12430
12431 #[tokio::test]
12432 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12433 let handler = ControlHandler::new(Arc::new(Registry::default()));
12434 let (module, mut rx) = route_ctx(ConnectionId::new(156));
12435 let ack = hello_via_sink(
12436 &handler,
12437 &module,
12438 &mut rx,
12439 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12440 )
12441 .await;
12442 assert!(parse_ack(&ack)
12443 .subc_capabilities
12444 .iter()
12445 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12446 assert!(parse_ack(&ack)
12447 .subc_capabilities
12448 .iter()
12449 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12450 let (client, _rx) = route_ctx(ConnectionId::new(157));
12451 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12452 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12453 let reply = handler
12454 .handle_control_frame(&client, frame)
12455 .await
12456 .unwrap()
12457 .pop()
12458 .unwrap();
12459 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12460 serde_json::from_slice(&reply.body).unwrap()
12461 else {
12462 panic!("server.describe reply")
12463 };
12464 assert!(capabilities
12465 .iter()
12466 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12467 assert!(capabilities
12468 .iter()
12469 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12470 }
12471
12472 #[tokio::test]
12473 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12474 let handler = ControlHandler::new(Arc::new(Registry::default()));
12475 let (module, mut rx) = route_ctx(ConnectionId::new(158));
12476 let ack = hello_via_sink(
12477 &handler,
12478 &module,
12479 &mut rx,
12480 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12481 )
12482 .await;
12483 assert!(parse_ack(&ack)
12484 .subc_capabilities
12485 .iter()
12486 .any(|cap| cap == "scope-workdir/v1"));
12487 let (client, _rx) = route_ctx(ConnectionId::new(159));
12488 let frame = Frame::build(
12489 FrameType::Request,
12490 control_flags(),
12491 0,
12492 0,
12493 5,
12494 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12495 )
12496 .unwrap();
12497 let reply = handler
12498 .handle_control_frame(&client, frame)
12499 .await
12500 .unwrap()
12501 .remove(0);
12502 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12503 serde_json::from_slice(&reply.body).unwrap()
12504 else {
12505 panic!("server.describe reply")
12506 };
12507 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12508 }
12509
12510 #[tokio::test]
12511 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12512 let handler = ControlHandler::new(Arc::new(Registry::default()));
12513 let (module, mut rx) = route_ctx(ConnectionId::new(154));
12514 let ack = hello_via_sink(
12515 &handler,
12516 &module,
12517 &mut rx,
12518 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12519 )
12520 .await;
12521 assert!(parse_ack(&ack)
12522 .subc_capabilities
12523 .iter()
12524 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12525 let (client, _rx) = route_ctx(ConnectionId::new(155));
12526 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12527 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12528 let reply = handler
12529 .handle_control_frame(&client, frame)
12530 .await
12531 .unwrap()
12532 .pop()
12533 .unwrap();
12534 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12535 serde_json::from_slice(&reply.body).unwrap()
12536 else {
12537 panic!("not a server.describe reply")
12538 };
12539 assert!(capabilities
12540 .iter()
12541 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12542 }
12543
12544 #[tokio::test]
12545 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12546 let registry = Arc::new(Registry::default());
12547 let forwarding = Arc::new(ForwardingTable::default());
12548 let handler =
12549 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12550 .with_health_probe_timeout(Duration::from_secs(5));
12551 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12552 hello_via_sink(
12553 &handler,
12554 &module_ctx,
12555 &mut module_rx,
12556 non_routable_hello_frame_with_control_ops(
12557 "mcp",
12558 300,
12559 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12560 ),
12561 )
12562 .await;
12563 assert!(registry
12564 .get_module("mcp")
12565 .unwrap()
12566 .unwrap()
12567 .manifest
12568 .provides
12569 .is_empty());
12570
12571 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12572 let route_response = handler
12573 .handle_control_frame(
12574 &route_client_ctx,
12575 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12576 )
12577 .await
12578 .unwrap();
12579 assert_eq!(route_response[0].header.ty, FrameType::Error);
12580 assert_eq!(
12581 parse_error(&route_response[0])["code"],
12582 "target_unavailable"
12583 );
12584 assert!(parse_error(&route_response[0])["message"]
12585 .as_str()
12586 .unwrap()
12587 .contains("does not provide the requested target"));
12588 assert!(module_rx.try_recv().is_err());
12589
12590 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12591 let health_handler = handler.clone();
12592 let health_task = tokio::spawn(async move {
12593 health_handler
12594 .handle_control_frame(
12595 &health_client_ctx,
12596 supervisor_health_probe_frame(302, "mcp"),
12597 )
12598 .await
12599 .unwrap()
12600 });
12601 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12602 .await
12603 .unwrap()
12604 .unwrap();
12605 assert_eq!(
12606 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12607 ModuleControlRequest::HealthCheck {}
12608 );
12609 handler
12610 .handle_control_frame(
12611 &module_ctx,
12612 health_response(health_frame.header.corr, HealthStatus::Ok),
12613 )
12614 .await
12615 .unwrap();
12616 let health_response = health_task.await.unwrap();
12617 assert_eq!(health_response[0].header.ty, FrameType::Response);
12618 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12619 ClientControlResponse::SupervisorHealthProbe {
12620 module_id, status, ..
12621 } => {
12622 assert_eq!(module_id, "mcp");
12623 assert_eq!(status, HealthStatus::Ok);
12624 }
12625 other => panic!("unexpected health response: {other:?}"),
12626 }
12627
12628 forwarding
12633 .cleanup_connection(module_ctx.connection_id)
12634 .unwrap();
12635 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12636 let cleanup_response = tokio::time::timeout(
12637 Duration::from_millis(200),
12638 handler.handle_control_frame(
12639 &cleanup_probe_ctx,
12640 supervisor_health_probe_frame(303, "mcp"),
12641 ),
12642 )
12643 .await
12644 .expect("probe should fail immediately when the control lane is gone")
12645 .unwrap();
12646 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12647 assert_eq!(
12648 parse_error(&cleanup_response[0])["code"],
12649 "target_unavailable"
12650 );
12651 assert!(parse_error(&cleanup_response[0])["message"]
12652 .as_str()
12653 .unwrap()
12654 .contains("no module connection"));
12655
12656 handler
12657 .cleanup_connection(module_ctx.connection_id)
12658 .unwrap();
12659 }
12660
12661 #[tokio::test]
12662 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12663 let registry = Arc::new(Registry::default());
12664 let supervisor_handle = SupervisorHandle::new();
12665 let supervisor =
12666 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12667 .with_handle(supervisor_handle.clone())
12668 .with_connection_file_path(
12669 std::env::temp_dir()
12670 .join(format!("subc-route-open-warming-{}", std::process::id())),
12671 );
12672 let module = supervisor
12673 .supervise_configured(
12674 ModuleSpec {
12675 module_id: "warming".to_string(),
12676 program: fake_aft_stub_path(),
12677 args: Vec::new(),
12678 env: Vec::new(),
12679 reserved: false,
12680 reserved_prefixes: Vec::new(),
12681 protocol: ModuleProtocol::Subc,
12682 overlap: Default::default(),
12683 },
12684 true,
12685 )
12686 .unwrap();
12687 assert_eq!(module.state().unwrap(), ModuleState::Running);
12688
12689 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12690 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12691 let response = handler
12692 .handle_control_frame(
12693 &ctx,
12694 route_open_frame(304, "warming", unique_project_root("warming")),
12695 )
12696 .await
12697 .unwrap();
12698 module.stop().await.unwrap();
12699
12700 assert_eq!(response[0].header.ty, FrameType::Error);
12701 let error = parse_error(&response[0]);
12702 assert_eq!(error["code"], "module_warming");
12703 assert!(error["message"]
12704 .as_str()
12705 .unwrap()
12706 .contains("state=running, enabled=true, live=false"));
12707 }
12708
12709 #[test]
12710 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12711 let handler = ControlHandler::new(Arc::new(Registry::default()));
12712 let capture = EventCapture::default();
12713 let _subscriber =
12714 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12715 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12716 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12717 let pending = (0..limit).collect::<Vec<_>>();
12718 let response = handler
12719 .route_open_capacity_refusal(
12720 &ctx,
12721 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12722 "busy",
12723 pending.len(),
12724 limit,
12725 )
12726 .unwrap();
12727 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12728 let event = capture
12729 .events()
12730 .into_iter()
12731 .find(|event| {
12732 event.target == "control"
12733 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12734 })
12735 .expect("connection admission refusal event");
12736 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12737 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12738 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
12741 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
12742 }
12743
12744 #[test]
12745 fn route_open_target_cap_logs_admission_reason_and_capacity() {
12746 let handler = ControlHandler::new(Arc::new(Registry::default()));
12747 let capture = EventCapture::default();
12748 let _subscriber =
12749 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12750 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12751 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12752 let guards = (0..limit)
12753 .map(|_| {
12754 handler
12755 .route_bind_concurrency
12756 .try_admit("busy", limit)
12757 .unwrap()
12758 })
12759 .collect::<Vec<_>>();
12760 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12761 Err(in_flight) => in_flight,
12762 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12763 };
12764 let response = handler
12765 .route_open_target_capacity_refusal(
12766 &ctx,
12767 &RouteOpenCaller::unknown(),
12768 &route_open_frame(397, "busy", unique_project_root("target-cap")),
12769 "busy",
12770 in_flight,
12771 )
12772 .unwrap();
12773 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12774 let event = capture
12775 .events()
12776 .into_iter()
12777 .find(|event| {
12778 event.target == "control"
12779 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12780 })
12781 .expect("target admission refusal event");
12782 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12783 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12784 drop(guards);
12785 }
12786
12787 #[tokio::test]
12788 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
12789 let registry = Arc::new(Registry::default());
12790 let handle = SupervisorHandle::new();
12791 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
12792 .with_handle(handle.clone());
12793 supervisor
12794 .supervise_configured(
12795 crate::ModuleSpec {
12796 module_id: "drift".into(),
12797 program: PathBuf::from("disabled-module"),
12798 args: Vec::new(),
12799 env: Vec::new(),
12800 reserved: false,
12801 reserved_prefixes: Vec::new(),
12802 protocol: ModuleProtocol::Subc,
12803 overlap: Default::default(),
12804 },
12805 false,
12806 )
12807 .unwrap();
12808 let handler = ControlHandler::new(registry).with_supervisor(handle);
12809 for configured in [None, Some(false), Some(true)] {
12810 handler.capability_evaluator.configure(
12811 configured.map(|enabled| ("drift".to_string(), enabled)),
12812 BTreeMap::new(),
12813 );
12814 let frame =
12815 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12816 let replies = handler
12817 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
12818 .await
12819 .unwrap();
12820 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12821 assert!(body.get("configured_enabled").is_none());
12822 assert_eq!(body["op"], "supervisor.ack");
12823 let frame =
12824 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12825 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12826 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12827 assert_eq!(
12828 body["modules"][0].get("configured_enabled"),
12829 configured.map(Value::Bool).as_ref()
12830 );
12831 }
12832 }
12833
12834 #[tokio::test(flavor = "current_thread")]
12835 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12836 let handler = ControlHandler::new(Arc::new(Registry::default()));
12837 let capture = EventCapture::default();
12838 let _subscriber =
12839 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12840 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12841 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12842
12843 let _ = handler
12844 .handle_client_control_request(
12845 &ctx,
12846 frame,
12847 ClientControlRequest::SupervisorSetEnabled {
12848 module_id: "broca".to_string(),
12849 enabled: false,
12850 },
12851 &mut None,
12852 )
12853 .await
12854 .unwrap();
12855
12856 let events = capture
12857 .events()
12858 .into_iter()
12859 .filter(|event| {
12860 event.target == "control"
12861 && event.fields.get("message").map(String::as_str)
12862 == Some("supervisor request received")
12863 })
12864 .collect::<Vec<_>>();
12865 assert_eq!(events.len(), 1, "one supervisor request log line");
12866 let event = &events[0];
12867 assert_eq!(event.level, tracing::Level::INFO);
12868 assert_eq!(
12869 event.fields.get("op"),
12870 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12871 );
12872 assert_eq!(
12873 event.fields.get("module_id"),
12874 Some(&"\"broca\"".to_string())
12875 );
12876 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12877 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12878 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12879 }
12880
12881 #[tokio::test(flavor = "current_thread")]
12882 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12883 let registry = Arc::new(Registry::default());
12884 let module_connection = ConnectionId::new(303);
12885 registry
12886 .register_with_control_ops(
12887 manifest("aft", PROTOCOL_VERSION),
12888 PROTOCOL_VERSION,
12889 module_connection,
12890 module_baseline_control_ops(),
12891 )
12892 .unwrap();
12893 let handler = ControlHandler::new(registry);
12894 let capture = EventCapture::default();
12895 let _subscriber =
12896 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12897 let (ctx, _rx) = route_ctx(module_connection);
12898 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12899
12900 let _ = handler
12901 .handle_client_control_request(
12902 &ctx,
12903 frame,
12904 ClientControlRequest::SupervisorSetEnabled {
12905 module_id: "broca".to_string(),
12906 enabled: false,
12907 },
12908 &mut None,
12909 )
12910 .await
12911 .unwrap();
12912
12913 let callers = capture
12914 .events()
12915 .into_iter()
12916 .filter(|event| {
12917 event.fields.get("message").map(String::as_str)
12918 == Some("supervisor request received")
12919 })
12920 .map(|event| event.fields.get("caller").cloned())
12921 .collect::<Vec<_>>();
12922 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12923 }
12924
12925 #[tokio::test(flavor = "current_thread")]
12926 async fn supervisor_list_does_not_log_request_received() {
12927 let handler = ControlHandler::new(Arc::new(Registry::default()));
12928 let capture = EventCapture::default();
12929 let _subscriber =
12930 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12931 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12932 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12933
12934 handler
12935 .handle_client_control_request(
12936 &ctx,
12937 frame,
12938 ClientControlRequest::SupervisorList {},
12939 &mut None,
12940 )
12941 .await
12942 .unwrap();
12943
12944 assert!(capture.events().into_iter().all(|event| {
12945 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12946 }));
12947 }
12948
12949 #[tokio::test(flavor = "current_thread")]
12950 async fn supervisor_rescan_preview_does_not_log_request_received() {
12951 let handler = ControlHandler::new(Arc::new(Registry::default()));
12952 let capture = EventCapture::default();
12953 let _subscriber =
12954 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12955 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
12956 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12957
12958 handler
12959 .handle_client_control_request(
12960 &ctx,
12961 frame,
12962 ClientControlRequest::SupervisorRescan { preview: true },
12963 &mut None,
12964 )
12965 .await
12966 .unwrap();
12967
12968 assert!(capture.events().into_iter().all(|event| {
12969 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12970 }));
12971 }
12972
12973 struct ApproveOperator;
12974 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
12975 fn prompt(
12976 &self,
12977 _: &str,
12978 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
12979 ) -> crate::operator_confirm::ProviderResult {
12980 crate::operator_confirm::ProviderResult::Approved
12981 }
12982 }
12983
12984 #[tokio::test]
12985 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
12986 assert_eq!(
12987 ControlHandler::default().privileged_operator_prompt,
12988 PrivilegedOperatorPrompt::Log
12989 );
12990 for (initial, changed, running) in [
12991 ("log", "enforce", PrivilegedOperatorPrompt::Log),
12992 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
12993 ] {
12994 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
12995 let path = dir.join("subc.jsonc");
12996 std::fs::write(
12997 &path,
12998 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
12999 )
13000 .unwrap();
13001 let config = crate::daemon_config::load(&path).unwrap().unwrap();
13002 let registry = Arc::new(Registry::default());
13003 let handle = SupervisorHandle::new();
13004 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13005 .with_handle(handle.clone());
13006 let handler = ControlHandler::new(registry)
13007 .with_supervisor(handle)
13008 .with_privileged_operator_prompt(config.privileged_operator_prompt)
13009 .with_supervisor_rescan(supervisor, &path, config.port);
13010 handler.forwarding.operator_confirms().configure(
13011 Arc::new(ApproveOperator),
13012 crate::operator_confirm::OperatorLimits::default(),
13013 );
13014 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13015
13016 for (disk_mode, needs_restart) in [
13017 (Some(initial), false),
13018 (Some(changed), true),
13019 (Some(changed), true),
13020 (None, running == PrivilegedOperatorPrompt::Enforce),
13021 (Some(initial), false),
13022 ] {
13023 let mut doc = json!({"version":1});
13024 if let Some(mode) = disk_mode {
13025 doc["privileged_operator_prompt"] = json!(mode);
13026 }
13027 std::fs::write(&path, doc.to_string()).unwrap();
13028 for preview in [true, false] {
13029 let request = ClientControlRequest::SupervisorRescan { preview };
13030 let frame = Frame::build(
13031 FrameType::Request,
13032 control_flags(),
13033 0,
13034 0,
13035 1,
13036 serde_json::to_vec(&request).unwrap(),
13037 )
13038 .unwrap();
13039 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13040 assert_eq!(frames.len(), 1);
13041 let ClientControlResponse::SupervisorRescan { result } =
13042 serde_json::from_slice(&frames[0].body).unwrap()
13043 else {
13044 panic!("expected a rescan response");
13045 };
13046 let expected = if needs_restart {
13047 vec!["privileged_operator_prompt"]
13048 } else {
13049 Vec::new()
13050 };
13051 assert_eq!(result.restart_required, expected);
13052 assert_eq!(handler.privileged_operator_prompt, running);
13053 assert_eq!(
13054 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13055 running
13056 );
13057 }
13058 }
13059 }
13060 }
13061
13062 #[tokio::test]
13063 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13064 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13065 let path = dir.join("subc.jsonc");
13066 let registry = Arc::new(Registry::default());
13067 let handle = SupervisorHandle::new();
13068 let supervisor =
13069 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13070 let handler = ControlHandler::new(registry)
13071 .with_supervisor(handle)
13072 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13073 .with_supervisor_rescan(supervisor, &path, None);
13074 handler.forwarding.operator_confirms().configure(
13075 Arc::new(ApproveOperator),
13076 crate::operator_confirm::OperatorLimits::default(),
13077 );
13078 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13079 for preview in [true, false] {
13080 std::fs::write(
13081 &path,
13082 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13083 )
13084 .unwrap();
13085 let request = ClientControlRequest::SupervisorRescan { preview };
13086 let frame = Frame::build(
13087 FrameType::Request,
13088 control_flags(),
13089 0,
13090 0,
13091 1,
13092 serde_json::to_vec(&request).unwrap(),
13093 )
13094 .unwrap();
13095 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13096 assert_eq!(frames.len(), 1);
13097 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13098 assert_eq!(
13099 handler.privileged_operator_prompt,
13100 PrivilegedOperatorPrompt::Enforce
13101 );
13102 }
13103 }
13104
13105 #[tokio::test]
13110 async fn route_open_refusal_names_the_check_that_refused() {
13111 let handler = ControlHandler::new(Arc::new(Registry::default()));
13112 let capture = EventCapture::default();
13113 let _subscriber =
13114 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13115 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13116 let response = handler
13117 .handle_control_frame(
13118 &ctx,
13119 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13120 )
13121 .await
13122 .unwrap();
13123
13124 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13125 let event = capture
13126 .events()
13127 .into_iter()
13128 .find(|event| {
13129 event.target == "control"
13130 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13131 })
13132 .expect("route.open refusal event");
13133 assert_eq!(
13134 event.fields.get("reason"),
13135 Some(&"\"not_registered\"".to_string())
13136 );
13137 }
13138
13139 #[tokio::test]
13140 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13141 let registry = Arc::new(Registry::default());
13142 let supervisor_handle = SupervisorHandle::new();
13143 let supervisor =
13144 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13145 .with_handle(supervisor_handle.clone())
13146 .with_connection_file_path(std::env::temp_dir().join(format!(
13147 "subc-route-open-refusal-info-{}",
13148 std::process::id()
13149 )));
13150 let module = supervisor
13151 .supervise_configured(
13152 ModuleSpec {
13153 module_id: "warming".to_string(),
13154 program: fake_aft_stub_path(),
13155 args: Vec::new(),
13156 env: Vec::new(),
13157 reserved: false,
13158 reserved_prefixes: Vec::new(),
13159 protocol: ModuleProtocol::Subc,
13160 overlap: Default::default(),
13161 },
13162 true,
13163 )
13164 .unwrap();
13165 assert_eq!(module.state().unwrap(), ModuleState::Running);
13166
13167 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13168 assert!(handler
13169 .counters()
13170 .snapshot()
13171 .get("route_open_refused_by_code")
13172 .is_none());
13173 let capture = EventCapture::default();
13174 let _subscriber =
13175 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13176 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13177 let response = handler
13178 .handle_control_frame(
13179 &ctx,
13180 route_open_frame(394, "warming", unique_project_root("refusal-info")),
13181 )
13182 .await
13183 .unwrap();
13184 module.stop().await.unwrap();
13185
13186 assert_eq!(parse_error(&response[0])["code"], "module_warming");
13187 let event = capture
13188 .events()
13189 .into_iter()
13190 .find(|event| {
13191 event.target == "control"
13192 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13193 })
13194 .expect("route.open refusal event");
13195 assert_eq!(
13196 event.fields.get("module_id"),
13197 Some(&"\"warming\"".to_string())
13198 );
13199 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13200 assert_eq!(
13201 event.fields.get("reason"),
13202 Some(&"\"supervised_not_registered\"".to_string())
13203 );
13204 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13205 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13206 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13207 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13208 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13209 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13210 assert_eq!(
13211 handler.counters().snapshot()["route_open_refused_by_code"],
13212 json!({ "module_warming": 1 })
13213 );
13214 }
13215
13216 const OUTAGE_START: &str = "route.open refusing module: not serving";
13217 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13218
13219 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13220 capture
13221 .events()
13222 .into_iter()
13223 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13224 .collect()
13225 }
13226
13227 fn supervise_stub(
13228 registry: &Arc<Registry>,
13229 module_id: &str,
13230 enabled: bool,
13231 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13232 let supervisor_handle = SupervisorHandle::new();
13233 let supervisor =
13234 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13235 .with_handle(supervisor_handle.clone())
13236 .with_connection_file_path(std::env::temp_dir().join(format!(
13237 "subc-route-outage-{module_id}-{}",
13238 std::process::id()
13239 )));
13240 let module = supervisor
13241 .supervise_configured(
13242 ModuleSpec {
13243 module_id: module_id.to_string(),
13244 program: fake_aft_stub_path(),
13245 args: Vec::new(),
13246 env: Vec::new(),
13247 reserved: false,
13248 reserved_prefixes: Vec::new(),
13249 protocol: ModuleProtocol::Subc,
13250 overlap: Default::default(),
13251 },
13252 enabled,
13253 )
13254 .unwrap();
13255 (supervisor_handle, module)
13256 }
13257
13258 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13259 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13260 module_id: module_id.to_string(),
13261 drain_timeout_ms: Some(50),
13262 })
13263 .unwrap();
13264 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13265 }
13266
13267 #[test]
13271 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13272 let registry = Arc::new(Registry::default());
13273 let forwarding = Arc::new(ForwardingTable::default());
13274 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13275 let second = ControlHandler::with_forwarding(registry, forwarding);
13276 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13277 }
13278
13279 #[tokio::test(flavor = "current_thread")]
13283 async fn route_open_unknown_module_refusals_add_no_outage_state() {
13284 let handler = ControlHandler::new(Arc::new(Registry::default()));
13285 let capture = EventCapture::default();
13286 let _subscriber =
13287 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13288 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13289 for corr in 0..8 {
13290 let response = handler
13291 .handle_control_frame(
13292 &ctx,
13293 route_open_frame(
13294 380 + corr,
13295 &format!("nobody-{corr}"),
13296 unique_project_root("outage-unknown"),
13297 ),
13298 )
13299 .await
13300 .unwrap();
13301 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13302 }
13303
13304 assert_eq!(handler.route_outages.tracked_module_count(), 0);
13305 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13306 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13307 }
13308
13309 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13314 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13315 let registry = Arc::new(Registry::default());
13316 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
13317 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13318 let capture = EventCapture::default();
13319 let _subscriber =
13320 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13321 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13322 handler.route_outages.record_accepted("outage-restart");
13325
13326 let response = handler
13327 .handle_control_frame(
13328 &ctx,
13329 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13330 )
13331 .await
13332 .unwrap();
13333 assert_eq!(response[0].header.ty, FrameType::Error);
13334 let starts = outage_lines(&capture, OUTAGE_START);
13335 assert_eq!(starts.len(), 1, "{starts:?}");
13336 assert_eq!(starts[0].level, tracing::Level::WARN);
13337 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13338 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13339 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13340 handler.route_outages.record_accepted("outage-restart");
13341 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13342
13343 let restart = handler
13344 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13345 .await
13346 .unwrap();
13347 assert_eq!(
13348 restart[0].header.ty,
13349 FrameType::Response,
13350 "{:?}",
13351 parse_error(&restart[0])
13352 );
13353 handler
13354 .handle_control_frame(
13355 &ctx,
13356 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13357 )
13358 .await
13359 .unwrap();
13360 module.stop().await.unwrap();
13361
13362 let starts = outage_lines(&capture, OUTAGE_START);
13363 assert_eq!(starts.len(), 2, "{starts:?}");
13364 assert_eq!(starts[1].level, tracing::Level::INFO);
13365 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13366 }
13367
13368 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13372 async fn failed_operator_restart_leaves_no_operator_mark() {
13373 let registry = Arc::new(Registry::default());
13374 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
13375 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13376 let capture = EventCapture::default();
13377 let _subscriber =
13378 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13379 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13380 handler.route_outages.record_accepted("outage-disabled");
13381
13382 let restart = handler
13383 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13384 .await
13385 .unwrap();
13386 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13387 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13388
13389 handler
13390 .handle_control_frame(
13391 &ctx,
13392 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13393 )
13394 .await
13395 .unwrap();
13396 let starts = outage_lines(&capture, OUTAGE_START);
13397 assert_eq!(starts.len(), 1, "{starts:?}");
13398 assert_eq!(starts[0].level, tracing::Level::WARN);
13399 }
13400
13401 #[tokio::test(flavor = "current_thread")]
13402 async fn route_open_unknown_module_escapes_target_module_id() {
13403 let handler = ControlHandler::new(Arc::new(Registry::default()));
13404 let capture = EventCapture::default();
13405 let _subscriber =
13406 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13407 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13408 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13409 let response = handler
13410 .handle_control_frame(
13411 &ctx,
13412 route_open_frame(
13413 395,
13414 hostile_module_id,
13415 unique_project_root("hostile-target-module-id"),
13416 ),
13417 )
13418 .await
13419 .unwrap();
13420
13421 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13422 let event = capture
13423 .events()
13424 .into_iter()
13425 .find(|event| {
13426 event.target == "control"
13427 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13428 })
13429 .expect("route.open unknown-module refusal event");
13430 let logged = event.fields.get("module_id").expect("module_id field");
13431 assert!(!logged.bytes().any(|byte| byte < 0x20));
13432 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13433 }
13434
13435 #[tokio::test(flavor = "current_thread")]
13436 async fn route_open_module_rejection_uses_daemon_counter_key() {
13437 let registry = Arc::new(Registry::default());
13438 let forwarding = Arc::new(ForwardingTable::default());
13439 let handler =
13440 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13441 let module_connection = ConnectionId::new(95);
13442 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13443 hello_via_sink(
13444 &handler,
13445 &module_ctx,
13446 &mut module_rx,
13447 hello_frame("aft", PROTOCOL_VERSION, 395),
13448 )
13449 .await;
13450
13451 let client_connection = ConnectionId::new(96);
13452 let (client_ctx, _client_rx) = route_ctx(client_connection);
13453 let capture = EventCapture::default();
13454 let _subscriber =
13455 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13456 let (route_task, bind) = relay_route_open(
13457 &handler,
13458 client_connection,
13459 &client_ctx.egress,
13460 &mut module_rx,
13461 396,
13462 "aft",
13463 "hostile-module-code",
13464 )
13465 .await;
13466 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13467 let rejection = Frame::build(
13468 FrameType::Error,
13469 control_flags(),
13470 0,
13471 0,
13472 bind.header.corr,
13473 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13474 )
13475 .unwrap();
13476 handler
13477 .handle_control_frame(&module_ctx, rejection)
13478 .await
13479 .unwrap();
13480
13481 let response = route_task.await.unwrap();
13482 assert_eq!(parse_error(&response[0])["code"], hostile_code);
13483 let counters = handler.counters().snapshot();
13484 assert_eq!(
13485 counters["route_open_refused_by_code"],
13486 json!({ "module_rejected": 1 })
13487 );
13488 assert!(counters["route_open_refused_by_code"]
13489 .get(hostile_code)
13490 .is_none());
13491
13492 let event = capture
13493 .events()
13494 .into_iter()
13495 .find(|event| {
13496 event.target == "control"
13497 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13498 })
13499 .expect("route.open module-rejection refusal event");
13500 let logged = event.fields.get("module_code").expect("module_code field");
13501 assert!(!logged.bytes().any(|byte| byte < 0x20));
13502 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13503 }
13504
13505 #[tokio::test]
13506 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13507 let registry = Arc::new(Registry::default());
13508 let supervisor_handle = SupervisorHandle::new();
13509 let missing_program = std::env::temp_dir().join(format!(
13510 "subc-route-open-missing-program-{}",
13511 std::process::id()
13512 ));
13513 let supervisor =
13514 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13515 .with_handle(supervisor_handle.clone());
13516 let module = supervisor
13517 .supervise_configured(
13518 ModuleSpec {
13519 module_id: "failed".to_string(),
13520 program: missing_program,
13521 args: Vec::new(),
13522 env: Vec::new(),
13523 reserved: false,
13524 reserved_prefixes: Vec::new(),
13525 protocol: ModuleProtocol::Subc,
13526 overlap: Default::default(),
13527 },
13528 true,
13529 )
13530 .unwrap();
13531 assert_eq!(module.state().unwrap(), ModuleState::Failed);
13532
13533 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13534 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13535 let response = handler
13536 .handle_control_frame(
13537 &ctx,
13538 route_open_frame(305, "failed", unique_project_root("failed")),
13539 )
13540 .await
13541 .unwrap();
13542
13543 assert_eq!(response[0].header.ty, FrameType::Error);
13544 let error = parse_error(&response[0]);
13545 assert_eq!(error["code"], "target_unavailable");
13546 assert!(error["message"]
13547 .as_str()
13548 .unwrap()
13549 .contains("state=failed, enabled=true, live=false"));
13550 }
13551
13552 #[tokio::test]
13553 async fn route_open_role_mismatch_remains_target_unavailable() {
13554 let registry = Arc::new(Registry::default());
13555 let handler = ControlHandler::new(Arc::clone(®istry));
13556 handler
13557 .handle_control(
13558 ConnectionId::new(41),
13559 non_routable_hello_frame_with_control_ops("health-only", 306, None),
13560 )
13561 .unwrap();
13562
13563 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13564 let response = handler
13565 .handle_control_frame(
13566 &ctx,
13567 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13568 )
13569 .await
13570 .unwrap();
13571
13572 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13573 assert!(parse_error(&response[0])["message"]
13574 .as_str()
13575 .unwrap()
13576 .contains("does not provide the requested target"));
13577 }
13578
13579 #[tokio::test]
13580 async fn route_open_inactive_registration_remains_target_unavailable() {
13581 let registry = Arc::new(Registry::default());
13582 let handler = ControlHandler::new(Arc::clone(®istry));
13583 handler
13584 .handle_control(
13585 ConnectionId::new(43),
13586 hello_frame("inactive", PROTOCOL_VERSION, 308),
13587 )
13588 .unwrap();
13589 assert!(registry
13590 .set_module_state_for_test("inactive", ChannelState::Closed)
13591 .unwrap());
13592
13593 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13594 let response = handler
13595 .handle_control_frame(
13596 &ctx,
13597 route_open_frame(309, "inactive", unique_project_root("inactive")),
13598 )
13599 .await
13600 .unwrap();
13601
13602 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13603 assert!(parse_error(&response[0])["message"]
13604 .as_str()
13605 .unwrap()
13606 .contains("is not active"));
13607 }
13608
13609 #[tokio::test]
13610 async fn late_health_reply_is_recorded_through_the_module_response_path() {
13611 let registry = Arc::new(Registry::default());
13612 let forwarding = Arc::new(ForwardingTable::default());
13613 let supervisor_handle = SupervisorHandle::new();
13614 let supervisor =
13615 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
13616 .with_forwarding(Arc::clone(&forwarding))
13617 .with_handle(supervisor_handle.clone());
13618 let module = supervisor
13619 .supervise_configured(
13620 crate::ModuleSpec {
13621 module_id: "late-health-response".to_string(),
13622 program: PathBuf::from("disabled-module"),
13623 args: Vec::new(),
13624 env: Vec::new(),
13625 reserved: false,
13626 reserved_prefixes: Vec::new(),
13627 protocol: ModuleProtocol::Subc,
13628 overlap: Default::default(),
13629 },
13630 false,
13631 )
13632 .unwrap();
13633 let handler =
13634 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13635 .with_supervisor(supervisor_handle);
13636 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13637 handler
13638 .handle_control_frame(
13639 &module_ctx,
13640 hello_frame_with_control_ops(
13641 "late-health-response",
13642 PROTOCOL_VERSION,
13643 7,
13644 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13645 ),
13646 )
13647 .await
13648 .unwrap();
13649 let probe_started_at = Instant::now() - Duration::from_millis(80);
13650 let pending = forwarding
13651 .begin_health_probe_rpc_for(
13652 "late-health-response",
13653 MODULE_CONTROL_OP_HEALTH_CHECK,
13654 probe_started_at,
13655 Instant::now() - Duration::from_millis(1),
13656 )
13657 .unwrap();
13658 assert!(forwarding
13659 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13660 .unwrap());
13661
13662 let responses = handler
13663 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13664 .await
13665 .unwrap();
13666
13667 assert!(responses.is_empty());
13668 let health = module.status().unwrap().health;
13669 assert_eq!(health.late_answer_count, 1);
13670 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13671 }
13672
13673 #[tokio::test]
13674 async fn health_probe_timeout_and_module_death_are_typed() {
13675 let registry = Arc::new(Registry::default());
13676 let forwarding = Arc::new(ForwardingTable::default());
13677 let handler =
13678 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13679 .with_health_probe_timeout(Duration::from_millis(50));
13680 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13681 hello_via_sink(
13682 &handler,
13683 &module_ctx,
13684 &mut module_rx,
13685 hello_frame_with_control_ops(
13686 "aft",
13687 PROTOCOL_VERSION,
13688 7,
13689 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13690 ),
13691 )
13692 .await;
13693
13694 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13695 let responses = handler
13696 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13697 .await
13698 .unwrap();
13699 assert_eq!(responses[0].header.ty, FrameType::Error);
13700 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13701 let _ = module_rx.try_recv();
13702
13703 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13704 let health_handler = handler.clone();
13705 let death_task = tokio::spawn(async move {
13706 health_handler
13707 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13708 .await
13709 .unwrap()
13710 });
13711 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13712 .await
13713 .unwrap()
13714 .unwrap();
13715 handler
13716 .cleanup_connection(module_ctx.connection_id)
13717 .unwrap();
13718 let responses = death_task.await.unwrap();
13719 assert_eq!(responses[0].header.ty, FrameType::Error);
13720 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13721 }
13722
13723 #[test]
13724 fn hello_requires_exact_protocol_version() {
13725 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13726 let registry = Arc::new(Registry::default());
13727 let handler = ControlHandler::new(Arc::clone(®istry));
13728 let responses = handler
13729 .handle_control(
13730 ConnectionId::new(connection),
13731 hello_frame("aft", offered, 9),
13732 )
13733 .unwrap();
13734
13735 assert_eq!(responses.len(), 1);
13736 assert_eq!(responses[0].header.ty, FrameType::Error);
13737 let error = parse_error(&responses[0]);
13738 assert_eq!(error["code"], "version_unsupported");
13739 assert!(registry.get_module("aft").unwrap().is_none());
13740 assert_eq!(registry.active_registration_count().unwrap(), 0);
13741 }
13742 }
13743
13744 #[test]
13745 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13746 let registry = Arc::new(Registry::default());
13747 let forwarding = Arc::new(ForwardingTable::default());
13748 let handler =
13749 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13750 let module_connection = ConnectionId::new(301);
13751 let registration = registry
13752 .register_with_control_ops(
13753 manifest("aft-push", PROTOCOL_VERSION),
13754 PROTOCOL_VERSION,
13755 module_connection,
13756 module_baseline_control_ops(),
13757 )
13758 .unwrap();
13759 let (module_tx, _module_rx) = mpsc::channel(8);
13760 let endpoint = forwarding
13761 .register_module_connection(
13762 module_connection,
13763 "aft-push".to_string(),
13764 PROTOCOL_VERSION,
13765 manifest_concurrency(®istration.manifest),
13766 FrameSink::new(module_tx),
13767 )
13768 .unwrap();
13769
13770 let unknown = Frame::build(
13772 FrameType::Push,
13773 control_flags(),
13774 0,
13775 0,
13776 5,
13777 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13778 )
13779 .unwrap();
13780 let out = handler.handle_status_update(endpoint, unknown).unwrap();
13781 assert!(
13782 out.is_empty(),
13783 "unknown push op must be ignored, got {out:?}"
13784 );
13785
13786 let malformed = Frame::build(
13788 FrameType::Push,
13789 control_flags(),
13790 0,
13791 0,
13792 6,
13793 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13794 )
13795 .unwrap();
13796 let out = handler.handle_status_update(endpoint, malformed).unwrap();
13797 assert_eq!(out.len(), 1);
13798 assert_eq!(out[0].header.ty, FrameType::Error);
13799 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13800 }
13801
13802 #[test]
13803 fn hello_rejected_when_connection_already_owns_client_routes() {
13804 let registry = Arc::new(Registry::default());
13805 let forwarding = Arc::new(ForwardingTable::default());
13806 let handler =
13807 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13808 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
13810 let client_connection = ConnectionId::new(202);
13811
13812 let responses = handler
13815 .handle_control(
13816 client_connection,
13817 hello_frame("aft-second", PROTOCOL_VERSION, 9),
13818 )
13819 .unwrap();
13820 assert_eq!(responses[0].header.ty, FrameType::Error);
13821 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13822 assert!(registry.get_module("aft-second").unwrap().is_none());
13823 }
13824
13825 #[tokio::test]
13826 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13827 let registry = Arc::new(Registry::default());
13828 let forwarding = Arc::new(ForwardingTable::default());
13829 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13830 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13831 hello_via_sink(
13832 &handler,
13833 &module_ctx,
13834 &mut module_rx,
13835 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13836 )
13837 .await;
13838 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13839 let pending = forwarding
13840 .begin_route_bind_relay_for_test(
13841 client_ctx.connection_id,
13842 client_ctx.egress.clone(),
13843 2,
13844 "alpha",
13845 )
13846 .unwrap();
13847 forwarding
13848 .complete_pending_relay(
13849 module_ctx.connection_id,
13850 pending.corr,
13851 RouteBindRelayOutcome::Accepted,
13852 )
13853 .unwrap();
13854 client_rx.try_recv().unwrap();
13855 for module_id in ["beta", "alpha"] {
13856 let replies = handler
13857 .handle_control_frame(
13858 &module_ctx,
13859 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13860 )
13861 .await
13862 .unwrap();
13863 assert_eq!(replies.len(), 1, "second HELLO must be refused");
13864 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13865 }
13866 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13867 assert!(registry.get_module("beta").unwrap().is_none());
13868 assert!(matches!(
13869 forwarding
13870 .lookup_data_route(
13871 client_ctx.connection_id,
13872 pending.client_channel,
13873 pending.client_epoch,
13874 )
13875 .unwrap(),
13876 DataRoute::Client(DataRouteState::Bound(_))
13877 ));
13878 assert!(handler
13879 .hello_launch_nonces
13880 .lock()
13881 .unwrap()
13882 .presented(module_ctx.connection_id, Some("alpha-nonce")));
13883 assert!(module_rx.try_recv().is_err());
13884 }
13885
13886 #[test]
13887 fn reserved_module_hello_requires_matching_launch_nonce() {
13888 let registry = Arc::new(Registry::default());
13889 let supervisor = SupervisorHandle::new();
13890 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13893 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13894
13895 let no_nonce = handler
13897 .handle_control(
13898 ConnectionId::new(1),
13899 hello_frame("vault", PROTOCOL_VERSION, 1),
13900 )
13901 .unwrap();
13902 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13903 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13904 assert!(registry.get_module("vault").unwrap().is_none());
13905
13906 let wrong = handler
13908 .handle_control(
13909 ConnectionId::new(2),
13910 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13911 )
13912 .unwrap();
13913 assert_eq!(wrong[0].header.ty, FrameType::Error);
13914 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13915 assert!(registry.get_module("vault").unwrap().is_none());
13916
13917 let ok = handler
13919 .handle_control(
13920 ConnectionId::new(3),
13921 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13922 )
13923 .unwrap();
13924 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13925 assert!(registry.get_module("vault").unwrap().is_some());
13926 }
13927
13928 #[test]
13929 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13930 let registry = Arc::new(Registry::default());
13931 let supervisor = SupervisorHandle::new();
13932 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13933 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13934 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13935
13936 let squat = handler
13937 .handle_control(
13938 ConnectionId::new(1),
13939 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13940 )
13941 .unwrap();
13942 assert_eq!(squat[0].header.ty, FrameType::Error);
13943 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
13944 assert!(parse_error(&squat[0])["message"]
13945 .as_str()
13946 .unwrap()
13947 .contains("fed:"));
13948
13949 let accepted_peer = handler
13950 .handle_control(
13951 ConnectionId::new(2),
13952 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
13953 )
13954 .unwrap();
13955 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
13956
13957 let accepted_short = handler
13958 .handle_control(
13959 ConnectionId::new(3),
13960 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
13961 )
13962 .unwrap();
13963 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
13964
13965 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
13966 let response = handler
13967 .handle_control(
13968 ConnectionId::new(conn),
13969 hello_frame(module_id, PROTOCOL_VERSION, conn),
13970 )
13971 .unwrap();
13972 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
13973 }
13974 }
13975
13976 #[test]
13977 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
13978 let registry = Arc::new(Registry::default());
13979 let supervisor = SupervisorHandle::new();
13980 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13981 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13982 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
13983 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13984
13985 let owner_nonce = handler
13986 .handle_control(
13987 ConnectionId::new(1),
13988 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
13989 )
13990 .unwrap();
13991 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
13992 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
13993 assert!(registry.get_module("fed:special").unwrap().is_none());
13994
13995 let exact_nonce = handler
13996 .handle_control(
13997 ConnectionId::new(2),
13998 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
13999 )
14000 .unwrap();
14001 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14002 assert!(registry.get_module("fed:special").unwrap().is_some());
14003 }
14004
14005 #[test]
14006 fn non_reserved_module_ignores_launch_nonce() {
14007 let registry = Arc::new(Registry::default());
14008 let handler = ControlHandler::new(Arc::clone(®istry));
14011 let no_nonce = handler
14012 .handle_control(
14013 ConnectionId::new(1),
14014 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14015 )
14016 .unwrap();
14017 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14018 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14019
14020 let echoed_nonce = handler
14021 .handle_control(
14022 ConnectionId::new(2),
14023 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14024 )
14025 .unwrap();
14026 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14027 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14028 }
14029
14030 #[test]
14031 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14032 let handler = ControlHandler::default();
14033 let conn = ConnectionId::new(1);
14034 let malformed = Frame::build(
14035 FrameType::Hello,
14036 control_flags(),
14037 0,
14038 0,
14039 3,
14040 b"{not json".to_vec(),
14041 )
14042 .unwrap();
14043
14044 let error = handler.handle_control(conn, malformed).unwrap();
14045 assert_eq!(error[0].header.ty, FrameType::Error);
14046 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14047
14048 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14049 let pong = handler.handle_control(conn, ping).unwrap();
14050 assert_eq!(pong[0].header.ty, FrameType::Pong);
14051 assert_eq!(pong[0].header.corr, 4);
14052 }
14053
14054 #[test]
14055 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14056 let registry = Arc::new(Registry::default());
14057 let handler = ControlHandler::new(Arc::clone(®istry));
14058
14059 handler
14060 .handle_control(
14061 ConnectionId::new(1),
14062 hello_frame("aft", PROTOCOL_VERSION, 1),
14063 )
14064 .unwrap();
14065 let duplicate = handler
14066 .handle_control(
14067 ConnectionId::new(2),
14068 hello_frame("aft", PROTOCOL_VERSION, 2),
14069 )
14070 .unwrap();
14071
14072 assert_eq!(duplicate[0].header.ty, FrameType::Error);
14073 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14074 let registration = registry.get_module("aft").unwrap().unwrap();
14075 assert_eq!(registration.connection_id, ConnectionId::new(1));
14076 }
14077
14078 #[test]
14079 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14080 let registry = Arc::new(Registry::default());
14081 let forwarding = Arc::new(ForwardingTable::default());
14082 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14083 let handler =
14084 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14085 .with_process_liveness(process_liveness);
14086 let (ctx, route_channel, route_epoch) =
14087 bind_liveness_route(®istry, &forwarding, "aft-dead");
14088 let responses = handler
14089 .handle_route_poll(
14090 &ctx,
14091 route_poll_frame(41, PollKind::Liveness, route_channel),
14092 route_channel,
14093 route_epoch,
14094 PollKind::Liveness,
14095 )
14096 .unwrap();
14097
14098 assert_eq!(responses.len(), 1);
14099 assert_eq!(responses[0].header.ty, FrameType::Response);
14100 assert_route_poll_liveness(&responses[0], false);
14101 }
14102
14103 #[test]
14104 fn liveness_poll_without_process_source_uses_bound_route() {
14105 let registry = Arc::new(Registry::default());
14106 let forwarding = Arc::new(ForwardingTable::default());
14107 let handler =
14108 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14109 let (ctx, route_channel, route_epoch) =
14110 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
14111 let responses = handler
14112 .handle_route_poll(
14113 &ctx,
14114 route_poll_frame(42, PollKind::Liveness, route_channel),
14115 route_channel,
14116 route_epoch,
14117 PollKind::Liveness,
14118 )
14119 .unwrap();
14120
14121 assert_route_poll_liveness(&responses[0], true);
14122 }
14123
14124 #[test]
14125 fn liveness_poll_untracked_process_source_uses_bound_route() {
14126 let registry = Arc::new(Registry::default());
14127 let forwarding = Arc::new(ForwardingTable::default());
14128 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14129 let handler =
14130 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14131 .with_process_liveness(process_liveness);
14132 let (ctx, route_channel, route_epoch) =
14133 bind_liveness_route(®istry, &forwarding, "aft-untracked");
14134 let responses = handler
14135 .handle_route_poll(
14136 &ctx,
14137 route_poll_frame(43, PollKind::Liveness, route_channel),
14138 route_channel,
14139 route_epoch,
14140 PollKind::Liveness,
14141 )
14142 .unwrap();
14143
14144 assert_route_poll_liveness(&responses[0], true);
14145 }
14146
14147 #[tokio::test]
14148 async fn unknown_op_returns_unknown_control_op() {
14149 let handler = ControlHandler::default();
14150 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14151 let request = Frame::build(
14152 FrameType::Request,
14153 control_flags(),
14154 0,
14155 0,
14156 55,
14157 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14158 )
14159 .unwrap();
14160
14161 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14162
14163 assert_eq!(response.len(), 1);
14164 assert_eq!(response[0].header.ty, FrameType::Error);
14165 assert_eq!(response[0].header.corr, 55);
14166 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14167 }
14168
14169 #[tokio::test]
14170 async fn supervisor_provenance_rejects_unknown_exact_module() {
14171 let handler = ControlHandler::default();
14172 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14173 let request = Frame::build(
14174 FrameType::Request,
14175 control_flags(),
14176 0,
14177 0,
14178 57,
14179 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14180 )
14181 .unwrap();
14182
14183 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14184
14185 assert_eq!(response.len(), 1);
14186 assert_eq!(response[0].header.ty, FrameType::Error);
14187 assert_eq!(response[0].header.corr, 57);
14188 let error = parse_error(&response[0]);
14189 assert_eq!(error["code"], "unknown_module");
14190 assert_eq!(error["message"], "module_id 'missing' is not supervised");
14191 }
14192
14193 #[test]
14194 fn provenance_probe_override_keeps_handler_tests_deterministic() {
14195 let expected = subc_control::RunningImageAgreement::Unavailable {
14196 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14197 };
14198 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14199 assert_eq!(handler.provenance_probe_override, Some(expected));
14200 }
14201
14202 #[test]
14203 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14204 let verdict = reload_verdict(
14205 std::path::Path::new("/bin/new"),
14206 Some(std::path::Path::new("/bin/old")),
14207 subc_control::RunningImageAgreement::Unavailable {
14208 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14209 },
14210 );
14211 assert!(matches!(
14212 verdict.path,
14213 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14214 if configured == std::path::Path::new("/bin/new")
14215 && spawned_from == std::path::Path::new("/bin/old")
14216 ));
14217 }
14218
14219 #[test]
14220 fn launch_spec_without_running_digest_is_unavailable_not_match() {
14221 use subc_control::{
14222 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14223 };
14224 let configured = crate::supervise::ModuleSpec {
14225 module_id: "legacy".into(),
14226 program: "/module".into(),
14227 args: Vec::new(),
14228 env: Vec::new(),
14229 reserved: false,
14230 reserved_prefixes: Vec::new(),
14231 protocol: subc_control::ModuleProtocol::Subc,
14232 overlap: Default::default(),
14233 };
14234 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14235 assert_eq!(
14236 launch_spec_agreement(&configured, false, Some(&digest)),
14237 Agreement::Unavailable {
14238 reason: Reason::NotRunning
14239 }
14240 );
14241 assert_eq!(
14242 launch_spec_agreement(&configured, true, None),
14243 Agreement::Unavailable {
14244 reason: Reason::SpawnedSpecUnavailable
14245 }
14246 );
14247 assert_eq!(
14248 launch_spec_agreement(&configured, true, Some(&digest)),
14249 Agreement::Match
14250 );
14251 }
14252
14253 #[cfg(windows)]
14254 #[test]
14255 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14256 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14257 let configured = dir.join("module.exe");
14258 std::fs::write(&configured, b"executable path fixture").unwrap();
14259 let spawned = std::fs::canonicalize(&configured).unwrap();
14260 assert_ne!(
14261 configured, spawned,
14262 "fixture must exercise different path spellings"
14263 );
14264 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14265 let lower_case = configured.to_string_lossy().to_lowercase();
14266 for spelling in [
14267 &configured,
14268 Path::new(&forward_slashes),
14269 Path::new(&lower_case),
14270 ] {
14271 let verdict = reload_verdict(
14272 spelling,
14273 Some(&spawned),
14274 subc_control::RunningImageAgreement::Unavailable {
14275 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14276 },
14277 );
14278 assert_eq!(
14279 verdict.path,
14280 ReloadPathAgreement::Match,
14281 "configured {spelling:?}, spawned {spawned:?}"
14282 );
14283 }
14284 }
14285
14286 #[cfg(windows)]
14287 #[test]
14288 fn windows_reload_path_keeps_different_existing_program_pending() {
14289 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14290 let old = dir.join("old.exe");
14291 let new = dir.join("new.exe");
14292 std::fs::write(&old, b"same executable content").unwrap();
14293 std::fs::copy(&old, &new).unwrap();
14294 let spawned = std::fs::canonicalize(&old).unwrap();
14295 let image = subc_control::RunningImageAgreement::Unavailable {
14296 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14297 };
14298 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14299 assert!(
14300 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14301 if configured == new && spawned_from == spawned)
14302 );
14303 let missing = dir.join("missing.exe");
14304 assert!(matches!(
14305 reload_verdict(&missing, Some(&spawned), image).path,
14306 ReloadPathAgreement::Mismatch { .. }
14307 ));
14308 }
14309
14310 #[test]
14311 fn reload_verdict_detects_replaced_image_at_same_path() {
14312 let image = subc_control::RunningImageAgreement::Mismatch {
14313 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14314 digest: "old".into(),
14315 },
14316 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14317 digest: "new".into(),
14318 },
14319 };
14320 let verdict = reload_verdict(
14321 std::path::Path::new("/bin/same"),
14322 Some(std::path::Path::new("/bin/same")),
14323 image.clone(),
14324 );
14325 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14326 assert_eq!(verdict.image, image);
14327 }
14328
14329 #[test]
14330 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14331 let image = subc_control::RunningImageAgreement::Unavailable {
14332 reason: subc_control::RunningImageUnavailableReason::NotRunning,
14333 };
14334 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14335 assert_eq!(
14336 verdict.path,
14337 subc_control::ReloadPathAgreement::Unavailable {
14338 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14339 }
14340 );
14341 assert_eq!(verdict.image, image);
14342
14343 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14344 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14345 };
14346 let verdict = reload_verdict(
14347 std::path::Path::new("/bin/same"),
14348 Some(std::path::Path::new("/bin/same")),
14349 unconfirmed.clone(),
14350 );
14351 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14352 assert_eq!(verdict.image, unconfirmed);
14353 }
14354
14355 #[test]
14356 fn reload_verdict_preserves_each_image_unavailability_reason() {
14357 use subc_control::RunningImageUnavailableReason as Reason;
14358
14359 for reason in [
14360 Reason::NotRunning,
14361 Reason::UnsupportedPlatform,
14362 Reason::RunningExecutableUnreadable,
14363 Reason::SpawnedPathUnreadable,
14364 Reason::HashFailed,
14365 Reason::ProcessIdentityUnconfirmed,
14366 Reason::Unknown("future_probe_reason".to_string()),
14367 ] {
14368 let image = subc_control::RunningImageAgreement::Unavailable {
14369 reason: reason.clone(),
14370 };
14371 let verdict = reload_verdict(
14372 std::path::Path::new("/bin/same"),
14373 Some(std::path::Path::new("/bin/same")),
14374 image.clone(),
14375 );
14376 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14377 assert_eq!(verdict.image, image, "{reason:?}");
14378 }
14379 }
14380
14381 #[tokio::test]
14382 async fn malformed_control_bodies_return_invalid_control_body() {
14383 let handler = ControlHandler::default();
14384 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14385
14386 for (corr, body) in [
14387 (56, br#"{"route_channel":1}"#.as_slice()),
14388 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14389 (
14390 58,
14391 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14392 ),
14393 ] {
14394 let request = Frame::build(
14395 FrameType::Request,
14396 control_flags(),
14397 0,
14398 0,
14399 corr,
14400 body.to_vec(),
14401 )
14402 .unwrap();
14403 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14404
14405 assert_eq!(response.len(), 1);
14406 assert_eq!(response[0].header.ty, FrameType::Error);
14407 assert_eq!(response[0].header.corr, corr);
14408 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14409 }
14410 }
14411
14412 #[tokio::test]
14413 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14414 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14415 let registry = Arc::new(Registry::default());
14416 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14417 let router = Router::with_control_handler(Arc::clone(&control));
14418 let connection = router.begin_connection();
14419 let (ctx, mut rx) = route_ctx(connection.id());
14420
14421 router
14422 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14423 .await
14424 .unwrap();
14425 let response = rx.recv().await.unwrap();
14426 let ack = parse_ack(&response);
14427 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14428 let channel = 1;
14429
14430 let goodbye =
14431 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14432 router.route_for_connection(&ctx, goodbye).await.unwrap();
14433 assert!(rx.try_recv().is_err());
14434 assert!(registry.get_module("aft").unwrap().is_none());
14435
14436 router
14437 .route_for_connection(&ctx, channel_request(channel, 13))
14438 .await
14439 .unwrap();
14440 let error_frame = rx.recv().await.unwrap();
14441 assert_eq!(error_frame.header.ty, FrameType::Error);
14442 assert_eq!(error_frame.header.channel, channel);
14443 let captured = crate::router::test_log::captured_logs(&logs);
14444 assert_eq!(
14445 captured
14446 .lines()
14447 .filter(|line| {
14448 line.contains("module_id=aft")
14449 && line.contains("reason=explicit_goodbye")
14450 && line.contains("module registration ended")
14451 })
14452 .count(),
14453 1,
14454 "unexpected GOODBYE registry log: {captured}"
14455 );
14456 }
14457
14458 #[tokio::test]
14459 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14460 let registry = Arc::new(Registry::default());
14461 let handler = ControlHandler::new(registry).with_capability_config(
14462 [("prov".to_string(), true), ("cons".to_string(), true)],
14463 BTreeMap::new(),
14464 );
14465 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14466 register_capability_manifest(
14467 &handler,
14468 &provider_ctx,
14469 &mut provider_rx,
14470 capability_manifest("prov", &["thing/v1"], &[]),
14471 1,
14472 )
14473 .await;
14474 let mut consumer = capability_manifest("cons", &[], &[]);
14475 consumer.capabilities.as_mut().unwrap().requires.push(
14476 subc_protocol::manifest::CapabilityRequirement {
14477 capability: "thing/v1".to_string(),
14478 need: subc_protocol::manifest::CapabilityNeed::Required,
14479 },
14480 );
14481 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14482 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14483 assert_eq!(
14484 handler.capability_evaluator.verdict("cons", "thing/v1"),
14485 Some(CapabilityVerdict::Provided)
14486 );
14487 let (mut client_rx, _) = open_route_for_capability_test(
14488 &handler,
14489 &provider_ctx,
14490 &mut provider_rx,
14491 703,
14492 3,
14493 "prov",
14494 None,
14495 )
14496 .await;
14497 let goodbye =
14498 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14499 handler
14500 .handle_control_frame(&provider_ctx, goodbye)
14501 .await
14502 .unwrap();
14503 assert_eq!(
14504 handler.capability_evaluator.verdict("cons", "thing/v1"),
14505 Some(CapabilityVerdict::NeverProvided)
14506 );
14507 let closed = client_rx
14508 .try_recv()
14509 .expect("GOODBYE pushes route.closed before route GOODBYE");
14510 assert!(
14511 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14512 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14513 );
14514 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14515 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14516 }
14517
14518 struct RouteClosedFixture {
14519 handler: ControlHandler,
14520 #[cfg(unix)]
14521 supervisor: Supervisor,
14522 module_ctx: RouteCtx,
14523 #[cfg(unix)]
14524 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14525 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14526 route: Arc<crate::forwarding::RouteBinding>,
14527 }
14528
14529 async fn route_closed_fixture() -> RouteClosedFixture {
14530 let registry = Arc::new(Registry::default());
14531 let handler = ControlHandler::new(Arc::clone(®istry));
14532 #[cfg(unix)]
14533 let supervisor = Supervisor::new(registry, RestartPolicy::default())
14534 .with_forwarding(Arc::clone(&handler.forwarding));
14535 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14536 register_capability_manifest(
14537 &handler,
14538 &module_ctx,
14539 &mut module_rx,
14540 capability_manifest("prov", &[], &[]),
14541 1,
14542 )
14543 .await;
14544 let (client_rx, bind) = open_route_for_capability_test(
14545 &handler,
14546 &module_ctx,
14547 &mut module_rx,
14548 702,
14549 2,
14550 "prov",
14551 None,
14552 )
14553 .await;
14554 let ModuleControlRequest::RouteBind {
14555 route_channel: channel,
14556 epoch: route_epoch,
14557 ..
14558 } = bind
14559 else {
14560 panic!("expected route.bind");
14561 };
14562 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14563 handler
14564 .forwarding
14565 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14566 .unwrap()
14567 else {
14568 panic!("the acknowledged route must be bound");
14569 };
14570 RouteClosedFixture {
14571 handler,
14572 #[cfg(unix)]
14573 supervisor,
14574 module_ctx,
14575 #[cfg(unix)]
14576 module_rx,
14577 client_rx,
14578 route,
14579 }
14580 }
14581
14582 impl RouteClosedFixture {
14583 #[cfg(unix)]
14584 async fn cut(&mut self) {
14585 let notices = async {
14586 let mut draining = self.module_rx.recv().await.unwrap();
14587 assert!(matches!(
14588 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14589 &draining.body
14590 )
14591 .unwrap(),
14592 subc_protocol::session::ModuleControlCommand::Draining {
14593 reason: RouteCloseReason::Restart,
14594 ..
14595 }
14596 ));
14597 draining.flushed.take().unwrap().send(()).unwrap();
14598 let mut closing = self.client_rx.recv().await.unwrap();
14599 assert!(matches!(
14600 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14601 ClientControlPush::RouteClosing {
14602 reason: RouteCloseReason::Restart,
14603 ..
14604 }
14605 ));
14606 closing.flushed.take().unwrap().send(()).unwrap();
14607 };
14608 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14609 result.unwrap();
14610 }
14611
14612 fn disconnect(&mut self) -> ClientControlPush {
14613 self.handler
14614 .cleanup_connection(self.module_ctx.connection_id)
14615 .unwrap();
14616 let closed = self.client_rx.try_recv().unwrap();
14617 assert_eq!(closed.header.ty, FrameType::Push);
14618 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14619 assert!(matches!(
14620 &push,
14621 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14622 if channels == &[self.route.client_channel]
14623 ));
14624 assert_eq!(
14625 self.client_rx.try_recv().unwrap().header.ty,
14626 FrameType::Goodbye
14627 );
14628 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14629 push
14630 }
14631 }
14632
14633 #[cfg(unix)]
14634 #[tokio::test(start_paused = true)]
14635 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14636 let mut fixture = route_closed_fixture().await;
14637 fixture.cut().await;
14638 assert!(matches!(
14639 fixture.disconnect(),
14640 ClientControlPush::RouteClosed {
14641 reason: RouteCloseReason::Restart,
14642 drained: true,
14643 ..
14644 }
14645 ));
14646 }
14647
14648 #[cfg(unix)]
14649 #[tokio::test(start_paused = true)]
14650 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14651 let mut fixture = route_closed_fixture().await;
14652 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14653 let started = tokio::time::Instant::now();
14654 fixture.cut().await;
14655 assert!(started.elapsed() >= Duration::from_secs(2));
14656 assert_eq!(fixture.route.flow.in_flight(), 1);
14657 assert!(matches!(
14658 fixture.disconnect(),
14659 ClientControlPush::RouteClosed {
14660 reason: RouteCloseReason::Restart,
14661 drained: false,
14662 ..
14663 }
14664 ));
14665 }
14666
14667 #[tokio::test]
14668 async fn module_connection_crash_reports_crash_and_not_drained() {
14669 let mut fixture = route_closed_fixture().await;
14670 assert!(matches!(
14674 fixture.disconnect(),
14675 ClientControlPush::RouteClosed {
14676 reason: RouteCloseReason::Crash,
14677 drained: false,
14678 ..
14679 }
14680 ));
14681 }
14682
14683 #[tokio::test]
14684 async fn dropping_router_connection_releases_registration() {
14685 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14686 let registry = Arc::new(Registry::default());
14687 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14688 let router = Router::with_control_handler(Arc::clone(&control));
14689 let connection = router.begin_connection();
14690 let connection_id = connection.id();
14691 let (ctx, mut rx) = route_ctx(connection_id);
14692
14693 router
14694 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14695 .await
14696 .unwrap();
14697 let response = rx.recv().await.unwrap();
14698 let ack = parse_ack(&response);
14699 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14700 assert!(registry.get_module("aft").unwrap().is_some());
14701
14702 drop(connection);
14703
14704 assert!(registry.get_module("aft").unwrap().is_none());
14705 assert_eq!(registry.active_registration_count().unwrap(), 0);
14706
14707 control
14708 .cleanup_connection(ConnectionId::new(u64::MAX))
14709 .unwrap();
14710 let captured = crate::router::test_log::captured_logs(&logs);
14711 println!("captured registry lifecycle logs:\n{captured}");
14712 let events: Vec<_> = captured
14713 .lines()
14714 .filter(|line| {
14715 line.contains("module registered module_id=aft ")
14716 || line.contains("module registration ended")
14717 })
14718 .collect();
14719 assert_eq!(
14720 events.len(),
14721 2,
14722 "unexpected registry lifecycle logs: {captured}"
14723 );
14724 assert!(events[0].contains("module registered module_id=aft "));
14725 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14726 assert!(events[1].contains(&format!(
14727 "module_id=aft connection_id={}",
14728 connection_id.get()
14729 )));
14730 assert!(events[1].contains("reason=connection_closed"));
14731 assert!(events[1].contains("module registration ended"));
14732 assert_eq!(
14733 captured
14734 .lines()
14735 .filter(|line| line.contains("module registered module_id=aft "))
14736 .count(),
14737 1,
14738 "legacy registration admission line must appear once: {captured}"
14739 );
14740 }
14741
14742 #[tokio::test]
14743 async fn hello_registration_keeps_legacy_module_registered_line_once() {
14744 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14745 let registry = Arc::new(Registry::default());
14746 let control = ControlHandler::new(Arc::clone(®istry));
14747 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14748 hello_via_sink(
14749 &control,
14750 &ctx,
14751 &mut rx,
14752 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14753 )
14754 .await;
14755
14756 let captured = crate::router::test_log::captured_logs(&logs);
14757 let admissions: Vec<_> = captured
14758 .lines()
14759 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14760 .collect();
14761 assert_eq!(
14762 admissions.len(),
14763 1,
14764 "legacy admission line must remain exactly once: {captured}"
14765 );
14766 assert!(admissions[0].contains("routable_provider=true"));
14767 assert!(admissions[0].contains("connection_id=777"));
14768 }
14769
14770 fn capability_manifest(
14771 module_id: &str,
14772 provides: &[&str],
14773 must_never_reach: &[&str],
14774 ) -> ModuleManifest {
14775 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14776 manifest.capabilities = Some(CapabilityDeclarations {
14777 provides: provides
14778 .iter()
14779 .map(|capability| (*capability).to_string())
14780 .collect(),
14781 requires: Vec::new(),
14782 must_never_reach: must_never_reach
14783 .iter()
14784 .map(|capability| (*capability).to_string())
14785 .collect(),
14786 });
14787 manifest
14788 }
14789
14790 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14791 Frame::build(
14792 FrameType::Hello,
14793 control_flags(),
14794 0,
14795 0,
14796 corr,
14797 serde_json::to_vec(&ModuleHelloBody {
14798 protocol_ver: manifest.protocol_ver,
14799 manifest,
14800 control_ops: None,
14801 launch_nonce: None,
14802 })
14803 .expect("capability test HELLO serializes"),
14804 )
14805 .expect("capability test HELLO frame builds")
14806 }
14807
14808 fn catalog_update_with_capabilities_frame(
14809 corr: u64,
14810 capabilities: CapabilityDeclarations,
14811 ) -> Frame {
14812 Frame::build(
14813 FrameType::Request,
14814 control_flags(),
14815 0,
14816 0,
14817 corr,
14818 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14819 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14820 capabilities: Some(capabilities),
14821 ready: None,
14822 })
14823 .expect("capability catalog.update serializes"),
14824 )
14825 .expect("capability catalog.update frame builds")
14826 }
14827
14828 async fn register_capability_manifest(
14829 handler: &ControlHandler,
14830 ctx: &RouteCtx,
14831 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14832 manifest: ModuleManifest,
14833 corr: u64,
14834 ) {
14835 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14836 }
14837
14838 async fn open_route_for_capability_test(
14839 handler: &ControlHandler,
14840 target_ctx: &RouteCtx,
14841 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14842 client_connection_id: u64,
14843 corr: u64,
14844 target_module_id: &str,
14845 consumer_identity: Option<ConsumerIdentity>,
14846 ) -> (
14847 mpsc::Receiver<crate::router::OutboundFrame>,
14848 ModuleControlRequest,
14849 ) {
14850 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14851 let route_handler = handler.clone();
14852 let target_module_id = target_module_id.to_string();
14853 let route_task = tokio::spawn(async move {
14854 route_handler
14855 .handle_control_frame(
14856 &client_ctx,
14857 route_open_frame_with_admission_facts(
14858 corr,
14859 &target_module_id,
14860 unique_project_root("admission-facts"),
14861 consumer_identity,
14862 None,
14863 ),
14864 )
14865 .await
14866 .expect("capability test route.open succeeds")
14867 });
14868 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14869 .await
14870 .expect("capability test route.open must reach route.bind")
14871 .expect("target control receiver stays open");
14872 let bind_request: ModuleControlRequest =
14873 serde_json::from_slice(&bind.body).expect("route.bind decodes");
14874 handler
14875 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14876 .await
14877 .expect("capability test route.bind ACK succeeds");
14878 assert!(route_task.await.expect("route.open task joins").is_empty());
14879 let opened = client_rx
14880 .recv()
14881 .await
14882 .expect("successful route.open publishes a response");
14883 assert!(matches!(
14884 serde_json::from_slice::<ClientControlResponse>(&opened.body),
14885 Ok(ClientControlResponse::RouteOpen { .. })
14886 ));
14887 (client_rx, bind_request)
14888 }
14889
14890 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
14891 assert_eq!(frame.header.ty, FrameType::Push);
14892 assert_eq!(frame.header.channel, 0);
14893 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14894 .expect("route.closed control push decodes");
14895 let ClientControlPush::RouteClosed { channels, .. } = &push else {
14896 panic!("expected route.closed");
14897 };
14898 assert_eq!(channels.len(), 1, "exactly one violating route closed");
14899 let channels = channels.clone();
14900 assert_eq!(
14901 push,
14902 ClientControlPush::RouteClosed {
14903 module_id: target_module_id.to_string(),
14904 channels,
14905 reason: RouteCloseReason::CapabilityDenied,
14906 drained: false,
14907 abandoned: 0,
14908 excluded_subscriptions: 0,
14909 terminal: Some(false),
14910 }
14911 );
14912 }
14913
14914 #[tokio::test]
14915 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14916 let registry = Arc::new(Registry::default());
14917 let forwarding = Arc::new(ForwardingTable::default());
14918 let supervisor = SupervisorHandle::new();
14919 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14920 let handler =
14921 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14922 .with_supervisor(supervisor);
14923 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14924 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14925 register_capability_manifest(
14926 &handler,
14927 &target_ctx,
14928 &mut target_rx,
14929 capability_manifest("target", &["credentials-provider/v1"], &[]),
14930 1,
14931 )
14932 .await;
14933 register_capability_manifest(
14934 &handler,
14935 &opener_ctx,
14936 &mut opener_rx,
14937 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14938 2,
14939 )
14940 .await;
14941
14942 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14943 let replies = handler
14944 .handle_control_frame(
14945 &client_ctx,
14946 route_open_frame_with_admission_facts(
14947 3,
14948 "target",
14949 unique_project_root("admission-facts"),
14950 Some(ConsumerIdentity {
14951 module_id: "opener".to_string(),
14952 launch_nonce: "opener-nonce".to_string(),
14953 }),
14954 None,
14955 ),
14956 )
14957 .await
14958 .expect("denied route.open returns a typed frame");
14959 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14960 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14961 assert!(
14962 target_rx.try_recv().is_err(),
14963 "forbidden route.open must not relay route.bind"
14964 );
14965 }
14966
14967 #[tokio::test]
14968 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
14969 let registry = Arc::new(Registry::default());
14970 let forwarding = Arc::new(ForwardingTable::default());
14971 let supervisor = SupervisorHandle::new();
14972 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14973 let handler =
14974 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14975 .with_supervisor(supervisor);
14976 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
14977 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
14978 register_capability_manifest(
14979 &handler,
14980 &target_ctx,
14981 &mut target_rx,
14982 capability_manifest("target", &["credentials-provider/v1"], &[]),
14983 1,
14984 )
14985 .await;
14986 register_capability_manifest(
14987 &handler,
14988 &old_opener_ctx,
14989 &mut old_opener_rx,
14990 capability_manifest("opener", &[], &[]),
14991 2,
14992 )
14993 .await;
14994 let (mut client_rx, _) = open_route_for_capability_test(
14995 &handler,
14996 &target_ctx,
14997 &mut target_rx,
14998 712,
14999 3,
15000 "target",
15001 Some(ConsumerIdentity {
15002 module_id: "opener".to_string(),
15003 launch_nonce: "opener-nonce".to_string(),
15004 }),
15005 )
15006 .await;
15007 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15008
15009 handler
15010 .cleanup_connection(old_opener_ctx.connection_id)
15011 .expect("old opener registration cleans up");
15012 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15013 register_capability_manifest(
15014 &handler,
15015 &new_opener_ctx,
15016 &mut new_opener_rx,
15017 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15018 4,
15019 )
15020 .await;
15021
15022 assert_capability_denied_push(
15023 client_rx
15024 .try_recv()
15025 .expect("HELLO deny addition must emit route.closed")
15026 .frame,
15027 "target",
15028 );
15029 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15030 assert!(matches!(
15031 target_rx.try_recv(),
15032 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15033 ));
15034 }
15035
15036 #[tokio::test]
15037 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15038 let registry = Arc::new(Registry::default());
15039 let forwarding = Arc::new(ForwardingTable::default());
15040 let supervisor = SupervisorHandle::new();
15041 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15042 let handler =
15043 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15044 .with_supervisor(supervisor);
15045 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15046 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15047 register_capability_manifest(
15048 &handler,
15049 &target_ctx,
15050 &mut target_rx,
15051 capability_manifest("target", &[], &[]),
15052 1,
15053 )
15054 .await;
15055 register_capability_manifest(
15056 &handler,
15057 &opener_ctx,
15058 &mut opener_rx,
15059 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15060 2,
15061 )
15062 .await;
15063 let (mut client_rx, _) = open_route_for_capability_test(
15064 &handler,
15065 &target_ctx,
15066 &mut target_rx,
15067 722,
15068 3,
15069 "target",
15070 Some(ConsumerIdentity {
15071 module_id: "opener".to_string(),
15072 launch_nonce: "opener-nonce".to_string(),
15073 }),
15074 )
15075 .await;
15076 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15077
15078 let replies = handler
15079 .handle_control_frame(
15080 &target_ctx,
15081 catalog_update_with_capabilities_frame(
15082 4,
15083 CapabilityDeclarations {
15084 provides: vec!["credentials-provider/v1".to_string()],
15085 requires: Vec::new(),
15086 must_never_reach: Vec::new(),
15087 },
15088 ),
15089 )
15090 .await
15091 .expect("claim catalog.update succeeds");
15092 assert!(matches!(
15093 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15094 Ok(ModuleControlResponseToModule::CatalogUpdate {})
15095 ));
15096 assert_capability_denied_push(
15097 client_rx
15098 .try_recv()
15099 .expect("claim addition must emit route.closed")
15100 .frame,
15101 "target",
15102 );
15103 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15104 assert!(matches!(
15105 target_rx.try_recv(),
15106 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15107 ));
15108 }
15109
15110 #[tokio::test]
15111 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15112 let registry = Arc::new(Registry::default());
15113 let forwarding = Arc::new(ForwardingTable::default());
15114 let supervisor = SupervisorHandle::new();
15115 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15116 let handler =
15117 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15118 .with_supervisor(supervisor);
15119 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15120 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15121 register_capability_manifest(
15122 &handler,
15123 &target_ctx,
15124 &mut target_rx,
15125 capability_manifest("target", &["credentials-provider/v1"], &[]),
15126 1,
15127 )
15128 .await;
15129 register_capability_manifest(
15130 &handler,
15131 &opener_ctx,
15132 &mut opener_rx,
15133 capability_manifest("opener", &[], &[]),
15134 2,
15135 )
15136 .await;
15137 let (mut client_rx, _) = open_route_for_capability_test(
15138 &handler,
15139 &target_ctx,
15140 &mut target_rx,
15141 732,
15142 3,
15143 "target",
15144 Some(ConsumerIdentity {
15145 module_id: "opener".to_string(),
15146 launch_nonce: "opener-nonce".to_string(),
15147 }),
15148 )
15149 .await;
15150 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15151
15152 handler
15153 .handle_control_frame(
15154 &target_ctx,
15155 catalog_update_with_capabilities_frame(
15156 4,
15157 CapabilityDeclarations {
15158 provides: Vec::new(),
15159 requires: Vec::new(),
15160 must_never_reach: Vec::new(),
15161 },
15162 ),
15163 )
15164 .await
15165 .expect("claim removal catalog.update succeeds");
15166 assert_eq!(
15167 forwarding.active_binding_count().unwrap(),
15168 1,
15169 "removing an attested target claim must leave the route census unchanged"
15170 );
15171 assert!(
15172 client_rx.try_recv().is_err(),
15173 "claim removal must not emit route.closed capability_denied"
15174 );
15175 assert!(
15176 target_rx.try_recv().is_err(),
15177 "claim removal must not send the target a route GOODBYE"
15178 );
15179 }
15180
15181 #[tokio::test]
15184 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15185 let registry = Arc::new(Registry::default());
15186 let forwarding = Arc::new(ForwardingTable::default());
15187 let supervisor = SupervisorHandle::new();
15188 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15189 let handler =
15190 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15191 .with_supervisor(supervisor);
15192 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15193 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15194 register_capability_manifest(
15195 &handler,
15196 &target_ctx,
15197 &mut target_rx,
15198 capability_manifest("target", &["credentials-provider/v1"], &[]),
15199 1,
15200 )
15201 .await;
15202 register_capability_manifest(
15203 &handler,
15204 &opener_ctx,
15205 &mut opener_rx,
15206 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15207 2,
15208 )
15209 .await;
15210
15211 let (_client_rx, bind) = open_route_for_capability_test(
15212 &handler,
15213 &target_ctx,
15214 &mut target_rx,
15215 742,
15216 3,
15217 "target",
15218 None,
15219 )
15220 .await;
15221 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15222 panic!("direct scope-honesty route must bind");
15223 };
15224 assert_eq!(principal, Some(Principal::Direct));
15225 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15226 }
15227
15228 #[tokio::test]
15231 async fn must_never_reach_self_route_is_capability_forbidden() {
15232 let registry = Arc::new(Registry::default());
15233 let forwarding = Arc::new(ForwardingTable::default());
15234 let supervisor = SupervisorHandle::new();
15235 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15236 let handler =
15237 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15238 .with_supervisor(supervisor);
15239 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15240 register_capability_manifest(
15241 &handler,
15242 &self_ctx,
15243 &mut self_rx,
15244 capability_manifest(
15245 "self-provider",
15246 &["credentials-provider/v1"],
15247 &["credentials-provider/v1"],
15248 ),
15249 1,
15250 )
15251 .await;
15252
15253 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15254 let replies = handler
15255 .handle_control_frame(
15256 &client_ctx,
15257 route_open_frame_with_admission_facts(
15258 2,
15259 "self-provider",
15260 unique_project_root("admission-facts"),
15261 Some(ConsumerIdentity {
15262 module_id: "self-provider".to_string(),
15263 launch_nonce: "self-nonce".to_string(),
15264 }),
15265 None,
15266 ),
15267 )
15268 .await
15269 .expect("self-route refusal returns a typed frame");
15270 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15271 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15272 assert!(
15273 self_rx.try_recv().is_err(),
15274 "self denial must not relay route.bind"
15275 );
15276 }
15277
15278 #[test]
15279 fn unsupported_channel_zero_frame_returns_error() {
15280 let handler = ControlHandler::default();
15281 let request = Frame::build(
15282 FrameType::Request,
15283 control_flags(),
15284 0,
15285 0,
15286 21,
15287 b"opaque".to_vec(),
15288 )
15289 .unwrap();
15290
15291 let response = handler
15292 .handle_control(ConnectionId::new(1), request)
15293 .unwrap();
15294
15295 assert_eq!(response[0].header.ty, FrameType::Error);
15296 assert_eq!(
15297 parse_error(&response[0])["code"],
15298 "unsupported_control_frame"
15299 );
15300 }
15301
15302 mod swap {
15307 use super::*;
15308
15309 const INCUMBENT: ConnectionId = ConnectionId::new(30);
15310 const CANDIDATE: ConnectionId = ConnectionId::new(40);
15311
15312 struct Swap {
15313 registry: Arc<Registry>,
15314 forwarding: Arc<ForwardingTable>,
15315 handler: ControlHandler,
15316 incumbent_ctx: RouteCtx,
15317 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15318 candidate_ctx: RouteCtx,
15319 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15320 }
15321
15322 async fn swap_with_incumbent() -> Swap {
15323 let registry = Arc::new(Registry::default());
15324 let forwarding = Arc::new(ForwardingTable::default());
15325 let handler =
15326 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15327 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15328 hello_via_sink(
15329 &handler,
15330 &incumbent_ctx,
15331 &mut incumbent_rx,
15332 hello_frame("aft", PROTOCOL_VERSION, 7),
15333 )
15334 .await;
15335 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15336 Swap {
15337 registry,
15338 forwarding,
15339 handler,
15340 incumbent_ctx,
15341 incumbent_rx,
15342 candidate_ctx,
15343 candidate_rx,
15344 }
15345 }
15346
15347 fn register_candidate(swap: &Swap, ready: Option<bool>) {
15348 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15349 candidate_manifest.ready = ready;
15350 let registration = swap
15351 .registry
15352 .register_candidate_with_control_ops(
15353 candidate_manifest,
15354 PROTOCOL_VERSION,
15355 CANDIDATE,
15356 module_baseline_control_ops(),
15357 )
15358 .unwrap();
15359 swap.forwarding
15360 .register_candidate_module_connection(
15361 CANDIDATE,
15362 "aft".to_string(),
15363 PROTOCOL_VERSION,
15364 manifest_concurrency(®istration.manifest),
15365 swap.candidate_ctx.egress.clone(),
15366 )
15367 .unwrap();
15368 }
15369
15370 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15371 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15372 swap.registry.promote_candidate("aft").unwrap().unwrap();
15373 cutover.incumbent.unwrap()
15374 }
15375
15376 #[tokio::test]
15385 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15386 let mut swap = swap_with_incumbent().await;
15387 let hold = Arc::new(std::sync::Barrier::new(2));
15388 let handler = swap
15389 .handler
15390 .clone()
15391 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15392 handler
15393 .supervisor
15394 .open_swap("aft", "candidate-nonce".to_string());
15395 let hello = {
15396 let handler = handler.clone();
15397 let candidate_ctx = swap.candidate_ctx.clone();
15398 std::thread::spawn(move || {
15399 handler
15400 .handle_hello(
15401 candidate_ctx.connection_id,
15402 Some(candidate_ctx.egress.clone()),
15403 hello_frame_with_nonce(
15404 "aft",
15405 PROTOCOL_VERSION,
15406 8,
15407 Some("candidate-nonce"),
15408 ),
15409 )
15410 .unwrap()
15411 })
15412 };
15413
15414 hold.wait();
15415 assert!(swap
15418 .registry
15419 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15420 .unwrap()
15421 .is_some());
15422 assert!(swap
15423 .registry
15424 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15425 .unwrap()
15426 .is_none());
15427 assert!(swap
15428 .forwarding
15429 .module_endpoint_for_connection(CANDIDATE)
15430 .unwrap()
15431 .is_none());
15432
15433 hold.wait();
15434 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15435 assert_eq!(
15436 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15437 FrameType::HelloAck
15438 );
15439 let published = swap
15440 .registry
15441 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15442 .unwrap()
15443 .expect("candidate is published once its HELLO completes");
15444 assert_eq!(published.connection_id, CANDIDATE);
15445 assert!(swap
15446 .forwarding
15447 .module_endpoint_for_connection(CANDIDATE)
15448 .unwrap()
15449 .is_some());
15450 }
15451
15452 fn keyed_total(counters: &Value, key: &str) -> u64 {
15453 counters[key]
15454 .as_object()
15455 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15456 .unwrap_or(0)
15457 }
15458
15459 #[tokio::test]
15460 async fn replacement_logs_old_end_and_new_admission_once() {
15461 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15462 let mut swap = swap_with_incumbent().await;
15463 swap.handler
15464 .supervisor
15465 .open_swap("aft", "candidate-nonce".to_string());
15466 hello_via_sink(
15467 &swap.handler,
15468 &swap.candidate_ctx,
15469 &mut swap.candidate_rx,
15470 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15471 )
15472 .await;
15473 cutover(&swap);
15474 swap.handler.cleanup_connection(INCUMBENT).unwrap();
15475
15476 let captured = crate::router::test_log::captured_logs(&logs);
15477 println!("captured replacement registry logs:\n{captured}");
15478 let new_admissions: Vec<_> = captured
15479 .lines()
15480 .filter(|line| {
15481 line.contains("connection_id=40")
15482 && line.contains("swap candidate registered; not routable until cutover")
15483 })
15484 .collect();
15485 assert_eq!(
15486 new_admissions.len(),
15487 1,
15488 "unexpected admission logs: {captured}"
15489 );
15490 assert!(new_admissions[0].contains("module_id=aft"));
15491 assert!(new_admissions[0].contains("connection_id=40"));
15492 assert!(
15493 new_admissions[0].contains("swap candidate registered; not routable until cutover")
15494 );
15495 assert!(new_admissions[0].contains("ready=true"));
15496
15497 let old_admissions: Vec<_> = captured
15498 .lines()
15499 .filter(|line| {
15500 line.contains("module registered module_id=aft ")
15501 && line.contains("connection_id=30")
15502 })
15503 .collect();
15504 assert_eq!(
15505 old_admissions.len(),
15506 1,
15507 "unexpected incumbent admission: {captured}"
15508 );
15509
15510 let old_ends: Vec<_> = captured
15511 .lines()
15512 .filter(|line| {
15513 line.contains("connection_id=30") && line.contains("module registration ended")
15514 })
15515 .collect();
15516 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15517 assert!(old_ends[0].contains("module_id=aft"));
15518 assert!(old_ends[0].contains("connection_id=30"));
15519 assert!(old_ends[0].contains("reason=replaced"));
15520 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15521 assert_eq!(
15522 captured
15523 .lines()
15524 .filter(|line| line.contains("module registration promoted"))
15525 .count(),
15526 1,
15527 "unexpected promotion logs: {captured}"
15528 );
15529 }
15530
15531 #[tokio::test]
15537 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15538 let mut swap = swap_with_incumbent().await;
15539 let handler = swap.handler.clone();
15540
15541 let cotenant = ConnectionId::new(31);
15543 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15544 let (cotenant_task, cotenant_bind) = relay_route_open(
15545 &handler,
15546 cotenant,
15547 &cotenant_ctx.egress,
15548 &mut swap.incumbent_rx,
15549 100,
15550 "aft",
15551 "swap-cotenant",
15552 )
15553 .await;
15554 handler
15555 .handle_control_frame(
15556 &swap.incumbent_ctx,
15557 route_bind_ack(cotenant_bind.header.corr),
15558 )
15559 .await
15560 .unwrap();
15561 assert!(cotenant_task.await.unwrap().is_empty());
15562 let (cotenant_channel, cotenant_epoch) =
15563 published_route(&cotenant_rx.recv().await.unwrap());
15564
15565 let caller = ConnectionId::new(32);
15567 let (caller_ctx, mut caller_rx) = route_ctx(caller);
15568 let (caller_task, caller_bind) = relay_route_open(
15569 &handler,
15570 caller,
15571 &caller_ctx.egress,
15572 &mut swap.incumbent_rx,
15573 101,
15574 "aft",
15575 "swap-caller",
15576 )
15577 .await;
15578 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15579
15580 register_candidate(&swap, None);
15581 cutover(&swap);
15582
15583 let ack = handler
15585 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15586 .await;
15587 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15588 if module_loop_error.is_some() {
15589 handler.cleanup_connection(INCUMBENT).unwrap();
15592 }
15593
15594 assert!(
15596 cotenant_rx.try_recv().is_err(),
15597 "the co-tenant route on the incumbent was torn down by one late ack: \
15598 {module_loop_error:?}"
15599 );
15600 assert!(matches!(
15601 swap.forwarding
15602 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15603 .unwrap(),
15604 DataRoute::Client(DataRouteState::Bound(_))
15605 ));
15606 assert_eq!(module_loop_error, None);
15607 assert!(swap
15608 .registry
15609 .get_module_by_connection(INCUMBENT)
15610 .unwrap()
15611 .is_some());
15612
15613 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15615 .await
15616 .expect("the incumbent is told to drop the abandoned binding")
15617 .unwrap()
15618 .frame;
15619 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15620 assert_eq!(goodbye.header.channel, abandoned_channel);
15621 assert_eq!(goodbye.header.epoch, abandoned_epoch);
15622 assert!(swap.incumbent_rx.try_recv().is_err());
15623
15624 let response = caller_task.await.unwrap();
15626 assert_eq!(response.len(), 1);
15627 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15628 assert!(caller_rx.try_recv().is_err());
15629
15630 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15634 let counters = handler.counters().snapshot();
15635 assert_eq!(
15636 keyed_total(&counters, "route_open_accepted_by_principal"),
15637 1
15638 );
15639 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15640 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15641 }
15642
15643 #[tokio::test]
15647 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15648 let mut swap = swap_with_incumbent().await;
15649 register_candidate(&swap, None);
15650 let incumbent = cutover(&swap);
15651 swap.forwarding
15652 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15653 .unwrap()
15654 .expect("the incumbent is still registered");
15655
15656 let client = ConnectionId::new(33);
15657 let (client_ctx, mut client_rx) = route_ctx(client);
15658 let route_handler = swap.handler.clone();
15659 let open_ctx = RouteCtx {
15660 connection_id: client,
15661 egress: client_ctx.egress.clone(),
15662 };
15663 let mut route_task = tokio::spawn(async move {
15664 route_handler
15665 .handle_control_frame(
15666 &open_ctx,
15667 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15668 )
15669 .await
15670 .unwrap()
15671 });
15672 let bind = tokio::select! {
15673 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15674 response = &mut route_task => {
15675 let response = response.unwrap();
15676 panic!(
15677 "post-cutover route.open was refused instead of relayed to the candidate: {}",
15678 parse_error(&response[0])["code"]
15679 );
15680 }
15681 };
15682 swap.handler
15683 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15684 .await
15685 .unwrap();
15686 assert!(route_task.await.unwrap().is_empty());
15687 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15688 match swap
15689 .forwarding
15690 .lookup_data_route(client, channel, epoch)
15691 .unwrap()
15692 {
15693 DataRoute::Client(DataRouteState::Bound(route)) => {
15694 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15695 }
15696 other => panic!("expected a bound route on the candidate, got {other:?}"),
15697 }
15698 assert!(swap.incumbent_rx.try_recv().is_err());
15699 }
15700
15701 #[tokio::test]
15706 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15707 let swap = swap_with_incumbent().await;
15708 register_candidate(&swap, Some(false));
15709 let update = Frame::build(
15710 FrameType::Request,
15711 control_flags(),
15712 0,
15713 0,
15714 55,
15715 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15716 provides: manifest("aft", PROTOCOL_VERSION).provides,
15717 capabilities: None,
15718 ready: Some(true),
15719 })
15720 .unwrap(),
15721 )
15722 .unwrap();
15723
15724 let replies = swap
15725 .handler
15726 .handle_control_frame(&swap.candidate_ctx, update)
15727 .await
15728 .unwrap();
15729
15730 assert_eq!(replies.len(), 1);
15731 assert_eq!(
15732 replies[0].header.ty,
15733 FrameType::Response,
15734 "candidate catalog.update was refused: {:?}",
15735 serde_json::from_slice::<Value>(&replies[0].body).ok()
15736 );
15737 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15738 assert_eq!(
15739 swap.registry
15740 .get_module("aft")
15741 .unwrap()
15742 .unwrap()
15743 .connection_id,
15744 INCUMBENT
15745 );
15746 }
15747 }
15748
15749 mod swap_admission {
15753 use super::*;
15754
15755 const INCUMBENT_NONCE: &str = "incumbent-nonce";
15756 const CANDIDATE_NONCE: &str = "candidate-nonce";
15757
15758 fn handler_with_incumbent(
15759 module_id: &str,
15760 reserved: bool,
15761 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15762 let registry = Arc::new(Registry::default());
15763 let supervisor = SupervisorHandle::new();
15764 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15765 if reserved {
15766 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15767 }
15768 let handler =
15769 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
15770 let incumbent = handler
15771 .handle_control(
15772 ConnectionId::new(1),
15773 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15774 )
15775 .unwrap();
15776 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15777 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15778 (registry, supervisor, handler)
15779 }
15780
15781 #[test]
15787 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15788 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15789
15790 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15791 let replies = handler
15792 .handle_control(
15793 ConnectionId::new(connection),
15794 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15795 )
15796 .unwrap();
15797 assert_eq!(replies[0].header.ty, FrameType::Error);
15798 assert_eq!(
15799 parse_error(&replies[0])["code"],
15800 "swap_token_invalid",
15801 "nonce {nonce:?}"
15802 );
15803 }
15804 assert!(registry.get_candidate("aft").unwrap().is_none());
15805 assert_eq!(
15806 registry.get_module("aft").unwrap().unwrap().connection_id,
15807 ConnectionId::new(1)
15808 );
15809
15810 let admitted = handler
15813 .handle_control(
15814 ConnectionId::new(4),
15815 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15816 )
15817 .unwrap();
15818 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15819 assert_eq!(
15820 registry
15821 .get_candidate("aft")
15822 .unwrap()
15823 .unwrap()
15824 .connection_id,
15825 ConnectionId::new(4)
15826 );
15827 assert_eq!(
15828 registry.get_module("aft").unwrap().unwrap().connection_id,
15829 ConnectionId::new(1),
15830 "the candidate must not take the active slot"
15831 );
15832 let replayed = handler
15833 .handle_control(
15834 ConnectionId::new(5),
15835 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15836 )
15837 .unwrap();
15838 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15839
15840 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15844 let squatter = handler
15845 .handle_control(
15846 ConnectionId::new(6),
15847 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15848 )
15849 .unwrap();
15850 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15851 assert!(
15852 registry.get_module("aft").unwrap().is_none(),
15853 "a squatter took the active slot of an id being swapped"
15854 );
15855 }
15856
15857 #[test]
15862 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15863 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15864
15865 let replies = handler
15866 .handle_control(
15867 ConnectionId::new(2),
15868 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15869 )
15870 .unwrap();
15871
15872 assert_eq!(
15873 replies[0].header.ty,
15874 FrameType::HelloAck,
15875 "reserved candidate refused: {:?}",
15876 serde_json::from_slice::<Value>(&replies[0].body).ok()
15877 );
15878 assert_eq!(
15879 registry
15880 .get_candidate("vault")
15881 .unwrap()
15882 .unwrap()
15883 .connection_id,
15884 ConnectionId::new(2)
15885 );
15886 }
15887
15888 #[test]
15891 fn without_an_open_swap_the_ordinary_gates_decide() {
15892 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15893 supervisor.close_swap("vault");
15894
15895 let candidate = handler
15896 .handle_control(
15897 ConnectionId::new(2),
15898 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15899 )
15900 .unwrap();
15901 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15902 let duplicate = handler
15903 .handle_control(
15904 ConnectionId::new(3),
15905 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15906 )
15907 .unwrap();
15908 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15909 assert!(registry.get_candidate("vault").unwrap().is_none());
15910 }
15911 }
15912
15913 mod scopes {
15917 mod idle_tests {
15918 use super::*;
15919 include!("scope_idle_tests.rs");
15920 }
15921 mod route_open_timings {
15922 use super::*;
15923 include!("route_open_timing_tests.rs");
15924 }
15925 mod route_open_refusal_log {
15926 use super::*;
15927 include!("route_open_refusal_log_tests.rs");
15928 }
15929 mod activity {
15930 include!("activity_scope_tests.rs");
15931 }
15932 mod delta {
15933 include!("scope_handler_delta_tests.rs");
15934 }
15935 mod call_modes {
15936 include!("scope_call_mode_tests.rs");
15937 }
15938 #[tokio::test]
15939 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15940 let rig = rig().await;
15941 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15942 "s".into(),
15943 1,
15944 "tool".into(),
15945 ))
15946 .unwrap();
15947 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
15948 let (direct, _rx) = wide_ctx(999);
15949 for (ctx, expected) in [
15950 (&rig.owner, FrameType::Response),
15951 (&direct, FrameType::Error),
15952 ] {
15953 let frame =
15954 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
15955 .unwrap();
15956 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
15957 assert_eq!(replies[0].header.ty, expected);
15958 if expected == FrameType::Response {
15959 let reply: subc_control::ScopeServedByReply =
15960 serde_json::from_slice(&replies[0].body).unwrap();
15961 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
15962 } else {
15963 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
15964 }
15965 }
15966 let reply = call(
15967 &rig.handler,
15968 &rig.owner,
15969 &ModuleControlRequestFromModule::ScopeDescribe {
15970 owner: Principal::Reserved {
15971 module_id: OWNER.into(),
15972 },
15973 scope_ref: "s".into(),
15974 },
15975 )
15976 .await;
15977 assert!(matches!(
15978 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
15979 ModuleControlResponseToModule::ScopeDescribe { .. }
15980 ));
15981 }
15982
15983 use subc_protocol::scope::{
15984 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
15985 ScopeStatus,
15986 };
15987
15988 use super::*;
15989
15990 const OWNER: &str = "prefrontal-core";
15991
15992 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
15993 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
15994 }
15995
15996 async fn call(
15997 handler: &ControlHandler,
15998 ctx: &RouteCtx,
15999 request: &ModuleControlRequestFromModule,
16000 ) -> Frame {
16001 let body = serde_json::to_vec(request).unwrap();
16002 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16003 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16004 assert_eq!(replies.len(), 1, "{replies:?}");
16005 replies.pop().unwrap()
16006 }
16007
16008 async fn sync(
16009 handler: &ControlHandler,
16010 ctx: &RouteCtx,
16011 generation: u64,
16012 scopes: Vec<ScopeRecord>,
16013 ) -> Result<ModuleControlResponseToModule, String> {
16014 let reply = call(
16015 handler,
16016 ctx,
16017 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16018 )
16019 .await;
16020 match reply.header.ty {
16021 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16022 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16023 }
16024 }
16025
16026 async fn describe(
16027 handler: &ControlHandler,
16028 ctx: &RouteCtx,
16029 owner: &str,
16030 scope_ref: &str,
16031 ) -> ModuleControlResponseToModule {
16032 let reply = call(
16033 handler,
16034 ctx,
16035 &ModuleControlRequestFromModule::ScopeDescribe {
16036 owner: Principal::Reserved {
16037 module_id: owner.to_string(),
16038 },
16039 scope_ref: scope_ref.to_string(),
16040 },
16041 )
16042 .await;
16043 assert_eq!(
16044 reply.header.ty,
16045 FrameType::Response,
16046 "{:?}",
16047 parse_error(&reply)
16048 );
16049 serde_json::from_slice(&reply.body).unwrap()
16050 }
16051
16052 #[tokio::test]
16053 async fn scope_describe_never_echoes_bind_proof_or_hash() {
16054 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16055 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16056 let mut rig = rig().await;
16057 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16058 .await;
16059 let _bound = rig
16060 .bound(
16061 Some(AFT),
16062 PLEXUS,
16063 Some(
16064 rig.selector("s", Some(1))
16065 .with_bind_proof(Some(PROOF.into())),
16066 ),
16067 )
16068 .await;
16069 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16070 let output = serde_json::to_string(&reply).unwrap();
16071 assert!(output.contains("\"status\":\"live\""));
16072 assert!(!output.contains(PROOF));
16073 assert!(!output.contains(HASH));
16074 }
16075
16076 async fn module(
16078 handler: &ControlHandler,
16079 connection: u64,
16080 module_id: &str,
16081 nonce: Option<&str>,
16082 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16083 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16084 hello_via_sink(
16085 handler,
16086 &ctx,
16087 &mut rx,
16088 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16089 )
16090 .await;
16091 (ctx, rx)
16092 }
16093
16094 #[tokio::test]
16097 async fn a_client_connection_cannot_sync_or_describe() {
16098 let handler = ControlHandler::new(Arc::new(Registry::default()));
16099 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16100 for request in [
16101 ModuleControlRequestFromModule::ScopeSync {
16102 generation: 1,
16103 scopes: vec![head("s", 1)],
16104 },
16105 ModuleControlRequestFromModule::ScopeDescribe {
16106 owner: Principal::Direct,
16107 scope_ref: "s".to_string(),
16108 },
16109 ] {
16110 let reply = call(&handler, &ctx, &request).await;
16111 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16112 }
16113 assert!(
16114 !handler
16115 .scopes
16116 .read()
16117 .unwrap()
16118 .describe(
16119 &Principal::Reserved {
16120 module_id: OWNER.to_string()
16121 },
16122 "s"
16123 )
16124 .owner_synced
16125 );
16126 }
16127
16128 #[tokio::test]
16131 async fn a_module_without_a_supervised_launch_cannot_sync() {
16132 let handler = ControlHandler::new(Arc::new(Registry::default()));
16133 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16134 assert_eq!(
16135 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16136 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16137 );
16138 }
16139
16140 #[tokio::test]
16141 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16142 let supervisor = SupervisorHandle::new();
16143 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16144 let handler = ControlHandler::new(Arc::new(Registry::default()))
16145 .with_supervisor(supervisor.clone());
16146 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16147 sync(&handler, &incumbent, 1, vec![head("s", 1)])
16148 .await
16149 .expect("the current launch syncs");
16150
16151 supervisor.open_swap(OWNER, "n2".to_string());
16154 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16155 assert_eq!(
16156 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16157 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16158 );
16159 sync(&handler, &incumbent, 2, vec![head("s", 1)])
16160 .await
16161 .expect("the serving owner syncs during the swap");
16162
16163 supervisor.close_swap(OWNER);
16166 assert_eq!(
16167 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16168 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16169 );
16170 sync(&handler, &incumbent, 3, vec![head("s", 1)])
16171 .await
16172 .expect("the serving owner syncs after the rollback");
16173 handler.cleanup_connection(candidate.connection_id).unwrap();
16174
16175 supervisor.open_swap(OWNER, "n3".to_string());
16180 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16181 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16182 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16183 .await
16184 .expect("the promoted launch takes authority");
16185 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16186 panic!("unexpected reply {reply:?}");
16187 };
16188 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16189 assert_eq!(
16190 sync(&handler, &incumbent, 4, Vec::new()).await,
16191 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16192 );
16193 }
16194
16195 #[tokio::test]
16198 async fn closing_the_authority_connection_frees_sync_authority() {
16199 let supervisor = SupervisorHandle::new();
16200 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16201 let handler = ControlHandler::new(Arc::new(Registry::default()))
16202 .with_supervisor(supervisor.clone());
16203 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16204 sync(&handler, &first, 10, vec![head("s", 1)])
16205 .await
16206 .unwrap();
16207 handler.cleanup_connection(first.connection_id).unwrap();
16208
16209 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16210 sync(&handler, &second, 1, vec![head("s", 1)])
16211 .await
16212 .expect("the next connection takes the released authority");
16213 }
16214
16215 #[tokio::test]
16216 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16217 let supervisor = SupervisorHandle::new();
16218 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16219 let handler =
16220 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16221 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16222 sync(&handler, &first, 10, vec![head("s", 1)])
16223 .await
16224 .unwrap();
16225 handler
16226 .handle_control_frame(
16227 &first,
16228 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16229 )
16230 .await
16231 .unwrap();
16232 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16233 sync(&handler, &second, 1, vec![head("s", 1)])
16234 .await
16235 .expect("GOODBYE releases authority even if the old socket remains open");
16236 }
16237
16238 #[tokio::test]
16239 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16240 let registry = Arc::new(Registry::default());
16241 let supervisor_handle = SupervisorHandle::new();
16242 let supervisor =
16243 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16244 .with_handle(supervisor_handle.clone())
16245 .with_daemon_incarnation("incarnation-7".to_string());
16246 supervisor
16249 .supervise_configured(
16250 ModuleSpec {
16251 module_id: OWNER.to_string(),
16252 program: PathBuf::from("/nonexistent/prefrontal-core"),
16253 args: Vec::new(),
16254 env: Vec::new(),
16255 reserved: false,
16256 reserved_prefixes: Vec::new(),
16257 protocol: ModuleProtocol::Subc,
16258 overlap: Default::default(),
16259 },
16260 false,
16261 )
16262 .unwrap();
16263 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16264 let handler =
16265 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16266 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16267
16268 let ModuleControlResponseToModule::ScopeDescribe {
16270 status,
16271 daemon_incarnation,
16272 owner_synced,
16273 owner_configured,
16274 scope,
16275 ..
16276 } = describe(&handler, &reader, OWNER, "s").await
16277 else {
16278 panic!("not a describe reply");
16279 };
16280 assert_eq!(status, ScopeStatus::NotLive);
16281 assert_eq!(daemon_incarnation, "incarnation-7");
16282 assert!(!owner_synced);
16283 assert!(owner_configured);
16284 assert!(scope.is_none());
16285
16286 let ModuleControlResponseToModule::ScopeDescribe {
16289 status,
16290 owner_configured,
16291 ..
16292 } = describe(&handler, &reader, "ghost", "s").await
16293 else {
16294 panic!("not a describe reply");
16295 };
16296 assert_eq!(status, ScopeStatus::NotLive);
16297 assert!(!owner_configured);
16298
16299 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16301 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16302 let ModuleControlResponseToModule::ScopeDescribe {
16303 status,
16304 scope_epoch,
16305 owner_synced,
16306 scope,
16307 ..
16308 } = describe(&handler, &reader, OWNER, "s").await
16309 else {
16310 panic!("not a describe reply");
16311 };
16312 assert_eq!(status, ScopeStatus::Live);
16313 assert_eq!(scope_epoch, Some(4));
16314 assert!(owner_synced);
16315 let stamp = scope.expect("a live scope carries its stamp");
16316 assert!(
16317 stamp.owner_authorized,
16318 "prefrontal-core is the default authority"
16319 );
16320 assert_eq!(stamp.kind, ScopeKind::Head);
16321 }
16322
16323 #[tokio::test]
16324 async fn scope_authority_owners_decides_owner_authorized() {
16325 let supervisor = SupervisorHandle::new();
16326 supervisor.set_spawn_nonce("broca", "b1".to_string());
16327 let handler = ControlHandler::new(Arc::new(Registry::default()))
16328 .with_supervisor(supervisor)
16329 .with_scope_authority_owners(vec!["broca".to_string()]);
16330 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16331 let mut gated = head("s", 1);
16332 gated.attributes.agent_id = Some("agent".to_string());
16333 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16334 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16335 describe(&handler, &broca, "broca", "s").await
16336 else {
16337 panic!("not a describe reply");
16338 };
16339 assert!(scope.unwrap().owner_authorized);
16340 }
16341
16342 #[tokio::test]
16346 async fn scope_ops_and_the_scopes_capability_are_advertised() {
16347 let handler = ControlHandler::new(Arc::new(Registry::default()));
16348 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16349 let ack = hello_via_sink(
16350 &handler,
16351 &ctx,
16352 &mut rx,
16353 hello_frame("m", PROTOCOL_VERSION, 1),
16354 )
16355 .await;
16356 let ack = parse_ack(&ack);
16357 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16358 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16359 }
16360 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16361
16362 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16363 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16364 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16365 let reply = handler
16366 .handle_control_frame(&client, frame)
16367 .await
16368 .unwrap()
16369 .pop()
16370 .unwrap();
16371 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16372 serde_json::from_slice(&reply.body).unwrap()
16373 else {
16374 panic!("not a server.describe reply");
16375 };
16376 assert!(
16377 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16378 "{capabilities:?}"
16379 );
16380 }
16381
16382 const PLEXUS: &str = "plexus";
16385 const OTHER: &str = "other";
16386 const AFT: &str = "aft";
16387 const BROCA: &str = "broca";
16388 const MAGIC: &str = "magic-context";
16389
16390 fn nonce(module_id: &str) -> String {
16391 format!("nonce-{module_id}")
16392 }
16393
16394 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16395 let (tx, rx) = mpsc::channel(64);
16396 (
16397 RouteCtx {
16398 connection_id: ConnectionId::new(connection),
16399 egress: FrameSink::new(tx),
16400 },
16401 rx,
16402 )
16403 }
16404
16405 struct Rig {
16409 handler: ControlHandler,
16410 forwarding: Arc<ForwardingTable>,
16411 owner: RouteCtx,
16412 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16413 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16414 generation: u64,
16415 next_connection: u64,
16416 _supervisor: Supervisor,
16417 }
16418
16419 async fn rig() -> Rig {
16420 rig_with_flow_support(true).await
16421 }
16422
16423 async fn rig_with_flow_support(flow_support: bool) -> Rig {
16424 rig_with_scope_capabilities(flow_support, false).await
16425 }
16426
16427 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16428 let mut provides = Vec::new();
16429 if flow_support {
16430 provides.push("flow-scopes/v1");
16431 }
16432 if run_support {
16433 provides.push("agent-run-scopes/v1");
16434 }
16435 rig_with_target_capabilities(&provides).await
16436 }
16437
16438 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16439 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16440 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16441 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16444 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16445 Frame::build(
16446 FrameType::Hello,
16447 control_flags(),
16448 0,
16449 0,
16450 connection,
16451 serde_json::to_vec(&body).unwrap(),
16452 )
16453 .unwrap()
16454 }
16455
16456 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16457 let registry = Arc::new(Registry::default());
16458 let forwarding = Arc::new(ForwardingTable::default());
16459 let supervisor_handle = SupervisorHandle::new();
16460 let supervisor =
16461 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16462 .with_handle(supervisor_handle.clone());
16463 supervisor
16464 .supervise_configured(
16465 ModuleSpec {
16466 module_id: OWNER.to_string(),
16467 program: PathBuf::from("/nonexistent/prefrontal-core"),
16468 args: Vec::new(),
16469 env: Vec::new(),
16470 reserved: false,
16471 reserved_prefixes: Vec::new(),
16472 protocol: ModuleProtocol::Subc,
16473 overlap: Default::default(),
16474 },
16475 false,
16476 )
16477 .unwrap();
16478 for module_id in [OWNER, AFT, BROCA, MAGIC] {
16479 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16480 }
16481 let handler =
16482 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16483 .with_supervisor(supervisor_handle);
16484 let (owner, mut owner_rx) = wide_ctx(1);
16485 hello_via_sink(
16486 &handler,
16487 &owner,
16488 &mut owner_rx,
16489 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16490 )
16491 .await;
16492 let mut modules = BTreeMap::new();
16493 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16494 let (ctx, mut rx) = wide_ctx(connection);
16495 let hello = hello_with_capabilities(module_id, connection, provides);
16496 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16497 modules.insert(module_id.to_string(), (ctx, rx));
16498 }
16499 Rig {
16500 handler,
16501 forwarding,
16502 owner,
16503 _owner_rx: owner_rx,
16504 modules,
16505 generation: 0,
16506 next_connection: 100,
16507 _supervisor: supervisor,
16508 }
16509 }
16510
16511 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16512 ScopeCarrier::new(Principal::Reserved {
16513 module_id: module_id.to_string(),
16514 })
16515 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16516 }
16517
16518 fn session(scope_epoch: u64) -> ScopeRecord {
16521 let mut record = head("s", scope_epoch);
16522 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16523 record.attributes.agent_id = Some("agent-1".to_string());
16524 record.attributes.delegates = true;
16525 record
16526 }
16527
16528 impl Rig {
16529 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16530 self.generation += 1;
16531 sync(&self.handler, &self.owner, self.generation, scopes)
16532 .await
16533 .expect("the owner's sync is accepted");
16534 }
16535
16536 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16537 ScopeSelector::new(
16538 Principal::Reserved {
16539 module_id: OWNER.to_string(),
16540 },
16541 scope_ref.to_string(),
16542 scope_epoch,
16543 )
16544 }
16545
16546 fn open_frame(
16547 &mut self,
16548 opener: Option<&str>,
16549 target: &str,
16550 scope: Option<ScopeSelector>,
16551 ) -> (
16552 RouteCtx,
16553 mpsc::Receiver<crate::router::OutboundFrame>,
16554 Frame,
16555 ) {
16556 self.next_connection += 1;
16557 let (ctx, rx) = wide_ctx(self.next_connection);
16558 let root = unique_project_root("scoped-open");
16559 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16560 target: RouteTarget::ToolProvider {
16561 module_id: target.to_string(),
16562 },
16563 identity: BindIdentity::new(
16564 root.path().to_path_buf(),
16565 "unit".to_string(),
16566 "session".to_string(),
16567 ),
16568 consumer_identity: opener.map(|module_id| ConsumerIdentity {
16569 module_id: module_id.to_string(),
16570 launch_nonce: nonce(module_id),
16571 }),
16572 consumer_capabilities: None,
16573 role_versions: None,
16574 admission_facts: None,
16575 scope,
16576 })
16577 .unwrap();
16578 let frame = Frame::build(
16579 FrameType::Request,
16580 control_flags(),
16581 0,
16582 0,
16583 self.next_connection,
16584 body,
16585 )
16586 .unwrap();
16587 (ctx, rx, frame)
16588 }
16589
16590 async fn refused(
16592 &mut self,
16593 opener: Option<&str>,
16594 target: &str,
16595 scope: Option<ScopeSelector>,
16596 ) -> String {
16597 self.refusal_body(opener, target, scope).await["code"]
16598 .as_str()
16599 .unwrap()
16600 .to_string()
16601 }
16602
16603 async fn refusal_body(
16604 &mut self,
16605 opener: Option<&str>,
16606 target: &str,
16607 scope: Option<ScopeSelector>,
16608 ) -> Value {
16609 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16610 let replies = tokio::time::timeout(
16611 Duration::from_secs(2),
16612 self.handler.handle_control_frame(&ctx, frame),
16613 )
16614 .await
16615 .expect("the open must be refused before waiting for a bind ack")
16616 .unwrap();
16617 assert_eq!(replies.len(), 1, "{replies:?}");
16618 assert_eq!(replies[0].header.ty, FrameType::Error);
16619 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16620 assert!(
16621 module_rx.try_recv().is_err(),
16622 "a refused open relays nothing"
16623 );
16624 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16625 parse_error(&replies[0])
16626 }
16627
16628 async fn relayed(
16630 &mut self,
16631 opener: Option<&str>,
16632 target: &str,
16633 scope: Option<ScopeSelector>,
16634 ) -> Relayed {
16635 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16636 let handler = self.handler.clone();
16637 let task_ctx = ctx.clone();
16638 let task = tokio::spawn(async move {
16639 handler
16640 .handle_control_frame(&task_ctx, frame)
16641 .await
16642 .unwrap()
16643 });
16644 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16645 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16646 .await
16647 .expect("the target receives the relayed route.bind")
16648 .unwrap()
16649 .frame;
16650 Relayed {
16651 target: target.to_string(),
16652 client: ctx,
16653 client_rx: rx,
16654 task,
16655 bind,
16656 }
16657 }
16658
16659 async fn ack(&self, relayed: &Relayed) {
16660 let (module, _) = &self.modules[&relayed.target];
16661 self.handler
16662 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16663 .await
16664 .unwrap();
16665 }
16666
16667 async fn bound(
16669 &mut self,
16670 opener: Option<&str>,
16671 target: &str,
16672 scope: Option<ScopeSelector>,
16673 ) -> Bound {
16674 let relayed = self.relayed(opener, target, scope).await;
16675 self.ack(&relayed).await;
16676 let Relayed {
16677 target,
16678 client,
16679 mut client_rx,
16680 task,
16681 bind,
16682 } = relayed;
16683 assert!(
16684 task.await.unwrap().is_empty(),
16685 "the open is answered by commit"
16686 );
16687 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16688 Bound {
16689 target,
16690 client,
16691 client_rx,
16692 channel,
16693 epoch,
16694 bind,
16695 }
16696 }
16697
16698 fn live(&self, route: &Bound) -> bool {
16699 matches!(
16700 self.forwarding
16701 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16702 .unwrap(),
16703 DataRoute::Client(DataRouteState::Bound(_))
16704 )
16705 }
16706 }
16707
16708 struct Relayed {
16709 target: String,
16710 client: RouteCtx,
16711 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16712 task: tokio::task::JoinHandle<Vec<Frame>>,
16713 bind: Frame,
16714 }
16715
16716 struct Bound {
16717 target: String,
16718 client: RouteCtx,
16719 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16720 channel: u16,
16721 epoch: u32,
16722 bind: Frame,
16723 }
16724
16725 impl Bound {
16726 fn closed_reason(&mut self) -> RouteCloseReason {
16729 let mut reason = None;
16730 let mut goodbye = false;
16731 while let Ok(outbound) = self.client_rx.try_recv() {
16732 let frame = outbound.frame;
16733 match frame.header.ty {
16734 FrameType::Goodbye => {
16735 assert_eq!(
16736 (frame.header.channel, frame.header.epoch),
16737 (self.channel, self.epoch)
16738 );
16739 goodbye = true;
16740 }
16741 FrameType::Push => {
16742 let ClientControlPush::RouteClosed {
16743 reason: r,
16744 module_id,
16745 ..
16746 } = serde_json::from_slice(&frame.body).unwrap()
16747 else {
16748 panic!("unexpected push");
16749 };
16750 assert_eq!(module_id, self.target);
16751 reason = Some(r);
16752 }
16753 other => panic!("unexpected frame {other:?}"),
16754 }
16755 }
16756 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16757 reason.expect("the client is told why the route closed")
16758 }
16759
16760 fn untouched(&mut self) -> bool {
16761 self.client_rx.try_recv().is_err()
16762 }
16763
16764 fn stamp(&self) -> Option<ScopeStamp> {
16765 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16766 ModuleControlRequest::RouteBind { scope, .. } => scope,
16767 other => panic!("expected a route.bind, got {other:?}"),
16768 }
16769 }
16770 }
16771
16772 #[tokio::test]
16773 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16774 ) {
16775 let mut rig = rig().await;
16776 let mut record = session(1);
16777 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16778 record.child_owners = vec![Principal::Reserved {
16779 module_id: MAGIC.to_string(),
16780 }];
16781 rig.sync(vec![record]).await;
16782 let scope = || Some(rig_selector("s", Some(1)));
16783
16784 rig.bound(Some(OWNER), PLEXUS, scope()).await;
16787 rig.bound(Some(AFT), OTHER, scope()).await;
16788 rig.bound(Some(BROCA), PLEXUS, scope()).await;
16789
16790 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16794 assert_eq!(
16795 rig.refused(opener, target, scope()).await,
16796 error_codes::SCOPE_NOT_CARRIER,
16797 "{opener:?} -> {target}"
16798 );
16799 }
16800 }
16801
16802 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16803 ScopeSelector::new(
16804 Principal::Reserved {
16805 module_id: OWNER.to_string(),
16806 },
16807 scope_ref.to_string(),
16808 scope_epoch,
16809 )
16810 }
16811
16812 #[tokio::test]
16813 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16814 let mut rig = rig().await;
16815 rig.sync(vec![session(1)]).await;
16816 for opener in [OWNER, AFT] {
16817 assert_eq!(
16818 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16819 .await,
16820 error_codes::SCOPE_EPOCH_REQUIRED,
16821 "{opener}"
16822 );
16823 }
16824 }
16825
16826 #[tokio::test]
16827 async fn admission_separates_not_synced_not_live_and_ended() {
16828 let mut rig = rig().await;
16829 let code = rig
16831 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16832 .await;
16833 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16834 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16835
16836 let ghost = ScopeSelector::new(
16838 Principal::Reserved {
16839 module_id: "ghost".to_string(),
16840 },
16841 "s".to_string(),
16842 Some(1),
16843 );
16844 assert_eq!(
16845 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16846 error_codes::SCOPE_NOT_LIVE
16847 );
16848
16849 rig.sync(vec![session(2)]).await;
16850 assert_eq!(
16851 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16852 .await,
16853 error_codes::SCOPE_NOT_LIVE
16854 );
16855 for epoch in [1, 3] {
16856 assert_eq!(
16857 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16858 .await,
16859 error_codes::SCOPE_ENDED,
16860 "epoch {epoch}"
16861 );
16862 }
16863 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16865 .await;
16866 }
16867
16868 #[tokio::test]
16869 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16870 let mut rig = rig().await;
16871 rig.sync(vec![session(1)]).await;
16872 let route = rig
16873 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16874 .await;
16875 let stamp = route.stamp().expect("a scoped bind carries the stamp");
16876 assert_eq!(stamp.scope_ref, "s");
16877 assert_eq!(stamp.scope_epoch, 1);
16878 assert_eq!(stamp.kind, ScopeKind::Head);
16879 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16880 assert!(stamp.attributes.delegates);
16881 assert!(stamp.owner_authorized);
16882 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16883 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16884
16885 let (broca, mut broca_rx) = wide_ctx(50);
16888 hello_via_sink(
16889 &rig.handler,
16890 &broca,
16891 &mut broca_rx,
16892 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16893 )
16894 .await;
16895 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16896 .await
16897 .unwrap();
16898 let own = ScopeSelector::new(
16899 Principal::Reserved {
16900 module_id: BROCA.to_string(),
16901 },
16902 "b".to_string(),
16903 Some(1),
16904 );
16905 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16906 assert!(!route.stamp().unwrap().owner_authorized);
16907 }
16908
16909 #[tokio::test]
16910 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16911 let mut rig = rig().await;
16912 let mut record = head("s", 1);
16913 record.carriers = vec![carrier(AFT, None)];
16914 let flow_id = "Flow:run-7/step_2!~";
16915 record.attributes.flow_id = Some(flow_id.to_string());
16916 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16917 .await
16918 .unwrap();
16919 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16920 panic!("not a sync reply");
16921 };
16922 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16923 let route = rig
16924 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16925 .await;
16926 assert!(rig.live(&route), "the stamped bind committed");
16927 let stamp = route.stamp().expect("a flow scope carries a stamp");
16928 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16929 assert_eq!(stamp.attributes.agent_id, None);
16930 assert!(!stamp.attributes.delegates);
16931 assert!(stamp.owner_authorized);
16932 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16933 assert_eq!(unscoped.stamp(), None);
16934 }
16935
16936 #[tokio::test]
16937 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16938 let mut rig = rig_with_flow_support(false).await;
16939 let mut record = session(1);
16940 record.attributes.flow_id = Some("flow:7".to_string());
16941 rig.sync(vec![record]).await;
16942 for opener in [OWNER, AFT] {
16943 let body = rig
16944 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16945 .await;
16946 assert_eq!(body["code"], "target_flow_unsupported");
16947 let message = body["message"].as_str().unwrap();
16948 for required in [PLEXUS, "flow-scopes/v1"] {
16949 assert!(message.contains(required), "{message}");
16950 }
16951 }
16952 }
16953
16954 #[tokio::test]
16955 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
16956 let mut rig = rig_with_flow_support(true).await;
16957 let mut record = session(1);
16958 record.attributes.flow_id = Some("flow:7".to_string());
16959 rig.sync(vec![record]).await;
16960 for opener in [OWNER, AFT] {
16961 let route = rig
16962 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16963 .await;
16964 assert!(rig.live(&route));
16965 assert_eq!(
16966 route.stamp().unwrap().attributes.flow_id.as_deref(),
16967 Some("flow:7")
16968 );
16969 }
16970 }
16971
16972 #[tokio::test]
16973 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
16974 use std::future::Future;
16975
16976 let mut rig = rig_with_flow_support(true).await;
16977 let mut record = session(1);
16978 record.attributes.flow_id = Some("flow:7".to_string());
16979 rig.sync(vec![record]).await;
16980 let (client, mut client_rx, frame) =
16981 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
16982 for _ in 0..64 {
16985 client.egress.try_send(route_bind_ack(1)).unwrap();
16986 }
16987 let handler = rig.handler.clone();
16988 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16989 std::future::poll_fn(|cx| {
16990 assert!(open.as_mut().poll(cx).is_pending());
16991 std::task::Poll::Ready(())
16992 })
16993 .await;
16994 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16995
16996 let old_connection = rig.modules[PLEXUS].0.connection_id;
16997 rig.handler.cleanup_connection(old_connection).unwrap();
16998 let (replacement, mut replacement_rx) = wide_ctx(200);
16999 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17000 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17001 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17002 let hello = Frame::build(
17003 FrameType::Hello,
17004 control_flags(),
17005 0,
17006 0,
17007 200,
17008 serde_json::to_vec(&body).unwrap(),
17009 )
17010 .unwrap();
17011 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17012
17013 client_rx.try_recv().unwrap();
17014 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17015 .await
17016 .expect("the replacement is refused without waiting for a bind ack")
17017 .unwrap();
17018 assert_eq!(replies.len(), 1);
17019 let body = parse_error(&replies[0]);
17020 assert_eq!(body["code"], "target_flow_unsupported");
17021 for required in [PLEXUS, "flow-scopes/v1"] {
17022 assert!(body["message"].as_str().unwrap().contains(required));
17023 }
17024 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17025 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17026 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17027 assert!(rig
17028 .handler
17029 .registry
17030 .get_module_by_connection(replacement.connection_id)
17031 .unwrap()
17032 .is_some());
17033 }
17034
17035 #[tokio::test]
17036 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17037 use std::future::Future;
17038 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17039 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17040 const NEXT_HASH: &str =
17041 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17042 let mut rig = rig().await;
17043 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17044 record.carriers.push(ScopeCarrier::new(Principal::Direct));
17045 rig.sync(vec![record.clone()]).await;
17046 let (client, mut rx, frame) = rig.open_frame(
17047 None,
17048 PLEXUS,
17049 Some(
17050 rig.selector("s", Some(1))
17051 .with_bind_proof(Some(PROOF.into())),
17052 ),
17053 );
17054 for _ in 0..64 {
17057 client.egress.try_send(route_bind_ack(1)).unwrap();
17058 }
17059 let handler = rig.handler.clone();
17060 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17061 std::future::poll_fn(|cx| {
17062 assert!(open.as_mut().poll(cx).is_pending());
17063 std::task::Poll::Ready(())
17064 })
17065 .await;
17066 record.bind_proof_sha256 = Some(NEXT_HASH.into());
17067 rig.sync(vec![record]).await;
17068 rx.try_recv().unwrap();
17069 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17070 .await
17071 .unwrap()
17072 .unwrap();
17073 assert_eq!(
17074 parse_error(&replies[0])["code"],
17075 error_codes::SCOPE_BIND_PROOF_MISMATCH
17076 );
17077 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17078 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17079 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17080 }
17081
17082 #[tokio::test]
17083 async fn scope_bind_proof_reserved_openers_ignore_proof() {
17084 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17085 let mut rig = rig().await;
17086 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17087 .await;
17088 for opener in [OWNER, AFT] {
17089 for proof in [None, Some("malformed+proof=")] {
17090 let route = rig
17091 .bound(
17092 Some(opener),
17093 PLEXUS,
17094 Some(
17095 rig.selector("s", Some(1))
17096 .with_bind_proof(proof.map(str::to_string)),
17097 ),
17098 )
17099 .await;
17100 assert!(rig.live(&route));
17101 }
17102 }
17103 }
17104
17105 #[tokio::test]
17106 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17107 use std::future::Future;
17108 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17109 const NEXT_HASH: &str =
17110 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17111 let mut rig = rig().await;
17112 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17113 .await;
17114 let (client, mut rx, frame) =
17115 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17116 for _ in 0..64 {
17117 client.egress.try_send(route_bind_ack(1)).unwrap();
17118 }
17119 let handler = rig.handler.clone();
17120 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17121 std::future::poll_fn(|cx| {
17122 assert!(open.as_mut().poll(cx).is_pending());
17123 std::task::Poll::Ready(())
17124 })
17125 .await;
17126 rig.sync(vec![
17127 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17128 ])
17129 .await;
17130 rx.try_recv().unwrap();
17131 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17132 let bind = std::future::poll_fn(|cx| {
17133 assert!(open.as_mut().poll(cx).is_pending());
17134 module_rx.poll_recv(cx)
17135 })
17136 .await
17137 .unwrap()
17138 .frame;
17139 handler
17140 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17141 .await
17142 .unwrap();
17143 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17144 .await
17145 .unwrap()
17146 .unwrap();
17147 assert_eq!(replies.len(), 1);
17148 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17151 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17152 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17153 let route = rig
17154 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17155 .await;
17156 assert!(rig.live(&route));
17157 }
17158
17159 #[tokio::test]
17160 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17161 {
17162 let mut rig = rig_with_flow_support(false).await;
17163 rig.sync(vec![session(1)]).await;
17164 let route = rig
17165 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17166 .await;
17167 assert!(rig.live(&route));
17168 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17169 let mut record = session(1);
17170 record.attributes.flow_id = Some("flow:7".to_string());
17171 rig.sync(vec![record]).await;
17172 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17173 assert!(rig.live(&unscoped));
17174 assert_eq!(unscoped.stamp(), None);
17175 }
17176
17177 #[tokio::test]
17178 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17179 let mut rig = rig().await;
17180 let mut record = session(1);
17181 record.attributes.flow_id = Some("flow:7".to_string());
17182 rig.sync(vec![record.clone()]).await;
17183 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17184 let mut owner_route = rig
17185 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17186 .await;
17187 let mut carrier_route = rig
17188 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17189 .await;
17190 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17191 rig.sync(vec![record.clone()]).await;
17192 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17193 assert!(rig.live(&owner_route) && owner_route.untouched());
17194 assert!(rig.live(&carrier_route) && carrier_route.untouched());
17195
17196 record.attributes.flow_id = Some("flow:8".to_string());
17197 rig.sync(vec![record]).await;
17198 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17199 let before = before.unwrap();
17200 assert_eq!(after.scope_epoch, before.scope_epoch);
17201 assert!(after.version > before.version);
17202 for route in [&mut owner_route, &mut carrier_route] {
17203 assert!(!rig.live(route));
17204 assert_eq!(
17205 route.closed_reason(),
17206 RouteCloseReason::ScopeDelegationChanged
17207 );
17208 }
17209 assert!(rig.live(&unscoped) && unscoped.untouched());
17210 for target in [PLEXUS, OTHER] {
17213 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17214 let goodbye = module_rx
17215 .try_recv()
17216 .expect("the provider sees the drain")
17217 .frame;
17218 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17219 assert!(module_rx.try_recv().is_err());
17220 }
17221 let rebound = rig
17222 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17223 .await;
17224 assert_eq!(
17225 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17226 Some("flow:8")
17227 );
17228 }
17229
17230 #[tokio::test]
17235 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17236 let mut rig = rig().await;
17237 rig.sync(vec![session(1)]).await;
17238 let mut cotenant = rig
17239 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17240 .await;
17241
17242 let mut changed = session(1);
17243 changed.child_owners.push(Principal::Reserved {
17244 module_id: MAGIC.to_string(),
17245 });
17246 let mut ended = None;
17247 for (code, next) in [
17248 (error_codes::SCOPE_CHANGED, vec![changed]),
17249 (error_codes::SCOPE_ENDED, Vec::new()),
17250 ] {
17251 let relayed = rig
17252 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17253 .await;
17254 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17255 ended = Some(next.is_empty());
17256 rig.sync(next).await;
17257 rig.ack(&relayed).await;
17258 let replies = relayed.task.await.unwrap();
17259 assert_eq!(replies.len(), 1, "{replies:?}");
17260 assert_eq!(parse_error(&replies[0])["code"], code);
17261 assert_eq!(
17262 subc_protocol::error_codes::is_retryable_route_open(code),
17263 code == error_codes::SCOPE_CHANGED
17264 );
17265 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17266 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17268 let mut goodbyes = Vec::new();
17271 while let Ok(outbound) = plexus_rx.try_recv() {
17272 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17273 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17274 }
17275 assert!(
17276 goodbyes.contains(&(bind_channel, bind_epoch)),
17277 "{goodbyes:?}"
17278 );
17279 assert!(rig
17280 .handler
17281 .registry
17282 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17283 .unwrap()
17284 .is_some());
17285 }
17286 assert_eq!(ended, Some(true));
17287 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17290 }
17291
17292 #[tokio::test]
17295 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17296 struct Case {
17297 name: &'static str,
17298 change: fn(&mut ScopeRecord),
17299 closed: [Option<RouteCloseReason>; 4],
17302 }
17303 use RouteCloseReason::*;
17304 let cases = [
17305 Case {
17306 name: "a carrier entry removed",
17307 change: |r| {
17308 r.carriers.retain(|c| {
17309 c.principal
17310 != Principal::Reserved {
17311 module_id: AFT.to_string(),
17312 }
17313 })
17314 },
17315 closed: [None, Some(ScopeCarrierRemoved), None, None],
17316 },
17317 Case {
17318 name: "a target removed from a carrier",
17319 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17320 closed: [None, None, None, Some(ScopeCarrierRemoved)],
17321 },
17322 Case {
17323 name: "a bare carrier narrowed to targets",
17324 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17325 closed: [None, Some(ScopeCarrierRemoved), None, None],
17326 },
17327 Case {
17328 name: "delegates turned off",
17329 change: |r| r.attributes.delegates = false,
17330 closed: [Some(ScopeDelegationChanged); 4],
17331 },
17332 Case {
17333 name: "agent_id changed",
17334 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17335 closed: [Some(ScopeDelegationChanged); 4],
17336 },
17337 Case {
17338 name: "a carrier added, child owners changed, the record re-sent",
17339 change: |r| {
17340 r.carriers.push(carrier(MAGIC, None));
17341 r.child_owners.push(Principal::Reserved {
17342 module_id: MAGIC.to_string(),
17343 });
17344 },
17345 closed: [None; 4],
17346 },
17347 Case {
17348 name: "a target added",
17349 change: |r| {
17350 r.carriers[1]
17351 .targets
17352 .as_mut()
17353 .unwrap()
17354 .push("third".to_string())
17355 },
17356 closed: [None; 4],
17357 },
17358 Case {
17359 name: "delegates turned on",
17360 change: |r| r.attributes.delegates = true,
17361 closed: [None; 4],
17362 },
17363 ];
17364 for case in cases {
17365 let mut rig = rig().await;
17366 rig.sync(vec![session(1)]).await;
17367 let scope = || Some(rig_selector("s", Some(1)));
17368 let mut routes = [
17369 rig.bound(Some(OWNER), PLEXUS, scope()).await,
17370 rig.bound(Some(AFT), PLEXUS, scope()).await,
17371 rig.bound(Some(BROCA), PLEXUS, scope()).await,
17372 rig.bound(Some(BROCA), OTHER, scope()).await,
17373 ];
17374 let mut record = session(1);
17375 (case.change)(&mut record);
17376 rig.sync(vec![record]).await;
17377 for (index, expected) in case.closed.iter().enumerate() {
17378 let route = &mut routes[index];
17379 match expected {
17380 Some(reason) => {
17381 assert!(!rig.live(route), "{}: route {index} still live", case.name);
17382 assert_eq!(
17383 route.closed_reason(),
17384 *reason,
17385 "{}: route {index}",
17386 case.name
17387 );
17388 }
17389 None => {
17390 assert!(rig.live(route), "{}: route {index} closed", case.name);
17391 assert!(
17392 route.untouched(),
17393 "{}: route {index} was told something",
17394 case.name
17395 );
17396 }
17397 }
17398 }
17399 }
17400 }
17401
17402 #[tokio::test]
17403 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17404 for next in [Vec::new(), vec![session(2)]] {
17406 let mut rig = rig().await;
17407 rig.sync(vec![session(1)]).await;
17408 let mut route = rig
17409 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17410 .await;
17411 rig.sync(next).await;
17412 assert!(!rig.live(&route));
17413 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17414 }
17415
17416 let mut rig = rig().await;
17419 let mut child = session(1);
17420 child.scope_ref = "child".to_string();
17421 child.kind = ScopeKind::Worker;
17422 child.parent = Some(ScopeParent::new(
17423 Principal::Reserved {
17424 module_id: OWNER.to_string(),
17425 },
17426 "s".to_string(),
17427 1,
17428 ));
17429 rig.sync(vec![session(1), child.clone()]).await;
17430 let mut child_route = rig
17431 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17432 .await;
17433 assert_eq!(
17434 child_route.stamp().unwrap().parent_state,
17435 Some(ParentState::Linked)
17436 );
17437 rig.sync(vec![child]).await;
17438 assert!(!rig.live(&child_route));
17439 assert_eq!(
17440 child_route.closed_reason(),
17441 RouteCloseReason::ScopeParentEnded
17442 );
17443 }
17444
17445 #[tokio::test]
17446 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17447 ) {
17448 let mut rig = rig().await;
17449 rig.sync(vec![session(1)]).await;
17450 let mut route = rig
17451 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17452 .await;
17453 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17454 rig.sync(vec![session(1)]).await;
17455 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17456 assert!(rig.live(&route) && route.untouched());
17457
17458 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17462 .forwarding
17463 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17464 .unwrap()
17465 else {
17466 panic!("the route is bound");
17467 };
17468 binding.flow.acquire_tagged(9, false).await.unwrap();
17469 let mut widened = session(1);
17470 widened.carriers.push(carrier(MAGIC, None));
17471 rig.sync(vec![widened]).await;
17472 assert!(rig.live(&route) && route.untouched());
17473 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17474 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17475 assert_eq!(binding.flow.in_flight(), 1);
17478 binding
17479 .flow
17480 .acquire_tagged(10, false)
17481 .await
17482 .expect("the flow is still open");
17483 }
17484
17485 #[tokio::test]
17488 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17489 let mut rig = rig().await;
17490 rig.sync(vec![session(1)]).await;
17491 let mut on_incumbent = rig
17492 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17493 .await;
17494
17495 let (candidate, _candidate_rx) = wide_ctx(9);
17498 let registration = rig
17499 .handler
17500 .registry
17501 .register_candidate_with_control_ops(
17502 manifest(PLEXUS, PROTOCOL_VERSION),
17503 PROTOCOL_VERSION,
17504 candidate.connection_id,
17505 module_baseline_control_ops(),
17506 )
17507 .unwrap();
17508 rig.forwarding
17509 .register_candidate_module_connection(
17510 candidate.connection_id,
17511 PLEXUS.to_string(),
17512 PROTOCOL_VERSION,
17513 manifest_concurrency(®istration.manifest),
17514 candidate.egress.clone(),
17515 )
17516 .unwrap();
17517 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17518 rig.handler
17519 .registry
17520 .promote_candidate(PLEXUS)
17521 .unwrap()
17522 .unwrap();
17523 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17524
17525 rig.sync(Vec::new()).await;
17526 assert!(!rig.live(&on_incumbent));
17527 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17528 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17529 let goodbye = incumbent_rx
17530 .try_recv()
17531 .expect("the superseded endpoint is told")
17532 .frame;
17533 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17534 }
17535 }
17536}
17537
17538#[cfg(test)]
17539mod concurrency_default_exposure_tests {
17540 use super::*;
17541
17542 fn hello_body(role_json: &str) -> Vec<u8> {
17543 format!(
17544 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17545 )
17546 .into_bytes()
17547 }
17548
17549 fn manifest_from(body: &[u8]) -> ModuleManifest {
17550 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17551 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17552 .expect("manifest parses")
17553 }
17554
17555 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17556
17557 #[test]
17558 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17559 let body = hello_body(&format!(
17560 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17561 ));
17562 let manifest = manifest_from(&body);
17563 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17566 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17567 }
17568
17569 #[test]
17570 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17571 let body = hello_body(&format!(
17572 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17573 ));
17574 let manifest = manifest_from(&body);
17575 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17576 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17577 }
17578
17579 #[test]
17580 fn non_management_roles_are_never_reported() {
17581 let body = hello_body(
17582 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17583 );
17584 let manifest = manifest_from(&body);
17585 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17586 }
17587}