1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_rich_ui_descriptor, validate_hello_self_signal_declarations,
26 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
27 ProviderRole,
28 },
29 scope::{
30 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32 SCOPE_SYNC_OP,
33 },
34 session::{
35 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38 ROLE_VERSIONS_FIELD,
39 },
40 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41 Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47 caller_identity::CallerIdentity,
48 capability_requirements::{
49 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51 RuntimeModule,
52 },
53 child_roster::DaemonShutdownFlag,
54 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55 forwarding::{
56 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59 },
60 observability::{
61 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62 },
63 privileged_prompt::{prompt_text, PrivilegedOperation},
64 provenance::{
65 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66 },
67 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68 router::{RouteCtx, RouterError},
69 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71 stderr_tail::{CaptureState, TailEntry},
72 supervise::{
73 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74 SupervisorHandle, SwapHelloAdmission,
75 },
76 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93 ops::ACTIVITY_RECORD,
94 ops::ACTIVITY_QUERY,
95 ops::ACTIVITY_SUBSCRIBE,
96 ops::SERVER_DESCRIBE,
97 ops::CATALOG_LIST,
98 ops::ROUTE_OPEN,
99 ops::ROUTE_POLL,
100 ops::ROUTE_CLOSING,
101 ops::ROUTE_CLOSED,
102 ops::SUPERVISOR_LIST,
103 ops::SUPERVISOR_RESTART,
104 ops::SUPERVISOR_SWAP,
105 ops::SUPERVISOR_RELOAD,
106 ops::SUPERVISOR_RESCAN,
107 ops::SUPERVISOR_RELEASE_RESERVED,
108 ops::SUPERVISOR_SET_ENABLED,
109 ops::SUPERVISOR_HEALTH_PROBE,
110 ops::SUPERVISOR_HEALTH,
111 ops::SUPERVISOR_STDERR_TAIL,
112 ops::SUPERVISOR_TERMINALS,
113 ops::SUPERVISOR_ROUTES,
114 ops::SUPERVISOR_PROVENANCE,
115 ops::SUPERVISOR_SPAWN_SNAPSHOT,
116 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132 "supervisor.live_roots",
133 SCOPE_SYNC_OP,
134 SCOPE_APPLY_OP,
135 SCOPE_DESCRIBE_OP,
136 "operator.confirm",
137];
138
139const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194 configured: &Path,
195 spawned_from: Option<&Path>,
196 image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198 let path = match spawned_from {
199 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200 ReloadPathAgreement::Match
201 }
202 Some(spawned_from) => ReloadPathAgreement::Mismatch {
203 configured: configured.to_path_buf(),
204 spawned_from: spawned_from.to_path_buf(),
205 },
206 None => ReloadPathAgreement::Unavailable {
207 reason: if matches!(
208 image,
209 subc_control::RunningImageAgreement::Unavailable {
210 reason: subc_control::RunningImageUnavailableReason::NotRunning
211 }
212 ) {
213 ReloadPathUnavailableReason::NotRunning
214 } else {
215 ReloadPathUnavailableReason::SpawnedPathUnavailable
216 },
217 },
218 };
219 PendingReloadVerdict {
220 path,
221 image,
222 spec: None,
223 }
224}
225
226fn launch_spec_agreement(
227 configured: &crate::supervise::ModuleSpec,
228 running: bool,
229 spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232 if !running {
233 return Agreement::Unavailable {
234 reason: Reason::NotRunning,
235 };
236 }
237 match spawned {
238 None => Agreement::Unavailable {
239 reason: Reason::SpawnedSpecUnavailable,
240 },
241 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242 Agreement::Match
243 }
244 Some(_) => Agreement::Mismatch,
245 }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249 if configured == spawned_from {
250 return true;
251 }
252 #[cfg(windows)]
253 {
254 match (
260 std::fs::canonicalize(configured),
261 std::fs::canonicalize(spawned_from),
262 ) {
263 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264 _ => false,
265 }
266 }
267 #[cfg(not(windows))]
268 {
269 false
270 }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275 build: DaemonBuildProvenance,
276 pid: Option<u32>,
277 started_at_ms: Option<u64>,
278 start_clock: Option<crate::clock::StartClock>,
279 executable_path: Option<PathBuf>,
280 executable_identity: Option<SpawnedFileIdentity>,
281 process_start_time: Option<u64>,
282 probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286 fn default() -> Self {
287 Self {
288 build: DaemonBuildProvenance {
289 build_git_sha: None,
290 build_lock_digest: None,
291 },
292 pid: None,
293 started_at_ms: None,
294 start_clock: None,
295 executable_path: None,
296 executable_identity: None,
297 process_start_time: None,
298 probe: ExecutableIdentityProbe::default(),
299 }
300 }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305 supervisor: Supervisor,
306 config_path: PathBuf,
307 configured_port: Option<u16>,
308 storage_config: Option<crate::daemon_config::StorageConfig>,
309 admission_facts_carrier_module_id: Option<String>,
310 admission_facts_targets: Option<Vec<String>>,
311 scope_authority_owners: Vec<String>,
312 privileged_operator_prompt: PrivilegedOperatorPrompt,
313 generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342 "reloading",
343 "supervisor_not_live",
344 "registration_not_active",
345 "no_forwarding_connection",
346 "relay_send_failed",
347];
348
349#[derive(Clone)]
351pub struct ControlHandler {
352 registry: Arc<Registry>,
353 activity: Option<crate::activity::ActivityLedger>,
354 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355 forwarding: Arc<ForwardingTable>,
356 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357 supervisor: SupervisorHandle,
358 subc_capabilities: Arc<[String]>,
359 route_bind_relay_timeout: Duration,
363 route_bind_relay_timeouts: BTreeMap<String, Duration>,
367 route_bind_breakers: RouteBindBreakers,
370 route_bind_concurrency: RouteBindConcurrency,
373 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378 route_bind_breaker_threshold: u32,
380 route_bind_breaker_cooldown: Duration,
382 health_probe_timeout: Duration,
383 storage_config: Option<crate::daemon_config::StorageConfig>,
386 machine_id: Option<crate::machine_id::MachineId>,
390 admission_facts_carrier_module_id: Option<String>,
391 admission_facts_targets: Option<Vec<String>>,
392 scopes: Arc<RwLock<ScopeTable>>,
396 scope_deadline_added: Arc<tokio::sync::Notify>,
400 scope_authority_owners: Vec<String>,
403 privileged_operator_prompt: PrivilegedOperatorPrompt,
405 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408 #[cfg(test)]
409 caller_lookup: Option<TestCallerLookup>,
410 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413 rescan: Option<SupervisorRescanContext>,
414 connected_clients: ConnectedClients,
415 counters: DaemonCounters,
416 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417 daemon_shutdown: DaemonShutdownFlag,
422 daemon_provenance: DaemonProvenanceFacts,
423 #[cfg(test)]
424 control_dispatch_delay: Option<Duration>,
425 #[cfg(test)]
426 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427 #[cfg(test)]
431 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432 #[cfg(test)]
438 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443 f.debug_struct("ControlHandler")
444 .field("registry", &self.registry)
445 .field("forwarding", &self.forwarding)
446 .field("process_liveness", &self.process_liveness.is_some())
447 .field("supervisor", &self.supervisor)
448 .field("subc_capabilities", &self.subc_capabilities)
449 .finish()
450 }
451}
452
453struct RouteOpenRequest {
454 target: RouteTarget,
455 identity: BindIdentity,
456 consumer_identity: Option<ConsumerIdentity>,
457 consumer_capabilities: Option<Vec<String>>,
458 role_versions: Option<BTreeMap<String, String>>,
459 admission_facts: Option<serde_json::Value>,
460 scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465 Admission,
466 ProjectRoot,
467 EgressReservation,
468 BindSend,
469 BindAckWait,
470}
471
472struct RouteOpenTimings {
476 module_id: String,
477 principal: String,
478 admission: Duration,
479 project_root: Duration,
480 egress_reservation: Duration,
481 bind_send: Duration,
482 bind_ack_wait: Duration,
483 phase: Option<RouteOpenPhase>,
484 phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488 fn new() -> Self {
489 Self {
490 module_id: String::new(),
491 principal: "unresolved".to_string(),
493 admission: Duration::ZERO,
494 project_root: Duration::ZERO,
495 egress_reservation: Duration::ZERO,
496 bind_send: Duration::ZERO,
497 bind_ack_wait: Duration::ZERO,
498 phase: Some(RouteOpenPhase::Admission),
499 phase_started_at: Instant::now(),
500 }
501 }
502
503 fn finish(&mut self) {
504 let elapsed = self.phase_started_at.elapsed();
505 match self.phase.take() {
506 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511 None => {}
512 }
513 }
514
515 fn enter(&mut self, phase: RouteOpenPhase) {
516 self.finish();
517 self.phase = Some(phase);
518 self.phase_started_at = Instant::now();
519 }
520}
521
522struct RouteBindReservationGuard {
523 forwarding: Arc<ForwardingTable>,
524 endpoint: ModuleEndpointId,
525 relay_corr: u64,
526 armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530 forwarding: Arc<ForwardingTable>,
531 endpoint: ModuleEndpointId,
532 corr: u64,
533 armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538 Self {
539 forwarding,
540 endpoint,
541 corr,
542 armed: true,
543 }
544 }
545
546 fn disarm(&mut self) {
547 self.armed = false;
548 }
549}
550
551impl Drop for ModuleControlRpcGuard {
552 fn drop(&mut self) {
553 if self.armed {
554 let _ = self
555 .forwarding
556 .cancel_module_control_rpc(self.endpoint, self.corr);
557 }
558 }
559}
560
561impl RouteBindReservationGuard {
562 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563 Self {
564 forwarding,
565 endpoint,
566 relay_corr,
567 armed: true,
568 }
569 }
570
571 fn release_and_disarm(&mut self) {
572 if !self.armed {
573 return;
574 }
575 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576 self.endpoint,
577 self.relay_corr,
578 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579 ) {
580 send_goodbye_target_best_effort(
581 &self.forwarding.counters(),
582 &target,
583 "canceled route.bind",
584 );
585 }
586 self.armed = false;
587 }
588
589 fn disarm(&mut self) {
590 self.armed = false;
591 }
592}
593
594impl Drop for RouteBindReservationGuard {
595 fn drop(&mut self) {
596 self.release_and_disarm();
597 }
598}
599
600#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636 modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640 concurrency: RouteBindConcurrency,
641 module_id: String,
642}
643
644impl RouteBindConcurrency {
645 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648 let mut modules = self
649 .modules
650 .lock()
651 .expect("route.bind concurrency mutex poisoned");
652 let in_flight = modules.entry(module_id.to_string()).or_default();
653 if *in_flight >= limit {
654 return Err(*in_flight);
655 }
656 *in_flight += 1;
657 Ok(RouteBindConcurrencyGuard {
658 concurrency: self.clone(),
659 module_id: module_id.to_string(),
660 })
661 }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665 fn drop(&mut self) {
666 let mut modules = self
667 .concurrency
668 .modules
669 .lock()
670 .expect("route.bind concurrency mutex poisoned");
671 let remove = {
672 let in_flight = modules
673 .get_mut(&self.module_id)
674 .expect("admitted route.bind has a concurrency entry");
675 *in_flight -= 1;
676 *in_flight == 0
677 };
678 if remove {
679 modules.remove(&self.module_id);
680 }
681 }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686 consecutive_timeouts: u32,
688 cooldown_until: Option<Instant>,
691 probe_in_flight: Option<Arc<()>>,
696}
697
698enum RouteBindAdmission<'a> {
700 Admitted {
701 guard: RouteBindBreakerGuard<'a>,
702 probe: bool,
705 },
706 Refused {
707 consecutive_timeouts: u32,
708 retry_in: Duration,
712 probe_in_flight: bool,
713 },
714}
715
716struct RouteBindBreakerGuard<'a> {
723 breakers: RouteBindBreakers,
724 module_id: &'a str,
725 probe_token: Option<Arc<()>>,
726 settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730 fn record_accepted(&mut self) -> bool {
734 self.settled = true;
735 self.breakers.record_accepted(self.module_id)
736 }
737
738 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741 self.settled = true;
742 self.breakers.record_timeout(
743 self.module_id,
744 self.probe_token.as_ref(),
745 threshold,
746 cooldown,
747 )
748 }
749
750 fn record_inconclusive(&mut self) {
759 self.settled = true;
760 self.breakers
761 .record_inconclusive(self.module_id, self.probe_token.as_ref());
762 }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766 fn drop(&mut self) {
767 if !self.settled {
768 self.breakers
769 .record_inconclusive(self.module_id, self.probe_token.as_ref());
770 }
771 }
772}
773
774struct BreakerOpened {
778 consecutive_timeouts: u32,
779 reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786 self.modules
787 .lock()
788 .expect("route.bind breaker mutex poisoned")
789 }
790
791 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795 probe: probe_token.is_some(),
796 guard: RouteBindBreakerGuard {
797 breakers: self.clone(),
798 module_id,
799 probe_token,
800 settled: false,
801 },
802 };
803
804 let mut modules = self.lock();
805 let Some(state) = modules.get_mut(module_id) else {
806 return admitted(None);
807 };
808 let Some(cooldown_until) = state.cooldown_until else {
809 return admitted(None);
810 };
811 if state.probe_in_flight.is_some() {
812 return RouteBindAdmission::Refused {
813 consecutive_timeouts: state.consecutive_timeouts,
814 retry_in: Duration::ZERO,
815 probe_in_flight: true,
816 };
817 }
818 let now = Instant::now();
819 if now < cooldown_until {
820 return RouteBindAdmission::Refused {
821 consecutive_timeouts: state.consecutive_timeouts,
822 retry_in: cooldown_until - now,
823 probe_in_flight: false,
824 };
825 }
826 let token = Arc::new(());
827 state.probe_in_flight = Some(Arc::clone(&token));
828 admitted(Some(token))
829 }
830
831 fn record_accepted(&self, module_id: &str) -> bool {
832 self.lock()
833 .remove(module_id)
834 .is_some_and(|state| state.cooldown_until.is_some())
835 }
836
837 fn record_timeout(
838 &self,
839 module_id: &str,
840 probe_token: Option<&Arc<()>>,
841 threshold: u32,
842 cooldown: Duration,
843 ) -> Option<BreakerOpened> {
844 let mut modules = self.lock();
845 let state = modules.entry(module_id.to_string()).or_default();
846 let was_open = state.cooldown_until.is_some();
847 let was_probe = Self::owns_probe(state, probe_token);
848 if was_probe {
849 state.probe_in_flight = None;
850 }
851 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852 if state.consecutive_timeouts < threshold {
853 return None;
854 }
855 state.cooldown_until = Some(Instant::now() + cooldown);
856 Some(BreakerOpened {
857 consecutive_timeouts: state.consecutive_timeouts,
858 reopened_after_probe: was_open && was_probe,
859 })
860 }
861
862 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863 state
867 .probe_in_flight
868 .as_ref()
869 .zip(token)
870 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871 }
872
873 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874 if let Some(state) = self.lock().get_mut(module_id) {
875 if Self::owns_probe(state, probe_token) {
876 state.probe_in_flight = None;
877 }
878 }
879 }
880
881 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896 self.lock()
897 .remove(module_id)
898 .map(|state| state.consecutive_timeouts)
899 .filter(|discarded| *discarded > 0)
900 }
901
902 fn open_snapshot(&self) -> Option<serde_json::Value> {
909 let now = Instant::now();
910 let modules = self.lock();
911 let open = modules
912 .iter()
913 .filter_map(|(module_id, state)| {
914 let cooldown_until = state.cooldown_until?;
915 Some((
916 module_id.clone(),
917 serde_json::json!({
918 "consecutive_timeouts": state.consecutive_timeouts,
919 "cooldown_remaining_ms":
920 cooldown_until.saturating_duration_since(now).as_millis() as u64,
921 "probe_in_flight": state.probe_in_flight.is_some(),
922 }),
923 ))
924 })
925 .collect::<serde_json::Map<String, serde_json::Value>>();
926 (!open.is_empty()).then_some(serde_json::Value::Object(open))
927 }
928}
929
930impl ControlHandler {
931 pub fn new(registry: Arc<Registry>) -> Self {
932 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933 }
934
935 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936 let counters = forwarding.counters();
937 let route_bind_breakers = forwarding.route_bind_breakers();
941 let route_bind_concurrency = forwarding.route_bind_concurrency();
942 let route_outages = forwarding.route_outages();
943 Self {
944 registry,
945 activity: None,
946 activity_streams: Arc::new(Mutex::new(HashMap::new())),
947 forwarding,
948 process_liveness: None,
949 supervisor: SupervisorHandle::new(),
950 subc_capabilities: Arc::from([
951 CAP_MANIFEST_REGISTRATION.to_string(),
952 CAP_CHANNEL_LIFECYCLE.to_string(),
953 CAP_PING_PONG.to_string(),
954 CAP_SESSION_ATTACH.to_string(),
955 CAP_ADMISSION_FACTS_RELAY.to_string(),
956 CAP_SCOPES_V1.to_string(),
957 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
964 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
965 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
966 ]),
967 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
968 route_bind_relay_timeouts: BTreeMap::new(),
969 route_bind_breakers,
970 route_bind_concurrency,
971 route_outages,
972 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
973 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
974 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
975 storage_config: None,
976 machine_id: None,
977 admission_facts_carrier_module_id: None,
978 admission_facts_targets: None,
979 scopes: Arc::new(RwLock::new(ScopeTable::new(
980 crate::daemon_config::default_scope_authority_owners(),
981 ))),
982 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
983 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
984 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
985 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
986 #[cfg(test)]
987 caller_lookup: None,
988 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
989 rescan: None,
990 connected_clients: ConnectedClients::new(),
991 counters,
992 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
993 daemon_shutdown: DaemonShutdownFlag::default(),
994 daemon_provenance: DaemonProvenanceFacts::default(),
995 #[cfg(test)]
996 control_dispatch_delay: None,
997 #[cfg(test)]
998 provenance_probe_override: None,
999 #[cfg(test)]
1000 route_bind_relay_sent_hold: None,
1001 #[cfg(test)]
1002 hello_registry_to_forwarding_hold: None,
1003 }
1004 }
1005
1006 pub fn with_storage_config(
1009 mut self,
1010 storage_config: Option<crate::daemon_config::StorageConfig>,
1011 ) -> Self {
1012 self.storage_config = storage_config;
1013 self
1014 }
1015
1016 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1019 self.activity = Some(crate::activity::ActivityLedger::open(
1020 run_dir.join("activity.jsonl"),
1021 incarnation,
1022 ));
1023 self.supervisor.set_activity(self.activity.clone());
1024 self
1025 }
1026
1027 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1030 self.machine_id = machine_id;
1031 self
1032 }
1033
1034 pub fn with_admission_facts_config(
1038 mut self,
1039 carrier_module_id: Option<String>,
1040 targets: Option<Vec<String>>,
1041 ) -> Self {
1042 self.admission_facts_carrier_module_id = carrier_module_id;
1043 self.admission_facts_targets = targets;
1044 self
1045 }
1046
1047 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1051 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1052 self.scope_authority_owners = owners;
1053 self
1054 }
1055
1056 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1058 self.privileged_operator_prompt = mode;
1059 self
1060 }
1061
1062 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1065 self.route_bind_relay_timeout = timeout;
1066 self
1067 }
1068
1069 pub fn with_route_bind_relay_timeouts(
1075 mut self,
1076 timeouts: impl IntoIterator<Item = (String, Duration)>,
1077 ) -> Self {
1078 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1079 self
1080 }
1081
1082 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1088 self.route_bind_relay_timeouts
1089 .get(module_id)
1090 .copied()
1091 .unwrap_or(self.route_bind_relay_timeout)
1092 }
1093
1094 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1102 self.route_bind_breaker_threshold = threshold.max(1);
1103 self.route_bind_breaker_cooldown = cooldown;
1104 self
1105 }
1106
1107 #[cfg(test)]
1108 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1109 self.health_probe_timeout = timeout;
1110 self
1111 }
1112
1113 #[cfg(test)]
1114 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1115 self.control_dispatch_delay = Some(delay);
1116 self
1117 }
1118
1119 #[cfg(test)]
1123 pub(crate) fn with_route_bind_relay_sent_hold(
1124 mut self,
1125 hold: Arc<tokio::sync::Barrier>,
1126 ) -> Self {
1127 self.route_bind_relay_sent_hold = Some(hold);
1128 self
1129 }
1130
1131 #[cfg(test)]
1136 pub(crate) fn with_hello_registry_to_forwarding_hold(
1137 mut self,
1138 hold: Arc<std::sync::Barrier>,
1139 ) -> Self {
1140 self.hello_registry_to_forwarding_hold = Some(hold);
1141 self
1142 }
1143
1144 #[cfg(test)]
1145 fn hold_between_registry_and_forwarding(&self) {
1146 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1147 hold.wait();
1148 hold.wait();
1149 }
1150 }
1151
1152 pub fn with_process_liveness(
1153 mut self,
1154 process_liveness: Arc<dyn ModuleProcessLiveness>,
1155 ) -> Self {
1156 self.process_liveness = Some(process_liveness);
1157 self
1158 }
1159
1160 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1161 self.supervisor = supervisor;
1162 self.supervisor.set_activity(self.activity.clone());
1163 self
1164 }
1165
1166 pub fn with_daemon_provenance(
1167 mut self,
1168 pid: u32,
1169 started_at_ms: u64,
1170 executable_path: Option<PathBuf>,
1171 build_git_sha: Option<String>,
1172 build_lock_digest: Option<String>,
1173 ) -> Self {
1174 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1175 let process_start_time = process_start_time(pid);
1176 self.daemon_provenance = DaemonProvenanceFacts {
1177 build: DaemonBuildProvenance {
1178 build_git_sha,
1179 build_lock_digest,
1180 },
1181 pid: Some(pid),
1182 started_at_ms: Some(started_at_ms),
1183 start_clock: None,
1184 executable_path,
1185 executable_identity,
1186 process_start_time,
1187 probe: ExecutableIdentityProbe::default(),
1188 };
1189 self
1190 }
1191
1192 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1193 self.daemon_provenance.start_clock = Some(clock);
1194 self
1195 }
1196
1197 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1199 self.scopes
1200 .write()
1201 .unwrap_or_else(|poisoned| poisoned.into_inner())
1202 .set_wall_clock(Arc::new(clock));
1203 self
1204 }
1205
1206 #[cfg(test)]
1207 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1208 self.provenance_probe_override = Some(result);
1209 self
1210 }
1211
1212 pub fn with_capability_config(
1216 self,
1217 modules: impl IntoIterator<Item = (String, bool)>,
1218 reserved_capabilities: BTreeMap<String, String>,
1219 ) -> Self {
1220 self.capability_evaluator
1221 .configure(modules, reserved_capabilities);
1222 self
1223 }
1224
1225 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1229 self.daemon_shutdown = flag;
1230 self
1231 }
1232
1233 pub fn with_supervisor_rescan(
1234 mut self,
1235 supervisor: Supervisor,
1236 config_path: impl Into<PathBuf>,
1237 configured_port: Option<u16>,
1238 ) -> Self {
1239 self.rescan = Some(SupervisorRescanContext {
1240 supervisor,
1241 config_path: config_path.into(),
1242 configured_port,
1243 storage_config: self.storage_config.clone(),
1244 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1245 admission_facts_targets: self.admission_facts_targets.clone(),
1246 scope_authority_owners: self.scope_authority_owners.clone(),
1247 privileged_operator_prompt: self.privileged_operator_prompt,
1248 generation: Arc::default(),
1249 });
1250 self
1251 }
1252
1253 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1254 self.connected_clients = connected_clients;
1255 self
1256 }
1257
1258 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1259 Arc::clone(&self.forwarding)
1260 }
1261
1262 pub(crate) fn counters(&self) -> DaemonCounters {
1263 self.counters.clone()
1264 }
1265
1266 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1269 tokio::spawn(async move {
1270 loop {
1271 self.capability_evaluator
1272 .wait_for_change_or_deadline()
1273 .await;
1274 self.refresh_capability_requirements();
1275 }
1276 });
1277 }
1278
1279 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1285 tokio::spawn(async move {
1286 loop {
1287 let has_deadlines = self
1288 .scopes
1289 .read()
1290 .unwrap_or_else(|poisoned| poisoned.into_inner())
1291 .has_deadlines();
1292 if !has_deadlines {
1293 self.scope_deadline_added.notified().await;
1294 continue;
1295 }
1296 tokio::time::sleep(Duration::from_secs(1)).await;
1297 if let Err(error) = self.sweep_expired_scopes() {
1298 warn!(%error, "scope expiry sweep failed");
1299 }
1300 }
1301 });
1302 }
1303
1304 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1305 let mut table = self
1306 .scopes
1307 .write()
1308 .unwrap_or_else(|poisoned| poisoned.into_inner());
1309 let swept = table.sweep_expired();
1310 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1311 drop(table);
1312 self.close_scope_drained_routes(drained);
1313 Ok(())
1314 }
1315
1316 fn publish_scope_changes(
1317 &self,
1318 changes: &[crate::scopes::ScopeTagChange],
1319 expired: &[crate::scopes::ScopeExpired],
1320 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1321 if changes.is_empty() {
1322 return Ok(Vec::new());
1323 }
1324 let drained = self
1325 .forwarding
1326 .publish_scope_changes(changes)
1327 .map_err(RouterError::Forwarding)?;
1328 let mut counts = HashMap::new();
1329 for route in &drained {
1330 *counts
1331 .entry((
1332 route.scope.owner.as_str(),
1333 route.scope.scope_ref.as_str(),
1334 route.scope.tag.scope_epoch,
1335 ))
1336 .or_insert(0usize) += 1;
1337 }
1338 for scope in expired {
1339 let routes_closed = counts
1340 .get(&(
1341 scope.owner.as_str(),
1342 scope.scope_ref.as_str(),
1343 scope.scope_epoch,
1344 ))
1345 .copied()
1346 .unwrap_or(0);
1347 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1348 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1349 routes_closed, "scope expired");
1350 }
1351 Ok(drained)
1352 }
1353
1354 fn runtime_capability_snapshot(
1355 &self,
1356 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1357 let runtime = self
1358 .supervisor
1359 .list()
1360 .into_iter()
1361 .map(|module| {
1362 let status = module.status().map_err(|err| {
1363 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1364 })?;
1365 Ok(RuntimeModule {
1366 module_id: status.module_id,
1367 state: status.state,
1368 enabled: status.enabled,
1369 })
1370 })
1371 .collect::<Result<Vec<_>, RouterError>>()?;
1372 let registrations = self.registry.list_registered_modules().map_err(|err| {
1378 RouterError::backend(
1379 0,
1380 0,
1381 format!("failed to list capability registrations: {err}"),
1382 )
1383 })?;
1384 let registrations = registrations
1385 .into_iter()
1386 .map(|registration| RegisteredModule {
1387 module_id: registration.manifest.module_id,
1388 module_version: registration.manifest.module_version,
1389 capabilities: registration.manifest.capabilities,
1390 })
1391 .collect();
1392 Ok((runtime, registrations))
1393 }
1394
1395 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1402 let cached_registration = RegisteredModule {
1403 module_id: registration.manifest.module_id.clone(),
1404 module_version: registration.manifest.module_version.clone(),
1405 capabilities: registration.manifest.capabilities.clone(),
1406 };
1407 if self.capability_evaluator.record_hello(&cached_registration) {
1408 warn!(
1409 module_id = %cached_registration.module_id,
1410 "capability claims drifted from the cached manifest"
1411 );
1412 }
1413 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1414 self.enforce_capability_denies();
1415 }
1416 self.refresh_capability_requirements();
1417 }
1418
1419 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1423 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1424 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1425 self.supervisor.set_swap_promotion_observer(observer);
1426 }
1427
1428 pub fn refresh_capability_requirements(&self) {
1429 match self.runtime_capability_snapshot() {
1430 Ok((runtime, registrations)) => {
1431 log_requirement_events(
1432 self.capability_evaluator
1433 .evaluate_now(&runtime, ®istrations),
1434 &self.daemon_shutdown,
1435 );
1436 }
1437 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1438 }
1439 }
1440
1441 fn enforce_capability_denies(&self) {
1445 let registrations = match self.registry.list_registered_modules() {
1446 Ok(snapshot) => snapshot,
1447 Err(err) => {
1448 warn!(error = %err, "failed to read registrations for capability deny census");
1449 return;
1450 }
1451 };
1452 let manifests = registrations
1453 .into_iter()
1454 .map(|registration| {
1455 (
1456 registration.manifest.module_id.clone(),
1457 registration.manifest,
1458 )
1459 })
1460 .collect::<BTreeMap<_, _>>();
1461 let census = match self.forwarding.route_census(None) {
1462 Ok(census) => census,
1463 Err(err) => {
1464 warn!(error = %err, "failed to read route census for capability deny enforcement");
1465 return;
1466 }
1467 };
1468
1469 for (target_module_id, routes) in census {
1470 let Some(target_manifest) = manifests.get(&target_module_id) else {
1471 continue;
1472 };
1473 let mut closed_routes = Vec::new();
1474 let mut module_goodbyes = Vec::new();
1475 for route in routes {
1476 let Principal::Reserved {
1477 module_id: opening_module_id,
1478 } = &route.principal
1479 else {
1480 continue;
1481 };
1482 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1483 continue;
1484 };
1485 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1486 continue;
1487 };
1488
1489 match self.forwarding.release_client_route(
1490 route.goodbye_target.connection_id,
1491 route.goodbye_target.channel,
1492 route.goodbye_target.epoch,
1493 ) {
1494 Ok(RouteRelease::Removed(module_goodbye)) => {
1495 warn!(
1496 opening_module_id,
1497 target_module_id,
1498 capability,
1499 "force-closing route because an attested capability deny edge now matches"
1500 );
1501 closed_routes.push(route);
1502 module_goodbyes.push(module_goodbye);
1503 }
1504 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1505 Err(err) => warn!(
1506 opening_module_id,
1507 target_module_id,
1508 capability,
1509 error = %err,
1510 "failed to force-close capability-denied route"
1511 ),
1512 }
1513 }
1514
1515 if closed_routes.is_empty() {
1516 continue;
1517 }
1518 send_route_control_pushes(
1519 &self.forwarding,
1520 closed_routes,
1521 ClientControlPush::RouteClosed {
1522 module_id: target_module_id,
1523 channels: Vec::new(),
1524 reason: RouteCloseReason::CapabilityDenied,
1525 drained: false,
1526 abandoned: 0,
1527 excluded_subscriptions: 0,
1528 terminal: Some(false),
1529 },
1530 );
1531 self.emit_route_goodbyes(module_goodbyes);
1532 }
1533 }
1534
1535 fn not_ready_reason(
1541 &self,
1542 registration: &crate::registry::ModuleRegistration,
1543 ) -> Option<NotReadyReason> {
1544 if !registration.ready {
1545 return Some(NotReadyReason {
1546 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1547 capability: None,
1548 });
1549 }
1550 self.first_unprovided_required_capability(registration)
1551 .map(|capability| NotReadyReason {
1552 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1553 capability: Some(capability),
1554 })
1555 }
1556
1557 fn first_unprovided_required_capability(
1575 &self,
1576 registration: &crate::registry::ModuleRegistration,
1577 ) -> Option<String> {
1578 let required = registration
1579 .manifest
1580 .capabilities
1581 .iter()
1582 .flat_map(|declarations| declarations.requires.iter())
1583 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1584 .map(|requirement| requirement.capability.as_str())
1585 .collect::<BTreeSet<_>>();
1586 if required.is_empty() {
1587 return None;
1588 }
1589 let module_id = registration.manifest.module_id.as_str();
1590 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1591 if required
1592 .iter()
1593 .any(|capability| verdict(capability).is_none())
1594 {
1595 self.refresh_capability_requirements();
1596 }
1597 required
1598 .into_iter()
1599 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1600 .map(str::to_string)
1601 }
1602
1603 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1604 self.capability_evaluator
1605 .statuses()
1606 .into_iter()
1607 .map(capability_requirement_status)
1608 .collect()
1609 }
1610
1611 fn deregister_connection(
1618 &self,
1619 connection_id: ConnectionId,
1620 reason: RegistrationEndReason,
1621 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1622 self.registry
1623 .deregister_connection_with_reason(connection_id, reason)
1624 }
1625
1626 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1627 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1628 return None;
1629 }
1630 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1631 parse_client_control_request(&frame.body)
1632 else {
1633 return None;
1634 };
1635 Some(target_module_id(&target).to_string())
1636 }
1637
1638 pub(crate) fn route_open_capacity_refusal(
1639 &self,
1640 ctx: &RouteCtx,
1641 frame: &Frame,
1642 target_module_id: &str,
1643 in_flight: usize,
1644 limit: usize,
1645 ) -> Result<Frame, RouterError> {
1646 let caller = self.route_open_caller_from_frame(frame);
1649 self.route_open_admission_refusal_frame(
1650 ctx,
1651 &caller,
1652 frame,
1653 target_module_id,
1654 "open_admission_full",
1655 (in_flight, limit),
1656 format!(
1657 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1658 ),
1659 )
1660 }
1661
1662 fn route_open_target_capacity_refusal(
1663 &self,
1664 ctx: &RouteCtx,
1665 caller: &RouteOpenCaller,
1666 frame: &Frame,
1667 target_module_id: &str,
1668 in_flight: usize,
1669 ) -> Result<Frame, RouterError> {
1670 self.route_open_admission_refusal_frame(
1671 ctx,
1672 caller,
1673 frame,
1674 target_module_id,
1675 "target_binds_full",
1676 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1677 format!(
1678 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1679 ),
1680 )
1681 }
1682
1683 #[allow(clippy::too_many_arguments)]
1691 fn route_open_admission_refusal_frame(
1692 &self,
1693 ctx: &RouteCtx,
1694 caller: &RouteOpenCaller,
1695 frame: &Frame,
1696 target_module_id: &str,
1697 reason: &'static str,
1698 (in_flight, limit): (usize, usize),
1699 message: impl Into<String>,
1700 ) -> Result<Frame, RouterError> {
1701 let code = error_codes::TARGET_UNAVAILABLE;
1702 self.counters.increment_route_open_refused(code);
1703 info!(
1704 target: "control",
1705 code,
1706 reason,
1707 module_id = ?target_module_id,
1708 connection_id = ctx.connection_id.get(),
1709 principal = %caller.principal,
1710 scope = %caller.scope,
1711 harness = %caller.harness,
1712 in_flight,
1713 limit,
1714 "route.open refused"
1715 );
1716 control_error_frame(frame, code, message.into())
1717 }
1718
1719 #[cfg(test)]
1725 pub fn handle_control(
1726 &self,
1727 connection_id: ConnectionId,
1728 frame: Frame,
1729 ) -> Result<Vec<Frame>, RouterError> {
1730 match frame.header.ty {
1731 FrameType::Ping => Ok(vec![pong(&frame)?]),
1732 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1733 FrameType::Goodbye => self.handle_goodbye(connection_id),
1734 ty => Ok(vec![control_error_frame(
1735 &frame,
1736 "unsupported_control_frame",
1737 format!("unsupported channel-0 frame {ty:?}"),
1738 )?]),
1739 }
1740 }
1741
1742 pub async fn handle_control_frame(
1743 &self,
1744 ctx: &RouteCtx,
1745 frame: Frame,
1746 ) -> Result<Vec<Frame>, RouterError> {
1747 self.handle_control_frame_timed(ctx, frame, None).await
1748 }
1749
1750 pub(crate) fn record_caller_endpoints(
1751 &self,
1752 connection: ConnectionId,
1753 endpoints: Option<(SocketAddr, SocketAddr)>,
1754 ) {
1755 let mut callers = self
1756 .caller_endpoints
1757 .lock()
1758 .unwrap_or_else(|p| p.into_inner());
1759 callers.remove(&connection);
1760 if let Some(endpoints) = endpoints {
1761 callers.insert(connection, endpoints);
1762 }
1763 }
1764
1765 #[cfg(test)]
1766 pub(crate) fn with_caller_lookup_for_test(
1767 mut self,
1768 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1769 ) -> Self {
1770 self.caller_lookup = Some(Arc::new(lookup));
1771 self
1772 }
1773
1774 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1775 let endpoints = self
1776 .caller_endpoints
1777 .lock()
1778 .unwrap_or_else(|p| p.into_inner())
1779 .get(&connection)
1780 .copied();
1781 #[cfg(test)]
1782 let lookup = self.caller_lookup.clone();
1783 tokio::task::spawn_blocking(move || {
1784 #[cfg(test)]
1785 if let Some(lookup) = lookup {
1786 return lookup(endpoints);
1787 }
1788 endpoints
1789 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1790 .unwrap_or_default()
1791 })
1792 .await
1793 .unwrap_or_default()
1794 }
1795
1796 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1797 frame.header.channel == 0
1798 && frame.header.ty == FrameType::Request
1799 && self
1800 .forwarding
1801 .module_endpoint_for_connection(ctx.connection_id)
1802 .is_ok_and(|endpoint| endpoint.is_none())
1803 && parse_client_control_request(&frame.body)
1804 .ok()
1805 .is_some_and(|request| privileged_operation(&request).is_some())
1806 }
1807
1808 pub(crate) async fn handle_control_frame_timed(
1809 &self,
1810 ctx: &RouteCtx,
1811 frame: Frame,
1812 dispatch_started_at: Option<StdInstant>,
1813 ) -> Result<Vec<Frame>, RouterError> {
1814 match frame.header.ty {
1815 FrameType::Ping => Ok(vec![pong(&frame)?]),
1816 FrameType::Hello => {
1817 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1818 }
1819 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1820 FrameType::Cancel => {
1821 if self
1824 .forwarding
1825 .operator_confirms()
1826 .cancel(ctx.connection_id, frame.header.corr)
1827 || self
1828 .supervisor
1829 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1830 || self.cancel_activity_subscription(ctx, &frame)?
1831 {
1832 Ok(Vec::new())
1833 } else {
1834 Ok(vec![control_error_frame(
1835 &frame,
1836 "unknown_subscription",
1837 "no subscription has this correlation id",
1838 )?])
1839 }
1840 }
1841 FrameType::Request => {
1842 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1845 if op
1846 .as_ref()
1847 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1848 {
1849 return self.handle_activity_record(ctx, frame).await;
1850 }
1851 if op.as_ref().is_some_and(|probe| {
1852 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1853 }) {
1854 return self.handle_activity_read(ctx, frame).await;
1855 }
1856 if op
1857 .as_ref()
1858 .is_some_and(|probe| probe.op == "operator.confirm")
1859 {
1860 return self.handle_operator_confirm(ctx, frame);
1861 }
1862 if op
1863 .as_ref()
1864 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1865 {
1866 return self.handle_scope_served_by(ctx.connection_id, frame);
1867 }
1868 if self
1869 .forwarding
1870 .module_endpoint_for_connection(ctx.connection_id)
1871 .map_err(RouterError::Forwarding)?
1872 .is_some()
1873 {
1874 if !is_known_module_request_op(&frame.body) {
1875 return Ok(vec![control_error_frame(
1876 &frame,
1877 "unsupported_control_frame",
1878 "module-originated channel-0 REQUEST is not supported",
1879 )?]);
1880 }
1881 let request = match parse_module_control_request_from_module(&frame.body) {
1882 Ok(request) => request,
1883 Err((err, ControlRequestBodyError::UnknownOp)) => {
1884 return Ok(vec![control_error_frame(
1885 &frame,
1886 "unsupported_control_frame",
1887 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1888 )?])
1889 }
1890 Err((err, ControlRequestBodyError::InvalidBody)) => {
1891 return Ok(vec![control_error_frame(
1892 &frame,
1893 "invalid_control_body",
1894 format!("malformed module control body: {err}"),
1895 )?])
1896 }
1897 };
1898 let op = module_control_request_op(&request);
1899 let corr = frame.header.corr;
1900 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1901 let result =
1902 self.handle_module_control_request(ctx.connection_id, frame, request);
1903 log_slow_control_dispatch(
1904 dispatch_started_at,
1905 op,
1906 ctx.connection_id,
1907 corr,
1908 None,
1909 );
1910 return result;
1911 }
1912
1913 if is_known_module_request_op(&frame.body) {
1914 return Ok(vec![control_error_frame(
1915 &frame,
1916 "not_registered",
1917 "catalog.update requires an active module registration owned by this connection",
1918 )?]);
1919 }
1920
1921 let request = match parse_client_control_request(&frame.body) {
1922 Ok(request) => request,
1923 Err((err, ControlRequestBodyError::UnknownOp)) => {
1924 return Ok(vec![control_error_frame(
1925 &frame,
1926 "unknown_control_op",
1927 format!("unknown client control op: {err}"),
1928 )?])
1929 }
1930 Err((err, ControlRequestBodyError::InvalidBody)) => {
1931 return Ok(vec![control_error_frame(
1932 &frame,
1933 "invalid_control_body",
1934 format!("malformed client control body: {err}"),
1935 )?])
1936 }
1937 };
1938 let op = client_control_request_op(&request);
1939 let corr = frame.header.corr;
1940 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1941 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1942 #[cfg(test)]
1943 if let Some(delay) = self.control_dispatch_delay {
1944 tokio::time::sleep(delay).await;
1945 }
1946 let result = self
1947 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1948 .await;
1949 if let Some(timings) = route_open_timings.as_mut() {
1950 timings.finish();
1951 }
1952 log_slow_control_dispatch(
1953 dispatch_started_at,
1954 op,
1955 ctx.connection_id,
1956 corr,
1957 route_open_timings.as_ref(),
1958 );
1959 result
1960 }
1961 FrameType::Push => {
1962 let Some(endpoint) = self
1963 .forwarding
1964 .module_endpoint_for_connection(ctx.connection_id)
1965 .map_err(RouterError::Forwarding)?
1966 else {
1967 return Ok(vec![control_error_frame(
1968 &frame,
1969 "unsupported_control_frame",
1970 "client-originated channel-0 PUSH is not supported",
1971 )?]);
1972 };
1973 self.handle_status_update(endpoint, frame)
1974 }
1975 FrameType::Response | FrameType::Error
1976 if self
1977 .forwarding
1978 .module_endpoint_for_connection(ctx.connection_id)
1979 .map_err(RouterError::Forwarding)?
1980 .is_some() =>
1981 {
1982 self.handle_module_relay_response(ctx.connection_id, frame)
1983 }
1984 ty => Ok(vec![control_error_frame(
1985 &frame,
1986 "unsupported_control_frame",
1987 format!("unsupported channel-0 frame {ty:?}"),
1988 )?]),
1989 }
1990 }
1991
1992 pub fn cleanup_connection(
1993 &self,
1994 connection_id: ConnectionId,
1995 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1996 self.caller_endpoints
1997 .lock()
1998 .unwrap_or_else(|p| p.into_inner())
1999 .remove(&connection_id);
2000 self.forwarding
2001 .operator_confirms()
2002 .connection_closed(connection_id);
2003 self.cleanup_connection_with_end_reason(
2004 connection_id,
2005 RegistrationEndReason::ConnectionClosed,
2006 )
2007 }
2008
2009 fn cleanup_connection_with_end_reason(
2010 &self,
2011 connection_id: ConnectionId,
2012 requested_reason: RegistrationEndReason,
2013 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2014 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2015 self.registry
2016 .get_module_by_connection(connection_id)?
2017 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2018 .and_then(|module| module.registration_end_reason().ok().flatten())
2019 .unwrap_or(requested_reason)
2020 } else {
2021 requested_reason
2022 };
2023 let crash_closed = self
2024 .registry
2025 .get_module_by_connection(connection_id)?
2026 .and_then(|registration| {
2027 self.forwarding
2028 .module_endpoint_for_connection(connection_id)
2029 .ok()
2030 .flatten()
2031 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2032 .map(|routes| (registration.manifest.module_id, routes))
2033 });
2034 let crash_closed = crash_closed.map(|(module_id, routes)| {
2035 let terminal = match self.supervisor.get(&module_id) {
2036 None => false,
2037 Some(module) => match module.will_recover_after_connection_loss() {
2038 Ok(will_recover) => !will_recover,
2039 Err(err) => {
2040 warn!(
2041 %module_id,
2042 error = %err,
2043 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2044 );
2045 false
2046 }
2047 },
2048 };
2049 let (reason, drained) = match self
2053 .forwarding
2054 .daemon_drain_result_for_connection(connection_id)
2055 {
2056 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2057 Ok(None) => (RouteCloseReason::Crash, false),
2058 Err(err) => {
2059 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2060 (RouteCloseReason::Crash, false)
2061 }
2062 };
2063 (module_id, routes, reason, drained, terminal)
2064 });
2065 let registrations = self.deregister_connection(connection_id, end_reason);
2066 let cleanup = if crash_closed.is_some() {
2067 self.forwarding.cleanup_connection_counted(connection_id)
2068 } else {
2069 self.forwarding
2075 .cleanup_connection(connection_id)
2076 .map(|released| crate::forwarding::ConnectionCleanup {
2077 released,
2078 abandoned_relays: 0,
2079 })
2080 };
2081 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2086 let abandoned = cleanup
2087 .as_ref()
2088 .map_or(0, |cleanup| cleanup.abandoned_relays);
2089 send_route_control_pushes(
2090 &self.forwarding,
2091 routes,
2092 ClientControlPush::RouteClosed {
2093 module_id,
2094 channels: Vec::new(),
2095 reason,
2096 drained,
2097 abandoned,
2098 excluded_subscriptions: 0,
2099 terminal: Some(terminal),
2100 },
2101 );
2102 }
2103 if let Ok(cleanup) = cleanup {
2104 self.emit_route_goodbyes(cleanup.released);
2105 }
2106 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2110 crate::supervise::notify_registration_release();
2111 self.capability_evaluator.wake_deadline_loop();
2112 self.refresh_capability_requirements();
2113 }
2114 self.supervisor.remove_spawn_subscribers(connection_id);
2115 self.activity_streams
2116 .lock()
2117 .unwrap_or_else(|p| p.into_inner())
2118 .retain(|(id, _), task| {
2119 if *id == connection_id {
2120 task.abort();
2121 false
2122 } else {
2123 true
2124 }
2125 });
2126 self.hello_launch_nonces
2129 .lock()
2130 .unwrap_or_else(|poisoned| poisoned.into_inner())
2131 .forget(connection_id);
2132 self.scopes
2133 .write()
2134 .unwrap_or_else(|poisoned| poisoned.into_inner())
2135 .release_connection(connection_id);
2136 registrations
2137 }
2138
2139 pub(crate) fn handle_route_goodbye(
2140 &self,
2141 connection_id: ConnectionId,
2142 route_channel: u16,
2143 route_epoch: u32,
2144 ) -> Result<bool, RouterError> {
2145 debug!(
2146 connection_id = connection_id.get(),
2147 route_channel, route_epoch, "handling route GOODBYE"
2148 );
2149 let RouteRelease::Removed(released_route) = self
2150 .forwarding
2151 .release_client_route(connection_id, route_channel, route_epoch)
2152 .map_err(RouterError::Forwarding)?
2153 else {
2154 return Ok(false);
2155 };
2156 self.emit_route_goodbyes(vec![released_route]);
2157 Ok(true)
2158 }
2159
2160 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2161 for released in released_routes {
2162 let frame = match Frame::build_with_version(
2163 released.negotiated_ver,
2164 FrameType::Goodbye,
2165 control_flags(),
2166 released.channel,
2167 released.epoch,
2168 0,
2169 Vec::new(),
2170 ) {
2171 Ok(frame) => frame,
2172 Err(err) => {
2173 warn!(
2174 route_channel = released.channel,
2175 error = %err,
2176 "failed to build route GOODBYE frame"
2177 );
2178 continue;
2179 }
2180 };
2181 if !released.close_on_delivery_failure() {
2182 crate::forwarding::send_module_route_goodbye(
2183 &self.counters,
2184 &released.sink,
2185 frame,
2186 released.module_id.as_deref(),
2187 "client route released",
2188 );
2189 continue;
2190 }
2191 if let Err(err) = released.sink.try_send(frame) {
2192 warn!(
2193 target_connection_id = released.connection_id.get(),
2194 route_channel = released.channel,
2195 error = %err,
2196 "route GOODBYE was not delivered to client; closing target connection"
2197 );
2198 if self
2199 .forwarding
2200 .escalate_client_delivery_failure(
2201 released.connection_id,
2202 released.channel,
2203 released.epoch,
2204 CloseReason::new(
2205 "route_goodbye_delivery_failed",
2206 format!(
2207 "failed to enqueue route GOODBYE for channel {}: {err}",
2208 released.channel
2209 ),
2210 ),
2211 crate::forwarding::UndeliveredFrame {
2212 module_id: released.module_id.as_deref(),
2213 sink: &released.sink,
2214 },
2215 )
2216 .unwrap_or(false)
2217 {
2218 self.counters.increment_goodbye_relay_client_failed();
2219 }
2220 }
2221 }
2222 }
2223
2224 fn send_abandoned_route_bind_goodbye(
2236 &self,
2237 module_sink: &crate::FrameSink,
2238 negotiated_ver: u8,
2239 module_channel: u16,
2240 module_epoch: u32,
2241 ) {
2242 let frame = match Frame::build_with_version(
2243 negotiated_ver,
2244 FrameType::Goodbye,
2245 control_flags(),
2246 module_channel,
2247 module_epoch,
2248 0,
2249 Vec::new(),
2250 ) {
2251 Ok(frame) => frame,
2252 Err(err) => {
2253 warn!(
2254 route_channel = module_channel,
2255 error = %err,
2256 "failed to build GOODBYE for abandoned route.bind"
2257 );
2258 return;
2259 }
2260 };
2261 crate::forwarding::send_module_route_goodbye(
2262 &self.counters,
2263 module_sink,
2264 frame,
2265 None,
2266 "abandoned route.bind",
2267 );
2268 }
2269
2270 fn handle_hello(
2271 &self,
2272 connection_id: ConnectionId,
2273 sink: Option<crate::FrameSink>,
2274 frame: Frame,
2275 ) -> Result<Vec<Frame>, RouterError> {
2276 debug!(
2277 connection_id = connection_id.get(),
2278 corr = frame.header.corr,
2279 "handling HELLO"
2280 );
2281 if self
2285 .registry
2286 .get_module_by_connection(connection_id)
2287 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2288 .is_some()
2289 {
2290 return Ok(vec![control_error_frame(
2291 &frame,
2292 "invalid_hello",
2293 "connection is already registered as a module",
2294 )?]);
2295 }
2296 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2297 Ok(value) => value,
2298 Err(err) => {
2299 return Ok(vec![control_error_frame(
2300 &frame,
2301 "invalid_hello",
2302 format!("malformed HELLO body: {err}"),
2303 )?])
2304 }
2305 };
2306 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2307 return Ok(vec![control_error_frame(
2308 &frame,
2309 "invalid_capability_grammar",
2310 err.to_string(),
2311 )?]);
2312 }
2313 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2314 return Ok(vec![control_error_frame(
2315 &frame,
2316 "invalid_manifest",
2317 err.to_string(),
2318 )?]);
2319 }
2320 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2321 return Ok(vec![control_error_frame(
2322 &frame,
2323 "invalid_event_declaration",
2324 err.to_string(),
2325 )?]);
2326 }
2327 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2328 return Ok(vec![control_error_frame(
2329 &frame,
2330 "invalid_manifest",
2331 err.to_string(),
2332 )?]);
2333 }
2334 if let Some(provenance) = hello_value
2335 .get("manifest")
2336 .and_then(|manifest| manifest.get("provenance"))
2337 {
2338 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2339 return Ok(vec![control_error_frame(
2340 &frame,
2341 "invalid_manifest",
2342 format!("malformed manifest provenance: {err}"),
2343 )?]);
2344 }
2345 }
2346 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2347 Ok(hello) => hello,
2348 Err(err) => {
2349 return Ok(vec![control_error_frame(
2350 &frame,
2351 "invalid_hello",
2352 format!("malformed HELLO body: {err}"),
2353 )?])
2354 }
2355 };
2356
2357 if hello.protocol_ver != hello.manifest.protocol_ver {
2358 return Ok(vec![control_error_frame(
2359 &frame,
2360 "invalid_manifest",
2361 format!(
2362 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2363 hello.protocol_ver, hello.manifest.protocol_ver
2364 ),
2365 )?]);
2366 }
2367
2368 if hello.manifest.module_id.trim().is_empty() {
2369 return Ok(vec![control_error_frame(
2370 &frame,
2371 "invalid_manifest",
2372 "manifest module_id must not be empty",
2373 )?]);
2374 }
2375
2376 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2377 Ok(negotiated_ver) => negotiated_ver,
2378 Err(message) => {
2379 return Ok(vec![control_error_frame(
2380 &frame,
2381 "version_unsupported",
2382 message,
2383 )?])
2384 }
2385 };
2386
2387 let swap_admission = self
2395 .supervisor
2396 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2397 if swap_admission == SwapHelloAdmission::Refused {
2398 warn!(
2399 module_id = %hello.manifest.module_id,
2400 connection_id = connection_id.get(),
2401 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2402 );
2403 return Ok(vec![control_error_frame(
2404 &frame,
2405 "swap_token_invalid",
2406 format!(
2407 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2408 hello.manifest.module_id
2409 ),
2410 )?]);
2411 }
2412 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2413
2414 let nonce_verified = if swap_candidate {
2419 true
2420 } else {
2421 match self.supervisor.reserved_hello_verification(
2422 &hello.manifest.module_id,
2423 hello.launch_nonce.as_deref(),
2424 ) {
2425 Ok(verified) => verified,
2426 Err(rejection) => {
2427 let message = match rejection {
2428 ReservedHelloRejection::Exact { module_id } => format!(
2429 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2430 ),
2431 ReservedHelloRejection::Prefix {
2432 prefix,
2433 owner_module_id,
2434 } => format!(
2435 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2436 hello.manifest.module_id
2437 ),
2438 };
2439 return Ok(vec![control_error_frame(
2440 &frame,
2441 "reserved_module",
2442 message,
2443 )?]);
2444 }
2445 }
2446 };
2447
2448 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2449 &hello.manifest.module_id,
2450 hello.manifest.capabilities.as_ref(),
2451 );
2452 if let Some(refusal) = reserved_capability_refusals.first() {
2453 let capability = refusal.capability.clone();
2454 let bound_module = refusal.claimants[0].clone();
2455 log_duplicate_claim_events(reserved_capability_refusals);
2456 return Ok(vec![control_error_frame(
2457 &frame,
2458 "reserved_capability",
2459 format!(
2460 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2461 capability, bound_module, hello.manifest.module_id
2462 ),
2463 )?]);
2464 }
2465
2466 if self
2469 .forwarding
2470 .connection_has_client_routes(connection_id)
2471 .map_err(RouterError::Forwarding)?
2472 {
2473 return Ok(vec![control_error_frame(
2474 &frame,
2475 "invalid_hello",
2476 "connection has open client routes and cannot also register as a module",
2477 )?]);
2478 }
2479
2480 self.hello_launch_nonces
2485 .lock()
2486 .unwrap_or_else(|poisoned| poisoned.into_inner())
2487 .record(connection_id, hello.launch_nonce.as_deref());
2488 let control_ops = effective_module_control_ops(hello.control_ops);
2489 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2492 if swap_candidate {
2493 return self.register_swap_candidate(
2494 connection_id,
2495 sink,
2496 &frame,
2497 hello.manifest,
2498 negotiated_ver,
2499 control_ops,
2500 hello_ack,
2501 );
2502 }
2503 let registration = match self
2510 .registry
2511 .register_unpublished_verified_with_control_ops(
2512 hello.manifest,
2513 negotiated_ver,
2514 connection_id,
2515 control_ops,
2516 nonce_verified,
2517 ) {
2518 Ok(registration) => registration,
2519 Err(RegistryError::DuplicateModuleId { module_id }) => {
2520 return Ok(vec![control_error_frame(
2521 &frame,
2522 "duplicate_module_id",
2523 format!(
2524 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2525 ),
2526 )?])
2527 }
2528 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2529 return Ok(vec![control_error_frame(
2530 &frame,
2531 "invalid_module_id",
2532 err.to_string(),
2533 )?])
2534 }
2535 Err(err) => {
2536 return Ok(vec![control_error_frame(
2537 &frame,
2538 "registry_error",
2539 err.to_string(),
2540 )?])
2541 }
2542 };
2543 #[cfg(test)]
2544 self.hold_between_registry_and_forwarding();
2545
2546 let reply = if let Some(sink) = sink {
2547 let concurrency = manifest_concurrency(®istration.manifest);
2563 if let Err(err) = self.forwarding.register_module_connection_acked(
2564 connection_id,
2565 registration.manifest.module_id.clone(),
2566 negotiated_ver,
2567 concurrency,
2568 sink,
2569 hello_ack,
2570 ) {
2571 if matches!(
2576 self.deregister_connection(
2577 connection_id,
2578 RegistrationEndReason::RegistrationFailed,
2579 ),
2580 Ok(r) if !r.is_empty()
2581 ) {
2582 crate::supervise::notify_registration_release();
2583 }
2584 return Ok(vec![control_error_frame(
2585 &frame,
2586 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2587 "invalid_hello"
2588 } else {
2589 forwarding_error_code(&err)
2590 },
2591 err.to_string(),
2592 )?]);
2593 }
2594 Vec::new()
2595 } else {
2596 vec![hello_ack]
2599 };
2600
2601 if !self
2606 .registry
2607 .publish_registration(connection_id)
2608 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2609 {
2610 debug!(
2611 module_id = %registration.manifest.module_id,
2612 connection_id = connection_id.get(),
2613 "registration was removed before it could be published"
2614 );
2615 }
2616
2617 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2626 info!(
2627 module_id = %registration.manifest.module_id,
2628 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2629 );
2630 }
2631
2632 self.apply_registration_capabilities(®istration);
2633
2634 self.supervisor
2635 .record_activity_registration(®istration.manifest.module_id);
2636
2637 info!(
2638 module_id = %registration.manifest.module_id,
2639 module_version = %registration.manifest.module_version,
2640 negotiated_ver,
2641 routable_provider = manifest_provides_routable_role(®istration.manifest),
2642 connection_id = connection_id.get(),
2643 "module registered"
2644 );
2645
2646 Ok(reply)
2647 }
2648
2649 #[allow(clippy::too_many_arguments)]
2666 fn register_swap_candidate(
2667 &self,
2668 connection_id: ConnectionId,
2669 sink: Option<crate::FrameSink>,
2670 frame: &Frame,
2671 manifest: ModuleManifest,
2672 negotiated_ver: u8,
2673 control_ops: Vec<String>,
2674 hello_ack: Frame,
2675 ) -> Result<Vec<Frame>, RouterError> {
2676 let module_id = manifest.module_id.clone();
2677 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2678 manifest,
2679 negotiated_ver,
2680 connection_id,
2681 control_ops,
2682 ) {
2683 Ok(registration) => registration,
2684 Err(RegistryError::DuplicateModuleId { module_id }) => {
2685 return Ok(vec![control_error_frame(
2686 frame,
2687 "duplicate_module_id",
2688 format!(
2689 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2690 ),
2691 )?])
2692 }
2693 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2694 return Ok(vec![control_error_frame(
2695 frame,
2696 "invalid_module_id",
2697 err.to_string(),
2698 )?])
2699 }
2700 Err(err) => {
2701 return Ok(vec![control_error_frame(
2702 frame,
2703 "registry_error",
2704 err.to_string(),
2705 )?])
2706 }
2707 };
2708 #[cfg(test)]
2709 self.hold_between_registry_and_forwarding();
2710 let reply = if let Some(sink) = sink {
2711 let concurrency = manifest_concurrency(®istration.manifest);
2715 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2716 connection_id,
2717 module_id.clone(),
2718 negotiated_ver,
2719 concurrency,
2720 sink,
2721 hello_ack,
2722 ) {
2723 if matches!(
2724 self.deregister_connection(
2725 connection_id,
2726 RegistrationEndReason::RegistrationFailed,
2727 ),
2728 Ok(r) if !r.is_empty()
2729 ) {
2730 crate::supervise::notify_registration_release();
2731 }
2732 return Ok(vec![control_error_frame(
2733 frame,
2734 forwarding_error_code(&err),
2735 err.to_string(),
2736 )?]);
2737 }
2738 Vec::new()
2739 } else {
2740 vec![hello_ack]
2741 };
2742 if !self
2743 .registry
2744 .publish_registration(connection_id)
2745 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2746 {
2747 debug!(
2748 module_id = %module_id,
2749 connection_id = connection_id.get(),
2750 "swap candidate registration was removed before it could be published"
2751 );
2752 }
2753 self.supervisor.mark_swap_candidate_admitted(&module_id);
2754 info!(
2755 module_id = %module_id,
2756 module_version = %registration.manifest.module_version,
2757 negotiated_ver,
2758 ready = registration.ready,
2759 connection_id = connection_id.get(),
2760 "swap candidate registered; not routable until cutover"
2761 );
2762 Ok(reply)
2763 }
2764
2765 fn build_hello_ack(
2766 &self,
2767 frame: &Frame,
2768 negotiated_ver: u8,
2769 module_id: &str,
2770 ) -> Result<Frame, RouterError> {
2771 let ack = ModuleHelloAckBody {
2772 negotiated_ver,
2773 subc_ops: module_subc_ops(),
2774 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2775 storage: self
2776 .storage_config
2777 .as_ref()
2778 .map(|cfg| cfg.descriptor_for(module_id)),
2779 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2780 };
2781 let body = serde_json::to_vec(&ack).map_err(|err| {
2782 RouterError::backend(
2783 0,
2784 frame.header.corr,
2785 format!("failed to encode HELLO_ACK: {err}"),
2786 )
2787 })?;
2788
2789 Frame::build_with_version(
2790 negotiated_ver,
2791 FrameType::HelloAck,
2792 control_flags(),
2793 0,
2794 0,
2795 frame.header.corr,
2796 body,
2797 )
2798 .map_err(RouterError::FrameBuild)
2799 }
2800
2801 async fn handle_client_control_request(
2802 &self,
2803 ctx: &RouteCtx,
2804 frame: Frame,
2805 request: ClientControlRequest,
2806 route_open_timings: &mut Option<RouteOpenTimings>,
2807 ) -> Result<Vec<Frame>, RouterError> {
2808 if let Some(operation) = privileged_operation(&request) {
2809 let caller = self.resolve_caller(ctx.connection_id).await;
2810 let op = client_control_request_op(&request);
2811 let text = prompt_text(operation, &caller);
2812 match self.privileged_operator_prompt {
2813 PrivilegedOperatorPrompt::Log => {
2814 info!(target: "subc_daemon::privileged_prompt", operation = op,
2815 target = ?operation, caller = %caller.description(), summary = %text,
2816 "privileged request would prompt the operator");
2817 }
2818 PrivilegedOperatorPrompt::Enforce => {
2819 let key =
2820 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2821 let outcome = self
2822 .forwarding
2823 .operator_confirms()
2824 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2825 .await;
2826 if outcome.code != "confirmed" {
2827 return Ok(vec![outcome.frame(&frame)]);
2828 }
2829 }
2830 }
2831 }
2832 match request {
2833 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2834 ClientControlRequest::CatalogList { module_id } => {
2835 self.handle_catalog_list(frame, module_id)
2836 }
2837 ClientControlRequest::RouteOpen {
2838 target,
2839 identity,
2840 consumer_identity,
2841 consumer_capabilities,
2842 role_versions,
2843 admission_facts,
2844 scope,
2845 } => {
2846 self.handle_route_open(
2847 ctx,
2848 frame,
2849 RouteOpenRequest {
2850 target,
2851 identity,
2852 consumer_identity,
2853 consumer_capabilities,
2854 role_versions,
2855 admission_facts,
2856 scope,
2857 },
2858 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2859 )
2860 .await
2861 }
2862 ClientControlRequest::RoutePoll {
2863 route_channel,
2864 route_epoch,
2865 kind,
2866 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2867 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2868 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2869 self.handle_supervisor_spawn_snapshot(frame)
2870 }
2871 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2872 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2873 }
2874 ClientControlRequest::SupervisorRestart {
2875 module_id,
2876 drain_timeout_ms,
2877 } => {
2878 self.log_supervisor_request_received(
2879 ctx,
2880 frame.header.corr,
2881 ops::SUPERVISOR_RESTART,
2882 Some(&module_id),
2883 None,
2884 )?;
2885 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2886 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2887 .await
2888 }
2889 ClientControlRequest::SupervisorSwap {
2890 module_id,
2891 ready_timeout_ms,
2892 } => {
2893 self.log_supervisor_request_received(
2894 ctx,
2895 frame.header.corr,
2896 ops::SUPERVISOR_SWAP,
2897 Some(&module_id),
2898 None,
2899 )?;
2900 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2901 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2902 .await
2903 }
2904 ClientControlRequest::SupervisorReload { module_id } => {
2905 self.log_supervisor_request_received(
2906 ctx,
2907 frame.header.corr,
2908 ops::SUPERVISOR_RELOAD,
2909 Some(&module_id),
2910 None,
2911 )?;
2912 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2913 self.handle_supervisor_reload(frame, module_id, requester)
2914 .await
2915 }
2916 ClientControlRequest::SupervisorRescan { preview } => {
2917 if !preview {
2918 self.log_supervisor_request_received(
2919 ctx,
2920 frame.header.corr,
2921 ops::SUPERVISOR_RESCAN,
2922 None,
2923 None,
2924 )?;
2925 }
2926 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2927 self.handle_supervisor_rescan(
2928 frame,
2929 if preview {
2930 RescanMode::Preview
2931 } else {
2932 RescanMode::Execute
2933 },
2934 requester,
2935 )
2936 .await
2937 }
2938 ClientControlRequest::SupervisorRescanChecked {
2939 preview,
2940 candidate,
2941 expect,
2942 } => {
2943 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2944 .await
2945 }
2946 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2947 self.log_supervisor_request_received(
2948 ctx,
2949 frame.header.corr,
2950 ops::SUPERVISOR_RELEASE_RESERVED,
2951 Some(&module_id),
2952 None,
2953 )?;
2954 self.handle_supervisor_release_reserved(frame, module_id)
2955 .await
2956 }
2957 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2958 self.log_supervisor_request_received(
2959 ctx,
2960 frame.header.corr,
2961 ops::SUPERVISOR_SET_ENABLED,
2962 Some(&module_id),
2963 Some(enabled),
2964 )?;
2965 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2966 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2967 .await
2968 }
2969 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2970 self.handle_supervisor_health_probe(frame, module_id).await
2971 }
2972 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2973 ClientControlRequest::SupervisorRoutes { module_id } => {
2974 self.handle_supervisor_routes(frame, module_id)
2975 }
2976 ClientControlRequest::SupervisorProvenance { module_id } => {
2977 self.handle_supervisor_provenance(frame, module_id).await
2978 }
2979 ClientControlRequest::SupervisorStderrTail {
2980 module_id,
2981 max_lines,
2982 max_bytes,
2983 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2984 ClientControlRequest::SupervisorTerminals { module_id } => {
2985 self.handle_supervisor_terminals(frame, module_id).await
2986 }
2987 }
2988 }
2989
2990 fn log_supervisor_request_received(
2991 &self,
2992 ctx: &RouteCtx,
2993 corr: u64,
2994 op: &'static str,
2995 module_id: Option<&str>,
2996 enabled: Option<bool>,
2997 ) -> Result<(), RouterError> {
2998 let caller = self.supervisor_requester(ctx, corr)?;
2999 let caller = principal_label(&caller);
3000 info!(
3003 target: "control",
3004 op,
3005 module_id,
3006 enabled,
3007 connection_id = ctx.connection_id.get(),
3008 caller = %caller,
3009 "supervisor request received"
3010 );
3011 Ok(())
3012 }
3013
3014 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3015 Ok(self
3016 .registry
3017 .get_module_by_connection(ctx.connection_id)
3018 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3019 .map(|registration| Principal::Reserved {
3020 module_id: registration.manifest.module_id,
3021 })
3022 .unwrap_or(Principal::Direct))
3023 }
3024
3025 fn handle_module_control_request(
3026 &self,
3027 connection_id: ConnectionId,
3028 frame: Frame,
3029 request: ModuleControlRequestFromModule,
3030 ) -> Result<Vec<Frame>, RouterError> {
3031 match request {
3032 ModuleControlRequestFromModule::CatalogUpdate {
3033 provides,
3034 capabilities,
3035 ready,
3036 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3037 ModuleControlRequestFromModule::LiveRoots {} => {
3038 let registered = self
3039 .registry
3040 .get_module_by_connection(connection_id)
3041 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3042 let Some(registration) = registered else {
3043 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3044 };
3045 let response = self
3046 .forwarding
3047 .live_roots(®istration.manifest.module_id)
3048 .map_err(RouterError::Forwarding)?;
3049 Ok(vec![control_response_body_frame(
3050 &frame,
3051 &response,
3052 "ModuleControlResponseToModule::LiveRoots",
3053 )?])
3054 }
3055 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3056 self.handle_scope_sync(connection_id, frame, generation, scopes)
3057 }
3058 ModuleControlRequestFromModule::ScopeApply {
3059 generation,
3060 upsert,
3061 end,
3062 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3063 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3064 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3065 }
3066 }
3067 }
3068
3069 fn handle_scope_served_by(
3070 &self,
3071 connection_id: ConnectionId,
3072 frame: Frame,
3073 ) -> Result<Vec<Frame>, RouterError> {
3074 let registered = self
3075 .registry
3076 .get_module_by_connection(connection_id)
3077 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3078 let Some(registered) = registered else {
3079 return Ok(vec![control_error_frame(
3080 &frame,
3081 "not_registered",
3082 "scope.served_by requires an active module registration owned by this connection",
3083 )?]);
3084 };
3085 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3086 {
3087 Ok(request) => request,
3088 Err(err) => {
3089 return Ok(vec![control_error_frame(
3090 &frame,
3091 "invalid_control_body",
3092 format!("malformed scope.served_by body: {err}"),
3093 )?])
3094 }
3095 };
3096 let answer = self
3097 .scopes
3098 .read()
3099 .unwrap_or_else(|poisoned| poisoned.into_inner())
3100 .served_by(
3101 ®istered.manifest.module_id,
3102 &request.scope_ref,
3103 request.scope_epoch,
3104 &request.module_id,
3105 );
3106 let reply = subc_control::ScopeServedByReply::new(
3107 answer,
3108 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3109 );
3110 Ok(vec![control_response_body_frame(
3111 &frame,
3112 &reply,
3113 "ScopeServedByReply",
3114 )?])
3115 }
3116
3117 fn handle_operator_confirm(
3118 &self,
3119 ctx: &RouteCtx,
3120 frame: Frame,
3121 ) -> Result<Vec<Frame>, RouterError> {
3122 use crate::operator_confirm::{audit, Outcome};
3123 let registration = self
3124 .registry
3125 .get_module_by_connection(ctx.connection_id)
3126 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3127 let Some(registration) = registration else {
3128 let outcome = Outcome::refusal("not_registered");
3129 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3130 return Ok(vec![outcome.frame(&frame)]);
3131 };
3132 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3133 Ok(request) => request,
3134 Err(_) => {
3135 let outcome = Outcome::refusal("invalid_control_body");
3136 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3137 return Ok(vec![outcome.frame(&frame)]);
3138 }
3139 };
3140 let module_id = registration.manifest.module_id;
3141 let nonce = self
3145 .hello_launch_nonces
3146 .lock()
3147 .unwrap_or_else(|p| p.into_inner())
3148 .nonce(ctx.connection_id)
3149 .map(str::to_owned);
3150 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3151 self.supervisor
3152 .spawned_consumer_authorized(&module_id, nonce)
3153 });
3154 let confirms = self.forwarding.operator_confirms();
3155 self.forwarding
3156 .with_operator_route(
3157 ctx.connection_id,
3158 request.route_channel,
3159 request.route_epoch,
3160 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3161 )
3162 .map_err(RouterError::Forwarding)
3163 }
3164
3165 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3168 let mut scopes = ScopeRecords::from(scopes);
3169 for (index, record) in scopes.records.iter_mut().enumerate() {
3170 if let Some(refusal) = Self::canonicalize_workdir(record) {
3171 scopes.refusals.insert(index, refusal);
3172 }
3173 }
3174 scopes
3175 }
3176
3177 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3178 let workdir = record.workdir.as_mut()?;
3179 let path = std::path::Path::new(workdir);
3180 if !path.is_absolute() {
3181 return Some((
3182 error_codes::INVALID_WORKDIR,
3183 "workdir must be absolute".to_string(),
3184 ));
3185 }
3186 match ProjectRootId::from_path_allowing_missing(path) {
3187 Ok(root) => match root.as_path().to_str() {
3188 Some(canonical) => {
3189 *workdir = canonical.to_string();
3190 None
3191 }
3192 None => Some((
3193 error_codes::INVALID_WORKDIR,
3194 "canonical workdir is not UTF-8".to_string(),
3195 )),
3196 },
3197 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3198 }
3199 }
3200
3201 fn handle_scope_sync(
3205 &self,
3206 connection_id: ConnectionId,
3207 frame: Frame,
3208 generation: u64,
3209 scopes: Vec<ScopeRecord>,
3210 ) -> Result<Vec<Frame>, RouterError> {
3211 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3212 }
3213
3214 fn handle_scope_change(
3215 &self,
3216 connection_id: ConnectionId,
3217 frame: Frame,
3218 generation: u64,
3219 scopes: Vec<ScopeRecord>,
3220 end: Option<Vec<ScopeEnd>>,
3221 ) -> Result<Vec<Frame>, RouterError> {
3222 let is_apply = end.is_some();
3223 let Some(registration) = self
3224 .registry
3225 .get_module_by_connection(connection_id)
3226 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3227 else {
3228 return Ok(vec![control_error_frame(
3229 &frame,
3230 "not_registered",
3231 "scope.sync and scope.apply require an active module registration owned by this connection",
3232 )?]);
3233 };
3234 let owner = registration.manifest.module_id;
3235 let scopes = Self::prepare_scope_records(scopes);
3236 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3237 let is_current_launch = |connection: ConnectionId| {
3238 self.hello_launch_nonces
3239 .lock()
3240 .unwrap_or_else(|poisoned| poisoned.into_inner())
3241 .presented(connection, current_nonce.as_deref())
3242 };
3243 let mut table = self
3248 .scopes
3249 .write()
3250 .unwrap_or_else(|poisoned| poisoned.into_inner());
3251 let outcome = match end {
3252 Some(end) => table.apply(
3253 &owner,
3254 connection_id,
3255 is_current_launch,
3256 generation,
3257 scopes,
3258 end,
3259 ),
3260 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3261 };
3262 let drained = match &outcome {
3263 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3264 Err(_) => Vec::new(),
3265 };
3266 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3267 drop(table);
3268 if has_deadlines {
3269 self.scope_deadline_added.notify_one();
3270 }
3271 match outcome {
3272 Ok(applied) => {
3273 let counts = ScopeOutcomeCounts::of(&applied.results);
3274 info!(
3275 owner = %owner,
3276 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3277 generation,
3278 records = applied.results.len(),
3279 created = counts.created,
3280 replaced = counts.replaced,
3281 updated = counts.updated,
3282 unchanged = counts.unchanged,
3283 refused = counts.refused,
3284 ended = applied.ended.len(),
3285 tag_changes = applied.tag_changes.len(),
3286 routes_closed = drained.len(),
3287 "scope change accepted"
3288 );
3289 for refused in applied
3295 .results
3296 .iter()
3297 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3298 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3299 {
3300 warn!(
3301 owner = %owner,
3302 generation,
3303 scope_ref = %refused.scope_ref,
3304 scope_epoch = refused.scope_epoch,
3305 code = refused.code.as_deref().unwrap_or(""),
3306 "scope record refused"
3307 );
3308 }
3309 self.close_scope_drained_routes(drained);
3310 let response = if is_apply {
3311 ModuleControlResponseToModule::ScopeApply {
3312 generation,
3313 results: applied.results,
3314 end_results: applied.end_results,
3315 ended: applied.ended,
3316 }
3317 } else {
3318 ModuleControlResponseToModule::ScopeSync {
3319 generation,
3320 results: applied.results,
3321 ended: applied.ended,
3322 }
3323 };
3324 Ok(vec![control_response_body_frame(
3325 &frame,
3326 &response,
3327 "scope change response",
3328 )?])
3329 }
3330 Err(refusal) => {
3331 info!(
3332 owner = %owner,
3333 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3334 generation,
3335 code = refusal.code,
3336 "scope change refused"
3337 );
3338 Ok(vec![control_error_frame(
3339 &frame,
3340 refusal.code,
3341 refusal.message,
3342 )?])
3343 }
3344 }
3345 }
3346
3347 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3353 if drained.is_empty() {
3354 return;
3355 }
3356 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3357 BTreeMap::new();
3358 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3359 for route in drained {
3360 warn!(
3361 module_id = %route.module_id,
3362 reason = ?route.reason,
3363 client_connection_id = route.client.connection_id.get(),
3364 route_channel = route.client.channel,
3365 "closing route because its scope changed"
3366 );
3367 pushes
3368 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3369 .or_insert_with(|| (route.reason, Vec::new()))
3370 .1
3371 .push(EndpointRoute {
3372 goodbye_target: route.client.clone(),
3373 principal: Principal::Unverified,
3374 bound_at: Instant::now(),
3375 draining: false,
3376 drain_reason: None,
3377 });
3378 goodbyes.push(route.module);
3379 goodbyes.push(route.client);
3380 }
3381 for ((module_id, _), (reason, routes)) in pushes {
3382 send_route_control_pushes(
3383 &self.forwarding,
3384 routes,
3385 ClientControlPush::RouteClosed {
3386 module_id,
3387 channels: Vec::new(),
3388 reason,
3389 drained: false,
3390 abandoned: 0,
3391 excluded_subscriptions: 0,
3392 terminal: Some(false),
3393 },
3394 );
3395 }
3396 self.emit_route_goodbyes(goodbyes);
3397 }
3398
3399 fn handle_scope_describe(
3402 &self,
3403 connection_id: ConnectionId,
3404 frame: Frame,
3405 owner: Principal,
3406 scope_ref: String,
3407 ) -> Result<Vec<Frame>, RouterError> {
3408 let registered = self
3409 .registry
3410 .get_module_by_connection(connection_id)
3411 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3412 let Some(registered) = registered else {
3413 return Ok(vec![control_error_frame(
3414 &frame,
3415 "not_registered",
3416 "scope.describe requires an active module registration owned by this connection",
3417 )?]);
3418 };
3419 let mut description = self
3420 .scopes
3421 .read()
3422 .unwrap_or_else(|poisoned| poisoned.into_inner())
3423 .describe(&owner, &scope_ref);
3424 if !registered
3427 .manifest
3428 .capabilities
3429 .as_ref()
3430 .is_some_and(|caps| {
3431 caps.provides
3432 .iter()
3433 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3434 })
3435 {
3436 if let Some(stamp) = description.stamp.as_mut() {
3437 stamp.attributes.on_behalf_of_agent = None;
3438 }
3439 }
3440 let owner_configured = match &owner {
3441 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3446 _ => false,
3447 };
3448 let response = ModuleControlResponseToModule::ScopeDescribe {
3449 status: description.status,
3450 scope_epoch: description.scope_epoch,
3451 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3452 owner_synced: description.owner_synced,
3453 owner_configured,
3454 scope: description.stamp,
3455 };
3456 Ok(vec![control_response_body_frame(
3457 &frame,
3458 &response,
3459 "ModuleControlResponseToModule::ScopeDescribe",
3460 )?])
3461 }
3462
3463 fn handle_catalog_update(
3464 &self,
3465 connection_id: ConnectionId,
3466 frame: Frame,
3467 provides: Vec<ProviderRole>,
3468 capabilities: Option<CapabilityDeclarations>,
3469 ready: Option<bool>,
3470 ) -> Result<Vec<Frame>, RouterError> {
3471 self.refresh_capability_requirements();
3472 let Some(registration) = self
3473 .registry
3474 .get_module_by_connection(connection_id)
3475 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3476 else {
3477 return Ok(vec![control_error_frame(
3478 &frame,
3479 "not_registered",
3480 "catalog.update requires an active module registration owned by this connection",
3481 )?]);
3482 };
3483
3484 if let Some(message) =
3485 catalog_update_frozen_field_message(®istration.manifest, &provides)
3486 {
3487 return Ok(vec![control_error_frame(
3488 &frame,
3489 "catalog_update_frozen_field",
3490 message,
3491 )?]);
3492 }
3493
3494 let mut candidate = registration.manifest.clone();
3495 candidate.provides = provides.clone();
3496 candidate.capabilities = capabilities
3497 .clone()
3498 .or_else(|| registration.manifest.capabilities.clone());
3499 if let Some(rich_ui) = &candidate.rich_ui {
3500 if let Err(err) = rich_ui.validate() {
3501 return Ok(vec![control_error_frame(
3502 &frame,
3503 "invalid_manifest",
3504 err.to_string(),
3505 )?]);
3506 }
3507 }
3508 if let Err(err) = candidate.validate_capability_grammar() {
3509 return Ok(vec![control_error_frame(
3510 &frame,
3511 "invalid_capability_grammar",
3512 err.to_string(),
3513 )?]);
3514 }
3515
3516 let mut conflicts = self
3519 .capability_evaluator
3520 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3521 if let Some(conflict) = conflicts.first() {
3522 let message = format!(
3523 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3524 conflict.capability, conflict.claimants[0], candidate.module_id
3525 );
3526 for conflict in &mut conflicts {
3527 conflict.source = DuplicateClaimSource::CatalogUpdate;
3528 }
3529 log_duplicate_claim_events(conflicts);
3530 return Ok(vec![control_error_frame(
3531 &frame,
3532 "reserved_capability",
3533 message,
3534 )?]);
3535 }
3536
3537 let updated = self
3538 .registry
3539 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3540 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3541 if updated.is_none() {
3542 return Ok(vec![control_error_frame(
3543 &frame,
3544 "not_registered",
3545 "catalog.update requires an active module registration owned by this connection",
3546 )?]);
3547 }
3548 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3549 log_duplicate_claim_events(
3550 self.capability_evaluator
3551 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3552 );
3553 }
3554 if capability_census_trigger(
3555 registration.manifest.capabilities.as_ref(),
3556 updated
3557 .as_ref()
3558 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3559 ) {
3560 self.enforce_capability_denies();
3561 }
3562 self.refresh_capability_requirements();
3563
3564 let response = ModuleControlResponseToModule::CatalogUpdate {};
3565 control_response_body_frame(
3566 &frame,
3567 &response,
3568 "ModuleControlResponseToModule::CatalogUpdate",
3569 )
3570 .map(|frame| vec![frame])
3571 }
3572
3573 pub(crate) fn with_activity_ledger_config(
3574 mut self,
3575 run_dir: &Path,
3576 incarnation: String,
3577 config: crate::activity::ActivityConfig,
3578 ) -> Self {
3579 self.activity = Some(crate::activity::ActivityLedger::open_config(
3580 run_dir.join("activity.jsonl"),
3581 incarnation,
3582 config,
3583 ));
3584 self.supervisor.set_activity(self.activity.clone());
3585 self
3586 }
3587
3588 pub(crate) async fn configure_activity(
3589 &self,
3590 config: crate::activity::ActivityConfig,
3591 ) -> Result<(), &'static str> {
3592 match &self.activity {
3593 Some(ledger) => ledger.configure(config).await,
3594 None => Ok(()),
3595 }
3596 }
3597
3598 fn cancel_activity_subscription(
3599 &self,
3600 ctx: &RouteCtx,
3601 frame: &Frame,
3602 ) -> Result<bool, RouterError> {
3603 let task = self
3604 .activity_streams
3605 .lock()
3606 .unwrap_or_else(|p| p.into_inner())
3607 .remove(&(ctx.connection_id, frame.header.corr));
3608 let Some(task) = task else {
3609 return Ok(false);
3610 };
3611 task.abort();
3612 let end = Frame::build_with_version(
3613 response_version(frame),
3614 FrameType::StreamEnd,
3615 control_flags(),
3616 0,
3617 0,
3618 frame.header.corr,
3619 Vec::new(),
3620 )
3621 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3622 let sink = ctx.egress.clone();
3623 tokio::spawn(async move {
3624 let _ = sink.send(end).await;
3625 });
3626 Ok(true)
3627 }
3628
3629 async fn handle_activity_read(
3630 &self,
3631 ctx: &RouteCtx,
3632 frame: Frame,
3633 ) -> Result<Vec<Frame>, RouterError> {
3634 use subc_control::{
3635 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3636 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3637 };
3638 #[derive(Deserialize)]
3639 struct Identity {
3640 #[serde(default)]
3641 consumer_identity: Option<ConsumerIdentity>,
3642 }
3643 let principal = match self
3647 .forwarding
3648 .module_id_for_connection(ctx.connection_id)
3649 .map_err(RouterError::Forwarding)?
3650 {
3651 Some(module_id) => Ok(Principal::Reserved { module_id }),
3652 None => {
3653 let identity: Identity = match serde_json::from_slice(&frame.body) {
3654 Ok(identity) => identity,
3655 Err(_) => {
3656 return Ok(vec![control_error_frame(
3657 &frame,
3658 ACTIVITY_MALFORMED,
3659 "malformed activity read",
3660 )?])
3661 }
3662 };
3663 self.route_open_principal(&frame, identity.consumer_identity)?
3664 }
3665 };
3666 let admitted = matches!(principal, Ok(Principal::Direct))
3667 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3668 if !admitted {
3669 return Ok(vec![control_error_frame(
3670 &frame,
3671 ACTIVITY_READ_FORBIDDEN,
3672 "activity reads require an operator or trusted mirror principal",
3673 )?]);
3674 }
3675 let Some(ledger) = &self.activity else {
3676 return Ok(vec![control_error_frame(
3677 &frame,
3678 ACTIVITY_UNAVAILABLE,
3679 "activity ledger unavailable",
3680 )?]);
3681 };
3682 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3683 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3684 {
3685 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3686 Ok(request) => ledger.query(request).await,
3687 Err(_) => Err(ACTIVITY_MALFORMED),
3688 };
3689 return match result {
3690 Ok(reply) => Ok(vec![control_response_body_frame(
3691 &frame,
3692 &reply,
3693 ops::ACTIVITY_QUERY,
3694 )?]),
3695 Err(code) => Ok(vec![control_error_frame(
3696 &frame,
3697 code,
3698 "activity query refused",
3699 )?]),
3700 };
3701 }
3702 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3703 Ok(request) => request,
3704 Err(_) => {
3705 return Ok(vec![control_error_frame(
3706 &frame,
3707 ACTIVITY_MALFORMED,
3708 "malformed activity subscription",
3709 )?])
3710 }
3711 };
3712 let mut subscription = match ledger.subscribe(request.since).await {
3713 Ok(subscription) => subscription,
3714 Err(code) => {
3715 return Ok(vec![control_error_frame(
3716 &frame,
3717 code,
3718 "activity subscription refused",
3719 )?])
3720 }
3721 };
3722 let key = (ctx.connection_id, frame.header.corr);
3723 let sink = ctx.egress.clone();
3724 let mut streams = self
3725 .activity_streams
3726 .lock()
3727 .unwrap_or_else(|p| p.into_inner());
3728 streams.retain(|_, task| !task.is_finished());
3729 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3730 return Ok(vec![control_error_frame(
3731 &frame,
3732 "duplicate_subscription",
3733 "activity subscription correlation id already in use",
3734 )?]);
3735 }
3736 let task = tokio::spawn(async move {
3737 loop {
3738 let entry = if let Some(entry) = subscription.replay.pop_front() {
3739 Some(entry)
3740 } else {
3741 subscription.receiver.recv().await
3742 };
3743 let Some(entry) = entry else {
3744 break;
3745 };
3746 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3747 return;
3748 };
3749 let Ok(next) = Frame::build_with_version(
3750 response_version(&frame),
3751 FrameType::StreamData,
3752 control_flags(),
3753 0,
3754 0,
3755 frame.header.corr,
3756 body,
3757 ) else {
3758 return;
3759 };
3760 if sink.send(next).await.is_err() {
3761 return;
3762 }
3763 }
3764 if let Some(code) = subscription.terminal_code() {
3767 if let Ok(end) =
3768 control_error_frame(&frame, code, "activity subscriber fell behind")
3769 {
3770 let _ = sink.send(end).await;
3771 }
3772 }
3773 });
3774 streams.insert(key, task.abort_handle());
3775 Ok(Vec::new())
3776 }
3777
3778 async fn handle_activity_record(
3779 &self,
3780 ctx: &RouteCtx,
3781 frame: Frame,
3782 ) -> Result<Vec<Frame>, RouterError> {
3783 use subc_control::{
3784 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3785 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3786 };
3787 let result = async {
3788 let module = self
3789 .forwarding
3790 .module_id_for_connection(ctx.connection_id)
3791 .ok()
3792 .flatten()
3793 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3794 let request: ActivityRecordRequest =
3795 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3796 crate::activity::validate(&module, &request)?;
3797 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3798 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3799 let at_ms = std::time::SystemTime::now()
3800 .duration_since(std::time::UNIX_EPOCH)
3801 .unwrap_or_default()
3802 .as_millis() as u64;
3803 ledger.record(module, request, actor, at_ms).await
3804 }
3805 .await;
3806 match result {
3807 Ok(entry) => Ok(vec![control_response_body_frame(
3808 &frame,
3809 &ActivityRecordResponse { seq: entry.seq },
3810 "activity.record",
3811 )?]),
3812 Err(code) => Ok(vec![control_error_frame(
3813 &frame,
3814 code,
3815 "activity recording refused",
3816 )?]),
3817 }
3818 }
3819
3820 fn activity_actor(
3821 &self,
3822 connection: ConnectionId,
3823 module: &str,
3824 on: Option<&subc_control::ActivityOn>,
3825 ) -> Result<subc_control::ActivityActor, &'static str> {
3826 use crate::forwarding::{DataRoute, DataRouteState};
3827 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3828 let principal = |id: &str| Principal::Reserved {
3829 module_id: id.to_owned(),
3830 };
3831 let mut actor = ActivityActor {
3832 principal: principal(module),
3833 scope_owner: None,
3834 agent_id: None,
3835 run_id: None,
3836 flow_id: None,
3837 project_root: None,
3838 workdir: None,
3839 project_id: None,
3840 };
3841 let scope = match on {
3842 None => return Ok(actor),
3843 Some(ActivityOn::Scope(scope)) => {
3844 let selector = subc_protocol::scope::ScopeSelector::new(
3845 principal(module),
3846 scope.scope_ref.clone(),
3847 Some(scope.epoch),
3848 );
3849 self.scopes
3850 .read()
3851 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3852 .attribute(&principal(module), module, &selector, true)
3853 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3854 }
3855 Some(ActivityOn::Route(route)) => {
3856 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3857 .forwarding
3858 .lookup_data_route(connection, route.channel, route.epoch)
3859 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3860 else {
3861 return Err(ACTIVITY_ACTOR_UNKNOWN);
3862 };
3863 if binding.module_id != module {
3864 return Err(ACTIVITY_ACTOR_UNKNOWN);
3865 }
3866 actor.principal = binding.principal.clone();
3867 actor.project_root = binding
3868 .project_root
3869 .as_ref()
3870 .map(|root| root.as_path().to_path_buf());
3871 let Some(bound) = &binding.scope else {
3872 return Ok(actor);
3873 };
3874 let owner = principal(&bound.owner);
3875 let selector = subc_protocol::scope::ScopeSelector::new(
3876 owner.clone(),
3877 bound.scope_ref.clone(),
3878 Some(bound.tag.scope_epoch),
3879 );
3880 let admitted = self
3881 .scopes
3882 .read()
3883 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3884 .attribute(&owner, module, &selector, true)
3885 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3886 if admitted.tag != bound.tag {
3887 return Err(ACTIVITY_ACTOR_UNKNOWN);
3888 }
3889 admitted
3890 }
3891 };
3892 actor.scope_owner = Some(scope.stamp.owner);
3893 actor.agent_id = scope.stamp.attributes.agent_id;
3894 actor.run_id = scope.stamp.attributes.run_id;
3895 actor.flow_id = scope.stamp.attributes.flow_id;
3896 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3897 actor.project_id = scope.stamp.project_id;
3898 Ok(actor)
3899 }
3900
3901 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3902 self.refresh_capability_requirements();
3903 let mut counters = self.counters.snapshot();
3909 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3910 obj.insert(
3911 "activity_unparseable_lines".into(),
3912 ledger.read_errors().into(),
3913 );
3914 obj.insert(
3915 "activity_lifecycle_append_failures".into(),
3916 ledger.lifecycle_failures().into(),
3917 );
3918 obj.insert(
3919 "activity_started_digest_skips".into(),
3920 ledger.lifecycle_digest_skips().into(),
3921 );
3922 }
3923 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3924 self.forwarding.client_route_concentration(),
3925 counters.as_object_mut(),
3926 ) {
3927 obj.insert(
3928 "client_connections_with_routes".into(),
3929 connections_with_routes.into(),
3930 );
3931 obj.insert("max_routes_on_one_connection".into(), max.into());
3932 }
3933 if let (Some(open_breakers), Some(obj)) = (
3939 self.route_bind_breakers.open_snapshot(),
3940 counters.as_object_mut(),
3941 ) {
3942 obj.insert("route_bind_breakers_open".into(), open_breakers);
3943 }
3944 let response = ClientControlResponse::ServerDescribe {
3945 protocol_ver: PROTOCOL_VERSION,
3946 subc_ops: subc_ops(),
3947 capabilities: self.describe_capabilities(),
3948 connected_clients: self.connected_clients.count(),
3949 counters: Some(counters),
3950 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3951 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3952 capability_requirements: self.capability_requirement_statuses(),
3953 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3954 config_path: self
3955 .rescan
3956 .as_ref()
3957 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3958 };
3959 Ok(vec![control_response_body_frame(
3960 &frame,
3961 &response,
3962 "ClientControlResponse::ServerDescribe",
3963 )?])
3964 }
3965
3966 fn handle_catalog_list(
3967 &self,
3968 frame: Frame,
3969 module_id: Option<String>,
3970 ) -> Result<Vec<Frame>, RouterError> {
3971 let (generation, modules) = self
3972 .registry
3973 .list_modules_with_nonce_verification()
3974 .map_err(|err| {
3975 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3976 })?;
3977 let entries = modules
3978 .into_iter()
3979 .filter(|(registration, _)| {
3980 module_id
3981 .as_deref()
3982 .map(|wanted| registration.manifest.module_id == wanted)
3983 .unwrap_or(true)
3984 })
3985 .map(|(registration, nonce_verified)| {
3986 let not_ready = self.not_ready_reason(®istration);
3987 let roles = registration.manifest.provides;
3988 CatalogEntry::new(
3989 registration.manifest.module_id,
3990 roles,
3991 registration.control_ops,
3992 )
3993 .with_ready(not_ready.is_none())
3994 .with_not_ready(not_ready)
3995 .with_module_version(Some(registration.manifest.module_version))
3996 .with_capabilities(registration.manifest.capabilities)
3997 .with_self_signals(registration.manifest.self_signals)
3998 .with_events(registration.manifest.events)
3999 .with_operation_delivery(registration.manifest.operation_delivery)
4000 .with_operation_payload(registration.manifest.operation_payload)
4001 .with_rich_ui(registration.manifest.rich_ui)
4002 .with_nonce_verified(nonce_verified)
4003 })
4004 .collect();
4005 let response = ClientControlResponse::CatalogList {
4006 generation,
4007 modules: entries,
4008 subc_ops: subc_ops(),
4009 };
4010 Ok(vec![control_response_body_frame(
4011 &frame,
4012 &response,
4013 "ClientControlResponse::CatalogList",
4014 )?])
4015 }
4016
4017 fn attest_route_open_consumer(
4022 &self,
4023 consumer_identity: Option<&ConsumerIdentity>,
4024 ) -> Option<Principal> {
4025 let Some(consumer_identity) = consumer_identity else {
4026 return Some(Principal::Direct);
4027 };
4028 self.supervisor
4029 .spawned_consumer_authorized(
4030 &consumer_identity.module_id,
4031 &consumer_identity.launch_nonce,
4032 )
4033 .then(|| Principal::Reserved {
4034 module_id: consumer_identity.module_id.clone(),
4035 })
4036 }
4037
4038 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4042 let Ok(ClientControlRequest::RouteOpen {
4043 identity,
4044 consumer_identity,
4045 scope,
4046 ..
4047 }) = parse_client_control_request(&frame.body)
4048 else {
4049 return RouteOpenCaller::unknown();
4050 };
4051 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4052 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4053 }
4054
4055 fn route_open_principal(
4056 &self,
4057 frame: &Frame,
4058 consumer_identity: Option<ConsumerIdentity>,
4059 ) -> Result<Result<Principal, Frame>, RouterError> {
4060 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4061 self.route_open_principal_attested(frame, consumer_identity, attested)
4062 }
4063
4064 fn route_open_principal_attested(
4068 &self,
4069 frame: &Frame,
4070 consumer_identity: Option<ConsumerIdentity>,
4071 attested: Option<Principal>,
4072 ) -> Result<Result<Principal, Frame>, RouterError> {
4073 if let Some(principal) = attested {
4074 return Ok(Ok(principal));
4075 }
4076 let Some(consumer_identity) = consumer_identity else {
4077 return Ok(Ok(Principal::Direct));
4078 };
4079
4080 Ok(Err(control_error_frame(
4081 frame,
4082 "bad_consumer_identity",
4083 format!(
4084 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4085 consumer_identity.module_id
4086 ),
4087 )?))
4088 }
4089
4090 #[allow(clippy::too_many_arguments)]
4097 fn route_open_refusal_frame(
4098 &self,
4099 ctx: &RouteCtx,
4100 caller: &RouteOpenCaller,
4101 frame: &Frame,
4102 module_id: &str,
4103 reason: &'static str,
4104 code: &'static str,
4105 message: impl Into<String>,
4106 ) -> Result<Frame, RouterError> {
4107 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4108 control_error_frame(frame, code, message.into())
4109 }
4110
4111 #[allow(clippy::too_many_arguments)]
4129 fn route_open_breaker_refusal_frame(
4130 &self,
4131 ctx: &RouteCtx,
4132 caller: &RouteOpenCaller,
4133 frame: &Frame,
4134 module_id: &str,
4135 consecutive_timeouts: u32,
4136 retry_in: Duration,
4137 probe_in_flight: bool,
4138 ) -> Result<Frame, RouterError> {
4139 self.counters
4140 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4141 debug!(
4142 target: "control",
4143 code = "module_timeout",
4144 module_id = ?module_id,
4145 connection_id = ctx.connection_id.get(),
4146 principal = %caller.principal,
4147 scope = %caller.scope,
4148 harness = %caller.harness,
4149 consecutive_timeouts,
4150 retry_in_ms = retry_in.as_millis() as u64,
4151 probe_in_flight,
4152 "route.open refused by open bind-relay breaker"
4153 );
4154 let detail = if probe_in_flight {
4161 "one test route is already being tried; retry once it settles".to_string()
4162 } else {
4163 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4164 };
4165 control_error_frame(
4166 frame,
4167 "module_timeout",
4168 format!(
4169 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4170 timed out in a row); {detail}; its established routes are unaffected"
4171 ),
4172 )
4173 }
4174
4175 fn observe_route_open_refusal(
4188 &self,
4189 ctx: &RouteCtx,
4190 caller: &RouteOpenCaller,
4191 module_id: &str,
4192 reason: &'static str,
4193 code: &'static str,
4194 ) {
4195 self.counters.increment_route_open_refused(code);
4196 info!(
4197 target: "control",
4198 code,
4199 reason,
4200 module_id = ?module_id,
4201 connection_id = ctx.connection_id.get(),
4202 principal = %caller.principal,
4203 scope = %caller.scope,
4204 harness = %caller.harness,
4205 "route.open refused"
4206 );
4207 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4208 self.route_outages.record_not_serving(module_id, reason);
4209 }
4210 }
4211
4212 fn observe_route_open_accept(
4257 &self,
4258 ctx: &RouteCtx,
4259 module_id: &str,
4260 principal: &str,
4261 scope: &str,
4262 ) {
4263 self.route_outages.record_accepted(module_id);
4264 self.counters.increment_route_open_accepted(principal);
4265 info!(
4266 target: "control",
4267 principal,
4268 module_id,
4269 connection_id = ctx.connection_id.get(),
4270 scope = %scope,
4271 "route.open accepted"
4272 );
4273 }
4274
4275 fn supervised_absent_route_open_refusal_frame(
4276 &self,
4277 ctx: &RouteCtx,
4278 caller: &RouteOpenCaller,
4279 frame: &Frame,
4280 module_id: &str,
4281 code: &'static str,
4282 status: &crate::supervise::ModuleStatus,
4283 ) -> Result<Frame, RouterError> {
4284 self.counters.increment_route_open_refused(code);
4285 info!(
4286 target: "control",
4287 code,
4288 reason = "supervised_not_registered",
4289 module_id = ?module_id,
4290 connection_id = ctx.connection_id.get(),
4291 principal = %caller.principal,
4292 scope = %caller.scope,
4293 harness = %caller.harness,
4294 state = %status.state,
4295 enabled = status.enabled,
4296 live = status.live,
4297 "route.open refused"
4298 );
4299 self.route_outages
4303 .record_not_serving(module_id, "supervised_not_registered");
4304 let message = if let Some(pid) = status.waiting_for_previous_pid {
4305 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4306 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4307 })
4308 } else {
4309 format!(
4310 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4311 status.state, status.enabled, status.live
4312 )
4313 };
4314 control_error_frame(frame, code, message)
4315 }
4316
4317 async fn handle_route_open(
4318 &self,
4319 ctx: &RouteCtx,
4320 frame: Frame,
4321 request: RouteOpenRequest,
4322 timings: &mut RouteOpenTimings,
4323 ) -> Result<Vec<Frame>, RouterError> {
4324 let RouteOpenRequest {
4325 target,
4326 mut identity,
4327 consumer_identity,
4328 consumer_capabilities,
4329 role_versions,
4330 admission_facts,
4331 scope,
4332 } = request;
4333 let target_module_id = target_module_id(&target).to_string();
4334 timings.module_id.clone_from(&target_module_id);
4335 if self
4336 .registry
4337 .get_module_by_connection(ctx.connection_id)
4338 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4339 .is_some()
4340 {
4341 return Ok(vec![control_error_frame(
4342 &frame,
4343 "invalid_request",
4344 "module connections cannot open client routes",
4345 )?]);
4346 }
4347 debug!(
4348 connection_id = ctx.connection_id.get(),
4349 corr = frame.header.corr,
4350 module_id = %target_module_id,
4351 "handling route.open"
4352 );
4353
4354 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4359 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4360
4361 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4367 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4368 self.observe_route_open_refusal(
4369 ctx,
4370 &caller,
4371 &target_module_id,
4372 "invalid_role_versions",
4373 error_codes::INVALID_REQUEST,
4374 );
4375 return Ok(vec![control_error_body_frame(
4376 &frame,
4377 ErrorBody {
4378 code: error_codes::INVALID_REQUEST.to_string(),
4379 message: error.to_string(),
4380 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4381 },
4382 )?]);
4383 }
4384
4385 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4386 if status.waiting_for_previous_pid.is_some() {
4387 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4388 ctx,
4389 &caller,
4390 &frame,
4391 &target_module_id,
4392 "module_warming",
4393 &status,
4394 )?]);
4395 }
4396 }
4397
4398 let Some(registration) = self
4417 .registry
4418 .get_module(&target_module_id)
4419 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4420 else {
4421 if let Some((status, warming)) =
4422 self.supervisor_status(&target_module_id, frame.header.corr)?
4423 {
4424 if status.protocol == ModuleProtocol::None {
4431 return Ok(vec![self.route_open_refusal_frame(
4432 ctx,
4433 &caller,
4434 &frame,
4435 &target_module_id,
4436 "protocol_none",
4437 error_codes::MODULE_NO_PROTOCOL,
4438 format!(
4439 "module_id '{target_module_id}' is declared protocol: none; \
4440 it speaks no subc wire and serves no routes"
4441 ),
4442 )?]);
4443 }
4444 let code = if warming {
4445 "module_warming"
4446 } else {
4447 "target_unavailable"
4448 };
4449 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4450 ctx,
4451 &caller,
4452 &frame,
4453 &target_module_id,
4454 code,
4455 &status,
4456 )?]);
4457 }
4458 if let Some(removed_ago_ms) =
4459 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4460 {
4461 return Ok(vec![self.route_open_refusal_frame(
4462 ctx,
4463 &caller,
4464 &frame,
4465 &target_module_id,
4466 "removed",
4467 error_codes::MODULE_REMOVED,
4468 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4469 )?]);
4470 }
4471 return Ok(vec![self.route_open_refusal_frame(
4472 ctx,
4473 &caller,
4474 &frame,
4475 &target_module_id,
4476 "not_registered",
4477 error_codes::UNKNOWN_MODULE,
4478 format!("module_id '{target_module_id}' is not registered"),
4479 )?]);
4480 };
4481
4482 if !registration.ready {
4486 self.counters
4487 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4488 info!(
4489 target: "control",
4490 code = error_codes::MODULE_WARMING,
4491 module_id = ?target_module_id,
4492 connection_id = ctx.connection_id.get(),
4493 principal = %caller.principal,
4494 scope = %caller.scope,
4495 harness = %caller.harness,
4496 reason = "declared_not_ready",
4497 "route.open refused"
4498 );
4499 self.route_outages
4502 .record_not_serving(&target_module_id, "declared_not_ready");
4503 return Ok(vec![control_error_body_frame(
4504 &frame,
4505 ErrorBody {
4506 code: error_codes::MODULE_WARMING.to_string(),
4507 message: format!(
4508 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4509 ),
4510 detail: Some(serde_json::json!({
4511 "reason": "declared_not_ready"
4512 })),
4513 },
4514 )?]);
4515 }
4516
4517 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4536 self.counters
4537 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4538 info!(
4539 target: "control",
4540 code = error_codes::MODULE_WARMING,
4541 module_id = ?target_module_id,
4542 connection_id = ctx.connection_id.get(),
4543 principal = %caller.principal,
4544 scope = %caller.scope,
4545 harness = %caller.harness,
4546 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4547 capability = %capability,
4548 "route.open refused"
4549 );
4550 return Ok(vec![control_error_body_frame(
4551 &frame,
4552 ErrorBody {
4553 code: error_codes::MODULE_WARMING.to_string(),
4554 message: format!(
4555 "module_id '{target_module_id}' requires capability '{capability}', \
4556 which no registered module provides; retry"
4557 ),
4558 detail: Some(serde_json::json!({
4559 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4560 "capability": capability,
4561 })),
4562 },
4563 )?]);
4564 }
4565
4566 if !target_has_required_role(&target, ®istration.manifest.provides) {
4567 return Ok(vec![self.route_open_refusal_frame(
4568 ctx,
4569 &caller,
4570 &frame,
4571 &target_module_id,
4572 "role_not_provided",
4573 "target_unavailable",
4574 format!("module_id '{target_module_id}' does not provide the requested target"),
4575 )?]);
4576 }
4577
4578 if registration.state != ChannelState::Active {
4579 return Ok(vec![self.route_open_refusal_frame(
4580 ctx,
4581 &caller,
4582 &frame,
4583 &target_module_id,
4584 "registration_not_active",
4585 "target_unavailable",
4586 format!("module_id '{target_module_id}' is not active"),
4587 )?]);
4588 }
4589
4590 if self
4591 .forwarding
4592 .module_is_draining(&target_module_id)
4593 .map_err(RouterError::Forwarding)?
4594 {
4595 return Ok(vec![self.route_open_refusal_frame(
4596 ctx,
4597 &caller,
4598 &frame,
4599 &target_module_id,
4600 "reloading",
4601 "module_reloading",
4602 format!("module_id '{target_module_id}' is reloading"),
4603 )?]);
4604 }
4605
4606 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4607 process_liveness.process_live(&target_module_id) == Some(false)
4608 }) {
4609 if process_liveness.process_replacing(&target_module_id) {
4616 return Ok(vec![self.route_open_refusal_frame(
4617 ctx,
4618 &caller,
4619 &frame,
4620 &target_module_id,
4621 "reloading",
4622 "module_reloading",
4623 format!("module_id '{target_module_id}' is reloading"),
4624 )?]);
4625 }
4626 return Ok(vec![self.route_open_refusal_frame(
4627 ctx,
4628 &caller,
4629 &frame,
4630 &target_module_id,
4631 "supervisor_not_live",
4632 "target_unavailable",
4633 format!("module_id '{target_module_id}' is not live"),
4634 )?]);
4635 }
4636
4637 if !self
4638 .forwarding
4639 .has_live_module_connection(&target_module_id)
4640 .map_err(RouterError::Forwarding)?
4641 {
4642 return Ok(vec![self.route_open_refusal_frame(
4643 ctx,
4644 &caller,
4645 &frame,
4646 &target_module_id,
4647 "no_forwarding_connection",
4648 "target_unavailable",
4649 format!("module_id '{target_module_id}' has no live forwarding connection"),
4650 )?]);
4651 }
4652
4653 if let Some(error) =
4654 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4655 {
4656 self.observe_route_open_refusal(
4657 ctx,
4658 &caller,
4659 &target_module_id,
4660 "op_not_allowed",
4661 "op_not_allowed",
4662 );
4663 return Ok(vec![error]);
4664 }
4665
4666 let principal =
4667 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4668 Ok(principal) => principal,
4669 Err(error) => {
4670 self.observe_route_open_refusal(
4671 ctx,
4672 &caller,
4673 &target_module_id,
4674 "bad_consumer_identity",
4675 "bad_consumer_identity",
4676 );
4677 return Ok(vec![error]);
4678 }
4679 };
4680 timings.principal = principal_label(&principal);
4681
4682 if let Principal::Reserved {
4686 module_id: opening_module_id,
4687 } = &principal
4688 {
4689 if let Some(opening_registration) = self
4690 .registry
4691 .get_module(opening_module_id)
4692 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4693 {
4694 if let Some(capability) =
4695 denied_capability(&opening_registration.manifest, ®istration.manifest)
4696 {
4697 warn!(
4698 opening_module_id,
4699 target_module_id,
4700 capability,
4701 "refusing route.open because an attested capability deny edge matches"
4702 );
4703 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4704 &frame,
4705 &target_module_id,
4706 "capability_deny_edge",
4707 "capability_forbidden",
4708 format!(
4709 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4710 ),
4711 )?]);
4712 }
4713 }
4714 }
4715
4716 if admission_facts.is_some() {
4717 let carrier_matches = matches!(
4718 &principal,
4719 Principal::Reserved { module_id }
4720 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4721 );
4722 if !carrier_matches {
4723 return Ok(vec![self.route_open_refusal_frame(
4724 ctx,
4725 &caller,
4726 &frame,
4727 &target_module_id,
4728 "admission_facts_carrier_not_permitted",
4729 "admission_facts_not_permitted",
4730 "admission facts may only be carried by the configured reserved module",
4731 )?]);
4732 }
4733
4734 let target_allowed = self
4735 .admission_facts_targets
4736 .as_ref()
4737 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4738 if !target_allowed {
4739 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4740 &frame,
4741 &target_module_id,
4742 "admission_facts_target_not_listed",
4743 "admission_facts_target_not_allowed",
4744 format!(
4745 "admission facts are not permitted for target module_id '{target_module_id}'"
4746 ),
4747 )?]);
4748 }
4749
4750 }
4754
4755 let (bound_scope, mut scope_stamp) = match &scope {
4761 None => (None, None),
4762 Some(selector) => {
4763 let owner_configured = match &selector.owner {
4764 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4770 _ => false,
4771 };
4772 let mut scopes = self
4773 .scopes
4774 .write()
4775 .unwrap_or_else(|poisoned| poisoned.into_inner());
4776 let admitted = scopes
4777 .admit(&principal, &target_module_id, selector, owner_configured)
4778 .and_then(|admission| {
4779 crate::scopes::check_target_flow_support(
4780 &admission.stamp,
4781 &target_module_id,
4782 registration.manifest.capabilities.as_ref(),
4783 )?;
4784 crate::scopes::check_target_agent_run_support(
4785 &admission.stamp,
4786 &target_module_id,
4787 registration.manifest.capabilities.as_ref(),
4788 )?;
4789 let bound = scopes.reserve_bind(
4790 &admission,
4791 &principal,
4792 ctx.connection_id,
4793 selector.hold_presence,
4794 );
4795 Ok((admission, bound))
4796 });
4797 match admitted {
4798 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4799 Err(refusal) => {
4800 return Ok(vec![self.route_open_refusal_frame(
4801 ctx,
4802 &caller,
4803 &frame,
4804 &target_module_id,
4805 refusal.code,
4806 refusal.code,
4807 refusal.message,
4808 )?]);
4809 }
4810 }
4811 }
4812 };
4813 if let Some(stamp) = &scope_stamp {
4816 caller.scope = route_open_scope_label(stamp);
4817 }
4818
4819 timings.enter(RouteOpenPhase::ProjectRoot);
4841 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4842 Ok(project_root) => project_root,
4843 Err(err) => {
4844 return Ok(vec![control_error_frame(
4845 &frame,
4846 "invalid_project_root",
4847 err.to_string(),
4848 )?])
4849 }
4850 };
4851 timings.enter(RouteOpenPhase::Admission);
4852 identity.project_root = project_root.as_path().to_path_buf();
4853
4854 let _concurrency_guard = match self
4865 .route_bind_concurrency
4866 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4867 {
4868 Ok(guard) => guard,
4869 Err(in_flight) => {
4870 return Ok(vec![self.route_open_target_capacity_refusal(
4871 ctx,
4872 &caller,
4873 &frame,
4874 &target_module_id,
4875 in_flight,
4876 )?]);
4877 }
4878 };
4879
4880 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4883 RouteBindAdmission::Admitted { guard, probe } => {
4884 if probe {
4885 info!(
4886 module_id = %target_module_id,
4887 connection_id = ctx.connection_id.get(),
4888 "route.bind breaker half-open: admitting one probe"
4889 );
4890 }
4891 guard
4892 }
4893 RouteBindAdmission::Refused {
4894 consecutive_timeouts,
4895 retry_in,
4896 probe_in_flight,
4897 } => {
4898 return Ok(vec![self.route_open_breaker_refusal_frame(
4899 ctx,
4900 &caller,
4901 &frame,
4902 &target_module_id,
4903 consecutive_timeouts,
4904 retry_in,
4905 probe_in_flight,
4906 )?]);
4907 }
4908 };
4909
4910 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4915 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4916 timings.enter(RouteOpenPhase::EgressReservation);
4917 let pending = match self
4918 .forwarding
4919 .begin_route_bind_relay_for(
4920 ctx.connection_id,
4921 ctx.egress.clone(),
4922 response_version(&frame),
4923 frame.header.corr,
4924 &target_module_id,
4925 principal.clone(),
4926 bound_scope,
4927 Some(project_root),
4928 relay_deadline,
4929 )
4930 .await
4931 {
4932 Ok(pending) => pending,
4933 Err(err) => {
4934 return Ok(vec![self.route_open_refusal_frame(
4935 ctx,
4936 &caller,
4937 &frame,
4938 &target_module_id,
4939 "relay_reservation_failed",
4940 forwarding_error_code(&err),
4941 err.to_string(),
4942 )?])
4943 }
4944 };
4945 timings.enter(RouteOpenPhase::Admission);
4946 let crate::forwarding::PendingRouteBindRelay {
4947 endpoint,
4948 module_sink,
4949 negotiated_ver,
4950 client_channel,
4951 client_epoch,
4952 module_channel,
4953 module_epoch,
4954 corr: relay_corr,
4955 receiver,
4956 } = pending;
4957 let mut reservation =
4958 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4959
4960 let relay_registration = self
4965 .registry
4966 .get_module_by_connection(endpoint.connection_id)
4967 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4968 let relay_capabilities = relay_registration
4969 .as_ref()
4970 .and_then(|registration| registration.manifest.capabilities.as_ref());
4971 if let Some(selector) = &scope {
4972 if let Err(refusal) = self
4975 .scopes
4976 .read()
4977 .unwrap_or_else(|poisoned| poisoned.into_inner())
4978 .check_bind_proof(&principal, selector)
4979 {
4980 reservation.release_and_disarm();
4981 return Ok(vec![self.route_open_refusal_frame(
4982 ctx,
4983 &caller,
4984 &frame,
4985 &target_module_id,
4986 refusal.code,
4987 refusal.code,
4988 refusal.message,
4989 )?]);
4990 }
4991 }
4992 if let Some(stamp) = scope_stamp
4993 .as_ref()
4994 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4995 {
4996 if let Err(refusal) = crate::scopes::check_target_flow_support(
4997 stamp,
4998 &target_module_id,
4999 relay_capabilities,
5000 )
5001 .and_then(|()| {
5002 crate::scopes::check_target_agent_run_support(
5003 stamp,
5004 &target_module_id,
5005 relay_capabilities,
5006 )
5007 }) {
5008 reservation.release_and_disarm();
5009 return Ok(vec![self.route_open_refusal_frame(
5010 ctx,
5011 &caller,
5012 &frame,
5013 &target_module_id,
5014 refusal.code,
5015 refusal.code,
5016 refusal.message,
5017 )?]);
5018 }
5019 }
5020
5021 if !relay_capabilities.is_some_and(|caps| {
5024 caps.provides
5025 .iter()
5026 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5027 }) {
5028 if let Some(stamp) = scope_stamp.as_mut() {
5029 stamp.workdir = None;
5030 stamp.project_id = None;
5031 }
5032 }
5033 if !relay_capabilities.is_some_and(|caps| {
5034 caps.provides
5035 .iter()
5036 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5037 }) {
5038 if let Some(stamp) = scope_stamp.as_mut() {
5039 stamp.attributes.on_behalf_of_agent = None;
5040 }
5041 }
5042 let call_mode = if relay_capabilities
5043 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5044 {
5045 Some(derive_call_mode(scope_stamp.as_ref()))
5046 } else {
5047 if let Some(stamp) = scope_stamp.as_mut() {
5048 stamp.expires_at_ms = None;
5049 }
5050 None
5051 };
5052
5053 debug!(
5054 connection_id = ctx.connection_id.get(),
5055 client_channel,
5056 client_epoch,
5057 module_channel,
5058 module_epoch,
5059 "reserved route handle pair"
5060 );
5061 let route_principal_label = principal_label(&principal);
5064 let relay = ModuleControlRequest::RouteBind {
5065 route_channel: module_channel,
5066 epoch: module_epoch,
5067 target,
5068 identity,
5069 principal: Some(principal),
5070 consumer_capabilities,
5071 role_versions,
5072 admission_facts,
5073 scope: scope_stamp,
5074 call_mode,
5075 };
5076 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5077 RouterError::backend(
5078 0,
5079 frame.header.corr,
5080 format!("failed to encode route.bind request: {err}"),
5081 )
5082 })?;
5083 let relay_frame = Frame::build_with_version(
5084 negotiated_ver,
5085 FrameType::Request,
5086 control_flags(),
5087 0,
5088 0,
5089 relay_corr,
5090 relay_body,
5091 )
5092 .map_err(RouterError::FrameBuild)?;
5093
5094 timings.enter(RouteOpenPhase::BindSend);
5095 let sent = module_sink.send(relay_frame).await;
5096 timings.enter(RouteOpenPhase::Admission);
5097 if let Err(err) = sent {
5098 reservation.release_and_disarm();
5099 return Ok(vec![self.route_open_refusal_frame(
5100 ctx,
5101 &caller,
5102 &frame,
5103 &target_module_id,
5104 "relay_send_failed",
5105 "target_unavailable",
5106 err.to_string(),
5107 )?]);
5108 }
5109
5110 #[cfg(test)]
5111 if let Some(hold) = &self.route_bind_relay_sent_hold {
5112 hold.wait().await;
5116 hold.wait().await;
5117 }
5118
5119 if self
5124 .forwarding
5125 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5126 .map_err(RouterError::Forwarding)?
5127 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5128 {
5129 self.send_abandoned_route_bind_goodbye(
5130 &module_sink,
5131 negotiated_ver,
5132 module_channel,
5133 module_epoch,
5134 );
5135 }
5136
5137 timings.enter(RouteOpenPhase::BindAckWait);
5138 let outcome = timeout_at(relay_deadline, receiver).await;
5139 timings.enter(RouteOpenPhase::Admission);
5140 match outcome {
5141 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5142 reservation.disarm();
5143 if breaker.record_accepted() {
5144 info!(
5145 module_id = %target_module_id,
5146 "route.bind breaker closed: the probe was accepted"
5147 );
5148 }
5149 self.observe_route_open_accept(
5150 ctx,
5151 &target_module_id,
5152 &route_principal_label,
5153 &caller.scope,
5154 );
5155 Ok(Vec::new())
5156 }
5157 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5158 reservation.release_and_disarm();
5159 breaker.record_inconclusive();
5163 let scope_code = match body.code.as_str() {
5167 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5168 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5169 _ => None,
5170 };
5171 if let Some(code) = scope_code {
5172 self.observe_route_open_refusal(
5173 ctx,
5174 &caller,
5175 &target_module_id,
5176 "scope_changed_before_commit",
5177 code,
5178 );
5179 return Ok(vec![control_error_body_frame(&frame, body)?]);
5180 }
5181 self.counters
5182 .increment_route_open_refused("module_rejected");
5183 info!(
5184 target: "control",
5185 code = "module_rejected",
5186 module_code = ?body.code,
5187 module_id = ?target_module_id,
5188 connection_id = ctx.connection_id.get(),
5189 principal = %caller.principal,
5190 scope = %caller.scope,
5191 harness = %caller.harness,
5192 "route.open refused"
5193 );
5194 Ok(vec![control_error_body_frame(&frame, body)?])
5195 }
5196 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5197 reservation.release_and_disarm();
5198 breaker.record_inconclusive();
5199 tracing::warn!(
5206 module_id = %target_module_id,
5207 "route.bind relay abandoned: {message}"
5208 );
5209 Ok(vec![self.route_open_refusal_frame(
5210 ctx,
5211 &caller,
5212 &frame,
5213 &target_module_id,
5214 "relay_abandoned",
5215 "target_unavailable",
5216 message,
5217 )?])
5218 }
5219 Ok(Err(_)) => {
5220 reservation.release_and_disarm();
5221 breaker.record_inconclusive();
5222 Ok(vec![self.route_open_refusal_frame(
5223 ctx,
5224 &caller,
5225 &frame,
5226 &target_module_id,
5227 "relay_waiter_canceled",
5228 "target_unavailable",
5229 "route.bind relay waiter was canceled before the module responded",
5230 )?])
5231 }
5232 Err(_) => {
5233 reservation.release_and_disarm();
5234 if let Some(opened) = breaker.record_timeout(
5238 self.route_bind_breaker_threshold,
5239 self.route_bind_breaker_cooldown,
5240 ) {
5241 warn!(
5242 module_id = %target_module_id,
5243 consecutive_timeouts = opened.consecutive_timeouts,
5244 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5245 reopened_after_probe = opened.reopened_after_probe,
5246 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5247 );
5248 }
5249 tracing::warn!(
5257 module_id = %target_module_id,
5258 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5259 "route.bind relay timed out: module did not ack within budget"
5260 );
5261 Ok(vec![self.route_open_refusal_frame(
5262 ctx,
5263 &caller,
5264 &frame,
5265 &target_module_id,
5266 "relay_timed_out",
5267 "module_timeout",
5268 format!(
5269 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5270 route_bind_relay_timeout
5271 ),
5272 )?])
5273 }
5274 }
5275 }
5276
5277 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5278 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5279 snapshot: self.supervisor.spawn_snapshot(),
5280 };
5281 Ok(vec![control_response_body_frame(
5282 &frame,
5283 &response,
5284 "ClientControlResponse::SupervisorSpawnSnapshot",
5285 )?])
5286 }
5287
5288 fn handle_supervisor_spawn_subscribe(
5289 &self,
5290 ctx: &RouteCtx,
5291 frame: Frame,
5292 since: Option<SpawnCursor>,
5293 ) -> Result<Vec<Frame>, RouterError> {
5294 match self.supervisor.subscribe_spawns(
5295 ctx.connection_id,
5296 frame.header.corr,
5297 response_version(&frame),
5298 since,
5299 ctx.egress.clone(),
5300 ) {
5301 Ok(()) => Ok(Vec::new()),
5302 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5303 Ok(vec![control_error_body_frame(
5304 &frame,
5305 ErrorBody {
5306 code: "spawn_cursor_incarnation_mismatch".to_string(),
5307 message: "spawn cursor belongs to a different daemon incarnation"
5308 .to_string(),
5309 detail: Some(serde_json::json!({
5310 "current_daemon_incarnation": current
5311 })),
5312 },
5313 )?])
5314 }
5315 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5316 &frame,
5317 ErrorBody {
5318 code: "spawn_cursor_too_old".to_string(),
5319 message: "spawn cursor predates the retained event ring".to_string(),
5320 detail: Some(serde_json::json!({
5321 "oldest_retained_cursor": oldest
5322 })),
5323 },
5324 )?]),
5325 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5326 0,
5327 frame.header.corr,
5328 format!("failed to open supervisor spawn subscription: {error}"),
5329 )),
5330 }
5331 }
5332
5333 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5334 let generation = self
5335 .registry
5336 .generation()
5337 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5338 let mut modules = Vec::new();
5339 for module in self.supervisor.list() {
5340 let status = module.status_for_control("list").map_err(|err| {
5341 RouterError::backend(
5342 0,
5343 frame.header.corr,
5344 format!("failed to read supervisor status: {err}"),
5345 )
5346 })?;
5347 let (configured, _) = module.configuration().map_err(|err| {
5348 RouterError::backend(
5349 0,
5350 frame.header.corr,
5351 format!("failed to read module configuration: {err}"),
5352 )
5353 })?;
5354 let image = module.running_image_agreement().await;
5356 let resources = Some(module.child_resource_usage());
5359 let mut verdict =
5360 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5361 verdict.spec = Some(launch_spec_agreement(
5362 &configured,
5363 status.process_alive,
5364 status.launch_spec_digest.as_ref(),
5365 ));
5366 let pending_reload = Some(verdict);
5367 let configured_enabled = self
5368 .capability_evaluator
5369 .configured_enabled(&status.module_id);
5370 modules.push(
5373 SupervisorEntry::new(
5374 status.module_id,
5375 status.state.to_string(),
5376 status.enabled,
5377 status.live,
5378 status.health.status,
5379 )
5380 .with_configured_enabled(configured_enabled)
5381 .with_launch_nonce_env(Some(!cfg!(unix)))
5382 .with_launch_nonce_source(status.launch_nonce_source)
5383 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5384 .with_protocol(status.protocol)
5385 .with_pending_reload(pending_reload)
5386 .with_last_probe_ms(status.health.last_probe_ms)
5387 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5388 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5389 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5390 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5391 .with_restart_count(Some(status.restart_count))
5392 .with_max_restarts(Some(status.max_restarts))
5393 .with_lifetime_restarts(Some(status.lifetime_restarts))
5394 .with_spawn_generation(Some(status.spawn_generation))
5395 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5396 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5397 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5398 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5399 .with_resources(resources),
5400 );
5401 }
5402 let response = ClientControlResponse::SupervisorList {
5403 generation,
5404 modules,
5405 };
5406 Ok(vec![control_response_body_frame(
5407 &frame,
5408 &response,
5409 "ClientControlResponse::SupervisorList",
5410 )?])
5411 }
5412
5413 fn handle_supervisor_stderr_tail(
5414 &self,
5415 frame: Frame,
5416 module_id: String,
5417 max_lines: Option<u32>,
5418 max_bytes: Option<u32>,
5419 ) -> Result<Vec<Frame>, RouterError> {
5420 let Some(module) = self.supervisor.get(&module_id) else {
5421 return Ok(vec![control_error_frame(
5422 &frame,
5423 "unknown_module",
5424 format!("module_id '{module_id}' is not supervised"),
5425 )?]);
5426 };
5427
5428 let snapshot = module.stderr_tail(
5429 max_lines.map(|value| value as usize),
5430 max_bytes.map(|value| value as usize),
5431 );
5432
5433 let response = ClientControlResponse::SupervisorStderrTail {
5434 module_id,
5435 tail: StderrTail {
5436 capture: match snapshot.capture {
5437 CaptureState::Captured => StderrCaptureState::Captured,
5438 CaptureState::Incomplete { reason } => {
5439 StderrCaptureState::Incomplete { reason }
5440 }
5441 CaptureState::NotCaptured { reason } => {
5442 StderrCaptureState::NotCaptured { reason }
5443 }
5444 },
5445 entries: snapshot
5446 .entries
5447 .into_iter()
5448 .map(|entry| match entry {
5449 TailEntry::Line {
5450 text,
5451 truncated,
5452 at_ms,
5453 } => StderrTailEntry::Line {
5454 text,
5455 truncated,
5456 at_ms,
5457 },
5458 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5459 })
5460 .collect(),
5461 dropped_lines: snapshot.dropped_lines,
5462 },
5463 };
5464 Ok(vec![control_response_body_frame(
5465 &frame,
5466 &response,
5467 "ClientControlResponse::SupervisorStderrTail",
5468 )?])
5469 }
5470
5471 async fn handle_supervisor_terminals(
5472 &self,
5473 frame: Frame,
5474 module_id: String,
5475 ) -> Result<Vec<Frame>, RouterError> {
5476 let Some(module) = self.supervisor.get(&module_id) else {
5477 return Ok(vec![control_error_frame(
5478 &frame,
5479 "unknown_module",
5480 format!("module_id '{module_id}' is not supervised"),
5481 )?]);
5482 };
5483
5484 let terminals = module
5487 .read_durable_terminal_history()
5488 .await
5489 .map_err(|error| {
5490 RouterError::backend(
5491 0,
5492 frame.header.corr,
5493 format!("failed to read terminal history: {error}"),
5494 )
5495 })?;
5496 let response = ClientControlResponse::SupervisorTerminals {
5497 module_id,
5498 terminals,
5499 };
5500 Ok(vec![control_response_body_frame(
5501 &frame,
5502 &response,
5503 "ClientControlResponse::SupervisorTerminals",
5504 )?])
5505 }
5506
5507 fn handle_supervisor_routes(
5508 &self,
5509 frame: Frame,
5510 module_id: Option<String>,
5511 ) -> Result<Vec<Frame>, RouterError> {
5512 let modules = self
5513 .forwarding
5514 .route_census(module_id.as_deref())
5515 .map_err(RouterError::Forwarding)?
5516 .into_iter()
5517 .map(|(module_id, routes)| SupervisorRouteModule {
5518 module_id,
5519 routes: routes
5520 .into_iter()
5521 .map(|route| SupervisorRoute {
5522 consumer: match route.principal {
5523 Principal::Reserved { module_id } => {
5524 SupervisorRouteConsumer::Reserved { module_id }
5525 }
5526 Principal::Direct | Principal::Unverified => {
5527 SupervisorRouteConsumer::Direct {
5528 connection_id: route.goodbye_target.connection_id.get(),
5529 }
5530 }
5531 },
5532 age_ms: Instant::now()
5533 .saturating_duration_since(route.bound_at)
5534 .as_millis()
5535 .try_into()
5536 .unwrap_or(u64::MAX),
5537 draining: route.draining,
5538 drain_reason: route.drain_reason,
5539 })
5540 .collect(),
5541 })
5542 .collect();
5543 let response = ClientControlResponse::SupervisorRoutes { modules };
5544 Ok(vec![control_response_body_frame(
5545 &frame,
5546 &response,
5547 "ClientControlResponse::SupervisorRoutes",
5548 )?])
5549 }
5550
5551 async fn handle_supervisor_provenance(
5552 &self,
5553 frame: Frame,
5554 module_id: Option<String>,
5555 ) -> Result<Vec<Frame>, RouterError> {
5556 let mut selected = if let Some(module_id) = module_id {
5557 let Some(module) = self.supervisor.get(&module_id) else {
5558 return Ok(vec![control_error_frame(
5559 &frame,
5560 "unknown_module",
5561 format!("module_id '{module_id}' is not supervised"),
5562 )?]);
5563 };
5564 vec![module]
5565 } else {
5566 self.supervisor.list()
5567 };
5568
5569 let mut modules = Vec::with_capacity(selected.len());
5570 for module in selected.drain(..) {
5571 let (status, observed_image) = module
5572 .status_and_running_image_agreement()
5573 .await
5574 .map_err(|err| {
5575 RouterError::backend(
5576 0,
5577 frame.header.corr,
5578 format!("failed to read supervisor status: {err}"),
5579 )
5580 })?;
5581 let module_declared = self
5582 .registry
5583 .get_module(&status.module_id)
5584 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5585 .and_then(|registration| registration.manifest.provenance)
5586 .map(|build| ModuleDeclaredProvenance::Reported { build })
5587 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5588 #[cfg(test)]
5589 let running_image = match &self.provenance_probe_override {
5590 Some(result) => result.clone(),
5591 None => observed_image,
5592 };
5593 #[cfg(not(test))]
5594 let running_image = observed_image;
5595 modules.push(SupervisorModuleProvenance {
5596 module_id: status.module_id,
5597 module_declared,
5598 daemon_observed: SupervisorObservedProcess {
5599 launch_nonce_source: status.launch_nonce_source,
5600 pid: status.pid,
5601 spawned_at_ms: status.spawned_at_ms,
5602 spawned_from: status.spawned_from,
5603 running_image,
5604 },
5605 });
5606 }
5607 let daemon = SupervisorDaemonProvenance {
5608 daemon_build: self.daemon_provenance.build.clone(),
5609 daemon_observed: DaemonObservedProcess {
5610 pid: self.daemon_provenance.pid,
5611 started_at_ms: self
5612 .daemon_provenance
5613 .start_clock
5614 .map(|clock| clock.started_at_ms())
5615 .or(self.daemon_provenance.started_at_ms),
5616 running_image: self
5617 .daemon_provenance
5618 .probe
5619 .observe(
5620 self.daemon_provenance.pid,
5621 self.daemon_provenance.executable_path.as_deref(),
5622 self.daemon_provenance.executable_identity.clone(),
5623 self.daemon_provenance.process_start_time,
5624 )
5625 .await,
5626 },
5627 };
5628 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5629 Ok(vec![control_response_body_frame(
5630 &frame,
5631 &response,
5632 "ClientControlResponse::SupervisorProvenance",
5633 )?])
5634 }
5635
5636 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5637 self.refresh_capability_requirements();
5638 let generation = self
5639 .registry
5640 .generation()
5641 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5642 let modules = self
5643 .supervisor
5644 .list()
5645 .into_iter()
5646 .map(|module| {
5647 let status = module.status_for_control("health").map_err(|err| {
5648 RouterError::backend(
5649 0,
5650 frame.header.corr,
5651 format!("failed to read supervisor health: {err}"),
5652 )
5653 })?;
5654 let module_id = status.module_id;
5655 let capability_detail = self
5656 .capability_evaluator
5657 .required_problem_detail(&module_id);
5658 Ok(SupervisorHealthEntry {
5659 module_id,
5660 status: status.health.status,
5661 detail: append_capability_problem_detail(
5662 status.health.detail,
5663 capability_detail,
5664 ),
5665 metrics: status.health.metrics,
5666 consecutive_failures: status.health.consecutive_failures,
5667 late_answer_count: status.health.late_answer_count,
5668 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5669 last_action: status.health.last_action,
5670 last_action_ms: status.health.last_action_ms,
5671 last_probe_ms: status.health.last_probe_ms,
5672 })
5673 })
5674 .collect::<Result<Vec<_>, RouterError>>()?;
5675 let response = ClientControlResponse::SupervisorHealth {
5676 generation,
5677 modules,
5678 };
5679 Ok(vec![control_response_body_frame(
5680 &frame,
5681 &response,
5682 "ClientControlResponse::SupervisorHealth",
5683 )?])
5684 }
5685
5686 async fn handle_supervisor_restart(
5687 &self,
5688 frame: Frame,
5689 module_id: String,
5690 drain_timeout_ms: Option<u64>,
5691 requester: Principal,
5692 ) -> Result<Vec<Frame>, RouterError> {
5693 let operation_lock = self.supervisor.operation_lock();
5694 let _operation_guard = operation_lock.lock().await;
5695 let Some(module) = self.supervisor.get(&module_id) else {
5696 return Ok(vec![control_error_frame(
5697 &frame,
5698 "unknown_module",
5699 format!("module_id '{module_id}' is not supervised"),
5700 )?]);
5701 };
5702
5703 self.route_outages.mark_operator_action(&module_id);
5704 if let Err(err) = module
5705 .restart_requested(drain_timeout_ms, Some(requester))
5706 .await
5707 {
5708 self.route_outages
5709 .operator_action_ended_unrefused(&module_id);
5710 let (code, message) = match err {
5711 crate::supervise::SuperviseError::Disabled { .. } => {
5712 ("module_disabled", err.to_string())
5713 }
5714 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5715 ("swap_in_progress", err.to_string())
5716 }
5717 _ => (
5718 "target_unavailable",
5719 format!("failed to restart module_id '{module_id}': {err}"),
5720 ),
5721 };
5722 return Ok(vec![control_error_frame(&frame, code, message)?]);
5723 }
5724
5725 let response = ClientControlResponse::SupervisorAck {
5726 module_id,
5727 applied: true,
5728 };
5729 Ok(vec![control_response_body_frame(
5730 &frame,
5731 &response,
5732 "ClientControlResponse::SupervisorAck",
5733 )?])
5734 }
5735
5736 async fn handle_supervisor_swap(
5740 &self,
5741 frame: Frame,
5742 module_id: String,
5743 ready_timeout_ms: Option<u64>,
5744 requester: Principal,
5745 ) -> Result<Vec<Frame>, RouterError> {
5746 let module = {
5753 let operation_lock = self.supervisor.operation_lock();
5754 let _operation_guard = operation_lock.lock().await;
5755 self.supervisor.get(&module_id)
5756 };
5757 let Some(module) = module else {
5758 return Ok(vec![control_error_frame(
5759 &frame,
5760 "unknown_module",
5761 format!("module_id '{module_id}' is not supervised"),
5762 )?]);
5763 };
5764
5765 self.route_outages.mark_operator_action(&module_id);
5766 if let Err(err) = module
5767 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5768 .await
5769 {
5770 self.route_outages
5771 .operator_action_ended_unrefused(&module_id);
5772 use crate::supervise::SuperviseError;
5773 let message = err.to_string();
5774 let error = match err {
5775 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5776 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5777 code: "swap_refused".to_string(),
5778 message,
5779 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5780 },
5781 SuperviseError::SwapFailed {
5782 arm,
5783 candidate_exit,
5784 ..
5785 } => ErrorBody {
5786 code: "swap_failed".to_string(),
5787 message,
5788 detail: Some(serde_json::json!({
5789 "arm": arm.as_str(),
5790 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5791 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5792 })),
5793 },
5794 _ => ErrorBody::new(
5795 "target_unavailable",
5796 format!("failed to swap module_id '{module_id}': {message}"),
5797 ),
5798 };
5799 return Ok(vec![control_error_body_frame(&frame, error)?]);
5800 }
5801 self.route_outages
5805 .operator_action_ended_unrefused(&module_id);
5806
5807 let response = ClientControlResponse::SupervisorAck {
5808 module_id,
5809 applied: true,
5810 };
5811 Ok(vec![control_response_body_frame(
5812 &frame,
5813 &response,
5814 "ClientControlResponse::SupervisorAck",
5815 )?])
5816 }
5817
5818 async fn handle_supervisor_reload(
5819 &self,
5820 frame: Frame,
5821 module_id: String,
5822 requester: Principal,
5823 ) -> Result<Vec<Frame>, RouterError> {
5824 let operation_lock = self.supervisor.operation_lock();
5825 let _operation_guard = operation_lock.lock().await;
5826 let Some(module) = self.supervisor.get(&module_id) else {
5827 return Ok(vec![control_error_frame(
5828 &frame,
5829 "unknown_module",
5830 format!("module_id '{module_id}' is not supervised"),
5831 )?]);
5832 };
5833
5834 self.route_outages.mark_operator_action(&module_id);
5835 if let Err(err) = module.reload_requested(Some(requester)).await {
5836 self.route_outages
5837 .operator_action_ended_unrefused(&module_id);
5838 let (code, message) = match err {
5839 crate::supervise::SuperviseError::Disabled { .. } => {
5840 ("module_disabled", err.to_string())
5841 }
5842 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5843 ("swap_in_progress", err.to_string())
5844 }
5845 _ => (
5846 "reload_failed",
5847 format!("failed to reload module_id '{module_id}': {err}"),
5848 ),
5849 };
5850 return Ok(vec![control_error_frame(&frame, code, message)?]);
5851 }
5852
5853 let response = ClientControlResponse::SupervisorAck {
5854 module_id,
5855 applied: true,
5856 };
5857 Ok(vec![control_response_body_frame(
5858 &frame,
5859 &response,
5860 "ClientControlResponse::SupervisorAck",
5861 )?])
5862 }
5863
5864 async fn handle_supervisor_rescan(
5868 &self,
5869 frame: Frame,
5870 mode: RescanMode,
5871 requester: Principal,
5872 ) -> Result<Vec<Frame>, RouterError> {
5873 let preview = mode.previews();
5874 let Some(context) = self.rescan.clone() else {
5875 return Ok(vec![control_error_frame(
5876 &frame,
5877 "rescan_unavailable",
5878 "the daemon was not started with a reloadable config path".to_string(),
5879 )?]);
5880 };
5881
5882 let operation_lock = self.supervisor.operation_lock();
5883 let _operation_guard = operation_lock.lock().await;
5884 let loaded = match self.load_rescan_config(&mode, &context) {
5888 Ok(config) => config,
5889 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5890 };
5891 let Some((config, activity_config)) = loaded else {
5899 return Ok(vec![control_error_frame(
5900 &frame,
5901 "invalid_daemon_config",
5902 format!(
5903 "daemon config not found at {}; refusing to rescan (an absent config would \
5904 retire every supervised module)",
5905 context.config_path.display()
5906 ),
5907 )?]);
5908 };
5909 let (
5910 configured_port,
5911 storage_config,
5912 admission_facts_carrier_module_id,
5913 admission_facts_targets,
5914 scope_authority_owners,
5915 privileged_operator_prompt,
5916 modules,
5917 reserved_capabilities,
5918 ) = (
5919 config.port,
5920 config.storage,
5921 config.admission_facts_carrier_module_id,
5922 config.admission_facts_targets,
5923 config.scope_authority_owners,
5924 config.privileged_operator_prompt,
5925 config.modules,
5926 config.reserved_capabilities,
5927 );
5928
5929 let mut restart_required = Vec::new();
5938 for section in RestartRequiredSection::ALL {
5939 let changed = match section {
5940 RestartRequiredSection::Port => configured_port != context.configured_port,
5941 RestartRequiredSection::Storage => storage_config != context.storage_config,
5942 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5943 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5944 }
5945 RestartRequiredSection::AdmissionFactsTargets => {
5946 admission_facts_targets != context.admission_facts_targets
5947 }
5948 RestartRequiredSection::ScopeAuthorityOwners => {
5949 scope_authority_owners != context.scope_authority_owners
5950 }
5951 RestartRequiredSection::PrivilegedOperatorPrompt => {
5952 privileged_operator_prompt != context.privileged_operator_prompt
5953 }
5954 };
5955 if changed {
5956 restart_required.push(section.label().to_string());
5957 }
5958 }
5959 if !restart_required.is_empty() {
5960 warn!(
5961 config_path = %context.config_path.display(),
5962 sections = %restart_required.join(", "),
5963 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5964 );
5965 }
5966
5967 for configured in &modules {
5968 if let Err(err) = validate_spec(&configured.module_spec()) {
5969 return Ok(vec![control_error_frame(
5970 &frame,
5971 "invalid_daemon_config",
5972 format!("supervisor rescan rejected daemon config: {err}"),
5973 )?]);
5974 }
5975 }
5976
5977 let configured_capabilities = modules
5978 .iter()
5979 .map(|module| (module.module_id.clone(), module.enabled))
5980 .collect::<Vec<_>>();
5981 let preview_capability_warnings = if preview {
5982 let (_, registrations) = self.runtime_capability_snapshot()?;
5983 let current_modules = self
5984 .supervisor
5985 .list()
5986 .into_iter()
5987 .map(|module| module.module_id().to_string())
5988 .collect::<BTreeSet<_>>();
5989 let resulting_modules = configured_capabilities.clone();
5990 let removed = current_modules
5991 .into_iter()
5992 .filter(|module_id| {
5993 !resulting_modules
5994 .iter()
5995 .any(|(configured_id, _)| configured_id == module_id)
5996 })
5997 .collect::<Vec<_>>();
5998 let disabled = match &mode {
6001 RescanMode::Candidate(_) => resulting_modules
6002 .iter()
6003 .filter(|(_, enabled)| !enabled)
6004 .map(|(module_id, _)| module_id.clone())
6005 .collect::<Vec<_>>(),
6006 _ => Vec::new(),
6007 };
6008 self.capability_evaluator.preview_warnings(
6009 resulting_modules,
6010 &removed,
6011 &disabled,
6012 ®istrations,
6013 )
6014 } else {
6015 Vec::new()
6016 };
6017 let reconciled = match &mode {
6018 RescanMode::Candidate(candidate) => self
6019 .preview_candidate(&context, modules, candidate)
6020 .map(|result| (result, Vec::new())),
6021 _ => {
6022 self.reconcile_supervised_modules(
6023 &context.supervisor,
6024 modules,
6025 preview,
6026 requester.clone(),
6027 )
6028 .await
6029 }
6030 };
6031 let (result, changed) = match reconciled {
6032 Ok(result) => result,
6033 Err(message) => {
6034 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6035 }
6036 };
6037 if !preview {
6038 if let Some(ledger) = &self.activity {
6039 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6040 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6041 }
6042 if let Err(code) = self.configure_activity(activity_config).await {
6043 warn!(
6044 code,
6045 "activity retention could not be applied during rescan"
6046 );
6047 }
6048 self.capability_evaluator
6049 .configure(configured_capabilities, reserved_capabilities);
6050 self.capability_evaluator.wake_deadline_loop();
6051 self.refresh_capability_requirements();
6052 }
6053 let mut result = result;
6054 result.restart_required = restart_required;
6055 result.capability_warnings = preview_capability_warnings;
6056 let response = ClientControlResponse::SupervisorRescan { result };
6057 Ok(vec![control_response_body_frame(
6058 &frame,
6059 &response,
6060 "ClientControlResponse::SupervisorRescan",
6061 )?])
6062 }
6063
6064 async fn handle_supervisor_release_reserved(
6065 &self,
6066 frame: Frame,
6067 module_id: String,
6068 ) -> Result<Vec<Frame>, RouterError> {
6069 let Some(context) = self.rescan.clone() else {
6070 return Ok(vec![control_error_frame(
6071 &frame,
6072 "release_unavailable",
6073 "reserved-id release requires a daemon started with a reloadable config path",
6074 )?]);
6075 };
6076 let operation_lock = self.supervisor.operation_lock();
6077 let _operation_guard = operation_lock.lock().await;
6078 let loaded = match crate::daemon_config::load(&context.config_path) {
6079 Ok(Some(config)) => config,
6080 Ok(None) => {
6081 return Ok(vec![control_error_frame(
6082 &frame,
6083 "invalid_daemon_config",
6084 format!(
6085 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6086 context.config_path.display()
6087 ),
6088 )?])
6089 }
6090 Err(err) => {
6091 return Ok(vec![control_error_frame(
6092 &frame,
6093 "invalid_daemon_config",
6094 format!("unable to verify reserved-id release against daemon config: {err}"),
6095 )?])
6096 }
6097 };
6098 if loaded
6099 .modules
6100 .iter()
6101 .any(|configured| configured.module_id == module_id)
6102 {
6103 return Ok(vec![control_error_frame(
6104 &frame,
6105 "reserved_module_configured",
6106 format!(
6107 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6108 ),
6109 )?]);
6110 }
6111 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6112 return Ok(vec![control_error_frame(
6113 &frame,
6114 "reserved_gate_not_retained",
6115 format!(
6116 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6117 ),
6118 )?]);
6119 }
6120
6121 let response = ClientControlResponse::SupervisorAck {
6122 module_id,
6123 applied: true,
6124 };
6125 Ok(vec![control_response_body_frame(
6126 &frame,
6127 &response,
6128 "ClientControlResponse::SupervisorAck",
6129 )?])
6130 }
6131
6132 fn plan_rescan(
6136 &self,
6137 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6138 ) -> Result<RescanPlan, String> {
6139 let mut current = BTreeMap::new();
6140 for module in self.supervisor.list() {
6141 let (spec, health) = module.configuration().map_err(|err| {
6142 format!(
6143 "failed to read configuration for module_id '{}': {err}",
6144 module.module_id()
6145 )
6146 })?;
6147 let enabled = module
6148 .status()
6149 .map_err(|err| {
6150 format!(
6151 "failed to read status for module_id '{}': {err}",
6152 module.module_id()
6153 )
6154 })?
6155 .enabled;
6156 current.insert(
6157 module.module_id().to_string(),
6158 (module, spec, health, enabled),
6159 );
6160 }
6161 let configured = configured_modules
6162 .into_iter()
6163 .map(|module| (module.module_id.clone(), module))
6164 .collect::<BTreeMap<_, _>>();
6165
6166 let added = configured
6167 .keys()
6168 .filter(|module_id| !current.contains_key(*module_id))
6169 .cloned()
6170 .collect::<Vec<_>>();
6171 let removed = current
6172 .keys()
6173 .filter(|module_id| !configured.contains_key(*module_id))
6174 .cloned()
6175 .collect::<Vec<_>>();
6176 let mut changed_pending_reload = Vec::new();
6177 let mut configuration_changes = BTreeSet::new();
6178 let mut enabled_changes = BTreeSet::new();
6179 let mut unchanged = 0_u32;
6180
6181 for (module_id, configured_module) in &configured {
6182 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6183 else {
6184 continue;
6185 };
6186 let launch_changed = *current_spec != configured_module.module_spec();
6190 let configuration_changed =
6191 launch_changed || *current_health != configured_module.health;
6192 let enabled_changed = *current_enabled != configured_module.enabled;
6193 if configuration_changed {
6194 configuration_changes.insert(module_id.clone());
6195 }
6196 if launch_changed {
6197 changed_pending_reload.push(module_id.clone());
6198 }
6199 if enabled_changed {
6200 enabled_changes.insert(module_id.clone());
6201 }
6202 if !configuration_changed && !enabled_changed {
6203 unchanged = unchanged.saturating_add(1);
6204 }
6205 }
6206 Ok(RescanPlan {
6207 current,
6208 configured,
6209 added,
6210 removed,
6211 changed_pending_reload,
6212 configuration_changes,
6213 enabled_changes,
6214 unchanged,
6215 })
6216 }
6217
6218 async fn reconcile_supervised_modules(
6227 &self,
6228 supervisor: &Supervisor,
6229 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6230 preview: bool,
6231 requester: Principal,
6232 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6233 let plan = self.plan_rescan(configured_modules)?;
6234
6235 if preview {
6241 return Ok((plan.preview_result(), Vec::new()));
6242 }
6243 let changed = plan
6244 .configuration_changes
6245 .union(&plan.enabled_changes)
6246 .cloned()
6247 .collect::<Vec<_>>();
6248 let RescanPlan {
6249 current,
6250 configured,
6251 added,
6252 removed,
6253 changed_pending_reload,
6254 configuration_changes,
6255 enabled_changes,
6256 unchanged,
6257 } = plan;
6258 if !(added.is_empty()
6263 && removed.is_empty()
6264 && configuration_changes.is_empty()
6265 && enabled_changes.is_empty())
6266 {
6267 if let Some(context) = &self.rescan {
6268 context
6269 .generation
6270 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6271 }
6272 }
6273
6274 for module_id in &removed {
6275 let module = ¤t
6276 .get(module_id)
6277 .expect("removed module came from current supervisor state")
6278 .0;
6279 module
6280 .retire_requested(Some(requester.clone()))
6281 .await
6282 .map_err(|err| {
6283 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6284 })?;
6285 self.supervisor.record_rescan_removal(module_id);
6319 self.supervisor.retire(module_id);
6320 self.route_outages.forget(module_id);
6321 }
6322
6323 for module_id in configured.keys() {
6324 let Some((module, _, _, _)) = current.get(module_id) else {
6325 continue;
6326 };
6327 let configured_module = configured
6328 .get(module_id)
6329 .expect("configured module id came from configured map");
6330 if configuration_changes.contains(module_id) {
6331 module
6332 .update_configuration(
6333 configured_module.module_spec(),
6334 configured_module.health.clone(),
6335 configured_module.drain_timeout_ms,
6336 )
6337 .await
6338 .map_err(|err| {
6339 format!(
6340 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6341 )
6342 })?;
6343 }
6344 if enabled_changes.contains(module_id) {
6345 self.route_outages.mark_operator_action(module_id);
6348 module
6349 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6350 .await
6351 .map_err(|err| {
6352 self.route_outages.operator_action_ended_unrefused(module_id);
6353 format!(
6354 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6355 configured_module.enabled
6356 )
6357 })?;
6358 }
6359 }
6360
6361 for module_id in &added {
6362 let configured_module = configured
6363 .get(module_id)
6364 .expect("added module id came from configured map");
6365 supervisor
6366 .supervise_configured_requested(
6367 configured_module.module_spec(),
6368 configured_module.enabled,
6369 configured_module.health.clone(),
6370 configured_module.drain_timeout_ms,
6371 configured_module.restart,
6372 Some(requester.clone()),
6373 )
6374 .map_err(|err| {
6375 format!("failed to add module_id '{module_id}' during rescan: {err}")
6376 })?;
6377 }
6378
6379 Ok((
6380 SupervisorRescanResult {
6381 added,
6382 removed,
6383 changed_pending_reload,
6384 enabled_changes: enabled_changes.iter().cloned().collect(),
6385 unchanged,
6386 preview: false,
6387 restart_required: Vec::new(),
6390 capability_warnings: Vec::new(),
6391 module_actions: Vec::new(),
6392 earlier_pending: Vec::new(),
6393 expect: None,
6394 },
6395 changed,
6396 ))
6397 }
6398
6399 async fn handle_supervisor_set_enabled(
6400 &self,
6401 frame: Frame,
6402 module_id: String,
6403 enabled: bool,
6404 requester: Principal,
6405 ) -> Result<Vec<Frame>, RouterError> {
6406 let operation_lock = self.supervisor.operation_lock();
6407 let _operation_guard = operation_lock.lock().await;
6408 let Some(module) = self.supervisor.get(&module_id) else {
6409 return Ok(vec![control_error_frame(
6410 &frame,
6411 "unknown_module",
6412 format!("module_id '{module_id}' is not supervised"),
6413 )?]);
6414 };
6415
6416 self.route_outages.mark_operator_action(&module_id);
6419 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6420 Ok(applied) => applied,
6421 Err(err) => {
6422 self.route_outages
6423 .operator_action_ended_unrefused(&module_id);
6424 return Ok(vec![control_error_frame(
6425 &frame,
6426 "target_unavailable",
6427 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6428 )?]);
6429 }
6430 };
6431 if !applied {
6432 self.route_outages
6435 .operator_action_ended_unrefused(&module_id);
6436 }
6437
6438 self.capability_evaluator.wake_deadline_loop();
6439 self.refresh_capability_requirements();
6440 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6441 Ok(vec![control_response_body_frame(
6442 &frame,
6443 &response,
6444 "ClientControlResponse::SupervisorAck",
6445 )?])
6446 }
6447
6448 async fn handle_supervisor_health_probe(
6449 &self,
6450 frame: Frame,
6451 module_id: String,
6452 ) -> Result<Vec<Frame>, RouterError> {
6453 self.refresh_capability_requirements();
6454 let Some(registration) = self
6455 .registry
6456 .get_module(&module_id)
6457 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6458 else {
6459 return Ok(vec![control_error_frame(
6460 &frame,
6461 "unknown_module",
6462 format!("module_id '{module_id}' is not registered"),
6463 )?]);
6464 };
6465
6466 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6479 {
6480 return Ok(vec![control_error_frame(
6481 &frame,
6482 "health_not_advertised",
6483 format!("module_id '{module_id}' did not advertise health.check"),
6484 )?]);
6485 }
6486
6487 let deadline = Instant::now() + self.health_probe_timeout;
6488 let pending = match self.forwarding.begin_module_control_rpc_for(
6489 &module_id,
6490 MODULE_CONTROL_OP_HEALTH_CHECK,
6491 deadline,
6492 ) {
6493 Ok(pending) => pending,
6494 Err(err) => {
6495 return Ok(vec![control_error_frame(
6496 &frame,
6497 forwarding_error_code(&err),
6498 err.to_string(),
6499 )?])
6500 }
6501 };
6502
6503 let PendingModuleControlRpc {
6504 endpoint,
6505 module_sink,
6506 negotiated_ver,
6507 corr: probe_corr,
6508 receiver,
6509 } = pending;
6510 let mut guard =
6511 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6512 let probe_body =
6513 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6514 RouterError::backend(
6515 0,
6516 frame.header.corr,
6517 format!("failed to encode health.check request: {err}"),
6518 )
6519 })?;
6520 let probe_frame = Frame::build_with_version(
6521 negotiated_ver,
6522 FrameType::Request,
6523 control_flags(),
6524 0,
6525 0,
6526 probe_corr,
6527 probe_body,
6528 )
6529 .map_err(RouterError::FrameBuild)?;
6530
6531 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6534 Ok(Ok(())) => {}
6535 Ok(Err(err)) => {
6536 return Ok(vec![control_error_frame(
6537 &frame,
6538 "target_unavailable",
6539 err.to_string(),
6540 )?]);
6541 }
6542 Err(_) => {
6543 return Ok(vec![control_error_frame(
6544 &frame,
6545 "module_timeout",
6546 format!(
6547 "module_id '{module_id}' did not answer health.check within {:?}",
6548 self.health_probe_timeout
6549 ),
6550 )?]);
6551 }
6552 }
6553
6554 match timeout_at(deadline, receiver).await {
6555 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6556 guard.disarm();
6557 let Some(report) = response.health_report() else {
6558 return Ok(vec![control_error_frame(
6559 &frame,
6560 "invalid_control_body",
6561 "health.check RPC returned a non-health response",
6562 )?]);
6563 };
6564 let HealthReport {
6569 status,
6570 detail,
6571 metrics,
6572 } = report;
6573 let capability_detail = self
6574 .capability_evaluator
6575 .required_problem_detail(&module_id);
6576 let response = ClientControlResponse::SupervisorHealthProbe {
6577 module_id,
6578 status,
6579 detail: append_capability_problem_detail(detail, capability_detail),
6580 metrics,
6581 };
6582 Ok(vec![control_response_body_frame(
6583 &frame,
6584 &response,
6585 "ClientControlResponse::SupervisorHealthProbe",
6586 )?])
6587 }
6588 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6589 guard.disarm();
6590 Ok(vec![control_error_body_frame(&frame, body)?])
6591 }
6592 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6593 guard.disarm();
6594 Ok(vec![control_error_frame(
6595 &frame,
6596 "target_unavailable",
6597 message,
6598 )?])
6599 }
6600 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6601 guard.disarm();
6602 Ok(vec![control_error_frame(
6603 &frame,
6604 "invalid_control_body",
6605 message,
6606 )?])
6607 }
6608 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6609 guard.disarm();
6610 Ok(vec![control_error_frame(
6611 &frame,
6612 "invalid_control_body",
6613 format!("expected module-control op '{expected}', got '{actual}'"),
6614 )?])
6615 }
6616 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6617 guard.disarm();
6618 Ok(vec![control_error_frame(
6619 &frame,
6620 "module_timeout",
6621 format!(
6622 "module_id '{module_id}' answered health.check after {:?}",
6623 self.health_probe_timeout
6624 ),
6625 )?])
6626 }
6627 Ok(Err(_)) => Ok(vec![control_error_frame(
6628 &frame,
6629 "target_unavailable",
6630 "health.check waiter was canceled before the module responded",
6631 )?]),
6632 Err(_) => Ok(vec![control_error_frame(
6633 &frame,
6634 "module_timeout",
6635 format!(
6636 "module_id '{module_id}' did not answer health.check within {:?}",
6637 self.health_probe_timeout
6638 ),
6639 )?]),
6640 }
6641 }
6642
6643 fn supervisor_status(
6644 &self,
6645 module_id: &str,
6646 corr: u64,
6647 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6648 self.supervisor
6649 .get(module_id)
6650 .map(|module| {
6651 let warming = module.is_warming_for_control("status").map_err(|err| {
6652 RouterError::backend(
6653 0,
6654 corr,
6655 format!(
6656 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6657 ),
6658 )
6659 })?;
6660 module.status_for_control("status").map_err(|err| {
6661 RouterError::backend(
6662 0,
6663 corr,
6664 format!(
6665 "failed to read supervisor status for module_id '{module_id}': {err}"
6666 ),
6667 )
6668 }).map(|status| (status, warming))
6669 })
6670 .transpose()
6671 }
6672
6673 fn guard_module_control_op(
6674 &self,
6675 frame: &Frame,
6676 module_id: &str,
6677 op: &str,
6678 ) -> Result<Option<Frame>, RouterError> {
6679 if self.module_grants_op(module_id, op, frame.header.corr)? {
6680 return Ok(None);
6681 }
6682
6683 Ok(Some(control_error_frame(
6684 frame,
6685 "op_not_allowed",
6686 format!("module_id '{module_id}' did not grant control op '{op}'"),
6687 )?))
6688 }
6689
6690 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6691 let Some(registration) = self
6692 .registry
6693 .get_module(module_id)
6694 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6695 else {
6696 return Ok(false);
6697 };
6698 Ok(module_registration_grants_op(®istration.control_ops, op))
6699 }
6700
6701 fn handle_status_update(
6702 &self,
6703 endpoint: ModuleEndpointId,
6704 frame: Frame,
6705 ) -> Result<Vec<Frame>, RouterError> {
6706 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6707 Ok(update) => update,
6708 Err(err) => {
6709 if is_known_module_push_op(&frame.body) {
6714 return Ok(vec![control_error_frame(
6715 &frame,
6716 "invalid_control_body",
6717 format!("malformed module control push body: {err}"),
6718 )?]);
6719 }
6720 return Ok(Vec::new());
6721 }
6722 };
6723
6724 match update {
6725 ModuleControlPush::RouteStatus {
6726 route_channel,
6727 route_epoch,
6728 status,
6729 } => {
6730 self.forwarding
6731 .cache_status(endpoint, route_channel, route_epoch, status)
6732 .map_err(RouterError::Forwarding)?;
6733 }
6734 }
6735 Ok(Vec::new())
6736 }
6737
6738 fn handle_route_poll(
6739 &self,
6740 ctx: &RouteCtx,
6741 frame: Frame,
6742 route_channel: u16,
6743 route_epoch: u32,
6744 kind: PollKind,
6745 ) -> Result<Vec<Frame>, RouterError> {
6746 let snapshot = self
6747 .forwarding
6748 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6749 .map_err(RouterError::Forwarding)?;
6750 let response = match (kind, snapshot) {
6751 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6752 ClientControlResponse::RoutePoll {
6753 route_channel,
6754 route_epoch,
6755 status,
6756 live: None,
6757 }
6758 }
6759 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6760 route_channel,
6761 route_epoch,
6762 status: None,
6763 live: None,
6764 },
6765 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6766 let live = self
6792 .process_liveness
6793 .as_ref()
6794 .and_then(|source| source.process_live(&module_id))
6795 .unwrap_or(true);
6796 ClientControlResponse::RoutePoll {
6797 route_channel,
6798 route_epoch,
6799 status: None,
6800 live: Some(live),
6801 }
6802 }
6803 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6804 route_channel,
6805 route_epoch,
6806 status: None,
6807 live: Some(false),
6808 },
6809 };
6810
6811 Ok(vec![control_response_body_frame(
6812 &frame,
6813 &response,
6814 "ClientControlResponse::RoutePoll",
6815 )?])
6816 }
6817
6818 pub(crate) fn observe_module_control_completion(
6819 &self,
6820 completion: ModuleControlRpcCompletion,
6821 ) -> bool {
6822 match completion {
6823 ModuleControlRpcCompletion::Unknown => false,
6824 ModuleControlRpcCompletion::Settled => true,
6825 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6826 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6827 info!(
6828 module_id = %module_id,
6829 latency_ms,
6830 "late health.check answer proves the module is alive"
6831 );
6832 match self
6833 .supervisor
6834 .record_late_health_answer(&module_id, latency_ms)
6835 {
6836 Ok(true) => {}
6837 Ok(false) => debug!(
6838 module_id = %module_id,
6839 latency_ms,
6840 "late health.check answer has no active supervisor snapshot"
6841 ),
6842 Err(err) => warn!(
6843 module_id = %module_id,
6844 latency_ms,
6845 error = %err,
6846 "failed to record late health.check answer"
6847 ),
6848 }
6849 true
6850 }
6851 }
6852 }
6853
6854 fn refuse_to_end_module_connection_for_a_client(
6875 &self,
6876 module_connection_id: ConnectionId,
6877 corr: u64,
6878 err: ForwardingError,
6879 ) -> Result<(), RouterError> {
6880 if let ForwardingError::ConnectionClosing { connection_id } = err {
6881 if connection_id != module_connection_id {
6882 warn!(
6883 module_connection_id = module_connection_id.get(),
6884 client_connection_id = connection_id.get(),
6885 corr,
6886 "dropping a route.bind response for a closing client; the module connection keeps serving"
6887 );
6888 return Ok(());
6889 }
6890 }
6891 Err(RouterError::Forwarding(err))
6892 }
6893
6894 fn handle_module_relay_response(
6895 &self,
6896 connection_id: ConnectionId,
6897 frame: Frame,
6898 ) -> Result<Vec<Frame>, RouterError> {
6899 let mut secondary_error = None;
6900 let outcome = match frame.header.ty {
6901 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6902 Ok(probe) if probe.op == "route.bind" => {
6903 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6904 Ok(ModuleControlResponse::RouteBindAck {}) => {
6905 RouteBindRelayOutcome::Accepted
6906 }
6907 Ok(other) => {
6908 let message =
6909 format!("route.bind response carried unexpected body: {other:?}");
6910 secondary_error = Some(control_error_frame(
6911 &frame,
6912 "invalid_control_body",
6913 message.clone(),
6914 )?);
6915 RouteBindRelayOutcome::ModuleGone(message)
6916 }
6917 Err(err) => {
6918 let message = format!("malformed route.bind response body: {err}");
6919 secondary_error = Some(control_error_frame(
6920 &frame,
6921 "invalid_control_body",
6922 message.clone(),
6923 )?);
6924 RouteBindRelayOutcome::ModuleGone(message)
6925 }
6926 }
6927 }
6928 Ok(probe) => {
6929 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6930 {
6931 Ok(response) => ModuleControlRpcOutcome::Response(response),
6932 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6933 "malformed {} response body: {err}",
6934 probe.op
6935 )),
6936 };
6937 let completion = self
6938 .forwarding
6939 .complete_module_control_rpc(
6940 connection_id,
6941 frame.header.corr,
6942 Some(&probe.op),
6943 outcome,
6944 )
6945 .map_err(RouterError::Forwarding)?;
6946 if !self.observe_module_control_completion(completion) {
6947 debug!(
6948 connection_id = connection_id.get(),
6949 corr = frame.header.corr,
6950 op = %probe.op,
6951 "dropping late or unknown module-control RPC response"
6952 );
6953 }
6954 return Ok(Vec::new());
6955 }
6956 Err(err) => {
6957 if let Some(expected_op) = self
6958 .forwarding
6959 .pending_module_control_op(connection_id, frame.header.corr)
6960 .map_err(RouterError::Forwarding)?
6961 {
6962 let completion = self
6963 .forwarding
6964 .complete_module_control_rpc(
6965 connection_id,
6966 frame.header.corr,
6967 None,
6968 ModuleControlRpcOutcome::MalformedResponse(format!(
6969 "malformed {expected_op} response body: {err}"
6970 )),
6971 )
6972 .map_err(RouterError::Forwarding)?;
6973 if !self.observe_module_control_completion(completion) {
6974 debug!(
6975 connection_id = connection_id.get(),
6976 corr = frame.header.corr,
6977 "dropping late malformed module-control RPC response"
6978 );
6979 }
6980 return Ok(Vec::new());
6981 }
6982 let message = format!("malformed route.bind response body: {err}");
6983 secondary_error = Some(control_error_frame(
6984 &frame,
6985 "invalid_control_body",
6986 message.clone(),
6987 )?);
6988 RouteBindRelayOutcome::ModuleGone(message)
6989 }
6990 },
6991 FrameType::Error => {
6992 if self
6993 .forwarding
6994 .pending_module_control_op(connection_id, frame.header.corr)
6995 .map_err(RouterError::Forwarding)?
6996 .is_some()
6997 {
6998 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6999 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7000 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7001 "malformed module-control ERROR body: {err}"
7002 )),
7003 };
7004 let completion = self
7005 .forwarding
7006 .complete_module_control_rpc(
7007 connection_id,
7008 frame.header.corr,
7009 None,
7010 outcome,
7011 )
7012 .map_err(RouterError::Forwarding)?;
7013 if !self.observe_module_control_completion(completion) {
7014 debug!(
7015 connection_id = connection_id.get(),
7016 corr = frame.header.corr,
7017 "dropping late or unknown module-control RPC error"
7018 );
7019 }
7020 return Ok(Vec::new());
7021 }
7022 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7023 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7024 Err(err) => {
7025 let message = format!("malformed route.bind ERROR body: {err}");
7026 secondary_error = Some(control_error_frame(
7027 &frame,
7028 "invalid_control_body",
7029 message.clone(),
7030 )?);
7031 RouteBindRelayOutcome::ModuleGone(message)
7032 }
7033 }
7034 }
7035 ty => {
7036 return Ok(vec![control_error_frame(
7037 &frame,
7038 "unsupported_control_frame",
7039 format!("unsupported module channel-0 frame {ty:?}"),
7040 )?])
7041 }
7042 };
7043
7044 let settled =
7045 self.forwarding
7046 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7047 let completion = match settled {
7048 Ok(completion) => completion,
7049 Err(err) => {
7050 self.refuse_to_end_module_connection_for_a_client(
7051 connection_id,
7052 frame.header.corr,
7053 err,
7054 )?;
7055 return Ok(secondary_error.into_iter().collect());
7056 }
7057 };
7058 if let Some(target) = completion.abandoned.as_ref() {
7059 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7060 }
7061 if !completion.settled {
7062 debug!(
7063 connection_id = connection_id.get(),
7064 corr = frame.header.corr,
7065 frame_type = ?frame.header.ty,
7066 "dropping late or unknown route.bind relay response"
7067 );
7068 }
7069 Ok(secondary_error.into_iter().collect())
7070 }
7071
7072 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7073 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7074 self.cleanup_connection_with_end_reason(
7078 connection_id,
7079 RegistrationEndReason::ExplicitGoodbye,
7080 )
7081 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7082 Ok(Vec::new())
7083 }
7084}
7085
7086impl Default for ControlHandler {
7087 fn default() -> Self {
7088 Self::new(Arc::new(Registry::default()))
7089 }
7090}
7091
7092impl crate::supervise::SwapPromotionObserver for ControlHandler {
7093 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7094 self.apply_registration_capabilities(registration);
7095 }
7096}
7097
7098fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7099 CapabilityRequirementStatus {
7100 consumer: status.consumer,
7101 capability: status.capability,
7102 need: match status.need {
7103 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7104 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7105 },
7106 verdict: status.verdict.as_str().to_string(),
7107 episode_seq: status.episode_seq,
7108 config_satisfiable: status.config_satisfiable,
7109 runtime_available: status.runtime_available,
7110 detail: status.detail,
7111 }
7112}
7113
7114fn append_capability_problem_detail(
7115 detail: Option<String>,
7116 capability_detail: Option<String>,
7117) -> Option<String> {
7118 match (detail, capability_detail) {
7119 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7120 (Some(detail), None) => Some(detail),
7121 (None, Some(capability_detail)) => Some(capability_detail),
7122 (None, None) => None,
7123 }
7124}
7125
7126fn subc_ops() -> Vec<String> {
7127 SUBC_CONTROL_OPS
7128 .iter()
7129 .map(|op| (*op).to_string())
7130 .collect()
7131}
7132
7133fn module_subc_ops() -> Vec<String> {
7134 SUBC_CONTROL_OPS
7135 .iter()
7136 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7137 .map(|op| (*op).to_string())
7138 .collect()
7139}
7140
7141#[cfg(test)]
7142fn module_baseline_control_ops() -> Vec<String> {
7143 MODULE_BASELINE_CONTROL_OPS
7144 .iter()
7145 .map(|op| (*op).to_string())
7146 .collect()
7147}
7148
7149fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7150 let mut seen = HashSet::new();
7151 let mut effective = Vec::new();
7152 for op in MODULE_BASELINE_CONTROL_OPS {
7153 if seen.insert((*op).to_string()) {
7154 effective.push((*op).to_string());
7155 }
7156 }
7157 for op in declared.unwrap_or_default() {
7158 if seen.insert(op.clone()) {
7159 effective.push(op);
7160 }
7161 }
7162 effective
7163}
7164
7165fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7166 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7167}
7168
7169fn target_module_id(target: &RouteTarget) -> &str {
7170 match target {
7171 RouteTarget::ToolProvider { module_id }
7172 | RouteTarget::ManagementSurface { module_id }
7173 | RouteTarget::InternalService { module_id, .. } => module_id,
7174 }
7175}
7176
7177fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7178 roles.iter().any(|role| match (target, role) {
7179 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7180 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7181 (
7182 RouteTarget::InternalService { service_id, .. },
7183 ProviderRole::InternalService {
7184 service_id: provided,
7185 ..
7186 },
7187 ) => service_id == provided,
7188 _ => false,
7189 })
7190}
7191
7192fn is_routable_role(role: &ProviderRole) -> bool {
7193 matches!(
7194 role,
7195 ProviderRole::ToolProvider { .. }
7196 | ProviderRole::ManagementSurface { .. }
7197 | ProviderRole::InternalService { .. }
7198 )
7199}
7200
7201#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7202enum ControlRequestBodyError {
7203 UnknownOp,
7204 InvalidBody,
7205}
7206
7207#[derive(Debug, Deserialize)]
7208struct ControlOpProbe {
7209 op: String,
7210}
7211
7212const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7215
7216fn is_known_module_push_op(body: &[u8]) -> bool {
7217 serde_json::from_slice::<ControlOpProbe>(body)
7218 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7219 .unwrap_or(false)
7220}
7221
7222fn is_known_module_request_op(body: &[u8]) -> bool {
7223 serde_json::from_slice::<ControlOpProbe>(body)
7224 .map(|probe| is_module_to_subc_op(&probe.op))
7225 .unwrap_or(false)
7226}
7227
7228fn is_module_to_subc_op(op: &str) -> bool {
7229 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7230}
7231
7232fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7233 debug!(
7234 op = %op,
7235 connection_id = connection_id.get(),
7236 corr,
7237 "control dispatch"
7238 );
7239}
7240
7241fn log_slow_control_dispatch(
7242 dispatch_started_at: Option<StdInstant>,
7243 op: &'static str,
7244 connection_id: ConnectionId,
7245 corr: u64,
7246 route_open_timings: Option<&RouteOpenTimings>,
7247) {
7248 let Some(dispatch_started_at) = dispatch_started_at else {
7249 return;
7250 };
7251 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7253 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7254 if let Some(timings) = route_open_timings {
7255 warn!(
7256 op = %op,
7257 connection_id = connection_id.get(),
7258 corr,
7259 elapsed_ms = elapsed.as_millis() as u64,
7260 module_id = ?timings.module_id,
7261 principal = %timings.principal,
7262 admission_ms = timings.admission.as_millis() as u64,
7263 project_root_ms = timings.project_root.as_millis() as u64,
7264 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7265 bind_send_ms = timings.bind_send.as_millis() as u64,
7266 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7267 "slow control dispatch"
7268 );
7269 return;
7270 }
7271 warn!(
7272 op = %op,
7273 connection_id = connection_id.get(),
7274 corr,
7275 elapsed_ms = elapsed.as_millis() as u64,
7276 "slow control dispatch"
7277 );
7278 }
7279}
7280
7281#[cfg(test)]
7283type TestCallerLookup =
7284 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7285
7286fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7288 match request {
7289 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7290 Some(PrivilegedOperation::Restart { module_id })
7291 }
7292 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7293 Some(PrivilegedOperation::Swap { module_id })
7294 }
7295 ClientControlRequest::SupervisorReload { module_id } => {
7296 Some(PrivilegedOperation::Reload { module_id })
7297 }
7298 ClientControlRequest::SupervisorRescan { preview } => {
7299 if *preview {
7300 None
7301 } else {
7302 Some(PrivilegedOperation::Rescan)
7303 }
7304 }
7305 ClientControlRequest::SupervisorRescanChecked {
7310 preview,
7311 candidate,
7312 expect,
7313 } => (!*preview && candidate.is_none() && expect.is_some())
7314 .then_some(PrivilegedOperation::Rescan),
7315 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7316 Some(PrivilegedOperation::SetEnabled {
7317 module_id,
7318 enabled: *enabled,
7319 })
7320 }
7321 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7322 Some(PrivilegedOperation::ReleaseReserved { module_id })
7323 }
7324 ClientControlRequest::ServerDescribe {}
7325 | ClientControlRequest::CatalogList { .. }
7326 | ClientControlRequest::RouteOpen { .. }
7327 | ClientControlRequest::RoutePoll { .. }
7328 | ClientControlRequest::SupervisorList {}
7329 | ClientControlRequest::SupervisorHealth {}
7330 | ClientControlRequest::SupervisorHealthProbe { .. }
7331 | ClientControlRequest::SupervisorRoutes { .. }
7332 | ClientControlRequest::SupervisorProvenance { .. }
7333 | ClientControlRequest::SupervisorStderrTail { .. }
7334 | ClientControlRequest::SupervisorTerminals { .. }
7335 | ClientControlRequest::SupervisorSpawnSnapshot {}
7336 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7337 }
7338}
7339
7340fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7341 match request {
7342 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7343 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7344 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7345 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7346 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7347 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7348 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7349 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7350 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7351 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7352 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7353 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7354 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7355 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7356 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7357 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7358 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7359 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7360 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7361 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7362 }
7363}
7364
7365struct RouteOpenCaller {
7373 principal: String,
7377 scope: String,
7380 harness: String,
7383}
7384
7385impl RouteOpenCaller {
7386 fn new(
7387 principal: Option<&Principal>,
7388 scope: Option<&ScopeSelector>,
7389 identity: &BindIdentity,
7390 ) -> Self {
7391 Self {
7392 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7393 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7394 harness: bounded_log_text(&identity.harness),
7395 }
7396 }
7397
7398 fn unknown() -> Self {
7401 Self {
7402 principal: "unknown".to_string(),
7403 scope: "unknown".to_string(),
7404 harness: "unknown".to_string(),
7405 }
7406 }
7407}
7408
7409const MAX_LOGGED_TEXT_CHARS: usize = 64;
7412
7413fn bounded_log_text(value: &str) -> String {
7417 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7418 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7419 bounded.push('…');
7420 }
7421 format!("{bounded:?}")
7422}
7423
7424fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7430 use subc_protocol::scope::ScopeKind;
7431 let kind = match stamp.kind {
7432 ScopeKind::Head => "head",
7433 ScopeKind::Worker => "worker",
7434 ScopeKind::Ephemeral => "ephemeral",
7435 };
7436 format!(
7437 "{kind}/{}/{}@{}",
7438 principal_label(&stamp.owner),
7439 bounded_log_text(&stamp.scope_ref),
7440 stamp.scope_epoch
7441 )
7442}
7443
7444fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7451 let epoch = selector
7452 .scope_epoch
7453 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7454 format!(
7455 "requested/{}/{}@{epoch}",
7456 bounded_log_text(&principal_label(&selector.owner)),
7457 bounded_log_text(&selector.scope_ref),
7458 )
7459}
7460
7461fn principal_label(principal: &Principal) -> String {
7462 match principal {
7463 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7464 Principal::Direct => "direct".to_string(),
7465 other => format!("{other:?}"),
7466 }
7467}
7468
7469fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7470 match request {
7471 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7472 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7473 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7474 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7475 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7476 }
7477}
7478
7479fn parse_client_control_request(
7480 body: &[u8],
7481) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7482 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7483 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7484 Ok(probe)
7485 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7486 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7487 {
7488 ControlRequestBodyError::InvalidBody
7489 }
7490 Ok(_) => ControlRequestBodyError::UnknownOp,
7491 Err(_) => ControlRequestBodyError::InvalidBody,
7492 };
7493 (err, classification)
7494 })
7495}
7496
7497fn parse_module_control_request_from_module(
7498 body: &[u8],
7499) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7500 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7501 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7502 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7503 Ok(_) => ControlRequestBodyError::UnknownOp,
7504 Err(_) => ControlRequestBodyError::InvalidBody,
7505 };
7506 (err, classification)
7507 })
7508}
7509
7510#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7511enum ProviderRoleKind {
7512 ToolProvider,
7513 PipelineStage,
7514 ManagementSurface,
7515 InternalService,
7516}
7517
7518fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7519 match role {
7520 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7521 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7522 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7523 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7524 }
7525}
7526
7527fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7528 roles.iter().map(provider_role_kind).collect()
7529}
7530
7531const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7534
7535#[derive(Debug, Default, PartialEq, Eq)]
7537struct ScopeOutcomeCounts {
7538 created: usize,
7539 replaced: usize,
7540 updated: usize,
7541 unchanged: usize,
7542 refused: usize,
7543}
7544
7545impl ScopeOutcomeCounts {
7546 fn of(results: &[ScopeRecordResult]) -> Self {
7547 let mut counts = Self::default();
7548 for result in results {
7549 let slot = match result.outcome {
7550 ScopeRecordOutcome::Created => &mut counts.created,
7551 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7552 ScopeRecordOutcome::Updated => &mut counts.updated,
7553 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7554 ScopeRecordOutcome::Refused => &mut counts.refused,
7555 };
7556 *slot += 1;
7557 }
7558 counts
7559 }
7560}
7561
7562#[cfg(test)]
7563mod scope_outcome_count_tests {
7564 use super::*;
7565
7566 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7567 ScopeRecordResult {
7568 scope_ref: "r".to_string(),
7569 scope_epoch: 1,
7570 outcome,
7571 code: None,
7572 message: None,
7573 version: None,
7574 parent_state: None,
7575 }
7576 }
7577
7578 #[test]
7581 fn every_outcome_is_counted_in_its_own_field() {
7582 let results = [
7583 result(ScopeRecordOutcome::Created),
7584 result(ScopeRecordOutcome::Created),
7585 result(ScopeRecordOutcome::Replaced),
7586 result(ScopeRecordOutcome::Updated),
7587 result(ScopeRecordOutcome::Unchanged),
7588 result(ScopeRecordOutcome::Refused),
7589 result(ScopeRecordOutcome::Refused),
7590 result(ScopeRecordOutcome::Refused),
7591 ];
7592 assert_eq!(
7593 ScopeOutcomeCounts::of(&results),
7594 ScopeOutcomeCounts {
7595 created: 2,
7596 replaced: 1,
7597 updated: 1,
7598 unchanged: 1,
7599 refused: 3,
7600 }
7601 );
7602 }
7603}
7604
7605fn capability_census_trigger(
7609 old: Option<&CapabilityDeclarations>,
7610 new: Option<&CapabilityDeclarations>,
7611) -> bool {
7612 let old_provides = old
7613 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7614 .unwrap_or_default();
7615 let old_denies = old
7616 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7617 .unwrap_or_default();
7618 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7619 provides: Vec::new(),
7620 requires: Vec::new(),
7621 must_never_reach: Vec::new(),
7622 });
7623
7624 new.provides
7625 .iter()
7626 .any(|capability| !old_provides.contains(capability))
7627 || new
7628 .must_never_reach
7629 .iter()
7630 .any(|capability| !old_denies.contains(capability))
7631}
7632
7633fn denied_capability<'a>(
7636 opening_manifest: &'a ModuleManifest,
7637 target_manifest: &ModuleManifest,
7638) -> Option<&'a str> {
7639 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7640 let target_capabilities = target_manifest.capabilities.as_ref()?;
7641 opening_capabilities
7642 .must_never_reach
7643 .iter()
7644 .find(|denied| {
7645 target_capabilities
7646 .provides
7647 .iter()
7648 .any(|provided| provided == *denied)
7649 })
7650 .map(String::as_str)
7651}
7652
7653fn catalog_update_frozen_field_message(
7654 registered: &ModuleManifest,
7655 provides: &[ProviderRole],
7656) -> Option<String> {
7657 let old_has_provides = !registered.provides.is_empty();
7658 let new_has_provides = !provides.is_empty();
7659 if old_has_provides != new_has_provides {
7660 return Some(format!(
7661 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7662 registered.module_id
7663 ));
7664 }
7665
7666 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7667 return Some(format!(
7668 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7669 registered.module_id
7670 ));
7671 }
7672
7673 let registered_concurrency = manifest_concurrency(registered);
7674 let mut candidate = registered.clone();
7675 candidate.provides = provides.to_vec();
7676 let candidate_concurrency = manifest_concurrency(&candidate);
7677 if candidate_concurrency != registered_concurrency {
7678 return Some(format!(
7679 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7680 registered.module_id, registered_concurrency, candidate_concurrency
7681 ));
7682 }
7683
7684 None
7687}
7688
7689fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7690 manifest.provides.iter().any(is_routable_role)
7691}
7692
7693fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7699 manifest
7700 .provides
7701 .iter()
7702 .find_map(|provider| match provider {
7703 ProviderRole::ToolProvider { concurrency, .. }
7704 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7705 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7706 })
7707 .unwrap_or(Concurrency::ModuleManaged)
7708}
7709
7710fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7717 let has_management_surface = manifest
7718 .provides
7719 .iter()
7720 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7721 if !has_management_surface {
7722 return false;
7723 }
7724 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7725 return false;
7726 };
7727 let Some(provides) = raw
7728 .get("manifest")
7729 .and_then(|manifest| manifest.get("provides"))
7730 .and_then(serde_json::Value::as_array)
7731 else {
7732 return false;
7733 };
7734 provides.iter().any(|role| {
7739 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7740 && role.get("concurrency").is_none()
7741 })
7742}
7743
7744fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7745 if peer_version != PROTOCOL_VERSION {
7746 return Err(format!(
7747 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7748 ));
7749 }
7750 Ok(PROTOCOL_VERSION)
7751}
7752
7753fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7754 Frame::build_with_version(
7755 response_version(frame),
7756 FrameType::Pong,
7757 frame.header.flags,
7758 0,
7759 0,
7760 frame.header.corr,
7761 Vec::new(),
7762 )
7763 .map_err(RouterError::FrameBuild)
7764}
7765
7766pub(crate) fn control_error_frame(
7767 frame: &Frame,
7768 code: &'static str,
7769 message: impl Into<String>,
7770) -> Result<Frame, RouterError> {
7771 control_error_body_frame(
7772 frame,
7773 ErrorBody {
7774 code: code.to_string(),
7775 message: message.into(),
7776 detail: None,
7777 },
7778 )
7779}
7780
7781fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7782 let body = serde_json::to_vec(&error).map_err(|err| {
7783 RouterError::backend(
7784 0,
7785 frame.header.corr,
7786 format!("failed to encode control ERROR: {err}"),
7787 )
7788 })?;
7789
7790 Frame::build_with_version(
7791 response_version(frame),
7792 FrameType::Error,
7793 control_flags(),
7794 0,
7795 0,
7796 frame.header.corr,
7797 body,
7798 )
7799 .map_err(RouterError::FrameBuild)
7800}
7801
7802fn control_response_body_frame<T: Serialize>(
7803 frame: &Frame,
7804 reply: &T,
7805 label: &'static str,
7806) -> Result<Frame, RouterError> {
7807 let body = serde_json::to_vec(reply).map_err(|err| {
7808 RouterError::backend(
7809 0,
7810 frame.header.corr,
7811 format!("failed to encode {label}: {err}"),
7812 )
7813 })?;
7814
7815 Frame::build_with_version(
7816 response_version(frame),
7817 FrameType::Response,
7818 control_flags(),
7819 0,
7820 0,
7821 frame.header.corr,
7822 body,
7823 )
7824 .map_err(RouterError::FrameBuild)
7825}
7826
7827fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7857 match err {
7858 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7859 ForwardingError::NoModuleConnection => "target_unavailable",
7860 ForwardingError::ModuleReloading { .. } => "module_reloading",
7861 ForwardingError::ClientRouteChannelExhausted { .. }
7862 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7863 ForwardingError::StaleModuleEndpoint
7864 | ForwardingError::UnknownReservation { .. }
7865 | ForwardingError::ConnectionClosing { .. }
7866 | ForwardingError::ClientEgressClosed { .. }
7867 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7868 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7871 ForwardingError::RelayCorrelationExhausted
7872 | ForwardingError::RouteOpenBuild(_)
7873 | ForwardingError::Poisoned => "forwarding_error",
7874 }
7875}
7876
7877fn response_version(frame: &Frame) -> u8 {
7878 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7879 frame.header.ver
7880 } else {
7881 PROTOCOL_VERSION
7882 }
7883}
7884
7885fn control_flags() -> Flags {
7886 Flags::new(false, Priority::Passive, false)
7887}
7888
7889fn send_goodbye_target_best_effort(
7894 counters: &DaemonCounters,
7895 target: &GoodbyeTarget,
7896 context: &'static str,
7897) {
7898 let Ok(frame) = Frame::build_with_version(
7899 target.negotiated_ver,
7900 FrameType::Goodbye,
7901 control_flags(),
7902 target.channel,
7903 target.epoch,
7904 0,
7905 Vec::new(),
7906 ) else {
7907 return;
7908 };
7909 crate::forwarding::send_module_route_goodbye(
7910 counters,
7911 &target.sink,
7912 frame,
7913 target.module_id.as_deref(),
7914 context,
7915 );
7916}
7917
7918pub(crate) fn send_route_control_pushes(
7919 forwarding: &ForwardingTable,
7920 routes: Vec<EndpointRoute>,
7921 push: ClientControlPush,
7922) {
7923 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7924 for route in routes {
7925 let target = route.goodbye_target;
7926 if let Some((existing, channels)) = targets
7927 .iter_mut()
7928 .find(|(existing, _)| existing.connection_id == target.connection_id)
7929 {
7930 debug_assert_eq!(
7931 existing.negotiated_ver, target.negotiated_ver,
7932 "one connection cannot negotiate multiple frame versions"
7933 );
7934 if !channels.contains(&target.channel) {
7935 channels.push(target.channel);
7936 }
7937 continue;
7938 }
7939 let channel = target.channel;
7940 targets.push((target, vec![channel]));
7941 }
7942 for (target, mut channels) in targets {
7943 channels.sort_unstable();
7944 let mut push = push.clone();
7945 match &mut push {
7946 ClientControlPush::RouteClosing {
7947 channels: covered, ..
7948 }
7949 | ClientControlPush::RouteClosed {
7950 channels: covered, ..
7951 } => *covered = channels,
7952 }
7953 let body = match serde_json::to_vec(&push) {
7954 Ok(body) => body,
7955 Err(err) => {
7956 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7957 continue;
7958 }
7959 };
7960 let frame = match Frame::build_with_version(
7961 target.negotiated_ver,
7962 FrameType::Push,
7963 control_flags(),
7964 0,
7965 0,
7966 0,
7967 body.clone(),
7968 ) {
7969 Ok(frame) => frame,
7970 Err(err) => {
7971 warn!(
7972 route_channel = target.channel,
7973 error = %err,
7974 "failed to build route lifecycle control PUSH frame"
7975 );
7976 continue;
7977 }
7978 };
7979 if let Err(err) = target.sink.try_send(frame) {
7980 if target.close_on_delivery_failure() {
7981 warn!(
7982 target_connection_id = target.connection_id.get(),
7983 route_channel = target.channel,
7984 error = %err,
7985 "route lifecycle control PUSH was not delivered to client; closing target connection"
7986 );
7987 let _ = forwarding.escalate_client_delivery_failure(
7988 target.connection_id,
7989 target.channel,
7990 target.epoch,
7991 CloseReason::new(
7992 "route_lifecycle_push_delivery_failed",
7993 format!(
7994 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7995 target.channel
7996 ),
7997 ),
7998 crate::forwarding::UndeliveredFrame {
7999 module_id: target.module_id.as_deref(),
8000 sink: &target.sink,
8001 },
8002 );
8003 }
8004 }
8005 }
8006}
8007
8008#[cfg(test)]
8009mod tests {
8010 mod activity_read {
8011 include!("activity_read_tests.rs");
8012 }
8013
8014 mod lifecycle_activity {
8015 include!("control_lifecycle_tests.rs");
8016 }
8017 #[cfg(unix)]
8018 #[tokio::test]
8019 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8020 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8021 let path = dir.join("subc.jsonc");
8022 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8023 "program":"/bin/sleep","args":["60"],"protocol":"none",
8024 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8025 }}}).to_string()).unwrap();
8026 let mut configured = crate::daemon_config::load(&path)
8027 .unwrap()
8028 .unwrap()
8029 .modules
8030 .pop()
8031 .unwrap();
8032 let registry = std::sync::Arc::new(crate::Registry::default());
8033 let handle = crate::SupervisorHandle::new();
8034 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8035 .with_handle(handle.clone());
8036 let module = supervisor
8037 .supervise_configured_with_health(
8038 configured.module_spec(),
8039 true,
8040 configured.health.clone(),
8041 None,
8042 configured.restart,
8043 )
8044 .unwrap();
8045 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8046 let before = module.status().unwrap().pid;
8047 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8048 configured.health.cadence = std::time::Duration::from_secs(3600);
8049 let health_only = handler
8050 .reconcile_supervised_modules(
8051 &supervisor,
8052 vec![configured.clone()],
8053 false,
8054 Principal::Direct,
8055 )
8056 .await
8057 .unwrap();
8058 assert!(
8059 health_only.0.changed_pending_reload.is_empty(),
8060 "health policy is already applied live"
8061 );
8062 assert_eq!(module.status().unwrap().pid, before);
8063 assert_eq!(
8064 module.configuration().unwrap().1.http,
8065 configured.health.http
8066 );
8067 configured.args = vec!["61".into()];
8068 let launch = handler
8069 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8070 .await
8071 .unwrap();
8072 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8073 assert_eq!(
8074 module.status().unwrap().pid,
8075 before,
8076 "a launch edit is stored until reload"
8077 );
8078 module.drain().await.unwrap();
8079 }
8080 use cortexkit_test_support::ScratchDir;
8081 use std::{
8082 collections::BTreeMap,
8083 fmt,
8084 path::PathBuf,
8085 sync::{Arc, Mutex},
8086 time::Duration,
8087 };
8088
8089 use serde_json::{json, Value};
8090 use subc_protocol::{
8091 manifest::{
8092 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8093 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8094 },
8095 session::HealthStatus,
8096 FrameType,
8097 };
8098
8099 use super::*;
8100 use crate::{
8101 forwarding::{DataRoute, DataRouteState},
8102 registry::ChannelState,
8103 router::FrameSink,
8104 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8105 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8106 RouteCtx, Router,
8107 };
8108 use tokio::{
8109 sync::mpsc,
8110 time::{sleep, Instant},
8111 };
8112 use tracing::{
8113 field::{Field, Visit},
8114 Event, Subscriber,
8115 };
8116 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8117
8118 fn fake_aft_stub_path() -> PathBuf {
8137 let mut path = std::env::current_exe().expect("current_exe available in tests");
8138 path.pop(); path.pop(); path.push(if cfg!(windows) {
8141 "fake-aft-stub.exe"
8142 } else {
8143 "fake-aft-stub"
8144 });
8145 assert!(
8146 path.exists(),
8147 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8148 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8149 path.display()
8150 );
8151 path
8152 }
8153
8154 fn client_retries(code: &str) -> bool {
8159 subc_protocol::error_codes::is_retryable_route_open(code)
8160 }
8161
8162 #[test]
8173 fn retryability_of_forwarding_codes_matches_the_failure() {
8174 let transient = [
8177 ForwardingError::NoModuleConnection,
8178 ForwardingError::ModuleReloading {
8179 module_id: "m".into(),
8180 },
8181 ForwardingError::StaleModuleEndpoint,
8182 ForwardingError::UnknownReservation {
8183 client_channel: 1,
8184 module_channel: 1,
8185 },
8186 ForwardingError::ConnectionClosing {
8187 connection_id: ConnectionId::new(1),
8188 },
8189 ForwardingError::ClientEgressClosed {
8190 connection_id: ConnectionId::new(1),
8191 },
8192 ForwardingError::ModuleEgressUnavailable {
8193 connection_id: ConnectionId::new(1),
8194 },
8195 ];
8196 for err in transient {
8197 let code = forwarding_error_code(&err);
8198 assert!(
8199 client_retries(code),
8200 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8201 );
8202 }
8203
8204 let permanent = [
8209 ForwardingError::ConnectionRoleConflict {
8210 connection_id: ConnectionId::new(1),
8211 },
8212 ForwardingError::ClientRouteChannelExhausted {
8213 connection_id: ConnectionId::new(1),
8214 },
8215 ForwardingError::ModuleRouteChannelExhausted {
8216 endpoint: ModuleEndpointId {
8217 connection_id: ConnectionId::new(1),
8218 generation: 1,
8219 },
8220 },
8221 ForwardingError::RelayCorrelationExhausted,
8222 ForwardingError::RouteOpenBuild("x".into()),
8223 ForwardingError::Poisoned,
8224 ];
8225 for err in permanent {
8226 let code = forwarding_error_code(&err);
8227 assert!(
8228 !client_retries(code),
8229 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8230 );
8231 }
8232 }
8233
8234 #[tokio::test]
8262 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8263 let handler = ControlHandler::default();
8264 let frame =
8265 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8266
8267 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8271 assert_eq!(
8272 stamped,
8273 Principal::Direct,
8274 "a caller that proved nothing must not be stamped as a supervised module"
8275 );
8276
8277 let forged = handler
8282 .route_open_principal(
8283 &frame,
8284 Some(ConsumerIdentity {
8285 module_id: "aft".to_string(),
8286 launch_nonce: "not-a-real-nonce".to_string(),
8287 }),
8288 )
8289 .unwrap();
8290 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8291 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8292 }
8293
8294 #[test]
8311 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8312 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8313 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8314 let ClientControlRequest::RouteOpen {
8315 consumer_identity, ..
8316 } = parsed
8317 else {
8318 panic!("route.open body must parse as RouteOpen");
8319 };
8320 assert_eq!(
8321 consumer_identity,
8322 Some(ConsumerIdentity {
8323 module_id: "aft".to_string(),
8324 launch_nonce: "n".to_string(),
8325 }),
8326 "the wire field name must reach the value route_open_principal reads"
8327 );
8328 }
8329
8330 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8331 ModuleManifest::builder(module_id, "0.1.0")
8332 .protocol_ver(protocol_ver)
8333 .provides(vec![ProviderRole::ToolProvider {
8334 tools: vec![Tool {
8335 name: "read".to_string(),
8336 description: None,
8337 execution_mode: ExecutionMode::Pure,
8338 schema: json!({"type": "object"}),
8339 }],
8340 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8341 concurrency: Concurrency::ModuleManaged,
8342 emits_push: true,
8343 sub_supervises: true,
8344 }])
8345 .build()
8346 }
8347
8348 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8349 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8350 }
8351
8352 fn hello_frame_with_control_ops(
8353 module_id: &str,
8354 protocol_ver: u8,
8355 corr: u64,
8356 control_ops: Option<Vec<String>>,
8357 ) -> Frame {
8358 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8359 }
8360
8361 fn hello_frame_with_nonce(
8362 module_id: &str,
8363 protocol_ver: u8,
8364 corr: u64,
8365 launch_nonce: Option<&str>,
8366 ) -> Frame {
8367 hello_frame_full(
8368 module_id,
8369 protocol_ver,
8370 corr,
8371 None,
8372 launch_nonce.map(ToOwned::to_owned),
8373 )
8374 }
8375
8376 fn hello_frame_full(
8377 module_id: &str,
8378 protocol_ver: u8,
8379 corr: u64,
8380 control_ops: Option<Vec<String>>,
8381 launch_nonce: Option<String>,
8382 ) -> Frame {
8383 let body = serde_json::to_vec(&ModuleHelloBody {
8384 manifest: manifest(module_id, protocol_ver),
8385 protocol_ver,
8386 control_ops,
8387 launch_nonce,
8388 })
8389 .unwrap();
8390 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8391 }
8392
8393 fn hello_frame_with_rich_ui(
8394 module_id: &str,
8395 protocol_ver: u8,
8396 corr: u64,
8397 launch_nonce: Option<&str>,
8398 rich_ui: Value,
8399 ) -> Frame {
8400 let mut body = serde_json::to_value(ModuleHelloBody {
8401 manifest: manifest(module_id, protocol_ver),
8402 protocol_ver,
8403 control_ops: None,
8404 launch_nonce: launch_nonce.map(str::to_string),
8405 })
8406 .unwrap();
8407 body["manifest"]["rich_ui"] = rich_ui;
8408 Frame::build(
8409 FrameType::Hello,
8410 control_flags(),
8411 0,
8412 0,
8413 corr,
8414 serde_json::to_vec(&body).unwrap(),
8415 )
8416 .unwrap()
8417 }
8418
8419 fn non_routable_hello_frame_with_control_ops(
8420 module_id: &str,
8421 corr: u64,
8422 control_ops: Option<Vec<String>>,
8423 ) -> Frame {
8424 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8425 manifest.provides.clear();
8426 let body = serde_json::to_vec(&ModuleHelloBody {
8427 manifest,
8428 protocol_ver: PROTOCOL_VERSION,
8429 control_ops,
8430 launch_nonce: None,
8431 })
8432 .unwrap();
8433 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8434 }
8435
8436 fn capability_grammar_hello_frame(
8437 capabilities: Value,
8438 runtime_computed: Option<Value>,
8439 corr: u64,
8440 ) -> Frame {
8441 let mut body = serde_json::to_value(ModuleHelloBody {
8442 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8443 protocol_ver: PROTOCOL_VERSION,
8444 control_ops: None,
8445 launch_nonce: None,
8446 })
8447 .expect("HELLO body serializes");
8448 body["manifest"]["capabilities"] = capabilities;
8449 if let Some(runtime_computed) = runtime_computed {
8450 body["runtime_computed"] = runtime_computed;
8451 }
8452 Frame::build(
8453 FrameType::Hello,
8454 control_flags(),
8455 0,
8456 0,
8457 corr,
8458 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8459 )
8460 .expect("HELLO frame builds")
8461 }
8462
8463 fn channel_request(channel: u16, corr: u64) -> Frame {
8464 Frame::build(
8465 FrameType::Request,
8466 Flags::new(true, Priority::Interactive, false),
8467 channel,
8468 0,
8469 corr,
8470 b"opaque".to_vec(),
8471 )
8472 .unwrap()
8473 }
8474
8475 fn route_ctx(
8476 connection_id: ConnectionId,
8477 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8478 let (tx, rx) = mpsc::channel(8);
8479 (
8480 RouteCtx {
8481 connection_id,
8482 egress: FrameSink::new(tx),
8483 },
8484 rx,
8485 )
8486 }
8487
8488 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8489 serde_json::from_slice(&frame.body).unwrap()
8490 }
8491
8492 async fn hello_via_sink(
8497 handler: &ControlHandler,
8498 ctx: &RouteCtx,
8499 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8500 hello: Frame,
8501 ) -> Frame {
8502 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8503 assert!(
8504 replies.is_empty(),
8505 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8506 );
8507 let ack = rx
8508 .try_recv()
8509 .expect("HELLO_ACK is queued on the module sink")
8510 .frame;
8511 assert_eq!(ack.header.ty, FrameType::HelloAck);
8512 ack
8513 }
8514
8515 fn parse_error(frame: &Frame) -> Value {
8516 serde_json::from_slice(&frame.body).unwrap()
8517 }
8518
8519 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8520 serde_json::from_slice(&frame.body).unwrap()
8521 }
8522
8523 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8524 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8525 route_channel,
8526 route_epoch: 0,
8527 kind,
8528 })
8529 .unwrap();
8530 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8531 }
8532
8533 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8534 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8535 module_id: module_id.to_string(),
8536 })
8537 .unwrap();
8538 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8539 }
8540
8541 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8542 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8543 }
8544
8545 fn route_open_frame_with_consumer_capabilities(
8546 corr: u64,
8547 module_id: &str,
8548 project_root: ScratchDir,
8549 consumer_capabilities: Option<Vec<String>>,
8550 ) -> Frame {
8551 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8552 target: RouteTarget::ToolProvider {
8553 module_id: module_id.to_string(),
8554 },
8555 identity: BindIdentity::new(
8556 project_root.path().to_path_buf(),
8557 "unit".to_string(),
8558 "session".to_string(),
8559 ),
8560 consumer_identity: None,
8561 consumer_capabilities,
8562 role_versions: None,
8563 admission_facts: None,
8564 scope: None,
8565 })
8566 .unwrap();
8567 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8568 }
8569
8570 fn route_open_frame_with_role_versions(
8571 corr: u64,
8572 module_id: &str,
8573 project_root: ScratchDir,
8574 role_versions: Option<BTreeMap<String, String>>,
8575 ) -> Frame {
8576 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8577 target: RouteTarget::ToolProvider {
8578 module_id: module_id.to_string(),
8579 },
8580 identity: BindIdentity::new(
8581 project_root.path().to_path_buf(),
8582 "unit".to_string(),
8583 format!("session-{corr}"),
8584 ),
8585 consumer_identity: None,
8586 consumer_capabilities: None,
8587 role_versions,
8588 admission_facts: None,
8589 scope: None,
8590 })
8591 .unwrap();
8592 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8593 }
8594
8595 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8596 entries
8597 .iter()
8598 .map(|(role, version)| (role.to_string(), version.to_string()))
8599 .collect()
8600 }
8601
8602 fn route_open_frame_with_admission_facts(
8603 corr: u64,
8604 module_id: &str,
8605 project_root: ScratchDir,
8606 consumer_identity: Option<subc_control::ConsumerIdentity>,
8607 facts: Option<Value>,
8608 ) -> Frame {
8609 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8610 target: RouteTarget::ToolProvider {
8611 module_id: module_id.to_string(),
8612 },
8613 identity: BindIdentity::new(
8614 project_root.path().to_path_buf(),
8615 "unit".to_string(),
8616 format!("session-{corr}"),
8617 ),
8618 consumer_identity,
8619 consumer_capabilities: None,
8620 role_versions: None,
8621 admission_facts: facts,
8622 scope: None,
8623 })
8624 .unwrap();
8625 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8626 }
8627
8628 #[derive(Clone, Default)]
8629 struct EventCapture {
8630 events: Arc<Mutex<Vec<CapturedEvent>>>,
8631 }
8632
8633 #[derive(Clone, Debug)]
8634 struct CapturedEvent {
8635 target: String,
8636 level: tracing::Level,
8637 fields: BTreeMap<String, String>,
8638 }
8639
8640 impl EventCapture {
8641 fn events(&self) -> Vec<CapturedEvent> {
8642 self.events.lock().unwrap().clone()
8643 }
8644 }
8645
8646 impl<S> Layer<S> for EventCapture
8647 where
8648 S: Subscriber,
8649 {
8650 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8651 let mut visitor = EventFieldVisitor::default();
8652 event.record(&mut visitor);
8653 self.events.lock().unwrap().push(CapturedEvent {
8654 target: event.metadata().target().to_string(),
8655 level: *event.metadata().level(),
8656 fields: visitor.fields,
8657 });
8658 }
8659 }
8660
8661 #[derive(Default)]
8662 struct EventFieldVisitor {
8663 fields: BTreeMap<String, String>,
8664 }
8665
8666 impl Visit for EventFieldVisitor {
8667 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8668 self.fields
8669 .insert(field.name().to_string(), format!("{value:?}"));
8670 }
8671 }
8672
8673 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8674 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8675 status,
8676 detail: Some("warming".to_string()),
8677 metrics: Some(json!({"queue_depth": 3})),
8678 })
8679 .unwrap();
8680 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8681 }
8682
8683 fn route_bind_ack(corr: u64) -> Frame {
8684 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8685 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8686 }
8687
8688 fn unique_project_root(label: &str) -> ScratchDir {
8689 ScratchDir::new(label)
8690 }
8691
8692 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8693 match parse_route_poll(frame) {
8694 ClientControlResponse::RoutePoll {
8695 status: None,
8696 live: Some(live),
8697 ..
8698 } => assert_eq!(live, expected_live),
8699 other => panic!("unexpected route.poll response: {other:?}"),
8700 }
8701 }
8702
8703 fn bind_liveness_route(
8704 registry: &Registry,
8705 forwarding: &ForwardingTable,
8706 module_id: &str,
8707 ) -> (RouteCtx, u16, u32) {
8708 let module_connection = ConnectionId::new(101);
8709 let client_connection = ConnectionId::new(202);
8710 let registration = registry
8711 .register_with_control_ops(
8712 manifest(module_id, PROTOCOL_VERSION),
8713 PROTOCOL_VERSION,
8714 module_connection,
8715 module_baseline_control_ops(),
8716 )
8717 .unwrap();
8718 let (module_tx, _module_rx) = mpsc::channel(8);
8719 let endpoint = forwarding
8720 .register_module_connection(
8721 module_connection,
8722 module_id.to_string(),
8723 PROTOCOL_VERSION,
8724 manifest_concurrency(®istration.manifest),
8725 FrameSink::new(module_tx),
8726 )
8727 .unwrap();
8728 let (client_ctx, _client_rx) = route_ctx(client_connection);
8729 let pending = forwarding
8730 .begin_route_bind_relay_for_test(
8731 client_connection,
8732 client_ctx.egress.clone(),
8733 1,
8734 module_id,
8735 )
8736 .unwrap();
8737 assert_eq!(pending.endpoint, endpoint);
8738 let route_channel = pending.client_channel;
8739 let route_epoch = pending.client_epoch;
8740 forwarding
8741 .complete_pending_relay(
8742 module_connection,
8743 pending.corr,
8744 RouteBindRelayOutcome::Accepted,
8745 )
8746 .unwrap();
8747 (client_ctx, route_channel, route_epoch)
8748 }
8749
8750 struct FakeProcessLiveness {
8751 live: Option<bool>,
8752 }
8753
8754 impl ModuleProcessLiveness for FakeProcessLiveness {
8755 fn process_live(&self, _module_id: &str) -> Option<bool> {
8756 self.live
8757 }
8758 }
8759
8760 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8761 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8762 {
8763 let registry = Arc::new(Registry::default());
8764 let supervisor_handle = SupervisorHandle::new();
8765 let supervisor = Supervisor::new_for_test(
8766 Arc::clone(®istry),
8767 RestartPolicy::new(1, Duration::from_millis(10)),
8768 )
8769 .with_handle(supervisor_handle.clone());
8770 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8771 let module = supervisor
8772 .spawn(ModuleSpec {
8773 module_id: "stderr-tail-wire".to_string(),
8774 program: fake_aft_stub_path(),
8775 args: Vec::new(),
8776 env: vec![
8777 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8778 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8779 ],
8780 reserved: false,
8781 reserved_prefixes: Vec::new(),
8782 protocol: ModuleProtocol::Subc,
8783 overlap: Default::default(),
8784 })
8785 .unwrap();
8786
8787 let deadline = Instant::now() + Duration::from_secs(5);
8788 loop {
8789 let tail = module.stderr_tail(None, None);
8790 if tail
8791 .entries
8792 .iter()
8793 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8794 && tail.entries.iter().any(|entry| {
8795 matches!(
8796 entry,
8797 TailEntry::Line {
8798 truncated: true,
8799 ..
8800 }
8801 )
8802 })
8803 {
8804 break;
8805 }
8806 assert!(
8807 Instant::now() < deadline,
8808 "module did not produce a truncated line and restart boundary: {tail:?}"
8809 );
8810 sleep(Duration::from_millis(10)).await;
8811 }
8812
8813 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8814 let request = ClientControlRequest::SupervisorStderrTail {
8815 module_id: "stderr-tail-wire".to_string(),
8816 max_lines: None,
8817 max_bytes: None,
8818 };
8819 let frame = Frame::build(
8820 FrameType::Request,
8821 control_flags(),
8822 0,
8823 0,
8824 1,
8825 serde_json::to_vec(&request).unwrap(),
8826 )
8827 .unwrap();
8828 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8829 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8830 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8831 serde_json::from_slice(&responses[0].body).unwrap()
8832 else {
8833 panic!("expected supervisor.stderr_tail response");
8834 };
8835
8836 assert!(
8837 tail.entries
8838 .iter()
8839 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8840 "the control response lost the restart boundary"
8841 );
8842 let Some(StderrTailEntry::Line {
8843 text,
8844 truncated,
8845 at_ms,
8846 }) = tail.entries.iter().find(|entry| {
8847 matches!(
8848 entry,
8849 StderrTailEntry::Line {
8850 truncated: true,
8851 ..
8852 }
8853 )
8854 })
8855 else {
8856 panic!("the control response lost the truncated line");
8857 };
8858 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8859 assert!(*truncated);
8860 assert!(
8861 at_ms.is_some(),
8862 "the control response lost the line's capture time"
8863 );
8864 }
8865
8866 #[tokio::test(flavor = "current_thread")]
8871 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8872 let dir = ScratchDir::new("terminals-off-worker");
8873 let journal_path = dir.join("terminals.jsonl");
8874 let registry = Arc::new(Registry::default());
8875 let supervisor_handle = SupervisorHandle::new();
8876 let supervisor =
8877 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8878 .with_handle(supervisor_handle.clone())
8879 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8880 let module = supervisor
8881 .spawn(ModuleSpec {
8882 module_id: "terminal-off-worker".to_string(),
8883 program: fake_aft_stub_path(),
8884 args: Vec::new(),
8885 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8886 reserved: false,
8887 reserved_prefixes: Vec::new(),
8888 protocol: ModuleProtocol::Subc,
8889 overlap: Default::default(),
8890 })
8891 .unwrap();
8892 let deadline = Instant::now() + Duration::from_secs(5);
8893 while module.terminal_history().entries.len() != 2 {
8894 assert!(Instant::now() < deadline, "module did not record two exits");
8895 sleep(Duration::from_millis(10)).await;
8896 }
8897
8898 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8899 let handler =
8900 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8901 let frame = Frame::build(
8902 FrameType::Request,
8903 control_flags(),
8904 0,
8905 0,
8906 1,
8907 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8908 module_id: "terminal-off-worker".to_string(),
8909 })
8910 .unwrap(),
8911 )
8912 .unwrap();
8913 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8914 let spawned_at = std::time::Instant::now();
8915 let read = tokio::spawn({
8916 let handler = Arc::clone(&handler);
8917 async move { handler.handle_control_frame(&ctx, frame).await }
8918 });
8919 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8922 .await
8923 .unwrap()
8924 .expect("the history read reached its pause");
8925 let elapsed = spawned_at.elapsed();
8926 assert!(
8927 elapsed < Duration::from_secs(2) && !read.is_finished(),
8928 "this task could not run while the history read was paused \
8929 (resumed after {elapsed:?}, read finished: {})",
8930 read.is_finished()
8931 );
8932
8933 drop(release);
8934 let responses = read.await.unwrap().unwrap();
8935 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8936 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8937 panic!("expected supervisor.terminals response");
8938 };
8939 assert_eq!(terminals.entries.len(), 2);
8940 assert_eq!(terminals.journal_skipped_lines, 0);
8941 assert_eq!(terminals.journal_read_errors, 0);
8942 }
8943
8944 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8945 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8946 let registry = Arc::new(Registry::default());
8947 let supervisor_handle = SupervisorHandle::new();
8948 let supervisor =
8949 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8950 .with_handle(supervisor_handle.clone());
8951 let module = supervisor
8952 .spawn(ModuleSpec {
8953 module_id: "terminal-golden".to_string(),
8954 program: fake_aft_stub_path(),
8955 args: Vec::new(),
8956 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8957 reserved: false,
8958 reserved_prefixes: Vec::new(),
8959 protocol: ModuleProtocol::Subc,
8960 overlap: Default::default(),
8961 })
8962 .unwrap();
8963
8964 let deadline = Instant::now() + Duration::from_secs(5);
8965 while module.terminal_history().entries.len() != 2 {
8966 assert!(
8967 Instant::now() < deadline,
8968 "module did not retain two terminal exits: {:?}",
8969 module.terminal_history()
8970 );
8971 sleep(Duration::from_millis(10)).await;
8972 }
8973
8974 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8975 let request = ClientControlRequest::SupervisorTerminals {
8976 module_id: "terminal-golden".to_string(),
8977 };
8978 let frame = Frame::build(
8979 FrameType::Request,
8980 control_flags(),
8981 0,
8982 0,
8983 1,
8984 serde_json::to_vec(&request).unwrap(),
8985 )
8986 .unwrap();
8987 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8988 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8989 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8990 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8991 panic!("expected supervisor.terminals response");
8992 };
8993 assert_eq!(terminals.entries.len(), 2);
8994 assert_eq!(terminals.dropped, 0);
8995
8996 let mut rendered = serde_json::to_value(response).unwrap();
8997 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9000 for (index, entry) in rendered["entries"]
9001 .as_array_mut()
9002 .expect("terminal response entries array")
9003 .iter_mut()
9004 .enumerate()
9005 {
9006 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9007 }
9008
9009 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9010 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9011 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9012 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9013 std::fs::write(&golden_path, &serialized).unwrap();
9014 }
9015 let expected: Value =
9016 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9017 assert_eq!(rendered, expected);
9018 }
9019
9020 #[test]
9021 fn hello_registers_manifest_and_returns_ack() {
9022 let registry = Arc::new(Registry::default());
9023 let handler = ControlHandler::new(Arc::clone(®istry));
9024 let conn = ConnectionId::new(1);
9025
9026 let responses = handler
9027 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9028 .unwrap();
9029
9030 assert_eq!(responses.len(), 1);
9031 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9032 assert_eq!(responses[0].header.channel, 0);
9033 assert_eq!(responses[0].header.corr, 7);
9034 let ack = parse_ack(&responses[0]);
9035 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9036 assert!(ack
9037 .subc_capabilities
9038 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9039 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9040 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9041 assert!(ack
9042 .subc_ops
9043 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9044 assert!(ack
9045 .subc_ops
9046 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9047
9048 let registration = registry.get_module("aft").unwrap().unwrap();
9049 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9050 assert_eq!(registration.state, ChannelState::Active);
9051 assert_eq!(registration.connection_id, conn);
9052 assert_eq!(registration.control_ops, module_baseline_control_ops());
9053 }
9054
9055 #[test]
9056 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9057 let invalid_identifiers = [
9058 ("case_change", "credentials-Provider/v1"),
9059 ("leading_zero", "credentials-provider/v01"),
9060 ("trailing_hyphen", "credentials-provider-/v1"),
9061 ("consecutive_hyphens", "credentials--provider/v1"),
9062 ("uppercase", "Credentials-provider/v1"),
9063 ("missing_v", "credentials-provider/1"),
9064 ("whitespace", "credentials provider/v1"),
9065 ("zero_version", "credentials-provider/v0"),
9066 ("out_of_range_version", "credentials-provider/v4294967296"),
9067 (
9068 "overlength_name",
9069 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9070 ),
9071 ];
9072 let mut cases = invalid_identifiers
9073 .into_iter()
9074 .map(|(name, identifier)| {
9075 (
9076 format!("identifier_{name}"),
9077 "capabilities.provides[0]".to_string(),
9078 identifier.to_string(),
9079 json!({ "provides": [identifier] }),
9080 None,
9081 )
9082 })
9083 .collect::<Vec<_>>();
9084 cases.extend([
9085 (
9086 "unknown_need".to_string(),
9087 "capabilities.requires[0].need".to_string(),
9088 "deferred".to_string(),
9089 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9090 None,
9091 ),
9092 (
9093 "duplicate_provides".to_string(),
9094 "capabilities.provides[1]".to_string(),
9095 "credentials-provider/v1".to_string(),
9096 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9097 None,
9098 ),
9099 (
9100 "duplicate_must_never_reach".to_string(),
9101 "capabilities.must_never_reach[1]".to_string(),
9102 "credentials-provider/v1".to_string(),
9103 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9104 None,
9105 ),
9106 (
9107 "duplicate_requires_same_need".to_string(),
9108 "capabilities.requires[1]".to_string(),
9109 "credentials-provider/v1".to_string(),
9110 json!({ "requires": [
9111 { "capability": "credentials-provider/v1", "need": "required" },
9112 { "capability": "credentials-provider/v1", "need": "required" }
9113 ] }),
9114 None,
9115 ),
9116 (
9117 "duplicate_requires_conflicting_need".to_string(),
9118 "capabilities.requires[1]".to_string(),
9119 "credentials-provider/v1".to_string(),
9120 json!({ "requires": [
9121 { "capability": "credentials-provider/v1", "need": "required" },
9122 { "capability": "credentials-provider/v1", "need": "optional" }
9123 ] }),
9124 None,
9125 ),
9126 (
9127 "capabilities_root_pointer".to_string(),
9128 "runtime_computed[0]".to_string(),
9129 "/capabilities".to_string(),
9130 json!({}),
9131 Some(json!(["/capabilities"])),
9132 ),
9133 (
9134 "capabilities_descendant_pointer".to_string(),
9135 "runtime_computed[0]".to_string(),
9136 "/capabilities/provides".to_string(),
9137 json!({}),
9138 Some(json!(["/capabilities/provides"])),
9139 ),
9140 (
9141 "malformed_pointer_without_leading_slash".to_string(),
9142 "runtime_computed[0]".to_string(),
9143 "capabilities".to_string(),
9144 json!({}),
9145 Some(json!(["capabilities"])),
9146 ),
9147 (
9148 "malformed_pointer_escape".to_string(),
9149 "runtime_computed[0]".to_string(),
9150 "/roles/~2/tools".to_string(),
9151 json!({}),
9152 Some(json!(["/roles/~2/tools"])),
9153 ),
9154 (
9155 "unknown_capabilities_field".to_string(),
9156 "capabilities.future".to_string(),
9157 "<array>".to_string(),
9158 json!({ "future": [] }),
9159 None,
9160 ),
9161 ]);
9162
9163 for (index, (name, field, value, capabilities, runtime_computed)) in
9164 cases.into_iter().enumerate()
9165 {
9166 let registry = Arc::new(Registry::default());
9167 let handler = ControlHandler::new(Arc::clone(®istry));
9168 let response = handler
9169 .handle_control(
9170 ConnectionId::new((index + 1) as u64),
9171 capability_grammar_hello_frame(
9172 capabilities,
9173 runtime_computed,
9174 index as u64 + 1,
9175 ),
9176 )
9177 .expect("invalid HELLO returns a refusal");
9178
9179 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9180 let error = parse_error(&response[0]);
9181 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9182 let message = error["message"]
9183 .as_str()
9184 .expect("error message is a string");
9185 assert!(
9186 message.contains(&field),
9187 "{name}: field missing from {message}"
9188 );
9189 assert!(
9190 message.contains(&value),
9191 "{name}: value missing from {message}"
9192 );
9193 assert_eq!(
9194 registry
9195 .active_registration_count()
9196 .expect("registry reads"),
9197 0,
9198 "{name}: refused HELLO must not create a catalog entry"
9199 );
9200 }
9201 }
9202
9203 #[test]
9204 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9205 let registry = Arc::new(Registry::default());
9206 let handler = ControlHandler::new(Arc::clone(®istry));
9207 let capabilities = json!({
9208 "provides": ["credentials-provider/v1"],
9209 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9210 "must_never_reach": ["federation-transport/v1"]
9211 });
9212 let response = handler
9213 .handle_control(
9214 ConnectionId::new(99),
9215 capability_grammar_hello_frame(
9216 capabilities.clone(),
9217 Some(json!(["/roles/0/tools"])),
9218 99,
9219 ),
9220 )
9221 .expect("valid HELLO registers");
9222 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9223
9224 let request = Frame::build(
9225 FrameType::Request,
9226 control_flags(),
9227 0,
9228 0,
9229 100,
9230 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9231 .expect("catalog request serializes"),
9232 )
9233 .expect("catalog request frame builds");
9234 let response = handler
9235 .handle_catalog_list(request, None)
9236 .expect("catalog list succeeds");
9237 let ClientControlResponse::CatalogList { modules, .. } =
9238 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9239 else {
9240 panic!("catalog request must return catalog.list");
9241 };
9242 assert_eq!(modules.len(), 1);
9243 assert_eq!(
9244 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9245 capabilities
9246 );
9247 }
9248
9249 #[test]
9250 fn catalog_list_mirrors_management_operation_description() {
9251 let registry = Arc::new(Registry::default());
9252 let handler = ControlHandler::new(Arc::clone(®istry));
9253 let description = "List managed records and return their identifiers and metadata.";
9254 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9255 manifest.provides = vec![ProviderRole::ManagementSurface {
9256 operations: vec![ManagementOperation {
9257 name: "records.list".to_string(),
9258 kind: ManagementOperationKind::Query,
9259 description: Some(description.to_string()),
9260 }],
9261 config_schema: json!({"type": "object"}),
9262 observability: vec![ObservabilitySurface {
9263 name: "records.stats".to_string(),
9264 kind: ObservabilityKind::Snapshot,
9265 }],
9266 identity_scope: vec![IdentityScope::Project],
9267 concurrency: Concurrency::ModuleManaged,
9268 }];
9269 registry
9270 .register_with_control_ops(
9271 manifest,
9272 PROTOCOL_VERSION,
9273 ConnectionId::new(99),
9274 Vec::new(),
9275 )
9276 .expect("described management manifest registers");
9277
9278 let request = Frame::build(
9279 FrameType::Request,
9280 control_flags(),
9281 0,
9282 0,
9283 100,
9284 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9285 .expect("catalog request serializes"),
9286 )
9287 .expect("catalog request frame builds");
9288 let response = handler
9289 .handle_catalog_list(request, None)
9290 .expect("catalog list succeeds");
9291 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9292 assert_eq!(
9293 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9294 "catalog.list must preserve the declared operation description verbatim"
9295 );
9296 }
9297
9298 #[test]
9299 fn catalog_list_mirrors_registered_operation_delivery() {
9300 let registry = Arc::new(Registry::default());
9301 let handler = ControlHandler::new(Arc::clone(®istry));
9302 let delivery =
9303 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9304 let mut body = serde_json::to_value(ModuleHelloBody {
9305 manifest: manifest("watch-management", PROTOCOL_VERSION),
9306 protocol_ver: PROTOCOL_VERSION,
9307 control_ops: None,
9308 launch_nonce: None,
9309 })
9310 .unwrap();
9311 body["manifest"]["provides"] = json!([{
9312 "role": "management_surface",
9313 "operations": [
9314 {"name": "status", "kind": "query"},
9315 {"name": "future", "kind": "query"},
9316 {"name": "reset", "kind": "mutate"}
9317 ],
9318 "config_schema": {}, "observability": [], "identity_scope": []
9319 }]);
9320 body["manifest"]["operation_delivery"] = delivery.clone();
9321 let hello = Frame::build(
9322 FrameType::Hello,
9323 control_flags(),
9324 0,
9325 0,
9326 99,
9327 serde_json::to_vec(&body).unwrap(),
9328 )
9329 .unwrap();
9330 let response = handler
9331 .handle_control(ConnectionId::new(99), hello)
9332 .unwrap();
9333 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9334
9335 let (_, registrations) = registry.list_modules().unwrap();
9336 let registered = ®istrations[0].manifest;
9337 assert!(registered.is_watch_operation("status"));
9338 assert!(!registered.is_watch_operation("future"));
9339 assert!(!registered.is_watch_operation("reset"));
9340 assert!(!registered.is_watch_operation("missing"));
9341
9342 let request = Frame::build(
9343 FrameType::Request,
9344 control_flags(),
9345 0,
9346 0,
9347 100,
9348 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9349 )
9350 .unwrap();
9351 let response = handler.handle_catalog_list(request, None).unwrap();
9352 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9353 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9354 let ClientControlResponse::CatalogList { modules, .. } =
9355 serde_json::from_slice(&response[0].body).unwrap()
9356 else {
9357 panic!("expected catalog.list");
9358 };
9359 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9360 }
9361
9362 #[test]
9363 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9364 let registry = Arc::new(Registry::default());
9365 let supervisor = SupervisorHandle::new();
9366 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9367 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9368 let descriptor = json!({
9369 "schema": "ckios.richui.manifest/1",
9370 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9371 "bytes": 4096,
9372 "chunks": 4
9373 });
9374 let connection = ConnectionId::new(201);
9375 let hello = handler
9376 .handle_control(
9377 connection,
9378 hello_frame_with_rich_ui(
9379 "vault",
9380 PROTOCOL_VERSION,
9381 201,
9382 Some("the-real-nonce"),
9383 descriptor.clone(),
9384 ),
9385 )
9386 .unwrap();
9387 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9388
9389 let request =
9390 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9391 let provides = registry
9392 .get_module_by_connection(connection)
9393 .unwrap()
9394 .unwrap()
9395 .manifest
9396 .provides;
9397 handler
9398 .handle_catalog_update(connection, request, provides, None, Some(false))
9399 .unwrap();
9400
9401 let request = Frame::build(
9402 FrameType::Request,
9403 control_flags(),
9404 0,
9405 0,
9406 203,
9407 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9408 )
9409 .unwrap();
9410 let response = handler.handle_catalog_list(request, None).unwrap();
9411 let ClientControlResponse::CatalogList { modules, .. } =
9412 serde_json::from_slice(&response[0].body).unwrap()
9413 else {
9414 panic!("expected catalog.list");
9415 };
9416 assert_eq!(
9417 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9418 descriptor
9419 );
9420 assert!(modules[0].nonce_verified);
9421 }
9422
9423 #[test]
9424 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9425 let registry = Arc::new(Registry::default());
9426 let handler = ControlHandler::new(registry.clone());
9427 let descriptor = json!({
9428 "schema": "ckios.richui.manifest/1",
9429 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9430 "bytes": 128,
9431 "chunks": 2
9432 });
9433 let malformed = [
9434 ("schema", json!("")),
9435 ("schema", json!("a".repeat(65))),
9436 ("schema", json!("Bad schema")),
9437 (
9438 "manifest_digest",
9439 json!(format!("sha256:{}", "A".repeat(64))),
9440 ),
9441 ("manifest_digest", json!("sha256:abc")),
9442 ("bytes", json!(0)),
9443 ("bytes", json!(16_777_217)),
9444 ("bytes", json!("128")),
9445 ("chunks", json!(0)),
9446 ("chunks", json!(1025)),
9447 ("chunks", json!("2")),
9448 ];
9449 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9450 let mut invalid = descriptor.clone();
9451 invalid[field] = invalid_value;
9452 let response = handler
9453 .handle_control(
9454 ConnectionId::new(210 + index as u64),
9455 hello_frame_with_rich_ui(
9456 &format!("bad-rich-ui-{index}"),
9457 PROTOCOL_VERSION,
9458 210 + index as u64,
9459 None,
9460 invalid,
9461 ),
9462 )
9463 .unwrap();
9464 assert_eq!(response[0].header.ty, FrameType::Error);
9465 let error = parse_error(&response[0]);
9466 assert_eq!(error["code"], "invalid_manifest");
9467 assert!(error["message"]
9468 .as_str()
9469 .unwrap()
9470 .contains(&format!("rich_ui.{field}")));
9471 assert!(registry
9472 .get_module(&format!("bad-rich-ui-{index}"))
9473 .unwrap()
9474 .is_none());
9475 }
9476 }
9477
9478 #[test]
9479 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9480 let registry = Arc::new(Registry::default());
9481 let handler = ControlHandler::new(Arc::clone(®istry));
9482 let connection = ConnectionId::new(219);
9483 registry
9484 .register_with_control_ops(
9485 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9486 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9487 "bad schema",
9488 format!("sha256:{}", "a".repeat(64)),
9489 128,
9490 2,
9491 )))
9492 .build(),
9493 PROTOCOL_VERSION,
9494 connection,
9495 Vec::new(),
9496 )
9497 .unwrap();
9498 let request =
9499 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9500 let response = handler
9501 .handle_catalog_update(connection, request, Vec::new(), None, None)
9502 .unwrap();
9503 assert_eq!(response[0].header.ty, FrameType::Error);
9504 let error = parse_error(&response[0]);
9505 assert_eq!(error["code"], "invalid_manifest");
9506 assert!(error["message"]
9507 .as_str()
9508 .unwrap()
9509 .contains("rich_ui.schema"));
9510 }
9511
9512 #[test]
9513 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9514 let registry = Arc::new(Registry::default());
9515 let supervisor = SupervisorHandle::new();
9516 supervisor.open_swap("provider", "candidate-nonce".to_string());
9517 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9518 let connection = ConnectionId::new(218);
9519 let hello = handler
9520 .handle_control(
9521 connection,
9522 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9523 )
9524 .unwrap();
9525 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9526 assert!(registry.list_modules().unwrap().1.is_empty());
9527
9528 registry.promote_candidate("provider").unwrap().unwrap();
9529 let request = Frame::build(
9530 FrameType::Request,
9531 control_flags(),
9532 0,
9533 0,
9534 219,
9535 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9536 )
9537 .unwrap();
9538 let response = handler.handle_catalog_list(request, None).unwrap();
9539 let ClientControlResponse::CatalogList { modules, .. } =
9540 serde_json::from_slice(&response[0].body).unwrap()
9541 else {
9542 panic!("expected catalog.list");
9543 };
9544 assert_eq!(modules.len(), 1);
9545 assert!(modules[0].nonce_verified);
9546 }
9547
9548 #[test]
9549 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9550 let registry = Arc::new(Registry::default());
9551 let handler = ControlHandler::new(Arc::clone(®istry));
9552 let hello = Frame::build(
9553 FrameType::Hello,
9554 control_flags(),
9555 0,
9556 0,
9557 220,
9558 serde_json::to_vec(&ModuleHelloBody {
9559 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9560 protocol_ver: PROTOCOL_VERSION,
9561 control_ops: None,
9562 launch_nonce: Some("unverified".to_string()),
9563 })
9564 .unwrap(),
9565 )
9566 .unwrap();
9567 assert_eq!(
9568 handler
9569 .handle_control(ConnectionId::new(220), hello)
9570 .unwrap()[0]
9571 .header
9572 .ty,
9573 FrameType::HelloAck
9574 );
9575 let request = Frame::build(
9576 FrameType::Request,
9577 control_flags(),
9578 0,
9579 0,
9580 221,
9581 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9582 )
9583 .unwrap();
9584 let response = handler.handle_catalog_list(request, None).unwrap();
9585 let ClientControlResponse::CatalogList { modules, .. } =
9586 serde_json::from_slice(&response[0].body).unwrap()
9587 else {
9588 panic!("expected catalog.list");
9589 };
9590 assert_eq!(
9591 serde_json::to_vec(&modules[0]).unwrap(),
9592 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9593 );
9594 assert!(!modules[0].nonce_verified);
9595 }
9596
9597 #[test]
9598 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9599 let registry = Arc::new(Registry::default());
9600 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9601 [("vault".to_string(), true), ("squatter".to_string(), true)],
9602 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9603 );
9604 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9605 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9606 provides: vec!["credentials-provider/v1".to_string()],
9607 requires: Vec::new(),
9608 must_never_reach: Vec::new(),
9609 });
9610 let frame = Frame::build(
9611 FrameType::Hello,
9612 control_flags(),
9613 0,
9614 0,
9615 77,
9616 serde_json::to_vec(&ModuleHelloBody {
9617 manifest: squatter,
9618 protocol_ver: PROTOCOL_VERSION,
9619 control_ops: None,
9620 launch_nonce: None,
9621 })
9622 .expect("HELLO serializes"),
9623 )
9624 .expect("HELLO frame builds");
9625 let response = handler
9626 .handle_control(ConnectionId::new(77), frame)
9627 .expect("reserved claim receives a typed refusal");
9628 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9629 assert_eq!(
9630 registry
9631 .active_registration_count()
9632 .expect("registry reads"),
9633 0,
9634 "a reserved capability refusal must not leave a catalog entry"
9635 );
9636 }
9637
9638 #[test]
9639 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9640 for settlement in ["timeout", "inconclusive", "drop"] {
9641 let breakers = RouteBindBreakers::default();
9642 let RouteBindAdmission::Admitted {
9643 guard: mut old,
9644 probe: false,
9645 } = breakers.admit("prov")
9646 else {
9647 panic!("ordinary relay admitted")
9648 };
9649 let RouteBindAdmission::Admitted {
9650 guard: mut opener, ..
9651 } = breakers.admit("prov")
9652 else {
9653 panic!("second relay admitted")
9654 };
9655 assert!(
9656 !opener
9657 .record_timeout(1, Duration::ZERO)
9658 .unwrap()
9659 .reopened_after_probe
9660 );
9661 let RouteBindAdmission::Admitted {
9662 guard: mut probe,
9663 probe: true,
9664 } = breakers.admit("prov")
9665 else {
9666 panic!("one cooldown probe admitted")
9667 };
9668 match settlement {
9669 "timeout" => assert!(
9670 !old.record_timeout(1, Duration::ZERO)
9671 .unwrap()
9672 .reopened_after_probe
9673 ),
9674 "inconclusive" => old.record_inconclusive(),
9675 "drop" => drop(old),
9676 _ => unreachable!(),
9677 }
9678 assert!(
9679 matches!(
9680 breakers.admit("prov"),
9681 RouteBindAdmission::Refused {
9682 probe_in_flight: true,
9683 ..
9684 }
9685 ),
9686 "{settlement} of a pre-open relay cannot release the real probe"
9687 );
9688 assert!(
9689 probe
9690 .record_timeout(1, Duration::ZERO)
9691 .unwrap()
9692 .reopened_after_probe
9693 );
9694 assert!(matches!(
9695 breakers.admit("prov"),
9696 RouteBindAdmission::Admitted { probe: true, .. }
9697 ));
9698 }
9699 let breakers = RouteBindBreakers::default();
9700 let admit = || match breakers.admit("prov") {
9701 RouteBindAdmission::Admitted { guard, .. } => guard,
9702 _ => panic!("relay admitted"),
9703 };
9704 admit().record_timeout(1, Duration::ZERO);
9705 let mut old_probe = admit();
9706 breakers.reset_for_new_module_connection("prov");
9707 admit().record_timeout(1, Duration::ZERO);
9708 let _new_probe = admit();
9709 old_probe.record_inconclusive();
9710 assert!(matches!(
9711 breakers.admit("prov"),
9712 RouteBindAdmission::Refused {
9713 probe_in_flight: true,
9714 ..
9715 }
9716 ));
9717 }
9718
9719 #[tokio::test]
9720 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9721 for candidate in [false, true] {
9722 let registry = Arc::new(Registry::default());
9723 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9724 [("vault".to_string(), true), ("squatter".to_string(), true)],
9725 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9726 );
9727 let conn = ConnectionId::new(77);
9728 let (ctx, mut rx) = route_ctx(conn);
9729 let initial = capability_manifest("squatter", &[], &[]);
9730 if candidate {
9731 registry
9732 .register_candidate_with_control_ops(
9733 initial.clone(),
9734 PROTOCOL_VERSION,
9735 conn,
9736 module_baseline_control_ops(),
9737 )
9738 .unwrap();
9739 handler
9740 .forwarding
9741 .register_candidate_module_connection(
9742 conn,
9743 "squatter".to_string(),
9744 PROTOCOL_VERSION,
9745 manifest_concurrency(&initial),
9746 ctx.egress.clone(),
9747 )
9748 .unwrap();
9749 } else {
9750 hello_via_sink(
9751 &handler,
9752 &ctx,
9753 &mut rx,
9754 hello_frame_with_manifest(initial.clone(), 1),
9755 )
9756 .await;
9757 }
9758 let response = handler
9759 .handle_control_frame(
9760 &ctx,
9761 catalog_update_with_capabilities_frame(
9762 2,
9763 capability_manifest("squatter", &["credentials-provider/v1"], &[])
9764 .capabilities
9765 .unwrap(),
9766 ),
9767 )
9768 .await
9769 .unwrap();
9770 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9771 assert_eq!(
9772 registry
9773 .get_module_by_connection(conn)
9774 .unwrap()
9775 .unwrap()
9776 .manifest,
9777 initial
9778 );
9779 }
9780 }
9781
9782 #[test]
9783 fn server_describe_surfaces_required_capability_verdict_fields() {
9784 let registry = Arc::new(Registry::default());
9785 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9786 [
9787 ("consumer".to_string(), true),
9788 ("provider".to_string(), false),
9789 ],
9790 BTreeMap::new(),
9791 );
9792 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9793 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9794 provides: Vec::new(),
9795 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9796 capability: "credentials-provider/v1".to_string(),
9797 need: subc_protocol::manifest::CapabilityNeed::Required,
9798 }],
9799 must_never_reach: Vec::new(),
9800 });
9801 let hello = Frame::build(
9802 FrameType::Hello,
9803 control_flags(),
9804 0,
9805 0,
9806 78,
9807 serde_json::to_vec(&ModuleHelloBody {
9808 manifest: consumer,
9809 protocol_ver: PROTOCOL_VERSION,
9810 control_ops: None,
9811 launch_nonce: None,
9812 })
9813 .expect("HELLO serializes"),
9814 )
9815 .expect("HELLO frame builds");
9816 handler
9817 .handle_control(ConnectionId::new(78), hello)
9818 .expect("consumer registers");
9819 let describe = Frame::build(
9820 FrameType::Request,
9821 control_flags(),
9822 0,
9823 0,
9824 79,
9825 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9826 .expect("request serializes"),
9827 )
9828 .expect("describe frame builds");
9829 let response = handler
9830 .handle_server_describe(describe)
9831 .expect("server.describe succeeds");
9832 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9833 let requirement = &rendered["capability_requirements"][0];
9834 assert_eq!(requirement["consumer"], "consumer");
9835 assert_eq!(requirement["verdict"], "never_provided");
9836 assert_eq!(requirement["episode_seq"], 1);
9837 assert_eq!(requirement["config_satisfiable"], false);
9838 assert_eq!(requirement["runtime_available"], false);
9839 assert!(requirement["detail"]
9840 .as_str()
9841 .expect("detail string")
9842 .contains("credentials-provider/v1"));
9843 }
9844
9845 fn deregister_required_provider(
9851 daemon_shutdown_begun: bool,
9852 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
9853 const CAPABILITY: &str = "project-identity/v1";
9854 let registry = Arc::new(Registry::default());
9855 let daemon_shutdown = DaemonShutdownFlag::default();
9856 let handler = ControlHandler::new(Arc::clone(®istry))
9857 .with_capability_config(
9858 [
9859 ("consumer".to_string(), true),
9860 ("provider".to_string(), true),
9861 ],
9862 BTreeMap::new(),
9863 )
9864 .with_daemon_shutdown_flag(daemon_shutdown.clone());
9865 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9866 consumer.capabilities = Some(CapabilityDeclarations {
9867 provides: Vec::new(),
9868 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9869 capability: CAPABILITY.to_string(),
9870 need: subc_protocol::manifest::CapabilityNeed::Required,
9871 }],
9872 must_never_reach: Vec::new(),
9873 });
9874 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
9875 let provider_conn = ConnectionId::new(182);
9876 for (conn, manifest, corr) in [
9877 (ConnectionId::new(181), consumer, 181),
9878 (provider_conn, provider, 182),
9879 ] {
9880 let hello = handler
9881 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
9882 .expect("module registers");
9883 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9884 }
9885 assert_eq!(
9886 handler.capability_evaluator.verdict("consumer", CAPABILITY),
9887 Some(CapabilityVerdict::Provided),
9888 "the registered provider must satisfy the requirement first"
9889 );
9890 if daemon_shutdown_begun {
9891 daemon_shutdown.set_for_test();
9892 }
9893
9894 let capture = EventCapture::default();
9895 let _subscriber =
9896 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9897 handler
9898 .cleanup_connection(provider_conn)
9899 .expect("provider deregisters");
9900 let events = capture
9901 .events()
9902 .into_iter()
9903 .filter(|event| {
9904 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
9905 })
9906 .collect();
9907 (
9908 events,
9909 handler.capability_evaluator.verdict("consumer", CAPABILITY),
9910 )
9911 }
9912
9913 #[test]
9914 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
9915 let (events, verdict) = deregister_required_provider(true);
9916 assert_eq!(
9917 verdict,
9918 Some(CapabilityVerdict::NeverProvided),
9919 "shutdown must not change the verdict route.open refusals read"
9920 );
9921 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9922 let event = &events[0];
9923 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
9924 assert_eq!(
9925 event.fields.get("cause").map(String::as_str),
9926 Some("\"daemon_shutdown\""),
9927 "{event:?}"
9928 );
9929 assert_eq!(
9930 event.fields.get("verdict").map(String::as_str),
9931 Some("\"never_provided\""),
9932 "{event:?}"
9933 );
9934 }
9935
9936 #[test]
9937 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
9938 let (events, verdict) = deregister_required_provider(false);
9939 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
9940 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9941 let event = &events[0];
9942 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
9943 assert_eq!(event.fields.get("cause"), None, "{event:?}");
9944 assert_eq!(
9945 event.fields.get("verdict").map(String::as_str),
9946 Some("\"never_provided\""),
9947 "{event:?}"
9948 );
9949 }
9950
9951 #[test]
9952 fn catalog_list_omits_capabilities_for_legacy_manifest() {
9953 let registry = Arc::new(Registry::default());
9954 let handler = ControlHandler::new(Arc::clone(®istry));
9955 let hello = handler
9956 .handle_control(
9957 ConnectionId::new(101),
9958 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9959 )
9960 .expect("legacy HELLO registers");
9961 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9962
9963 let request = Frame::build(
9964 FrameType::Request,
9965 control_flags(),
9966 0,
9967 0,
9968 102,
9969 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9970 .expect("catalog request serializes"),
9971 )
9972 .expect("catalog request frame builds");
9973 let response = handler
9974 .handle_catalog_list(request, None)
9975 .expect("catalog list succeeds");
9976 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9977 assert!(
9978 body["modules"][0].get("capabilities").is_none(),
9979 "legacy manifest must retain an absent capabilities field on catalog.list"
9980 );
9981 }
9982
9983 #[test]
9984 fn hello_ack_omits_storage_when_no_storage_config() {
9985 let registry = Arc::new(Registry::default());
9986 let handler = ControlHandler::new(Arc::clone(®istry));
9987 let responses = handler
9988 .handle_control(
9989 ConnectionId::new(1),
9990 hello_frame("aft", PROTOCOL_VERSION, 7),
9991 )
9992 .unwrap();
9993 let ack = parse_ack(&responses[0]);
9994 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9995 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9996 }
9997
9998 #[tokio::test]
9999 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10000 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10001 let registry = Arc::new(Registry::default());
10002 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
10003 let responses = handler
10004 .handle_control(
10005 ConnectionId::new(1),
10006 hello_frame("aft", PROTOCOL_VERSION, 7),
10007 )
10008 .unwrap();
10009 let ack = parse_ack(&responses[0]);
10010 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10011
10012 let described = handler
10013 .handle_control_frame(
10014 &route_ctx(ConnectionId::new(2)).0,
10015 Frame::build(
10016 FrameType::Request,
10017 control_flags(),
10018 0,
10019 0,
10020 9,
10021 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10022 )
10023 .unwrap(),
10024 )
10025 .await
10026 .unwrap();
10027 let ClientControlResponse::ServerDescribe { machine_id, .. } =
10028 serde_json::from_slice(&described[0].body).unwrap()
10029 else {
10030 panic!("server.describe answered with another shape");
10031 };
10032 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10033 }
10034
10035 #[test]
10036 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10037 let registry = Arc::new(Registry::default());
10040 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
10041 crate::daemon_config::StorageConfig::Sqlite {
10042 data_home: std::path::PathBuf::from("/data"),
10043 },
10044 ));
10045
10046 let responses = handler
10047 .handle_control(
10048 ConnectionId::new(1),
10049 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10050 )
10051 .unwrap();
10052 let ack = parse_ack(&responses[0]);
10053 assert_eq!(
10054 ack.storage,
10055 Some(serde_json::json!({
10056 "module_id": "alfonso-routing",
10057 "storage_namespace": "default",
10058 "isolation": { "kind": "module" },
10059 "backend": {
10060 "backend": "sqlite",
10061 "path": "/data/cortexkit/alfonso-routing/store.db"
10062 }
10063 })),
10064 "the delivered descriptor is the module's own sqlite store path"
10065 );
10066 }
10067
10068 #[test]
10069 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10070 let registry = Arc::new(Registry::default());
10071 let handler = ControlHandler::new(Arc::clone(®istry));
10072 let conn = ConnectionId::new(1);
10073 let responses = handler
10074 .handle_control(
10075 conn,
10076 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10077 )
10078 .unwrap();
10079 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10080 let registration = registry.get_module("aft").unwrap().unwrap();
10081 assert_eq!(registration.control_ops, module_baseline_control_ops());
10082
10083 let frame =
10084 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10085 assert!(handler
10086 .guard_module_control_op(&frame, "aft", "route.bind")
10087 .unwrap()
10088 .is_none());
10089 let error = handler
10090 .guard_module_control_op(&frame, "aft", "test.synthetic")
10091 .unwrap()
10092 .expect("synthetic ungranted op should be rejected");
10093 assert_eq!(error.header.ty, FrameType::Error);
10094 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10095 }
10096
10097 #[test]
10098 fn hello_control_ops_some_adds_optional_grants() {
10099 let registry = Arc::new(Registry::default());
10100 let handler = ControlHandler::new(Arc::clone(®istry));
10101 handler
10102 .handle_control(
10103 ConnectionId::new(1),
10104 hello_frame_with_control_ops(
10105 "aft",
10106 PROTOCOL_VERSION,
10107 7,
10108 Some(vec![
10109 "future.synthetic".to_string(),
10110 "route.bind".to_string(),
10111 ]),
10112 ),
10113 )
10114 .unwrap();
10115 let registration = registry.get_module("aft").unwrap().unwrap();
10116 assert_eq!(
10117 registration.control_ops,
10118 vec![
10119 "route.bind".to_string(),
10120 "route.status".to_string(),
10121 "future.synthetic".to_string(),
10122 ]
10123 );
10124 let frame =
10125 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10126 assert!(handler
10127 .guard_module_control_op(&frame, "aft", "future.synthetic")
10128 .unwrap()
10129 .is_none());
10130 }
10131
10132 #[tokio::test]
10133 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10134 let registry = Arc::new(Registry::default());
10135 let forwarding = Arc::new(ForwardingTable::default());
10136 let handler =
10137 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10138 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10139 hello_via_sink(
10140 &handler,
10141 &module_ctx,
10142 &mut module_rx,
10143 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10144 )
10145 .await;
10146
10147 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10148 let responses = handler
10149 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10150 .await
10151 .unwrap();
10152 assert_eq!(responses.len(), 1);
10153 assert_eq!(responses[0].header.ty, FrameType::Error);
10154 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10155 assert!(module_rx.try_recv().is_err());
10156 }
10157
10158 #[tokio::test]
10159 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10160 let registry = Arc::new(Registry::default());
10161 let forwarding = Arc::new(ForwardingTable::default());
10162 let handler =
10163 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10164 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10165 hello_via_sink(
10166 &handler,
10167 &module_ctx,
10168 &mut module_rx,
10169 hello_frame_with_control_ops(
10170 "aft",
10171 PROTOCOL_VERSION,
10172 7,
10173 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10174 ),
10175 )
10176 .await;
10177
10178 let project_root = unique_project_root("demux");
10179 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10180 let route_handler = handler.clone();
10181 let route_task = tokio::spawn(async move {
10182 route_handler
10183 .handle_control_frame(
10184 &route_client_ctx,
10185 route_open_frame(100, "aft", project_root),
10186 )
10187 .await
10188 .unwrap()
10189 });
10190 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10191 .await
10192 .unwrap()
10193 .unwrap();
10194 assert!(matches!(
10195 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10196 ModuleControlRequest::RouteBind { .. }
10197 ));
10198
10199 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10200 let health_handler = handler.clone();
10201 let health_task = tokio::spawn(async move {
10202 health_handler
10203 .handle_control_frame(
10204 &health_client_ctx,
10205 supervisor_health_probe_frame(101, "aft"),
10206 )
10207 .await
10208 .unwrap()
10209 });
10210 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10211 .await
10212 .unwrap()
10213 .unwrap();
10214 assert_eq!(
10215 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10216 ModuleControlRequest::HealthCheck {}
10217 );
10218
10219 handler
10220 .handle_control_frame(
10221 &module_ctx,
10222 health_response(health_frame.header.corr, HealthStatus::Degraded),
10223 )
10224 .await
10225 .unwrap();
10226 let health_response = health_task.await.unwrap();
10227 assert_eq!(health_response.len(), 1);
10228 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10229 ClientControlResponse::SupervisorHealthProbe {
10230 module_id,
10231 status,
10232 detail,
10233 metrics,
10234 } => {
10235 assert_eq!(module_id, "aft");
10236 assert_eq!(status, HealthStatus::Degraded);
10237 assert_eq!(detail.as_deref(), Some("warming"));
10238 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10239 }
10240 other => panic!("unexpected health response: {other:?}"),
10241 }
10242
10243 handler
10244 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10245 .await
10246 .unwrap();
10247 let route_response = route_task.await.unwrap();
10248 assert!(route_response.is_empty());
10249 let published = route_client_rx.recv().await.unwrap();
10250 assert!(matches!(
10251 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10252 ClientControlResponse::RouteOpen { .. }
10253 ));
10254 }
10255
10256 async fn relay_route_open(
10261 handler: &ControlHandler,
10262 client_connection: ConnectionId,
10263 client_egress: &FrameSink,
10264 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10265 corr: u64,
10266 module_id: &str,
10267 project_root_label: &str,
10268 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10269 let ctx = RouteCtx {
10270 connection_id: client_connection,
10271 egress: client_egress.clone(),
10272 };
10273 let handler = handler.clone();
10274 let project_root = unique_project_root(project_root_label);
10275 let module_id = module_id.to_string();
10276 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10277 let task = tokio::spawn(async move {
10278 let _guard = tracing::dispatcher::set_default(&dispatch);
10279 handler
10280 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10281 .await
10282 .unwrap()
10283 });
10284 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10285 .await
10286 .expect("module receives the relayed route.bind")
10287 .expect("module egress is open");
10288 (task, bind.frame)
10289 }
10290
10291 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10292 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10293 ModuleControlRequest::RouteBind {
10294 route_channel,
10295 epoch,
10296 ..
10297 } => (route_channel, epoch),
10298 other => panic!("expected a route.bind request, got {other:?}"),
10299 }
10300 }
10301
10302 fn published_route(frame: &Frame) -> (u16, u32) {
10303 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10304 ClientControlResponse::RouteOpen {
10305 route_channel,
10306 route_epoch,
10307 } => (route_channel, route_epoch),
10308 other => panic!("expected a route.open response, got {other:?}"),
10309 }
10310 }
10311
10312 #[tokio::test]
10332 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10333 let registry = Arc::new(Registry::default());
10334 let forwarding = Arc::new(ForwardingTable::default());
10335 let handler =
10336 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10337
10338 let module_connection = ConnectionId::new(30);
10339 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10340 hello_via_sink(
10341 &handler,
10342 &module_ctx,
10343 &mut module_rx,
10344 hello_frame("aft", PROTOCOL_VERSION, 7),
10345 )
10346 .await;
10347
10348 let dying_client = ConnectionId::new(31);
10349 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10350
10351 let (first_task, first_bind) = relay_route_open(
10354 &handler,
10355 dying_client,
10356 &dying_ctx.egress,
10357 &mut module_rx,
10358 100,
10359 "aft",
10360 "closing-first",
10361 )
10362 .await;
10363 handler
10364 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10365 .await
10366 .unwrap();
10367 assert!(first_task.await.unwrap().is_empty());
10368 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10369
10370 let (second_task, second_bind) = relay_route_open(
10372 &handler,
10373 dying_client,
10374 &dying_ctx.egress,
10375 &mut module_rx,
10376 101,
10377 "aft",
10378 "closing-second",
10379 )
10380 .await;
10381 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10382
10383 assert!(forwarding
10386 .escalate_client_delivery_failure(
10387 dying_client,
10388 first_channel,
10389 first_epoch,
10390 CloseReason::new(
10391 "module_to_client_delivery_failed",
10392 "client egress refused a module frame",
10393 ),
10394 crate::forwarding::UndeliveredFrame {
10395 module_id: None,
10396 sink: &dying_ctx.egress,
10397 },
10398 )
10399 .unwrap());
10400 assert!(!dying_ctx.egress.is_closed());
10401
10402 let ack = handler
10404 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10405 .await;
10406 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10407 if module_loop_error.is_some() {
10408 handler.cleanup_connection(module_connection).unwrap();
10412 }
10413 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10417 .await
10418 .ok()
10419 .flatten();
10420
10421 assert!(
10423 registry
10424 .get_module_by_connection(module_connection)
10425 .unwrap()
10426 .is_some(),
10427 "one client's closing connection ended the shared module connection: \
10428 {module_loop_error:?}"
10429 );
10430 let cotenant = ConnectionId::new(32);
10432 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10433 let (cotenant_task, cotenant_bind) = relay_route_open(
10434 &handler,
10435 cotenant,
10436 &cotenant_ctx.egress,
10437 &mut module_rx,
10438 102,
10439 "aft",
10440 "closing-cotenant",
10441 )
10442 .await;
10443 handler
10444 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10445 .await
10446 .unwrap();
10447 assert!(cotenant_task.await.unwrap().is_empty());
10448 let (cotenant_channel, cotenant_epoch) =
10449 published_route(&cotenant_rx.recv().await.unwrap());
10450 assert!(matches!(
10451 forwarding
10452 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10453 .unwrap(),
10454 DataRoute::Client(DataRouteState::Bound(_))
10455 ));
10456
10457 let goodbye = post_ack_module_frame
10460 .expect("module receives a GOODBYE for the abandoned route channel");
10461 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10462 assert_eq!(goodbye.header.channel, abandoned_channel);
10463 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10464
10465 assert!(dying_rx.try_recv().is_err());
10469 let second_response = second_task.await.unwrap();
10470 assert_eq!(second_response.len(), 1);
10471 assert_eq!(
10472 parse_error(&second_response[0])["code"],
10473 "target_unavailable"
10474 );
10475 }
10476
10477 async fn catalog_snapshot(
10480 handler: &ControlHandler,
10481 client_ctx: &RouteCtx,
10482 corr: u64,
10483 ) -> (u64, Vec<(String, bool)>) {
10484 let body =
10485 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10486 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10487 let reply = handler
10488 .handle_control_frame(client_ctx, request)
10489 .await
10490 .unwrap()
10491 .pop()
10492 .unwrap();
10493 let ClientControlResponse::CatalogList {
10494 generation,
10495 modules,
10496 ..
10497 } = serde_json::from_slice(&reply.body).unwrap()
10498 else {
10499 panic!("not a catalog.list reply");
10500 };
10501 (
10502 generation,
10503 modules
10504 .into_iter()
10505 .map(|entry| (entry.module_id, entry.ready))
10506 .collect(),
10507 )
10508 }
10509
10510 fn hello_held_before_forwarding() -> (
10516 ControlHandler,
10517 Arc<std::sync::Barrier>,
10518 std::thread::JoinHandle<Vec<Frame>>,
10519 RouteCtx,
10520 mpsc::Receiver<crate::router::OutboundFrame>,
10521 ) {
10522 let hold = Arc::new(std::sync::Barrier::new(2));
10523 let handler = ControlHandler::with_forwarding(
10524 Arc::new(Registry::default()),
10525 Arc::new(ForwardingTable::default()),
10526 )
10527 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10528 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10529 let hello = {
10530 let handler = handler.clone();
10531 let module_ctx = module_ctx.clone();
10532 std::thread::spawn(move || {
10533 handler
10534 .handle_hello(
10535 module_ctx.connection_id,
10536 Some(module_ctx.egress.clone()),
10537 hello_frame("aft", PROTOCOL_VERSION, 7),
10538 )
10539 .unwrap()
10540 })
10541 };
10542 (handler, hold, hello, module_ctx, module_rx)
10543 }
10544
10545 #[tokio::test]
10554 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10555 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10556 let client = ConnectionId::new(41);
10557 let (client_ctx, mut client_rx) = route_ctx(client);
10558
10559 hold.wait();
10560 assert!(handler.registry.get_module("aft").unwrap().is_some());
10562 assert!(!handler
10563 .forwarding
10564 .has_live_module_connection("aft")
10565 .unwrap());
10566 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10567 assert!(
10568 listed.is_empty(),
10569 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10570 );
10571 let refused = handler
10572 .handle_control_frame(
10573 &client_ctx,
10574 route_open_frame(301, "aft", unique_project_root("listed-window")),
10575 )
10576 .await
10577 .unwrap();
10578 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10579 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10580 let duplicate = handler
10581 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10582 .await
10583 .unwrap();
10584 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10585
10586 hold.wait();
10587 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10588 let ack = module_rx.try_recv().unwrap().frame;
10589 assert_eq!(ack.header.ty, FrameType::HelloAck);
10590 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10591 assert_eq!(listed, vec![("aft".to_string(), true)]);
10592
10593 let (open_task, bind) = relay_route_open(
10594 &handler,
10595 client,
10596 &client_ctx.egress,
10597 &mut module_rx,
10598 303,
10599 "aft",
10600 "listed-after",
10601 )
10602 .await;
10603 assert!(handler
10604 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10605 .await
10606 .unwrap()
10607 .is_empty());
10608 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10609 published_route(&client_rx.try_recv().unwrap().frame);
10610 }
10611
10612 #[tokio::test]
10619 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10620 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10621 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10622 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10623
10624 hold.wait();
10625 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10626 assert!(listed.is_empty());
10627 assert_eq!(
10628 during, before,
10629 "the generation moved while the module was not listed"
10630 );
10631
10632 hold.wait();
10633 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10634 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10635 assert_eq!(listed, vec![("aft".to_string(), true)]);
10636 assert_eq!(
10637 after,
10638 before + 1,
10639 "the module appeared without exactly one generation bump"
10640 );
10641 }
10642
10643 #[tokio::test]
10648 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10649 let registry = Arc::new(Registry::default());
10650 let handler = ControlHandler::with_forwarding(
10651 Arc::clone(®istry),
10652 Arc::new(ForwardingTable::default()),
10653 );
10654 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10655 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10656 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10657 drop(module_rx);
10658
10659 let replies = handler
10660 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10661 .await
10662 .unwrap();
10663 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10664 assert!(registry.get_module("aft").unwrap().is_none());
10665 assert!(registry
10666 .get_module_by_connection(module_ctx.connection_id)
10667 .unwrap()
10668 .is_none());
10669 assert_eq!(
10670 catalog_snapshot(&handler, &client_ctx, 301).await,
10671 (before, Vec::new())
10672 );
10673 }
10674
10675 async fn route_open_held_after_bind_send() -> (
10679 ControlHandler,
10680 Arc<tokio::sync::Barrier>,
10681 RouteCtx,
10682 mpsc::Receiver<crate::router::OutboundFrame>,
10683 ) {
10684 let hold = Arc::new(tokio::sync::Barrier::new(2));
10685 let handler = ControlHandler::with_forwarding(
10686 Arc::new(Registry::default()),
10687 Arc::new(ForwardingTable::default()),
10688 )
10689 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10690 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10691 hello_via_sink(
10692 &handler,
10693 &module_ctx,
10694 &mut module_rx,
10695 hello_frame("aft", PROTOCOL_VERSION, 7),
10696 )
10697 .await;
10698 (handler, hold, module_ctx, module_rx)
10699 }
10700
10701 #[tokio::test]
10708 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10709 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10710 let client = ConnectionId::new(41);
10711 let (client_ctx, mut client_rx) = route_ctx(client);
10712
10713 let (open_task, bind) = relay_route_open(
10714 &handler,
10715 client,
10716 &client_ctx.egress,
10717 &mut module_rx,
10718 200,
10719 "aft",
10720 "acked-before-mark",
10721 )
10722 .await;
10723 let (module_channel, module_epoch) = route_bind_channel(&bind);
10724 hold.wait().await;
10726
10727 assert!(handler
10729 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10730 .await
10731 .unwrap()
10732 .is_empty());
10733 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10734
10735 hold.wait().await;
10736 assert!(
10737 open_task.await.unwrap().is_empty(),
10738 "route.open answers accepted"
10739 );
10740
10741 match module_rx.try_recv() {
10744 Err(mpsc::error::TryRecvError::Empty) => {}
10745 Ok(outbound) => panic!(
10746 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10747 outbound.frame.header.ty,
10748 outbound.frame.header.channel,
10749 outbound.frame.header.epoch
10750 ),
10751 Err(err) => panic!("module egress closed: {err}"),
10752 }
10753
10754 let router = Router::with_control_handler(Arc::new(handler.clone()));
10757 let request = Frame::build(
10758 FrameType::Request,
10759 Flags::new(true, Priority::Interactive, false),
10760 client_channel,
10761 client_epoch,
10762 201,
10763 b"first request".to_vec(),
10764 )
10765 .unwrap();
10766 router
10767 .route_for_connection(&client_ctx, request)
10768 .await
10769 .unwrap();
10770 let delivered = module_rx.try_recv().unwrap().frame;
10771 assert_eq!(delivered.header.ty, FrameType::Request);
10772 assert_eq!(
10773 (delivered.header.channel, delivered.header.epoch),
10774 (module_channel, module_epoch)
10775 );
10776 assert_eq!(delivered.body, b"first request");
10777 }
10778
10779 #[tokio::test]
10785 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10786 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10787 let client = ConnectionId::new(42);
10788 let (client_ctx, _client_rx) = route_ctx(client);
10789
10790 let (open_task, bind) = relay_route_open(
10791 &handler,
10792 client,
10793 &client_ctx.egress,
10794 &mut module_rx,
10795 300,
10796 "aft",
10797 "released-before-mark",
10798 )
10799 .await;
10800 let (module_channel, module_epoch) = route_bind_channel(&bind);
10801 hold.wait().await;
10802
10803 handler.cleanup_connection(client).unwrap();
10804 assert!(
10805 module_rx.try_recv().is_err(),
10806 "client teardown must not GOODBYE a bind route.open is still queueing"
10807 );
10808
10809 hold.wait().await;
10810 let response = open_task.await.unwrap();
10811 assert_eq!(response.len(), 1);
10812 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10813
10814 let goodbye = module_rx
10815 .try_recv()
10816 .expect("module is told to drop the binding the daemon abandoned")
10817 .frame;
10818 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10819 assert_eq!(
10820 (goodbye.header.channel, goodbye.header.epoch),
10821 (module_channel, module_epoch)
10822 );
10823 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10824 }
10825
10826 #[tokio::test]
10831 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10832 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10833 let client = ConnectionId::new(43);
10834 let (client_ctx, client_rx) = route_ctx(client);
10835
10836 let (open_task, bind) = relay_route_open(
10837 &handler,
10838 client,
10839 &client_ctx.egress,
10840 &mut module_rx,
10841 400,
10842 "aft",
10843 "gone-client-before-mark",
10844 )
10845 .await;
10846 let (module_channel, module_epoch) = route_bind_channel(&bind);
10847 hold.wait().await;
10848
10849 drop(client_rx);
10850 assert!(client_ctx.egress.is_closed());
10851 handler
10852 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10853 .await
10854 .unwrap();
10855
10856 hold.wait().await;
10857 let response = open_task.await.unwrap();
10858 assert_eq!(response.len(), 1);
10859 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10860
10861 let goodbye = module_rx
10862 .try_recv()
10863 .expect("module is told to drop the binding it acked")
10864 .frame;
10865 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10866 assert_eq!(
10867 (goodbye.header.channel, goodbye.header.epoch),
10868 (module_channel, module_epoch)
10869 );
10870 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10871 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10872 }
10873
10874 #[test]
10881 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10882 let handler = ControlHandler::default();
10883 let module_connection = ConnectionId::new(30);
10884 let client_connection = ConnectionId::new(31);
10885
10886 handler
10887 .refuse_to_end_module_connection_for_a_client(
10888 module_connection,
10889 77,
10890 ForwardingError::ConnectionClosing {
10891 connection_id: client_connection,
10892 },
10893 )
10894 .expect("a closing client must never end the module connection");
10895
10896 assert!(matches!(
10897 handler.refuse_to_end_module_connection_for_a_client(
10898 module_connection,
10899 78,
10900 ForwardingError::ConnectionClosing {
10901 connection_id: module_connection,
10902 },
10903 ),
10904 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10905 connection_id
10906 })) if connection_id == module_connection
10907 ));
10908 assert!(matches!(
10909 handler.refuse_to_end_module_connection_for_a_client(
10910 module_connection,
10911 79,
10912 ForwardingError::Poisoned,
10913 ),
10914 Err(RouterError::Forwarding(ForwardingError::Poisoned))
10915 ));
10916 assert!(matches!(
10917 handler.refuse_to_end_module_connection_for_a_client(
10918 module_connection,
10919 80,
10920 ForwardingError::StaleModuleEndpoint,
10921 ),
10922 Err(RouterError::Forwarding(
10923 ForwardingError::StaleModuleEndpoint
10924 ))
10925 ));
10926 }
10927
10928 #[tokio::test]
10941 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10942 let registry = Arc::new(Registry::default());
10943 let forwarding = Arc::new(ForwardingTable::default());
10944 let supervisor = SupervisorHandle::new();
10945 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10946 let handler =
10947 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10948 .with_supervisor(supervisor);
10949
10950 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10951 hello_via_sink(
10952 &handler,
10953 &target_ctx,
10954 &mut target_rx,
10955 hello_frame("target", PROTOCOL_VERSION, 1),
10956 )
10957 .await;
10958
10959 let wrong_nonce = handler
10963 .handle_control_frame(
10964 &route_ctx(ConnectionId::new(91)).0,
10965 route_open_frame_with_admission_facts(
10966 20,
10967 "target",
10968 unique_project_root("admission-facts"),
10969 Some(subc_control::ConsumerIdentity {
10970 module_id: "fed".to_string(),
10971 launch_nonce: "not-the-real-nonce".to_string(),
10972 }),
10973 None,
10974 ),
10975 )
10976 .await
10977 .unwrap();
10978 assert_eq!(
10979 parse_error(&wrong_nonce[0])["code"],
10980 "bad_consumer_identity",
10981 "a mismatched launch nonce must be refused, not stamped Reserved"
10982 );
10983
10984 let never_spawned = handler
10988 .handle_control_frame(
10989 &route_ctx(ConnectionId::new(92)).0,
10990 route_open_frame_with_admission_facts(
10991 21,
10992 "target",
10993 unique_project_root("admission-facts"),
10994 Some(subc_control::ConsumerIdentity {
10995 module_id: "never-spawned".to_string(),
10996 launch_nonce: "any-nonce".to_string(),
10997 }),
10998 None,
10999 ),
11000 )
11001 .await
11002 .unwrap();
11003 assert_eq!(
11004 parse_error(&never_spawned[0])["code"],
11005 "bad_consumer_identity",
11006 "an unspawned module_id must be refused rather than accepted for lack of a record"
11007 );
11008 }
11009
11010 #[tokio::test]
11033 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11034 let registry = Arc::new(Registry::default());
11035 let forwarding = Arc::new(ForwardingTable::default());
11036 let supervisor = SupervisorHandle::new();
11037 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11038 let handler =
11039 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11040 .with_supervisor(supervisor);
11041
11042 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11043 hello_via_sink(
11044 &handler,
11045 &target_ctx,
11046 &mut target_rx,
11047 hello_frame("target", PROTOCOL_VERSION, 1),
11048 )
11049 .await;
11050
11051 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11052 let route_handler = handler.clone();
11053 let route_task = tokio::spawn(async move {
11054 route_handler
11055 .handle_control_frame(
11056 &client_ctx,
11057 route_open_frame_with_admission_facts(
11058 30,
11059 "target",
11060 unique_project_root("admission-facts"),
11061 Some(subc_control::ConsumerIdentity {
11062 module_id: "fed".to_string(),
11063 launch_nonce: "fed-nonce".to_string(),
11064 }),
11065 None,
11066 ),
11067 )
11068 .await
11069 .unwrap()
11070 });
11071
11072 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11079 .await
11080 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11081 .expect("module control channel closed before route.bind");
11082 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11083 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11084 panic!("expected route.bind")
11085 };
11086 assert_eq!(
11087 principal,
11088 Some(Principal::Reserved {
11089 module_id: "fed".to_string()
11090 }),
11091 "a correctly attested consumer must be stamped Reserved for its own id"
11092 );
11093
11094 handler
11095 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11096 .await
11097 .unwrap();
11098 assert!(route_task.await.unwrap().is_empty());
11099 assert!(
11100 matches!(
11101 serde_json::from_slice::<ClientControlResponse>(
11102 &client_rx.recv().await.unwrap().body
11103 )
11104 .unwrap(),
11105 ClientControlResponse::RouteOpen { .. }
11106 ),
11107 "the route must actually open, not merely avoid an error"
11108 );
11109 }
11110
11111 #[tokio::test(start_paused = true)]
11112 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11113 let registry = Arc::new(Registry::default());
11114 let forwarding = Arc::new(ForwardingTable::default());
11115 let supervisor = SupervisorHandle::new();
11116 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11117 let handler =
11118 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11119 .with_supervisor(supervisor);
11120
11121 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11122 hello_via_sink(
11123 &handler,
11124 &target_ctx,
11125 &mut target_rx,
11126 hello_frame("target", PROTOCOL_VERSION, 1),
11127 )
11128 .await;
11129
11130 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11131 let direct_handler = handler.clone();
11132 let direct_open = tokio::spawn(async move {
11133 direct_handler
11134 .handle_control_frame(
11135 &direct_ctx,
11136 route_open_frame(2, "target", unique_project_root("route-census-direct")),
11137 )
11138 .await
11139 .unwrap()
11140 });
11141 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11142 .await
11143 .expect("no direct route.bind within 5s")
11144 .expect("target control channel closed before direct route.bind");
11145 handler
11146 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11147 .await
11148 .unwrap();
11149 assert!(direct_open.await.unwrap().is_empty());
11150 let _ = direct_rx.recv().await.unwrap();
11151
11152 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11153 let reserved_handler = handler.clone();
11154 let reserved_open = tokio::spawn(async move {
11155 reserved_handler
11156 .handle_control_frame(
11157 &reserved_ctx,
11158 route_open_frame_with_admission_facts(
11159 3,
11160 "target",
11161 unique_project_root("admission-facts"),
11162 Some(ConsumerIdentity {
11163 module_id: "fed".to_string(),
11164 launch_nonce: "fed-nonce".to_string(),
11165 }),
11166 None,
11167 ),
11168 )
11169 .await
11170 .unwrap()
11171 });
11172 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11173 .await
11174 .expect("no reserved route.bind within 5s")
11175 .expect("target control channel closed before reserved route.bind");
11176 handler
11177 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11178 .await
11179 .unwrap();
11180 assert!(reserved_open.await.unwrap().is_empty());
11181 let _ = reserved_rx.recv().await.unwrap();
11182
11183 forwarding
11184 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11185 .unwrap();
11186 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11187 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11188 module_id: Some("target".to_string()),
11189 })
11190 .unwrap();
11191 let census_frame =
11192 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11193 let response = handler
11194 .handle_control_frame(&census_ctx, census_frame)
11195 .await
11196 .unwrap()
11197 .pop()
11198 .unwrap();
11199 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11200 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11201 assert!(matches!(
11202 decoded,
11203 ClientControlResponse::SupervisorRoutes { .. }
11204 ));
11205 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11206 assert_eq!(routes.len(), 2);
11207 assert!(routes.iter().all(|route| route["draining"] == true));
11208 assert!(
11211 routes.iter().all(|route| route["drain_reason"] == "reload"),
11212 "draining routes must name the drain's reason: {routes:?}"
11213 );
11214 assert!(routes.iter().any(|route| {
11215 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11216 }));
11217 assert!(routes.iter().any(|route| {
11218 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11219 }));
11220
11221 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11222 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11223 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11224 std::fs::write(
11225 &golden_path,
11226 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11227 )
11228 .unwrap();
11229 }
11230 let expected: Value =
11231 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11232 assert_eq!(actual, expected);
11233 }
11234
11235 async fn query_live_roots(
11236 handler: &ControlHandler,
11237 module_ctx: &RouteCtx,
11238 ) -> ModuleControlResponseToModule {
11239 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11240 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11241 let response = handler
11242 .handle_control_frame(module_ctx, frame)
11243 .await
11244 .unwrap()
11245 .pop()
11246 .unwrap();
11247 serde_json::from_slice(&response.body).unwrap()
11248 }
11249
11250 #[tokio::test(start_paused = true)]
11251 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11252 let registry = Arc::new(Registry::default());
11253 let forwarding = Arc::new(ForwardingTable::default());
11254 let handler = ControlHandler::with_forwarding(registry, forwarding);
11255 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11256 hello_via_sink(
11257 &handler,
11258 &target_ctx,
11259 &mut target_rx,
11260 hello_frame("target", PROTOCOL_VERSION, 1),
11261 )
11262 .await;
11263 let root = unique_project_root("live-roots-known");
11264 let path = ProjectRootId::from_path_allowing_missing(root.path())
11265 .unwrap()
11266 .as_path()
11267 .to_path_buf();
11268 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11269 let open_handler = handler.clone();
11270 let opened = tokio::spawn(async move {
11271 open_handler
11272 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11273 .await
11274 .unwrap()
11275 });
11276 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11277 .await
11278 .unwrap()
11279 .unwrap();
11280 handler
11281 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11282 .await
11283 .unwrap();
11284 assert!(opened.await.unwrap().is_empty());
11285 let _ = client_rx.recv().await.unwrap();
11286
11287 let root = unique_project_root("live-roots-pending");
11288 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11289 .unwrap()
11290 .as_path()
11291 .to_path_buf();
11292 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11293 let open_handler = handler.clone();
11294 let pending = tokio::spawn(async move {
11295 open_handler
11296 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11297 .await
11298 .unwrap()
11299 });
11300 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11301 .await
11302 .unwrap()
11303 .unwrap();
11304 let actual = query_live_roots(&handler, &target_ctx).await;
11305 let ModuleControlResponseToModule::LiveRoots {
11306 roots,
11307 unknown_root_bindings,
11308 total_bindings,
11309 } = actual
11310 else {
11311 panic!("expected live roots")
11312 };
11313 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11314 assert_eq!(unknown_root_bindings, 0);
11315 assert_eq!(
11316 roots.len(),
11317 2,
11318 "root-known arm must retain each canonical root"
11319 );
11320 assert_eq!(
11321 total_bindings,
11322 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11323 );
11324 let counts = roots
11325 .iter()
11326 .map(|root| (root.project_root.clone(), root.bound, root.pending))
11327 .collect::<Vec<_>>();
11328 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11329 expected.sort_by(|a, b| a.0.cmp(&b.0));
11330 assert_eq!(
11331 counts, expected,
11332 "roots must sort by path and count pending separately"
11333 );
11334 handler
11335 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11336 .await
11337 .unwrap();
11338 assert!(pending.await.unwrap().is_empty());
11339 }
11340
11341 #[tokio::test(start_paused = true)]
11342 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11343 let forwarding = Arc::new(ForwardingTable::default());
11344 let handler =
11345 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11346 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11347 hello_via_sink(
11348 &handler,
11349 &target_ctx,
11350 &mut target_rx,
11351 hello_frame("target", PROTOCOL_VERSION, 1),
11352 )
11353 .await;
11354 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11355 let pending = forwarding
11356 .begin_route_bind_relay_for_test(
11357 client_ctx.connection_id,
11358 client_ctx.egress.clone(),
11359 2,
11360 "target",
11361 )
11362 .unwrap();
11363 forwarding
11364 .complete_pending_relay(
11365 target_ctx.connection_id,
11366 pending.corr,
11367 RouteBindRelayOutcome::Accepted,
11368 )
11369 .unwrap();
11370 let actual = query_live_roots(&handler, &target_ctx).await;
11371 let ModuleControlResponseToModule::LiveRoots {
11372 roots,
11373 unknown_root_bindings,
11374 total_bindings,
11375 } = actual
11376 else {
11377 panic!("expected live roots")
11378 };
11379 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11380 assert_eq!(
11381 unknown_root_bindings, 1,
11382 "unknown-root arm must not read as no bindings"
11383 );
11384 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11385 assert_eq!(
11386 total_bindings,
11387 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11388 );
11389 }
11390
11391 #[tokio::test(start_paused = true)]
11399 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11400 let forwarding = Arc::new(ForwardingTable::default());
11401 let handler =
11402 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11403 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11404 let replies = handler
11405 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11406 .await
11407 .unwrap();
11408 let queued_by_hello = module_rx.len();
11409
11410 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11411 let open_handler = handler.clone();
11412 let open = tokio::spawn(async move {
11413 open_handler
11414 .handle_control_frame(
11415 &client_ctx,
11416 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11417 )
11418 .await
11419 .unwrap()
11420 });
11421 let mut spins = 0;
11423 while module_rx.len() == queued_by_hello {
11424 spins += 1;
11425 assert!(spins < 10_000, "route.open never queued a route.bind");
11426 tokio::task::yield_now().await;
11427 }
11428
11429 for reply in replies {
11431 module_ctx.egress.send(reply).await.unwrap();
11432 }
11433
11434 let first = module_rx.recv().await.unwrap().frame;
11435 assert_eq!(
11436 first.header.ty,
11437 FrameType::HelloAck,
11438 "the first frame a registering module reads must be its HELLO_ACK"
11439 );
11440 assert_eq!(first.header.corr, 7);
11441 let second = module_rx.recv().await.unwrap().frame;
11442 assert_eq!(second.header.ty, FrameType::Request);
11443 assert!(
11444 matches!(
11445 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11446 ModuleControlRequest::RouteBind { .. }
11447 ),
11448 "the route.bind follows the ack"
11449 );
11450 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11451
11452 handler
11453 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11454 .await
11455 .unwrap();
11456 assert!(open.await.unwrap().is_empty());
11457 let _ = client_rx.recv().await.unwrap();
11458 }
11459
11460 #[tokio::test(start_paused = true)]
11461 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11462 let handler = ControlHandler::with_forwarding(
11463 Arc::new(Registry::default()),
11464 Arc::new(ForwardingTable::default()),
11465 );
11466 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11467 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11468 hello_via_sink(
11469 &handler,
11470 &first_ctx,
11471 &mut first_rx,
11472 hello_frame("first", PROTOCOL_VERSION, 1),
11473 )
11474 .await;
11475 hello_via_sink(
11476 &handler,
11477 &second_ctx,
11478 &mut second_rx,
11479 hello_frame("second", PROTOCOL_VERSION, 2),
11480 )
11481 .await;
11482 let root = unique_project_root("second-only");
11483 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11484 let cloned = handler.clone();
11485 let open = tokio::spawn(async move {
11486 cloned
11487 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11488 .await
11489 .unwrap()
11490 });
11491 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11492 .await
11493 .unwrap()
11494 .unwrap();
11495 let first = query_live_roots(&handler, &first_ctx).await;
11496 let second = query_live_roots(&handler, &second_ctx).await;
11497 assert!(
11498 matches!(
11499 first,
11500 ModuleControlResponseToModule::LiveRoots {
11501 total_bindings: 0,
11502 ..
11503 }
11504 ),
11505 "cross-module scope must not expose another module's roots"
11506 );
11507 assert!(
11508 matches!(
11509 second,
11510 ModuleControlResponseToModule::LiveRoots {
11511 total_bindings: 1,
11512 ..
11513 }
11514 ),
11515 "second module must see its pending route"
11516 );
11517 handler
11518 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11519 .await
11520 .unwrap();
11521 assert!(open.await.unwrap().is_empty());
11522 }
11523
11524 #[tokio::test(start_paused = true)]
11525 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11526 let handler = ControlHandler::with_forwarding(
11527 Arc::new(Registry::default()),
11528 Arc::new(ForwardingTable::default()),
11529 );
11530 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11531 hello_via_sink(
11532 &handler,
11533 &target_ctx,
11534 &mut target_rx,
11535 hello_frame("target", PROTOCOL_VERSION, 1),
11536 )
11537 .await;
11538 let actual = query_live_roots(&handler, &target_ctx).await;
11539 let ModuleControlResponseToModule::LiveRoots {
11540 roots,
11541 unknown_root_bindings,
11542 total_bindings,
11543 } = actual
11544 else {
11545 panic!("expected live roots")
11546 };
11547 assert!(roots.is_empty());
11548 assert_eq!(unknown_root_bindings, 0);
11549 assert_eq!(total_bindings, 0);
11550 assert_eq!(
11551 total_bindings,
11552 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11553 );
11554 }
11555
11556 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11563 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11564 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11565 let text = std::fs::read_to_string(&path)
11566 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11567 let vectors: Vec<(String, Value)> = text
11568 .lines()
11569 .filter(|line| !line.trim().is_empty())
11570 .map(|line| {
11571 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11572 let id = entry["corpus_id"]
11573 .as_str()
11574 .expect("every vector carries a corpus_id")
11575 .to_string();
11576 (id, entry["package"].clone())
11577 })
11578 .collect();
11579 assert_eq!(
11583 vectors.len(),
11584 3,
11585 "vendored fed corpus changed size; re-sync from subc-federation"
11586 );
11587
11588 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11603 let richest = vectors
11604 .iter()
11605 .filter_map(|(_, package)| package.as_object())
11606 .map(|object| {
11607 object
11608 .keys()
11609 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11610 .count()
11611 })
11612 .max()
11613 .unwrap_or(0);
11614 assert!(
11615 richest >= 2,
11616 "vendored corpus no longer carries a package with unmodelled fields, \
11617 so the relay test can no longer distinguish a verbatim relay from a lossy one"
11618 );
11619
11620 vectors
11621 }
11622
11623 #[tokio::test]
11631 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11632 for (corpus_id, package) in fed_admission_facts_vectors() {
11633 let registry = Arc::new(Registry::default());
11634 let forwarding = Arc::new(ForwardingTable::default());
11635 let supervisor = SupervisorHandle::new();
11636 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11637 let handler =
11638 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11639 .with_supervisor(supervisor)
11640 .with_admission_facts_config(
11641 Some("fed".to_string()),
11642 Some(vec!["target".to_string()]),
11643 );
11644
11645 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11646 hello_via_sink(
11647 &handler,
11648 &target_ctx,
11649 &mut target_rx,
11650 hello_frame("target", PROTOCOL_VERSION, 1),
11651 )
11652 .await;
11653
11654 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11655 let route_handler = handler.clone();
11656 let expected = package.clone();
11657 let route_task = tokio::spawn(async move {
11658 route_handler
11659 .handle_control_frame(
11660 &client_ctx,
11661 route_open_frame_with_admission_facts(
11662 20,
11663 "target",
11664 unique_project_root("admission-facts"),
11665 Some(subc_control::ConsumerIdentity {
11666 module_id: "fed".to_string(),
11667 launch_nonce: "fed-nonce".to_string(),
11668 }),
11669 Some(package),
11670 ),
11671 )
11672 .await
11673 .unwrap()
11674 });
11675
11676 let bind_frame = target_rx.recv().await.unwrap();
11677 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11678 let ModuleControlRequest::RouteBind {
11679 admission_facts, ..
11680 } = bind
11681 else {
11682 panic!("{corpus_id}: expected route.bind")
11683 };
11684 assert_eq!(
11685 admission_facts,
11686 Some(expected),
11687 "{corpus_id}: relay must not add, drop or reshape any field"
11688 );
11689
11690 handler
11691 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11692 .await
11693 .unwrap();
11694 route_task.await.unwrap();
11695 }
11696 }
11697
11698 #[tokio::test]
11699 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11700 let registry = Arc::new(Registry::default());
11701 let forwarding = Arc::new(ForwardingTable::default());
11702 let supervisor = SupervisorHandle::new();
11703 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11704 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11705 let handler =
11706 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11707 .with_supervisor(supervisor)
11708 .with_admission_facts_config(
11709 Some("fed".to_string()),
11710 Some(vec!["target".to_string()]),
11711 );
11712
11713 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11714 hello_via_sink(
11715 &handler,
11716 &target_ctx,
11717 &mut target_rx,
11718 hello_frame("target", PROTOCOL_VERSION, 1),
11719 )
11720 .await;
11721 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11722 hello_via_sink(
11723 &handler,
11724 &other_ctx,
11725 &mut other_rx,
11726 hello_frame("other", PROTOCOL_VERSION, 2),
11727 )
11728 .await;
11729
11730 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11731 let expected_facts = facts.clone();
11732 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11733 let route_handler = handler.clone();
11734 let route_task = tokio::spawn(async move {
11735 route_handler
11736 .handle_control_frame(
11737 &client_ctx,
11738 route_open_frame_with_admission_facts(
11739 10,
11740 "target",
11741 unique_project_root("admission-facts"),
11742 Some(subc_control::ConsumerIdentity {
11743 module_id: "fed".to_string(),
11744 launch_nonce: "fed-nonce".to_string(),
11745 }),
11746 Some(facts.clone()),
11747 ),
11748 )
11749 .await
11750 .unwrap()
11751 });
11752 let bind_frame = target_rx.recv().await.unwrap();
11753 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11754 let ModuleControlRequest::RouteBind {
11755 admission_facts, ..
11756 } = bind
11757 else {
11758 panic!("expected route.bind")
11759 };
11760 assert_eq!(admission_facts, Some(expected_facts));
11761 handler
11762 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11763 .await
11764 .unwrap();
11765 assert!(route_task.await.unwrap().is_empty());
11766 assert!(matches!(
11767 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11768 .unwrap(),
11769 ClientControlResponse::RouteOpen { .. }
11770 ));
11771
11772 let direct = handler
11773 .handle_control_frame(
11774 &route_ctx(ConnectionId::new(73)).0,
11775 route_open_frame_with_admission_facts(
11776 11,
11777 "target",
11778 unique_project_root("admission-facts"),
11779 None,
11780 Some(json!({"x": 1})),
11781 ),
11782 )
11783 .await
11784 .unwrap();
11785 assert_eq!(
11786 parse_error(&direct[0])["code"],
11787 "admission_facts_not_permitted"
11788 );
11789
11790 let different_reserved = handler
11791 .handle_control_frame(
11792 &route_ctx(ConnectionId::new(77)).0,
11793 route_open_frame_with_admission_facts(
11794 15,
11795 "target",
11796 unique_project_root("admission-facts"),
11797 Some(subc_control::ConsumerIdentity {
11798 module_id: "other".to_string(),
11799 launch_nonce: "other-nonce".to_string(),
11800 }),
11801 Some(json!({"x": 1})),
11802 ),
11803 )
11804 .await
11805 .unwrap();
11806 assert_eq!(
11807 parse_error(&different_reserved[0])["code"],
11808 "admission_facts_not_permitted"
11809 );
11810
11811 let other_target = handler
11812 .handle_control_frame(
11813 &route_ctx(ConnectionId::new(74)).0,
11814 route_open_frame_with_admission_facts(
11815 12,
11816 "other",
11817 unique_project_root("admission-facts"),
11818 Some(subc_control::ConsumerIdentity {
11819 module_id: "fed".to_string(),
11820 launch_nonce: "fed-nonce".to_string(),
11821 }),
11822 Some(json!({"x": 1})),
11823 ),
11824 )
11825 .await
11826 .unwrap();
11827 assert_eq!(
11828 parse_error(&other_target[0])["code"],
11829 "admission_facts_target_not_allowed"
11830 );
11831
11832 let nonexistent = handler
11833 .handle_control_frame(
11834 &route_ctx(ConnectionId::new(75)).0,
11835 route_open_frame_with_admission_facts(
11836 13,
11837 "missing",
11838 unique_project_root("admission-facts"),
11839 None,
11840 Some(json!({"x": 1})),
11841 ),
11842 )
11843 .await
11844 .unwrap();
11845 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
11846
11847 let described = handler
11848 .handle_control_frame(
11849 &route_ctx(ConnectionId::new(76)).0,
11850 Frame::build(
11851 FrameType::Request,
11852 control_flags(),
11853 0,
11854 0,
11855 14,
11856 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11857 )
11858 .unwrap(),
11859 )
11860 .await
11861 .unwrap();
11862 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11863 serde_json::from_slice(&described[0].body).unwrap()
11864 else {
11865 panic!("expected server.describe response")
11866 };
11867 assert!(capabilities
11868 .iter()
11869 .any(|cap| cap == "admission_facts_relay_v1"));
11870 }
11871
11872 #[tokio::test]
11873 async fn admission_facts_without_configured_carrier_are_rejected() {
11874 let registry = Arc::new(Registry::default());
11875 let forwarding = Arc::new(ForwardingTable::default());
11876 let handler = ControlHandler::with_forwarding(registry, forwarding);
11877 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11878 hello_via_sink(
11879 &handler,
11880 &target_ctx,
11881 &mut target_rx,
11882 hello_frame("target", PROTOCOL_VERSION, 1),
11883 )
11884 .await;
11885
11886 let responses = handler
11887 .handle_control_frame(
11888 &route_ctx(ConnectionId::new(79)).0,
11889 route_open_frame_with_admission_facts(
11890 16,
11891 "target",
11892 unique_project_root("admission-facts"),
11893 None,
11894 Some(json!({"x": 1})),
11895 ),
11896 )
11897 .await
11898 .unwrap();
11899 assert_eq!(
11900 parse_error(&responses[0])["code"],
11901 "admission_facts_not_permitted"
11902 );
11903 }
11904
11905 #[tokio::test]
11906 async fn route_open_relays_consumer_capabilities_verbatim() {
11907 let registry = Arc::new(Registry::default());
11908 let forwarding = Arc::new(ForwardingTable::default());
11909 let handler =
11910 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11911 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11912 hello_via_sink(
11913 &handler,
11914 &module_ctx,
11915 &mut module_rx,
11916 hello_frame("aft", PROTOCOL_VERSION, 7),
11917 )
11918 .await;
11919
11920 let expected = vec!["elicitation".to_string(), "roots".to_string()];
11921 let expected_for_request = expected.clone();
11922 let project_root = unique_project_root("consumer-capabilities-present");
11923 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11924 let route_handler = handler.clone();
11925 let route_task = tokio::spawn(async move {
11926 route_handler
11927 .handle_control_frame(
11928 &client_ctx,
11929 route_open_frame_with_consumer_capabilities(
11930 401,
11931 "aft",
11932 project_root,
11933 Some(expected_for_request),
11934 ),
11935 )
11936 .await
11937 .unwrap()
11938 });
11939 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11940 .await
11941 .unwrap()
11942 .unwrap();
11943 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11944 let ModuleControlRequest::RouteBind {
11945 consumer_capabilities,
11946 ..
11947 } = bind
11948 else {
11949 panic!("expected route.bind request, got {bind:?}");
11950 };
11951 assert_eq!(consumer_capabilities, Some(expected.clone()));
11952
11953 handler
11954 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11955 .await
11956 .unwrap();
11957 let route_response = route_task.await.unwrap();
11958 assert!(route_response.is_empty());
11959 let published = client_rx.recv().await.unwrap();
11960 assert!(matches!(
11961 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11962 ClientControlResponse::RouteOpen { .. }
11963 ));
11964 }
11965
11966 #[tokio::test]
11967 async fn route_open_without_consumer_capabilities_relays_none() {
11968 let registry = Arc::new(Registry::default());
11969 let forwarding = Arc::new(ForwardingTable::default());
11970 let handler =
11971 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11972 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11973 hello_via_sink(
11974 &handler,
11975 &module_ctx,
11976 &mut module_rx,
11977 hello_frame("aft", PROTOCOL_VERSION, 7),
11978 )
11979 .await;
11980
11981 let project_root = unique_project_root("consumer-capabilities-absent");
11982 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11983 let route_handler = handler.clone();
11984 let route_task = tokio::spawn(async move {
11985 route_handler
11986 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11987 .await
11988 .unwrap()
11989 });
11990 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11991 .await
11992 .unwrap()
11993 .unwrap();
11994 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11995 let ModuleControlRequest::RouteBind {
11996 consumer_capabilities,
11997 ..
11998 } = bind
11999 else {
12000 panic!("expected route.bind request, got {bind:?}");
12001 };
12002 assert_eq!(consumer_capabilities, None);
12003
12004 handler
12005 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12006 .await
12007 .unwrap();
12008 let route_response = route_task.await.unwrap();
12009 assert!(route_response.is_empty());
12010 let published = client_rx.recv().await.unwrap();
12011 assert!(matches!(
12012 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12013 ClientControlResponse::RouteOpen { .. }
12014 ));
12015 }
12016
12017 async fn bind_role_versions_for(
12021 sent: Option<BTreeMap<String, String>>,
12022 connection: u64,
12023 ) -> Option<BTreeMap<String, String>> {
12024 let registry = Arc::new(Registry::default());
12025 let forwarding = Arc::new(ForwardingTable::default());
12026 let handler =
12027 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12028 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12029 hello_via_sink(
12030 &handler,
12031 &module_ctx,
12032 &mut module_rx,
12033 hello_frame("aft", PROTOCOL_VERSION, 7),
12034 )
12035 .await;
12036 let project_root = unique_project_root("role-versions");
12037 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12038 let route_handler = handler.clone();
12039 let route_task = tokio::spawn(async move {
12040 route_handler
12041 .handle_control_frame(
12042 &client_ctx,
12043 route_open_frame_with_role_versions(403, "aft", project_root, sent),
12044 )
12045 .await
12046 .unwrap()
12047 });
12048 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12049 .await
12050 .expect("a well-formed route.open reaches the module as a bind")
12051 .unwrap();
12052 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12053 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12054 panic!("expected route.bind request, got {bind:?}");
12055 };
12056 handler
12057 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12058 .await
12059 .unwrap();
12060 assert!(route_task.await.unwrap().is_empty());
12061 let published = client_rx.recv().await.unwrap();
12062 assert!(matches!(
12063 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12064 ClientControlResponse::RouteOpen { .. }
12065 ));
12066 role_versions
12067 }
12068
12069 #[tokio::test]
12070 async fn route_open_relays_role_versions_verbatim() {
12071 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12072 assert_eq!(
12073 bind_role_versions_for(Some(sent.clone()), 141).await,
12074 Some(sent)
12075 );
12076 }
12077
12078 #[tokio::test]
12081 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12082 assert_eq!(bind_role_versions_for(None, 143).await, None);
12083 assert_eq!(
12084 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12085 None
12086 );
12087 }
12088
12089 #[tokio::test]
12090 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12091 let registry = Arc::new(Registry::default());
12092 let forwarding = Arc::new(ForwardingTable::default());
12093 let handler =
12094 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12095 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12096 hello_via_sink(
12097 &handler,
12098 &module_ctx,
12099 &mut module_rx,
12100 hello_frame("aft", PROTOCOL_VERSION, 7),
12101 )
12102 .await;
12103
12104 let nine: BTreeMap<String, String> = (0..9)
12105 .map(|index| (format!("role-{index}"), "v1".to_string()))
12106 .collect();
12107 for (label, malformed) in [
12108 (
12109 "invalid role name",
12110 role_versions(&[("Tool_Provider", "v1")]),
12111 ),
12112 ("invalid version", role_versions(&[("tool-provider", "v0")])),
12113 ("nine entries", nine),
12114 ] {
12115 let responses = tokio::time::timeout(
12118 Duration::from_secs(1),
12119 handler.handle_control_frame(
12120 &route_ctx(ConnectionId::new(148)).0,
12121 route_open_frame_with_role_versions(
12122 404,
12123 "aft",
12124 unique_project_root("role-versions-malformed"),
12125 Some(malformed),
12126 ),
12127 ),
12128 )
12129 .await
12130 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12131 .unwrap();
12132 assert_eq!(responses.len(), 1, "{label}");
12133 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12134 let error = parse_error(&responses[0]);
12135 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12136 assert_eq!(
12137 error["detail"]["field"], "role_versions",
12138 "{label}: {error}"
12139 );
12140 assert!(
12141 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12142 "{label}: a malformed declaration is terminal"
12143 );
12144 assert!(
12145 module_rx.try_recv().is_err(),
12146 "{label}: the module must never see a bind"
12147 );
12148 }
12149 }
12150
12151 #[tokio::test]
12155 async fn route_role_versions_capability_is_advertised() {
12156 let handler = ControlHandler::new(Arc::new(Registry::default()));
12157 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12158 let ack = hello_via_sink(
12159 &handler,
12160 &ctx,
12161 &mut rx,
12162 hello_frame("m", PROTOCOL_VERSION, 1),
12163 )
12164 .await;
12165 let ack = parse_ack(&ack);
12166 assert!(
12167 ack.subc_capabilities
12168 .iter()
12169 .any(|c| c == "route-role-versions/v1"),
12170 "{:?}",
12171 ack.subc_capabilities
12172 );
12173
12174 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12175 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12176 let reply = handler
12177 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12178 .await
12179 .unwrap()
12180 .pop()
12181 .unwrap();
12182 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12183 serde_json::from_slice(&reply.body).unwrap()
12184 else {
12185 panic!("not a server.describe reply");
12186 };
12187 assert!(
12188 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12189 "{capabilities:?}"
12190 );
12191 }
12192
12193 #[tokio::test]
12197 async fn catalog_events_capability_is_advertised() {
12198 let handler = ControlHandler::new(Arc::new(Registry::default()));
12199 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12200 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12201 let reply = handler
12202 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12203 .await
12204 .unwrap()
12205 .pop()
12206 .unwrap();
12207 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12208 serde_json::from_slice(&reply.body).unwrap()
12209 else {
12210 panic!("not a server.describe reply");
12211 };
12212 assert!(
12213 capabilities
12214 .iter()
12215 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12216 "{capabilities:?}"
12217 );
12218 }
12219
12220 #[tokio::test]
12221 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12222 let handler = ControlHandler::new(Arc::new(Registry::default()));
12223 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12224 let ack = hello_via_sink(
12225 &handler,
12226 &module,
12227 &mut module_rx,
12228 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12229 )
12230 .await;
12231 let ack = parse_ack(&ack);
12232 assert!(
12233 ack.subc_capabilities
12234 .iter()
12235 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12236 "{:?}",
12237 ack.subc_capabilities
12238 );
12239 assert!(ack
12240 .subc_capabilities
12241 .iter()
12242 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12243
12244 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12245 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12246 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12247 let reply = handler
12248 .handle_control_frame(&client, frame)
12249 .await
12250 .unwrap()
12251 .pop()
12252 .unwrap();
12253 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12254 serde_json::from_slice(&reply.body).unwrap()
12255 else {
12256 panic!("not a server.describe reply");
12257 };
12258 assert!(
12259 capabilities
12260 .iter()
12261 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12262 "{capabilities:?}"
12263 );
12264 assert!(capabilities
12265 .iter()
12266 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12267 }
12268
12269 #[tokio::test]
12270 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12271 let handler = ControlHandler::new(Arc::new(Registry::default()));
12272 let (module, mut rx) = route_ctx(ConnectionId::new(156));
12273 let ack = hello_via_sink(
12274 &handler,
12275 &module,
12276 &mut rx,
12277 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12278 )
12279 .await;
12280 assert!(parse_ack(&ack)
12281 .subc_capabilities
12282 .iter()
12283 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12284 assert!(parse_ack(&ack)
12285 .subc_capabilities
12286 .iter()
12287 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12288 let (client, _rx) = route_ctx(ConnectionId::new(157));
12289 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12290 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12291 let reply = handler
12292 .handle_control_frame(&client, frame)
12293 .await
12294 .unwrap()
12295 .pop()
12296 .unwrap();
12297 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12298 serde_json::from_slice(&reply.body).unwrap()
12299 else {
12300 panic!("server.describe reply")
12301 };
12302 assert!(capabilities
12303 .iter()
12304 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12305 assert!(capabilities
12306 .iter()
12307 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12308 }
12309
12310 #[tokio::test]
12311 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12312 let handler = ControlHandler::new(Arc::new(Registry::default()));
12313 let (module, mut rx) = route_ctx(ConnectionId::new(158));
12314 let ack = hello_via_sink(
12315 &handler,
12316 &module,
12317 &mut rx,
12318 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12319 )
12320 .await;
12321 assert!(parse_ack(&ack)
12322 .subc_capabilities
12323 .iter()
12324 .any(|cap| cap == "scope-workdir/v1"));
12325 let (client, _rx) = route_ctx(ConnectionId::new(159));
12326 let frame = Frame::build(
12327 FrameType::Request,
12328 control_flags(),
12329 0,
12330 0,
12331 5,
12332 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12333 )
12334 .unwrap();
12335 let reply = handler
12336 .handle_control_frame(&client, frame)
12337 .await
12338 .unwrap()
12339 .remove(0);
12340 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12341 serde_json::from_slice(&reply.body).unwrap()
12342 else {
12343 panic!("server.describe reply")
12344 };
12345 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12346 }
12347
12348 #[tokio::test]
12349 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12350 let handler = ControlHandler::new(Arc::new(Registry::default()));
12351 let (module, mut rx) = route_ctx(ConnectionId::new(154));
12352 let ack = hello_via_sink(
12353 &handler,
12354 &module,
12355 &mut rx,
12356 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12357 )
12358 .await;
12359 assert!(parse_ack(&ack)
12360 .subc_capabilities
12361 .iter()
12362 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12363 let (client, _rx) = route_ctx(ConnectionId::new(155));
12364 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12365 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12366 let reply = handler
12367 .handle_control_frame(&client, frame)
12368 .await
12369 .unwrap()
12370 .pop()
12371 .unwrap();
12372 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12373 serde_json::from_slice(&reply.body).unwrap()
12374 else {
12375 panic!("not a server.describe reply")
12376 };
12377 assert!(capabilities
12378 .iter()
12379 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12380 }
12381
12382 #[tokio::test]
12383 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12384 let registry = Arc::new(Registry::default());
12385 let forwarding = Arc::new(ForwardingTable::default());
12386 let handler =
12387 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12388 .with_health_probe_timeout(Duration::from_secs(5));
12389 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12390 hello_via_sink(
12391 &handler,
12392 &module_ctx,
12393 &mut module_rx,
12394 non_routable_hello_frame_with_control_ops(
12395 "mcp",
12396 300,
12397 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12398 ),
12399 )
12400 .await;
12401 assert!(registry
12402 .get_module("mcp")
12403 .unwrap()
12404 .unwrap()
12405 .manifest
12406 .provides
12407 .is_empty());
12408
12409 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12410 let route_response = handler
12411 .handle_control_frame(
12412 &route_client_ctx,
12413 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12414 )
12415 .await
12416 .unwrap();
12417 assert_eq!(route_response[0].header.ty, FrameType::Error);
12418 assert_eq!(
12419 parse_error(&route_response[0])["code"],
12420 "target_unavailable"
12421 );
12422 assert!(parse_error(&route_response[0])["message"]
12423 .as_str()
12424 .unwrap()
12425 .contains("does not provide the requested target"));
12426 assert!(module_rx.try_recv().is_err());
12427
12428 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12429 let health_handler = handler.clone();
12430 let health_task = tokio::spawn(async move {
12431 health_handler
12432 .handle_control_frame(
12433 &health_client_ctx,
12434 supervisor_health_probe_frame(302, "mcp"),
12435 )
12436 .await
12437 .unwrap()
12438 });
12439 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12440 .await
12441 .unwrap()
12442 .unwrap();
12443 assert_eq!(
12444 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12445 ModuleControlRequest::HealthCheck {}
12446 );
12447 handler
12448 .handle_control_frame(
12449 &module_ctx,
12450 health_response(health_frame.header.corr, HealthStatus::Ok),
12451 )
12452 .await
12453 .unwrap();
12454 let health_response = health_task.await.unwrap();
12455 assert_eq!(health_response[0].header.ty, FrameType::Response);
12456 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12457 ClientControlResponse::SupervisorHealthProbe {
12458 module_id, status, ..
12459 } => {
12460 assert_eq!(module_id, "mcp");
12461 assert_eq!(status, HealthStatus::Ok);
12462 }
12463 other => panic!("unexpected health response: {other:?}"),
12464 }
12465
12466 forwarding
12471 .cleanup_connection(module_ctx.connection_id)
12472 .unwrap();
12473 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12474 let cleanup_response = tokio::time::timeout(
12475 Duration::from_millis(200),
12476 handler.handle_control_frame(
12477 &cleanup_probe_ctx,
12478 supervisor_health_probe_frame(303, "mcp"),
12479 ),
12480 )
12481 .await
12482 .expect("probe should fail immediately when the control lane is gone")
12483 .unwrap();
12484 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12485 assert_eq!(
12486 parse_error(&cleanup_response[0])["code"],
12487 "target_unavailable"
12488 );
12489 assert!(parse_error(&cleanup_response[0])["message"]
12490 .as_str()
12491 .unwrap()
12492 .contains("no module connection"));
12493
12494 handler
12495 .cleanup_connection(module_ctx.connection_id)
12496 .unwrap();
12497 }
12498
12499 #[tokio::test]
12500 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12501 let registry = Arc::new(Registry::default());
12502 let supervisor_handle = SupervisorHandle::new();
12503 let supervisor =
12504 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12505 .with_handle(supervisor_handle.clone())
12506 .with_connection_file_path(
12507 std::env::temp_dir()
12508 .join(format!("subc-route-open-warming-{}", std::process::id())),
12509 );
12510 let module = supervisor
12511 .supervise_configured(
12512 ModuleSpec {
12513 module_id: "warming".to_string(),
12514 program: fake_aft_stub_path(),
12515 args: Vec::new(),
12516 env: Vec::new(),
12517 reserved: false,
12518 reserved_prefixes: Vec::new(),
12519 protocol: ModuleProtocol::Subc,
12520 overlap: Default::default(),
12521 },
12522 true,
12523 )
12524 .unwrap();
12525 assert_eq!(module.state().unwrap(), ModuleState::Running);
12526
12527 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12528 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12529 let response = handler
12530 .handle_control_frame(
12531 &ctx,
12532 route_open_frame(304, "warming", unique_project_root("warming")),
12533 )
12534 .await
12535 .unwrap();
12536 module.stop().await.unwrap();
12537
12538 assert_eq!(response[0].header.ty, FrameType::Error);
12539 let error = parse_error(&response[0]);
12540 assert_eq!(error["code"], "module_warming");
12541 assert!(error["message"]
12542 .as_str()
12543 .unwrap()
12544 .contains("state=running, enabled=true, live=false"));
12545 }
12546
12547 #[test]
12548 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12549 let handler = ControlHandler::new(Arc::new(Registry::default()));
12550 let capture = EventCapture::default();
12551 let _subscriber =
12552 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12553 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12554 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12555 let pending = (0..limit).collect::<Vec<_>>();
12556 let response = handler
12557 .route_open_capacity_refusal(
12558 &ctx,
12559 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12560 "busy",
12561 pending.len(),
12562 limit,
12563 )
12564 .unwrap();
12565 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12566 let event = capture
12567 .events()
12568 .into_iter()
12569 .find(|event| {
12570 event.target == "control"
12571 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12572 })
12573 .expect("connection admission refusal event");
12574 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12575 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12576 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
12579 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
12580 }
12581
12582 #[test]
12583 fn route_open_target_cap_logs_admission_reason_and_capacity() {
12584 let handler = ControlHandler::new(Arc::new(Registry::default()));
12585 let capture = EventCapture::default();
12586 let _subscriber =
12587 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12588 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12589 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12590 let guards = (0..limit)
12591 .map(|_| {
12592 handler
12593 .route_bind_concurrency
12594 .try_admit("busy", limit)
12595 .unwrap()
12596 })
12597 .collect::<Vec<_>>();
12598 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12599 Err(in_flight) => in_flight,
12600 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12601 };
12602 let response = handler
12603 .route_open_target_capacity_refusal(
12604 &ctx,
12605 &RouteOpenCaller::unknown(),
12606 &route_open_frame(397, "busy", unique_project_root("target-cap")),
12607 "busy",
12608 in_flight,
12609 )
12610 .unwrap();
12611 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12612 let event = capture
12613 .events()
12614 .into_iter()
12615 .find(|event| {
12616 event.target == "control"
12617 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12618 })
12619 .expect("target admission refusal event");
12620 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12621 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12622 drop(guards);
12623 }
12624
12625 #[tokio::test]
12626 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
12627 let registry = Arc::new(Registry::default());
12628 let handle = SupervisorHandle::new();
12629 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
12630 .with_handle(handle.clone());
12631 supervisor
12632 .supervise_configured(
12633 crate::ModuleSpec {
12634 module_id: "drift".into(),
12635 program: PathBuf::from("disabled-module"),
12636 args: Vec::new(),
12637 env: Vec::new(),
12638 reserved: false,
12639 reserved_prefixes: Vec::new(),
12640 protocol: ModuleProtocol::Subc,
12641 overlap: Default::default(),
12642 },
12643 false,
12644 )
12645 .unwrap();
12646 let handler = ControlHandler::new(registry).with_supervisor(handle);
12647 for configured in [None, Some(false), Some(true)] {
12648 handler.capability_evaluator.configure(
12649 configured.map(|enabled| ("drift".to_string(), enabled)),
12650 BTreeMap::new(),
12651 );
12652 let frame =
12653 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12654 let replies = handler
12655 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
12656 .await
12657 .unwrap();
12658 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12659 assert!(body.get("configured_enabled").is_none());
12660 assert_eq!(body["op"], "supervisor.ack");
12661 let frame =
12662 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12663 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12664 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12665 assert_eq!(
12666 body["modules"][0].get("configured_enabled"),
12667 configured.map(Value::Bool).as_ref()
12668 );
12669 }
12670 }
12671
12672 #[tokio::test(flavor = "current_thread")]
12673 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12674 let handler = ControlHandler::new(Arc::new(Registry::default()));
12675 let capture = EventCapture::default();
12676 let _subscriber =
12677 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12678 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12679 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12680
12681 let _ = handler
12682 .handle_client_control_request(
12683 &ctx,
12684 frame,
12685 ClientControlRequest::SupervisorSetEnabled {
12686 module_id: "broca".to_string(),
12687 enabled: false,
12688 },
12689 &mut None,
12690 )
12691 .await
12692 .unwrap();
12693
12694 let events = capture
12695 .events()
12696 .into_iter()
12697 .filter(|event| {
12698 event.target == "control"
12699 && event.fields.get("message").map(String::as_str)
12700 == Some("supervisor request received")
12701 })
12702 .collect::<Vec<_>>();
12703 assert_eq!(events.len(), 1, "one supervisor request log line");
12704 let event = &events[0];
12705 assert_eq!(event.level, tracing::Level::INFO);
12706 assert_eq!(
12707 event.fields.get("op"),
12708 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12709 );
12710 assert_eq!(
12711 event.fields.get("module_id"),
12712 Some(&"\"broca\"".to_string())
12713 );
12714 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12715 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12716 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12717 }
12718
12719 #[tokio::test(flavor = "current_thread")]
12720 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12721 let registry = Arc::new(Registry::default());
12722 let module_connection = ConnectionId::new(303);
12723 registry
12724 .register_with_control_ops(
12725 manifest("aft", PROTOCOL_VERSION),
12726 PROTOCOL_VERSION,
12727 module_connection,
12728 module_baseline_control_ops(),
12729 )
12730 .unwrap();
12731 let handler = ControlHandler::new(registry);
12732 let capture = EventCapture::default();
12733 let _subscriber =
12734 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12735 let (ctx, _rx) = route_ctx(module_connection);
12736 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12737
12738 let _ = handler
12739 .handle_client_control_request(
12740 &ctx,
12741 frame,
12742 ClientControlRequest::SupervisorSetEnabled {
12743 module_id: "broca".to_string(),
12744 enabled: false,
12745 },
12746 &mut None,
12747 )
12748 .await
12749 .unwrap();
12750
12751 let callers = capture
12752 .events()
12753 .into_iter()
12754 .filter(|event| {
12755 event.fields.get("message").map(String::as_str)
12756 == Some("supervisor request received")
12757 })
12758 .map(|event| event.fields.get("caller").cloned())
12759 .collect::<Vec<_>>();
12760 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12761 }
12762
12763 #[tokio::test(flavor = "current_thread")]
12764 async fn supervisor_list_does_not_log_request_received() {
12765 let handler = ControlHandler::new(Arc::new(Registry::default()));
12766 let capture = EventCapture::default();
12767 let _subscriber =
12768 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12769 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12770 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12771
12772 handler
12773 .handle_client_control_request(
12774 &ctx,
12775 frame,
12776 ClientControlRequest::SupervisorList {},
12777 &mut None,
12778 )
12779 .await
12780 .unwrap();
12781
12782 assert!(capture.events().into_iter().all(|event| {
12783 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12784 }));
12785 }
12786
12787 #[tokio::test(flavor = "current_thread")]
12788 async fn supervisor_rescan_preview_does_not_log_request_received() {
12789 let handler = ControlHandler::new(Arc::new(Registry::default()));
12790 let capture = EventCapture::default();
12791 let _subscriber =
12792 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12793 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
12794 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12795
12796 handler
12797 .handle_client_control_request(
12798 &ctx,
12799 frame,
12800 ClientControlRequest::SupervisorRescan { preview: true },
12801 &mut None,
12802 )
12803 .await
12804 .unwrap();
12805
12806 assert!(capture.events().into_iter().all(|event| {
12807 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12808 }));
12809 }
12810
12811 struct ApproveOperator;
12812 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
12813 fn prompt(
12814 &self,
12815 _: &str,
12816 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
12817 ) -> crate::operator_confirm::ProviderResult {
12818 crate::operator_confirm::ProviderResult::Approved
12819 }
12820 }
12821
12822 #[tokio::test]
12823 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
12824 assert_eq!(
12825 ControlHandler::default().privileged_operator_prompt,
12826 PrivilegedOperatorPrompt::Log
12827 );
12828 for (initial, changed, running) in [
12829 ("log", "enforce", PrivilegedOperatorPrompt::Log),
12830 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
12831 ] {
12832 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
12833 let path = dir.join("subc.jsonc");
12834 std::fs::write(
12835 &path,
12836 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
12837 )
12838 .unwrap();
12839 let config = crate::daemon_config::load(&path).unwrap().unwrap();
12840 let registry = Arc::new(Registry::default());
12841 let handle = SupervisorHandle::new();
12842 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
12843 .with_handle(handle.clone());
12844 let handler = ControlHandler::new(registry)
12845 .with_supervisor(handle)
12846 .with_privileged_operator_prompt(config.privileged_operator_prompt)
12847 .with_supervisor_rescan(supervisor, &path, config.port);
12848 handler.forwarding.operator_confirms().configure(
12849 Arc::new(ApproveOperator),
12850 crate::operator_confirm::OperatorLimits::default(),
12851 );
12852 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
12853
12854 for (disk_mode, needs_restart) in [
12855 (Some(initial), false),
12856 (Some(changed), true),
12857 (Some(changed), true),
12858 (None, running == PrivilegedOperatorPrompt::Enforce),
12859 (Some(initial), false),
12860 ] {
12861 let mut doc = json!({"version":1});
12862 if let Some(mode) = disk_mode {
12863 doc["privileged_operator_prompt"] = json!(mode);
12864 }
12865 std::fs::write(&path, doc.to_string()).unwrap();
12866 for preview in [true, false] {
12867 let request = ClientControlRequest::SupervisorRescan { preview };
12868 let frame = Frame::build(
12869 FrameType::Request,
12870 control_flags(),
12871 0,
12872 0,
12873 1,
12874 serde_json::to_vec(&request).unwrap(),
12875 )
12876 .unwrap();
12877 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12878 assert_eq!(frames.len(), 1);
12879 let ClientControlResponse::SupervisorRescan { result } =
12880 serde_json::from_slice(&frames[0].body).unwrap()
12881 else {
12882 panic!("expected a rescan response");
12883 };
12884 let expected = if needs_restart {
12885 vec!["privileged_operator_prompt"]
12886 } else {
12887 Vec::new()
12888 };
12889 assert_eq!(result.restart_required, expected);
12890 assert_eq!(handler.privileged_operator_prompt, running);
12891 assert_eq!(
12892 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
12893 running
12894 );
12895 }
12896 }
12897 }
12898 }
12899
12900 #[tokio::test]
12901 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
12902 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
12903 let path = dir.join("subc.jsonc");
12904 let registry = Arc::new(Registry::default());
12905 let handle = SupervisorHandle::new();
12906 let supervisor =
12907 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
12908 let handler = ControlHandler::new(registry)
12909 .with_supervisor(handle)
12910 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
12911 .with_supervisor_rescan(supervisor, &path, None);
12912 handler.forwarding.operator_confirms().configure(
12913 Arc::new(ApproveOperator),
12914 crate::operator_confirm::OperatorLimits::default(),
12915 );
12916 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
12917 for preview in [true, false] {
12918 std::fs::write(
12919 &path,
12920 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
12921 )
12922 .unwrap();
12923 let request = ClientControlRequest::SupervisorRescan { preview };
12924 let frame = Frame::build(
12925 FrameType::Request,
12926 control_flags(),
12927 0,
12928 0,
12929 1,
12930 serde_json::to_vec(&request).unwrap(),
12931 )
12932 .unwrap();
12933 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12934 assert_eq!(frames.len(), 1);
12935 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
12936 assert_eq!(
12937 handler.privileged_operator_prompt,
12938 PrivilegedOperatorPrompt::Enforce
12939 );
12940 }
12941 }
12942
12943 #[tokio::test]
12948 async fn route_open_refusal_names_the_check_that_refused() {
12949 let handler = ControlHandler::new(Arc::new(Registry::default()));
12950 let capture = EventCapture::default();
12951 let _subscriber =
12952 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12953 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12954 let response = handler
12955 .handle_control_frame(
12956 &ctx,
12957 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
12958 )
12959 .await
12960 .unwrap();
12961
12962 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12963 let event = capture
12964 .events()
12965 .into_iter()
12966 .find(|event| {
12967 event.target == "control"
12968 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12969 })
12970 .expect("route.open refusal event");
12971 assert_eq!(
12972 event.fields.get("reason"),
12973 Some(&"\"not_registered\"".to_string())
12974 );
12975 }
12976
12977 #[tokio::test]
12978 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12979 let registry = Arc::new(Registry::default());
12980 let supervisor_handle = SupervisorHandle::new();
12981 let supervisor =
12982 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12983 .with_handle(supervisor_handle.clone())
12984 .with_connection_file_path(std::env::temp_dir().join(format!(
12985 "subc-route-open-refusal-info-{}",
12986 std::process::id()
12987 )));
12988 let module = supervisor
12989 .supervise_configured(
12990 ModuleSpec {
12991 module_id: "warming".to_string(),
12992 program: fake_aft_stub_path(),
12993 args: Vec::new(),
12994 env: Vec::new(),
12995 reserved: false,
12996 reserved_prefixes: Vec::new(),
12997 protocol: ModuleProtocol::Subc,
12998 overlap: Default::default(),
12999 },
13000 true,
13001 )
13002 .unwrap();
13003 assert_eq!(module.state().unwrap(), ModuleState::Running);
13004
13005 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13006 assert!(handler
13007 .counters()
13008 .snapshot()
13009 .get("route_open_refused_by_code")
13010 .is_none());
13011 let capture = EventCapture::default();
13012 let _subscriber =
13013 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13014 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13015 let response = handler
13016 .handle_control_frame(
13017 &ctx,
13018 route_open_frame(394, "warming", unique_project_root("refusal-info")),
13019 )
13020 .await
13021 .unwrap();
13022 module.stop().await.unwrap();
13023
13024 assert_eq!(parse_error(&response[0])["code"], "module_warming");
13025 let event = capture
13026 .events()
13027 .into_iter()
13028 .find(|event| {
13029 event.target == "control"
13030 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13031 })
13032 .expect("route.open refusal event");
13033 assert_eq!(
13034 event.fields.get("module_id"),
13035 Some(&"\"warming\"".to_string())
13036 );
13037 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13038 assert_eq!(
13039 event.fields.get("reason"),
13040 Some(&"\"supervised_not_registered\"".to_string())
13041 );
13042 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13043 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13044 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13045 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13046 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13047 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13048 assert_eq!(
13049 handler.counters().snapshot()["route_open_refused_by_code"],
13050 json!({ "module_warming": 1 })
13051 );
13052 }
13053
13054 const OUTAGE_START: &str = "route.open refusing module: not serving";
13055 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13056
13057 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13058 capture
13059 .events()
13060 .into_iter()
13061 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13062 .collect()
13063 }
13064
13065 fn supervise_stub(
13066 registry: &Arc<Registry>,
13067 module_id: &str,
13068 enabled: bool,
13069 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13070 let supervisor_handle = SupervisorHandle::new();
13071 let supervisor =
13072 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13073 .with_handle(supervisor_handle.clone())
13074 .with_connection_file_path(std::env::temp_dir().join(format!(
13075 "subc-route-outage-{module_id}-{}",
13076 std::process::id()
13077 )));
13078 let module = supervisor
13079 .supervise_configured(
13080 ModuleSpec {
13081 module_id: module_id.to_string(),
13082 program: fake_aft_stub_path(),
13083 args: Vec::new(),
13084 env: Vec::new(),
13085 reserved: false,
13086 reserved_prefixes: Vec::new(),
13087 protocol: ModuleProtocol::Subc,
13088 overlap: Default::default(),
13089 },
13090 enabled,
13091 )
13092 .unwrap();
13093 (supervisor_handle, module)
13094 }
13095
13096 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13097 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13098 module_id: module_id.to_string(),
13099 drain_timeout_ms: Some(50),
13100 })
13101 .unwrap();
13102 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13103 }
13104
13105 #[test]
13109 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13110 let registry = Arc::new(Registry::default());
13111 let forwarding = Arc::new(ForwardingTable::default());
13112 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13113 let second = ControlHandler::with_forwarding(registry, forwarding);
13114 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13115 }
13116
13117 #[tokio::test(flavor = "current_thread")]
13121 async fn route_open_unknown_module_refusals_add_no_outage_state() {
13122 let handler = ControlHandler::new(Arc::new(Registry::default()));
13123 let capture = EventCapture::default();
13124 let _subscriber =
13125 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13126 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13127 for corr in 0..8 {
13128 let response = handler
13129 .handle_control_frame(
13130 &ctx,
13131 route_open_frame(
13132 380 + corr,
13133 &format!("nobody-{corr}"),
13134 unique_project_root("outage-unknown"),
13135 ),
13136 )
13137 .await
13138 .unwrap();
13139 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13140 }
13141
13142 assert_eq!(handler.route_outages.tracked_module_count(), 0);
13143 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13144 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13145 }
13146
13147 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13152 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13153 let registry = Arc::new(Registry::default());
13154 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
13155 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13156 let capture = EventCapture::default();
13157 let _subscriber =
13158 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13159 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13160 handler.route_outages.record_accepted("outage-restart");
13163
13164 let response = handler
13165 .handle_control_frame(
13166 &ctx,
13167 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13168 )
13169 .await
13170 .unwrap();
13171 assert_eq!(response[0].header.ty, FrameType::Error);
13172 let starts = outage_lines(&capture, OUTAGE_START);
13173 assert_eq!(starts.len(), 1, "{starts:?}");
13174 assert_eq!(starts[0].level, tracing::Level::WARN);
13175 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13176 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13177 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13178 handler.route_outages.record_accepted("outage-restart");
13179 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13180
13181 let restart = handler
13182 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13183 .await
13184 .unwrap();
13185 assert_eq!(
13186 restart[0].header.ty,
13187 FrameType::Response,
13188 "{:?}",
13189 parse_error(&restart[0])
13190 );
13191 handler
13192 .handle_control_frame(
13193 &ctx,
13194 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13195 )
13196 .await
13197 .unwrap();
13198 module.stop().await.unwrap();
13199
13200 let starts = outage_lines(&capture, OUTAGE_START);
13201 assert_eq!(starts.len(), 2, "{starts:?}");
13202 assert_eq!(starts[1].level, tracing::Level::INFO);
13203 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13204 }
13205
13206 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13210 async fn failed_operator_restart_leaves_no_operator_mark() {
13211 let registry = Arc::new(Registry::default());
13212 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
13213 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13214 let capture = EventCapture::default();
13215 let _subscriber =
13216 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13217 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13218 handler.route_outages.record_accepted("outage-disabled");
13219
13220 let restart = handler
13221 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13222 .await
13223 .unwrap();
13224 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13225 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13226
13227 handler
13228 .handle_control_frame(
13229 &ctx,
13230 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13231 )
13232 .await
13233 .unwrap();
13234 let starts = outage_lines(&capture, OUTAGE_START);
13235 assert_eq!(starts.len(), 1, "{starts:?}");
13236 assert_eq!(starts[0].level, tracing::Level::WARN);
13237 }
13238
13239 #[tokio::test(flavor = "current_thread")]
13240 async fn route_open_unknown_module_escapes_target_module_id() {
13241 let handler = ControlHandler::new(Arc::new(Registry::default()));
13242 let capture = EventCapture::default();
13243 let _subscriber =
13244 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13245 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13246 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13247 let response = handler
13248 .handle_control_frame(
13249 &ctx,
13250 route_open_frame(
13251 395,
13252 hostile_module_id,
13253 unique_project_root("hostile-target-module-id"),
13254 ),
13255 )
13256 .await
13257 .unwrap();
13258
13259 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13260 let event = capture
13261 .events()
13262 .into_iter()
13263 .find(|event| {
13264 event.target == "control"
13265 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13266 })
13267 .expect("route.open unknown-module refusal event");
13268 let logged = event.fields.get("module_id").expect("module_id field");
13269 assert!(!logged.bytes().any(|byte| byte < 0x20));
13270 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13271 }
13272
13273 #[tokio::test(flavor = "current_thread")]
13274 async fn route_open_module_rejection_uses_daemon_counter_key() {
13275 let registry = Arc::new(Registry::default());
13276 let forwarding = Arc::new(ForwardingTable::default());
13277 let handler =
13278 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13279 let module_connection = ConnectionId::new(95);
13280 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13281 hello_via_sink(
13282 &handler,
13283 &module_ctx,
13284 &mut module_rx,
13285 hello_frame("aft", PROTOCOL_VERSION, 395),
13286 )
13287 .await;
13288
13289 let client_connection = ConnectionId::new(96);
13290 let (client_ctx, _client_rx) = route_ctx(client_connection);
13291 let capture = EventCapture::default();
13292 let _subscriber =
13293 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13294 let (route_task, bind) = relay_route_open(
13295 &handler,
13296 client_connection,
13297 &client_ctx.egress,
13298 &mut module_rx,
13299 396,
13300 "aft",
13301 "hostile-module-code",
13302 )
13303 .await;
13304 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13305 let rejection = Frame::build(
13306 FrameType::Error,
13307 control_flags(),
13308 0,
13309 0,
13310 bind.header.corr,
13311 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13312 )
13313 .unwrap();
13314 handler
13315 .handle_control_frame(&module_ctx, rejection)
13316 .await
13317 .unwrap();
13318
13319 let response = route_task.await.unwrap();
13320 assert_eq!(parse_error(&response[0])["code"], hostile_code);
13321 let counters = handler.counters().snapshot();
13322 assert_eq!(
13323 counters["route_open_refused_by_code"],
13324 json!({ "module_rejected": 1 })
13325 );
13326 assert!(counters["route_open_refused_by_code"]
13327 .get(hostile_code)
13328 .is_none());
13329
13330 let event = capture
13331 .events()
13332 .into_iter()
13333 .find(|event| {
13334 event.target == "control"
13335 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13336 })
13337 .expect("route.open module-rejection refusal event");
13338 let logged = event.fields.get("module_code").expect("module_code field");
13339 assert!(!logged.bytes().any(|byte| byte < 0x20));
13340 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13341 }
13342
13343 #[tokio::test]
13344 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13345 let registry = Arc::new(Registry::default());
13346 let supervisor_handle = SupervisorHandle::new();
13347 let missing_program = std::env::temp_dir().join(format!(
13348 "subc-route-open-missing-program-{}",
13349 std::process::id()
13350 ));
13351 let supervisor =
13352 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13353 .with_handle(supervisor_handle.clone());
13354 let module = supervisor
13355 .supervise_configured(
13356 ModuleSpec {
13357 module_id: "failed".to_string(),
13358 program: missing_program,
13359 args: Vec::new(),
13360 env: Vec::new(),
13361 reserved: false,
13362 reserved_prefixes: Vec::new(),
13363 protocol: ModuleProtocol::Subc,
13364 overlap: Default::default(),
13365 },
13366 true,
13367 )
13368 .unwrap();
13369 assert_eq!(module.state().unwrap(), ModuleState::Failed);
13370
13371 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13372 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13373 let response = handler
13374 .handle_control_frame(
13375 &ctx,
13376 route_open_frame(305, "failed", unique_project_root("failed")),
13377 )
13378 .await
13379 .unwrap();
13380
13381 assert_eq!(response[0].header.ty, FrameType::Error);
13382 let error = parse_error(&response[0]);
13383 assert_eq!(error["code"], "target_unavailable");
13384 assert!(error["message"]
13385 .as_str()
13386 .unwrap()
13387 .contains("state=failed, enabled=true, live=false"));
13388 }
13389
13390 #[tokio::test]
13391 async fn route_open_role_mismatch_remains_target_unavailable() {
13392 let registry = Arc::new(Registry::default());
13393 let handler = ControlHandler::new(Arc::clone(®istry));
13394 handler
13395 .handle_control(
13396 ConnectionId::new(41),
13397 non_routable_hello_frame_with_control_ops("health-only", 306, None),
13398 )
13399 .unwrap();
13400
13401 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13402 let response = handler
13403 .handle_control_frame(
13404 &ctx,
13405 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13406 )
13407 .await
13408 .unwrap();
13409
13410 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13411 assert!(parse_error(&response[0])["message"]
13412 .as_str()
13413 .unwrap()
13414 .contains("does not provide the requested target"));
13415 }
13416
13417 #[tokio::test]
13418 async fn route_open_inactive_registration_remains_target_unavailable() {
13419 let registry = Arc::new(Registry::default());
13420 let handler = ControlHandler::new(Arc::clone(®istry));
13421 handler
13422 .handle_control(
13423 ConnectionId::new(43),
13424 hello_frame("inactive", PROTOCOL_VERSION, 308),
13425 )
13426 .unwrap();
13427 assert!(registry
13428 .set_module_state_for_test("inactive", ChannelState::Closed)
13429 .unwrap());
13430
13431 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13432 let response = handler
13433 .handle_control_frame(
13434 &ctx,
13435 route_open_frame(309, "inactive", unique_project_root("inactive")),
13436 )
13437 .await
13438 .unwrap();
13439
13440 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13441 assert!(parse_error(&response[0])["message"]
13442 .as_str()
13443 .unwrap()
13444 .contains("is not active"));
13445 }
13446
13447 #[tokio::test]
13448 async fn late_health_reply_is_recorded_through_the_module_response_path() {
13449 let registry = Arc::new(Registry::default());
13450 let forwarding = Arc::new(ForwardingTable::default());
13451 let supervisor_handle = SupervisorHandle::new();
13452 let supervisor =
13453 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
13454 .with_forwarding(Arc::clone(&forwarding))
13455 .with_handle(supervisor_handle.clone());
13456 let module = supervisor
13457 .supervise_configured(
13458 crate::ModuleSpec {
13459 module_id: "late-health-response".to_string(),
13460 program: PathBuf::from("disabled-module"),
13461 args: Vec::new(),
13462 env: Vec::new(),
13463 reserved: false,
13464 reserved_prefixes: Vec::new(),
13465 protocol: ModuleProtocol::Subc,
13466 overlap: Default::default(),
13467 },
13468 false,
13469 )
13470 .unwrap();
13471 let handler =
13472 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13473 .with_supervisor(supervisor_handle);
13474 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13475 handler
13476 .handle_control_frame(
13477 &module_ctx,
13478 hello_frame_with_control_ops(
13479 "late-health-response",
13480 PROTOCOL_VERSION,
13481 7,
13482 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13483 ),
13484 )
13485 .await
13486 .unwrap();
13487 let probe_started_at = Instant::now() - Duration::from_millis(80);
13488 let pending = forwarding
13489 .begin_health_probe_rpc_for(
13490 "late-health-response",
13491 MODULE_CONTROL_OP_HEALTH_CHECK,
13492 probe_started_at,
13493 Instant::now() - Duration::from_millis(1),
13494 )
13495 .unwrap();
13496 assert!(forwarding
13497 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13498 .unwrap());
13499
13500 let responses = handler
13501 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13502 .await
13503 .unwrap();
13504
13505 assert!(responses.is_empty());
13506 let health = module.status().unwrap().health;
13507 assert_eq!(health.late_answer_count, 1);
13508 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13509 }
13510
13511 #[tokio::test]
13512 async fn health_probe_timeout_and_module_death_are_typed() {
13513 let registry = Arc::new(Registry::default());
13514 let forwarding = Arc::new(ForwardingTable::default());
13515 let handler =
13516 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13517 .with_health_probe_timeout(Duration::from_millis(50));
13518 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13519 hello_via_sink(
13520 &handler,
13521 &module_ctx,
13522 &mut module_rx,
13523 hello_frame_with_control_ops(
13524 "aft",
13525 PROTOCOL_VERSION,
13526 7,
13527 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13528 ),
13529 )
13530 .await;
13531
13532 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13533 let responses = handler
13534 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13535 .await
13536 .unwrap();
13537 assert_eq!(responses[0].header.ty, FrameType::Error);
13538 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13539 let _ = module_rx.try_recv();
13540
13541 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13542 let health_handler = handler.clone();
13543 let death_task = tokio::spawn(async move {
13544 health_handler
13545 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13546 .await
13547 .unwrap()
13548 });
13549 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13550 .await
13551 .unwrap()
13552 .unwrap();
13553 handler
13554 .cleanup_connection(module_ctx.connection_id)
13555 .unwrap();
13556 let responses = death_task.await.unwrap();
13557 assert_eq!(responses[0].header.ty, FrameType::Error);
13558 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13559 }
13560
13561 #[test]
13562 fn hello_requires_exact_protocol_version() {
13563 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13564 let registry = Arc::new(Registry::default());
13565 let handler = ControlHandler::new(Arc::clone(®istry));
13566 let responses = handler
13567 .handle_control(
13568 ConnectionId::new(connection),
13569 hello_frame("aft", offered, 9),
13570 )
13571 .unwrap();
13572
13573 assert_eq!(responses.len(), 1);
13574 assert_eq!(responses[0].header.ty, FrameType::Error);
13575 let error = parse_error(&responses[0]);
13576 assert_eq!(error["code"], "version_unsupported");
13577 assert!(registry.get_module("aft").unwrap().is_none());
13578 assert_eq!(registry.active_registration_count().unwrap(), 0);
13579 }
13580 }
13581
13582 #[test]
13583 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13584 let registry = Arc::new(Registry::default());
13585 let forwarding = Arc::new(ForwardingTable::default());
13586 let handler =
13587 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13588 let module_connection = ConnectionId::new(301);
13589 let registration = registry
13590 .register_with_control_ops(
13591 manifest("aft-push", PROTOCOL_VERSION),
13592 PROTOCOL_VERSION,
13593 module_connection,
13594 module_baseline_control_ops(),
13595 )
13596 .unwrap();
13597 let (module_tx, _module_rx) = mpsc::channel(8);
13598 let endpoint = forwarding
13599 .register_module_connection(
13600 module_connection,
13601 "aft-push".to_string(),
13602 PROTOCOL_VERSION,
13603 manifest_concurrency(®istration.manifest),
13604 FrameSink::new(module_tx),
13605 )
13606 .unwrap();
13607
13608 let unknown = Frame::build(
13610 FrameType::Push,
13611 control_flags(),
13612 0,
13613 0,
13614 5,
13615 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13616 )
13617 .unwrap();
13618 let out = handler.handle_status_update(endpoint, unknown).unwrap();
13619 assert!(
13620 out.is_empty(),
13621 "unknown push op must be ignored, got {out:?}"
13622 );
13623
13624 let malformed = Frame::build(
13626 FrameType::Push,
13627 control_flags(),
13628 0,
13629 0,
13630 6,
13631 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13632 )
13633 .unwrap();
13634 let out = handler.handle_status_update(endpoint, malformed).unwrap();
13635 assert_eq!(out.len(), 1);
13636 assert_eq!(out[0].header.ty, FrameType::Error);
13637 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13638 }
13639
13640 #[test]
13641 fn hello_rejected_when_connection_already_owns_client_routes() {
13642 let registry = Arc::new(Registry::default());
13643 let forwarding = Arc::new(ForwardingTable::default());
13644 let handler =
13645 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13646 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
13648 let client_connection = ConnectionId::new(202);
13649
13650 let responses = handler
13653 .handle_control(
13654 client_connection,
13655 hello_frame("aft-second", PROTOCOL_VERSION, 9),
13656 )
13657 .unwrap();
13658 assert_eq!(responses[0].header.ty, FrameType::Error);
13659 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13660 assert!(registry.get_module("aft-second").unwrap().is_none());
13661 }
13662
13663 #[tokio::test]
13664 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13665 let registry = Arc::new(Registry::default());
13666 let forwarding = Arc::new(ForwardingTable::default());
13667 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13668 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13669 hello_via_sink(
13670 &handler,
13671 &module_ctx,
13672 &mut module_rx,
13673 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13674 )
13675 .await;
13676 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13677 let pending = forwarding
13678 .begin_route_bind_relay_for_test(
13679 client_ctx.connection_id,
13680 client_ctx.egress.clone(),
13681 2,
13682 "alpha",
13683 )
13684 .unwrap();
13685 forwarding
13686 .complete_pending_relay(
13687 module_ctx.connection_id,
13688 pending.corr,
13689 RouteBindRelayOutcome::Accepted,
13690 )
13691 .unwrap();
13692 client_rx.try_recv().unwrap();
13693 for module_id in ["beta", "alpha"] {
13694 let replies = handler
13695 .handle_control_frame(
13696 &module_ctx,
13697 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13698 )
13699 .await
13700 .unwrap();
13701 assert_eq!(replies.len(), 1, "second HELLO must be refused");
13702 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13703 }
13704 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13705 assert!(registry.get_module("beta").unwrap().is_none());
13706 assert!(matches!(
13707 forwarding
13708 .lookup_data_route(
13709 client_ctx.connection_id,
13710 pending.client_channel,
13711 pending.client_epoch,
13712 )
13713 .unwrap(),
13714 DataRoute::Client(DataRouteState::Bound(_))
13715 ));
13716 assert!(handler
13717 .hello_launch_nonces
13718 .lock()
13719 .unwrap()
13720 .presented(module_ctx.connection_id, Some("alpha-nonce")));
13721 assert!(module_rx.try_recv().is_err());
13722 }
13723
13724 #[test]
13725 fn reserved_module_hello_requires_matching_launch_nonce() {
13726 let registry = Arc::new(Registry::default());
13727 let supervisor = SupervisorHandle::new();
13728 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13731 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13732
13733 let no_nonce = handler
13735 .handle_control(
13736 ConnectionId::new(1),
13737 hello_frame("vault", PROTOCOL_VERSION, 1),
13738 )
13739 .unwrap();
13740 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13741 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13742 assert!(registry.get_module("vault").unwrap().is_none());
13743
13744 let wrong = handler
13746 .handle_control(
13747 ConnectionId::new(2),
13748 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13749 )
13750 .unwrap();
13751 assert_eq!(wrong[0].header.ty, FrameType::Error);
13752 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13753 assert!(registry.get_module("vault").unwrap().is_none());
13754
13755 let ok = handler
13757 .handle_control(
13758 ConnectionId::new(3),
13759 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13760 )
13761 .unwrap();
13762 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13763 assert!(registry.get_module("vault").unwrap().is_some());
13764 }
13765
13766 #[test]
13767 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13768 let registry = Arc::new(Registry::default());
13769 let supervisor = SupervisorHandle::new();
13770 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13771 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13772 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13773
13774 let squat = handler
13775 .handle_control(
13776 ConnectionId::new(1),
13777 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13778 )
13779 .unwrap();
13780 assert_eq!(squat[0].header.ty, FrameType::Error);
13781 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
13782 assert!(parse_error(&squat[0])["message"]
13783 .as_str()
13784 .unwrap()
13785 .contains("fed:"));
13786
13787 let accepted_peer = handler
13788 .handle_control(
13789 ConnectionId::new(2),
13790 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
13791 )
13792 .unwrap();
13793 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
13794
13795 let accepted_short = handler
13796 .handle_control(
13797 ConnectionId::new(3),
13798 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
13799 )
13800 .unwrap();
13801 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
13802
13803 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
13804 let response = handler
13805 .handle_control(
13806 ConnectionId::new(conn),
13807 hello_frame(module_id, PROTOCOL_VERSION, conn),
13808 )
13809 .unwrap();
13810 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
13811 }
13812 }
13813
13814 #[test]
13815 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
13816 let registry = Arc::new(Registry::default());
13817 let supervisor = SupervisorHandle::new();
13818 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13819 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13820 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
13821 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13822
13823 let owner_nonce = handler
13824 .handle_control(
13825 ConnectionId::new(1),
13826 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
13827 )
13828 .unwrap();
13829 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
13830 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
13831 assert!(registry.get_module("fed:special").unwrap().is_none());
13832
13833 let exact_nonce = handler
13834 .handle_control(
13835 ConnectionId::new(2),
13836 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
13837 )
13838 .unwrap();
13839 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
13840 assert!(registry.get_module("fed:special").unwrap().is_some());
13841 }
13842
13843 #[test]
13844 fn non_reserved_module_ignores_launch_nonce() {
13845 let registry = Arc::new(Registry::default());
13846 let handler = ControlHandler::new(Arc::clone(®istry));
13849 let no_nonce = handler
13850 .handle_control(
13851 ConnectionId::new(1),
13852 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
13853 )
13854 .unwrap();
13855 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
13856 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
13857
13858 let echoed_nonce = handler
13859 .handle_control(
13860 ConnectionId::new(2),
13861 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
13862 )
13863 .unwrap();
13864 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
13865 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
13866 }
13867
13868 #[test]
13869 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
13870 let handler = ControlHandler::default();
13871 let conn = ConnectionId::new(1);
13872 let malformed = Frame::build(
13873 FrameType::Hello,
13874 control_flags(),
13875 0,
13876 0,
13877 3,
13878 b"{not json".to_vec(),
13879 )
13880 .unwrap();
13881
13882 let error = handler.handle_control(conn, malformed).unwrap();
13883 assert_eq!(error[0].header.ty, FrameType::Error);
13884 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
13885
13886 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13887 let pong = handler.handle_control(conn, ping).unwrap();
13888 assert_eq!(pong[0].header.ty, FrameType::Pong);
13889 assert_eq!(pong[0].header.corr, 4);
13890 }
13891
13892 #[test]
13893 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
13894 let registry = Arc::new(Registry::default());
13895 let handler = ControlHandler::new(Arc::clone(®istry));
13896
13897 handler
13898 .handle_control(
13899 ConnectionId::new(1),
13900 hello_frame("aft", PROTOCOL_VERSION, 1),
13901 )
13902 .unwrap();
13903 let duplicate = handler
13904 .handle_control(
13905 ConnectionId::new(2),
13906 hello_frame("aft", PROTOCOL_VERSION, 2),
13907 )
13908 .unwrap();
13909
13910 assert_eq!(duplicate[0].header.ty, FrameType::Error);
13911 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13912 let registration = registry.get_module("aft").unwrap().unwrap();
13913 assert_eq!(registration.connection_id, ConnectionId::new(1));
13914 }
13915
13916 #[test]
13917 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
13918 let registry = Arc::new(Registry::default());
13919 let forwarding = Arc::new(ForwardingTable::default());
13920 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
13921 let handler =
13922 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13923 .with_process_liveness(process_liveness);
13924 let (ctx, route_channel, route_epoch) =
13925 bind_liveness_route(®istry, &forwarding, "aft-dead");
13926 let responses = handler
13927 .handle_route_poll(
13928 &ctx,
13929 route_poll_frame(41, PollKind::Liveness, route_channel),
13930 route_channel,
13931 route_epoch,
13932 PollKind::Liveness,
13933 )
13934 .unwrap();
13935
13936 assert_eq!(responses.len(), 1);
13937 assert_eq!(responses[0].header.ty, FrameType::Response);
13938 assert_route_poll_liveness(&responses[0], false);
13939 }
13940
13941 #[test]
13942 fn liveness_poll_without_process_source_uses_bound_route() {
13943 let registry = Arc::new(Registry::default());
13944 let forwarding = Arc::new(ForwardingTable::default());
13945 let handler =
13946 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13947 let (ctx, route_channel, route_epoch) =
13948 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
13949 let responses = handler
13950 .handle_route_poll(
13951 &ctx,
13952 route_poll_frame(42, PollKind::Liveness, route_channel),
13953 route_channel,
13954 route_epoch,
13955 PollKind::Liveness,
13956 )
13957 .unwrap();
13958
13959 assert_route_poll_liveness(&responses[0], true);
13960 }
13961
13962 #[test]
13963 fn liveness_poll_untracked_process_source_uses_bound_route() {
13964 let registry = Arc::new(Registry::default());
13965 let forwarding = Arc::new(ForwardingTable::default());
13966 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13967 let handler =
13968 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13969 .with_process_liveness(process_liveness);
13970 let (ctx, route_channel, route_epoch) =
13971 bind_liveness_route(®istry, &forwarding, "aft-untracked");
13972 let responses = handler
13973 .handle_route_poll(
13974 &ctx,
13975 route_poll_frame(43, PollKind::Liveness, route_channel),
13976 route_channel,
13977 route_epoch,
13978 PollKind::Liveness,
13979 )
13980 .unwrap();
13981
13982 assert_route_poll_liveness(&responses[0], true);
13983 }
13984
13985 #[tokio::test]
13986 async fn unknown_op_returns_unknown_control_op() {
13987 let handler = ControlHandler::default();
13988 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13989 let request = Frame::build(
13990 FrameType::Request,
13991 control_flags(),
13992 0,
13993 0,
13994 55,
13995 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13996 )
13997 .unwrap();
13998
13999 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14000
14001 assert_eq!(response.len(), 1);
14002 assert_eq!(response[0].header.ty, FrameType::Error);
14003 assert_eq!(response[0].header.corr, 55);
14004 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14005 }
14006
14007 #[tokio::test]
14008 async fn supervisor_provenance_rejects_unknown_exact_module() {
14009 let handler = ControlHandler::default();
14010 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14011 let request = Frame::build(
14012 FrameType::Request,
14013 control_flags(),
14014 0,
14015 0,
14016 57,
14017 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14018 )
14019 .unwrap();
14020
14021 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14022
14023 assert_eq!(response.len(), 1);
14024 assert_eq!(response[0].header.ty, FrameType::Error);
14025 assert_eq!(response[0].header.corr, 57);
14026 let error = parse_error(&response[0]);
14027 assert_eq!(error["code"], "unknown_module");
14028 assert_eq!(error["message"], "module_id 'missing' is not supervised");
14029 }
14030
14031 #[test]
14032 fn provenance_probe_override_keeps_handler_tests_deterministic() {
14033 let expected = subc_control::RunningImageAgreement::Unavailable {
14034 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14035 };
14036 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14037 assert_eq!(handler.provenance_probe_override, Some(expected));
14038 }
14039
14040 #[test]
14041 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14042 let verdict = reload_verdict(
14043 std::path::Path::new("/bin/new"),
14044 Some(std::path::Path::new("/bin/old")),
14045 subc_control::RunningImageAgreement::Unavailable {
14046 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14047 },
14048 );
14049 assert!(matches!(
14050 verdict.path,
14051 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14052 if configured == std::path::Path::new("/bin/new")
14053 && spawned_from == std::path::Path::new("/bin/old")
14054 ));
14055 }
14056
14057 #[test]
14058 fn launch_spec_without_running_digest_is_unavailable_not_match() {
14059 use subc_control::{
14060 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14061 };
14062 let configured = crate::supervise::ModuleSpec {
14063 module_id: "legacy".into(),
14064 program: "/module".into(),
14065 args: Vec::new(),
14066 env: Vec::new(),
14067 reserved: false,
14068 reserved_prefixes: Vec::new(),
14069 protocol: subc_control::ModuleProtocol::Subc,
14070 overlap: Default::default(),
14071 };
14072 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14073 assert_eq!(
14074 launch_spec_agreement(&configured, false, Some(&digest)),
14075 Agreement::Unavailable {
14076 reason: Reason::NotRunning
14077 }
14078 );
14079 assert_eq!(
14080 launch_spec_agreement(&configured, true, None),
14081 Agreement::Unavailable {
14082 reason: Reason::SpawnedSpecUnavailable
14083 }
14084 );
14085 assert_eq!(
14086 launch_spec_agreement(&configured, true, Some(&digest)),
14087 Agreement::Match
14088 );
14089 }
14090
14091 #[cfg(windows)]
14092 #[test]
14093 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14094 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14095 let configured = dir.join("module.exe");
14096 std::fs::write(&configured, b"executable path fixture").unwrap();
14097 let spawned = std::fs::canonicalize(&configured).unwrap();
14098 assert_ne!(
14099 configured, spawned,
14100 "fixture must exercise different path spellings"
14101 );
14102 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14103 let lower_case = configured.to_string_lossy().to_lowercase();
14104 for spelling in [
14105 &configured,
14106 Path::new(&forward_slashes),
14107 Path::new(&lower_case),
14108 ] {
14109 let verdict = reload_verdict(
14110 spelling,
14111 Some(&spawned),
14112 subc_control::RunningImageAgreement::Unavailable {
14113 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14114 },
14115 );
14116 assert_eq!(
14117 verdict.path,
14118 ReloadPathAgreement::Match,
14119 "configured {spelling:?}, spawned {spawned:?}"
14120 );
14121 }
14122 }
14123
14124 #[cfg(windows)]
14125 #[test]
14126 fn windows_reload_path_keeps_different_existing_program_pending() {
14127 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14128 let old = dir.join("old.exe");
14129 let new = dir.join("new.exe");
14130 std::fs::write(&old, b"same executable content").unwrap();
14131 std::fs::copy(&old, &new).unwrap();
14132 let spawned = std::fs::canonicalize(&old).unwrap();
14133 let image = subc_control::RunningImageAgreement::Unavailable {
14134 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14135 };
14136 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14137 assert!(
14138 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14139 if configured == new && spawned_from == spawned)
14140 );
14141 let missing = dir.join("missing.exe");
14142 assert!(matches!(
14143 reload_verdict(&missing, Some(&spawned), image).path,
14144 ReloadPathAgreement::Mismatch { .. }
14145 ));
14146 }
14147
14148 #[test]
14149 fn reload_verdict_detects_replaced_image_at_same_path() {
14150 let image = subc_control::RunningImageAgreement::Mismatch {
14151 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14152 digest: "old".into(),
14153 },
14154 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14155 digest: "new".into(),
14156 },
14157 };
14158 let verdict = reload_verdict(
14159 std::path::Path::new("/bin/same"),
14160 Some(std::path::Path::new("/bin/same")),
14161 image.clone(),
14162 );
14163 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14164 assert_eq!(verdict.image, image);
14165 }
14166
14167 #[test]
14168 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14169 let image = subc_control::RunningImageAgreement::Unavailable {
14170 reason: subc_control::RunningImageUnavailableReason::NotRunning,
14171 };
14172 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14173 assert_eq!(
14174 verdict.path,
14175 subc_control::ReloadPathAgreement::Unavailable {
14176 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14177 }
14178 );
14179 assert_eq!(verdict.image, image);
14180
14181 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14182 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14183 };
14184 let verdict = reload_verdict(
14185 std::path::Path::new("/bin/same"),
14186 Some(std::path::Path::new("/bin/same")),
14187 unconfirmed.clone(),
14188 );
14189 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14190 assert_eq!(verdict.image, unconfirmed);
14191 }
14192
14193 #[test]
14194 fn reload_verdict_preserves_each_image_unavailability_reason() {
14195 use subc_control::RunningImageUnavailableReason as Reason;
14196
14197 for reason in [
14198 Reason::NotRunning,
14199 Reason::UnsupportedPlatform,
14200 Reason::RunningExecutableUnreadable,
14201 Reason::SpawnedPathUnreadable,
14202 Reason::HashFailed,
14203 Reason::ProcessIdentityUnconfirmed,
14204 Reason::Unknown("future_probe_reason".to_string()),
14205 ] {
14206 let image = subc_control::RunningImageAgreement::Unavailable {
14207 reason: reason.clone(),
14208 };
14209 let verdict = reload_verdict(
14210 std::path::Path::new("/bin/same"),
14211 Some(std::path::Path::new("/bin/same")),
14212 image.clone(),
14213 );
14214 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14215 assert_eq!(verdict.image, image, "{reason:?}");
14216 }
14217 }
14218
14219 #[tokio::test]
14220 async fn malformed_control_bodies_return_invalid_control_body() {
14221 let handler = ControlHandler::default();
14222 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14223
14224 for (corr, body) in [
14225 (56, br#"{"route_channel":1}"#.as_slice()),
14226 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14227 (
14228 58,
14229 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14230 ),
14231 ] {
14232 let request = Frame::build(
14233 FrameType::Request,
14234 control_flags(),
14235 0,
14236 0,
14237 corr,
14238 body.to_vec(),
14239 )
14240 .unwrap();
14241 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14242
14243 assert_eq!(response.len(), 1);
14244 assert_eq!(response[0].header.ty, FrameType::Error);
14245 assert_eq!(response[0].header.corr, corr);
14246 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14247 }
14248 }
14249
14250 #[tokio::test]
14251 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14252 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14253 let registry = Arc::new(Registry::default());
14254 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14255 let router = Router::with_control_handler(Arc::clone(&control));
14256 let connection = router.begin_connection();
14257 let (ctx, mut rx) = route_ctx(connection.id());
14258
14259 router
14260 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14261 .await
14262 .unwrap();
14263 let response = rx.recv().await.unwrap();
14264 let ack = parse_ack(&response);
14265 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14266 let channel = 1;
14267
14268 let goodbye =
14269 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14270 router.route_for_connection(&ctx, goodbye).await.unwrap();
14271 assert!(rx.try_recv().is_err());
14272 assert!(registry.get_module("aft").unwrap().is_none());
14273
14274 router
14275 .route_for_connection(&ctx, channel_request(channel, 13))
14276 .await
14277 .unwrap();
14278 let error_frame = rx.recv().await.unwrap();
14279 assert_eq!(error_frame.header.ty, FrameType::Error);
14280 assert_eq!(error_frame.header.channel, channel);
14281 let captured = crate::router::test_log::captured_logs(&logs);
14282 assert_eq!(
14283 captured
14284 .lines()
14285 .filter(|line| {
14286 line.contains("module_id=aft")
14287 && line.contains("reason=explicit_goodbye")
14288 && line.contains("module registration ended")
14289 })
14290 .count(),
14291 1,
14292 "unexpected GOODBYE registry log: {captured}"
14293 );
14294 }
14295
14296 #[tokio::test]
14297 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14298 let registry = Arc::new(Registry::default());
14299 let handler = ControlHandler::new(registry).with_capability_config(
14300 [("prov".to_string(), true), ("cons".to_string(), true)],
14301 BTreeMap::new(),
14302 );
14303 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14304 register_capability_manifest(
14305 &handler,
14306 &provider_ctx,
14307 &mut provider_rx,
14308 capability_manifest("prov", &["thing/v1"], &[]),
14309 1,
14310 )
14311 .await;
14312 let mut consumer = capability_manifest("cons", &[], &[]);
14313 consumer.capabilities.as_mut().unwrap().requires.push(
14314 subc_protocol::manifest::CapabilityRequirement {
14315 capability: "thing/v1".to_string(),
14316 need: subc_protocol::manifest::CapabilityNeed::Required,
14317 },
14318 );
14319 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14320 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14321 assert_eq!(
14322 handler.capability_evaluator.verdict("cons", "thing/v1"),
14323 Some(CapabilityVerdict::Provided)
14324 );
14325 let (mut client_rx, _) = open_route_for_capability_test(
14326 &handler,
14327 &provider_ctx,
14328 &mut provider_rx,
14329 703,
14330 3,
14331 "prov",
14332 None,
14333 )
14334 .await;
14335 let goodbye =
14336 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14337 handler
14338 .handle_control_frame(&provider_ctx, goodbye)
14339 .await
14340 .unwrap();
14341 assert_eq!(
14342 handler.capability_evaluator.verdict("cons", "thing/v1"),
14343 Some(CapabilityVerdict::NeverProvided)
14344 );
14345 let closed = client_rx
14346 .try_recv()
14347 .expect("GOODBYE pushes route.closed before route GOODBYE");
14348 assert!(
14349 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14350 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14351 );
14352 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14353 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14354 }
14355
14356 struct RouteClosedFixture {
14357 handler: ControlHandler,
14358 #[cfg(unix)]
14359 supervisor: Supervisor,
14360 module_ctx: RouteCtx,
14361 #[cfg(unix)]
14362 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14363 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14364 route: Arc<crate::forwarding::RouteBinding>,
14365 }
14366
14367 async fn route_closed_fixture() -> RouteClosedFixture {
14368 let registry = Arc::new(Registry::default());
14369 let handler = ControlHandler::new(Arc::clone(®istry));
14370 #[cfg(unix)]
14371 let supervisor = Supervisor::new(registry, RestartPolicy::default())
14372 .with_forwarding(Arc::clone(&handler.forwarding));
14373 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14374 register_capability_manifest(
14375 &handler,
14376 &module_ctx,
14377 &mut module_rx,
14378 capability_manifest("prov", &[], &[]),
14379 1,
14380 )
14381 .await;
14382 let (client_rx, bind) = open_route_for_capability_test(
14383 &handler,
14384 &module_ctx,
14385 &mut module_rx,
14386 702,
14387 2,
14388 "prov",
14389 None,
14390 )
14391 .await;
14392 let ModuleControlRequest::RouteBind {
14393 route_channel: channel,
14394 epoch: route_epoch,
14395 ..
14396 } = bind
14397 else {
14398 panic!("expected route.bind");
14399 };
14400 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14401 handler
14402 .forwarding
14403 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14404 .unwrap()
14405 else {
14406 panic!("the acknowledged route must be bound");
14407 };
14408 RouteClosedFixture {
14409 handler,
14410 #[cfg(unix)]
14411 supervisor,
14412 module_ctx,
14413 #[cfg(unix)]
14414 module_rx,
14415 client_rx,
14416 route,
14417 }
14418 }
14419
14420 impl RouteClosedFixture {
14421 #[cfg(unix)]
14422 async fn cut(&mut self) {
14423 let notices = async {
14424 let mut draining = self.module_rx.recv().await.unwrap();
14425 assert!(matches!(
14426 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14427 &draining.body
14428 )
14429 .unwrap(),
14430 subc_protocol::session::ModuleControlCommand::Draining {
14431 reason: RouteCloseReason::Restart,
14432 ..
14433 }
14434 ));
14435 draining.flushed.take().unwrap().send(()).unwrap();
14436 let mut closing = self.client_rx.recv().await.unwrap();
14437 assert!(matches!(
14438 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14439 ClientControlPush::RouteClosing {
14440 reason: RouteCloseReason::Restart,
14441 ..
14442 }
14443 ));
14444 closing.flushed.take().unwrap().send(()).unwrap();
14445 };
14446 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14447 result.unwrap();
14448 }
14449
14450 fn disconnect(&mut self) -> ClientControlPush {
14451 self.handler
14452 .cleanup_connection(self.module_ctx.connection_id)
14453 .unwrap();
14454 let closed = self.client_rx.try_recv().unwrap();
14455 assert_eq!(closed.header.ty, FrameType::Push);
14456 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14457 assert!(matches!(
14458 &push,
14459 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14460 if channels == &[self.route.client_channel]
14461 ));
14462 assert_eq!(
14463 self.client_rx.try_recv().unwrap().header.ty,
14464 FrameType::Goodbye
14465 );
14466 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14467 push
14468 }
14469 }
14470
14471 #[cfg(unix)]
14472 #[tokio::test(start_paused = true)]
14473 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14474 let mut fixture = route_closed_fixture().await;
14475 fixture.cut().await;
14476 assert!(matches!(
14477 fixture.disconnect(),
14478 ClientControlPush::RouteClosed {
14479 reason: RouteCloseReason::Restart,
14480 drained: true,
14481 ..
14482 }
14483 ));
14484 }
14485
14486 #[cfg(unix)]
14487 #[tokio::test(start_paused = true)]
14488 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14489 let mut fixture = route_closed_fixture().await;
14490 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14491 let started = tokio::time::Instant::now();
14492 fixture.cut().await;
14493 assert!(started.elapsed() >= Duration::from_secs(2));
14494 assert_eq!(fixture.route.flow.in_flight(), 1);
14495 assert!(matches!(
14496 fixture.disconnect(),
14497 ClientControlPush::RouteClosed {
14498 reason: RouteCloseReason::Restart,
14499 drained: false,
14500 ..
14501 }
14502 ));
14503 }
14504
14505 #[tokio::test]
14506 async fn module_connection_crash_reports_crash_and_not_drained() {
14507 let mut fixture = route_closed_fixture().await;
14508 assert!(matches!(
14512 fixture.disconnect(),
14513 ClientControlPush::RouteClosed {
14514 reason: RouteCloseReason::Crash,
14515 drained: false,
14516 ..
14517 }
14518 ));
14519 }
14520
14521 #[tokio::test]
14522 async fn dropping_router_connection_releases_registration() {
14523 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14524 let registry = Arc::new(Registry::default());
14525 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14526 let router = Router::with_control_handler(Arc::clone(&control));
14527 let connection = router.begin_connection();
14528 let connection_id = connection.id();
14529 let (ctx, mut rx) = route_ctx(connection_id);
14530
14531 router
14532 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14533 .await
14534 .unwrap();
14535 let response = rx.recv().await.unwrap();
14536 let ack = parse_ack(&response);
14537 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14538 assert!(registry.get_module("aft").unwrap().is_some());
14539
14540 drop(connection);
14541
14542 assert!(registry.get_module("aft").unwrap().is_none());
14543 assert_eq!(registry.active_registration_count().unwrap(), 0);
14544
14545 control
14546 .cleanup_connection(ConnectionId::new(u64::MAX))
14547 .unwrap();
14548 let captured = crate::router::test_log::captured_logs(&logs);
14549 println!("captured registry lifecycle logs:\n{captured}");
14550 let events: Vec<_> = captured
14551 .lines()
14552 .filter(|line| {
14553 line.contains("module registered module_id=aft ")
14554 || line.contains("module registration ended")
14555 })
14556 .collect();
14557 assert_eq!(
14558 events.len(),
14559 2,
14560 "unexpected registry lifecycle logs: {captured}"
14561 );
14562 assert!(events[0].contains("module registered module_id=aft "));
14563 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14564 assert!(events[1].contains(&format!(
14565 "module_id=aft connection_id={}",
14566 connection_id.get()
14567 )));
14568 assert!(events[1].contains("reason=connection_closed"));
14569 assert!(events[1].contains("module registration ended"));
14570 assert_eq!(
14571 captured
14572 .lines()
14573 .filter(|line| line.contains("module registered module_id=aft "))
14574 .count(),
14575 1,
14576 "legacy registration admission line must appear once: {captured}"
14577 );
14578 }
14579
14580 #[tokio::test]
14581 async fn hello_registration_keeps_legacy_module_registered_line_once() {
14582 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14583 let registry = Arc::new(Registry::default());
14584 let control = ControlHandler::new(Arc::clone(®istry));
14585 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14586 hello_via_sink(
14587 &control,
14588 &ctx,
14589 &mut rx,
14590 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14591 )
14592 .await;
14593
14594 let captured = crate::router::test_log::captured_logs(&logs);
14595 let admissions: Vec<_> = captured
14596 .lines()
14597 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14598 .collect();
14599 assert_eq!(
14600 admissions.len(),
14601 1,
14602 "legacy admission line must remain exactly once: {captured}"
14603 );
14604 assert!(admissions[0].contains("routable_provider=true"));
14605 assert!(admissions[0].contains("connection_id=777"));
14606 }
14607
14608 fn capability_manifest(
14609 module_id: &str,
14610 provides: &[&str],
14611 must_never_reach: &[&str],
14612 ) -> ModuleManifest {
14613 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14614 manifest.capabilities = Some(CapabilityDeclarations {
14615 provides: provides
14616 .iter()
14617 .map(|capability| (*capability).to_string())
14618 .collect(),
14619 requires: Vec::new(),
14620 must_never_reach: must_never_reach
14621 .iter()
14622 .map(|capability| (*capability).to_string())
14623 .collect(),
14624 });
14625 manifest
14626 }
14627
14628 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14629 Frame::build(
14630 FrameType::Hello,
14631 control_flags(),
14632 0,
14633 0,
14634 corr,
14635 serde_json::to_vec(&ModuleHelloBody {
14636 protocol_ver: manifest.protocol_ver,
14637 manifest,
14638 control_ops: None,
14639 launch_nonce: None,
14640 })
14641 .expect("capability test HELLO serializes"),
14642 )
14643 .expect("capability test HELLO frame builds")
14644 }
14645
14646 fn catalog_update_with_capabilities_frame(
14647 corr: u64,
14648 capabilities: CapabilityDeclarations,
14649 ) -> Frame {
14650 Frame::build(
14651 FrameType::Request,
14652 control_flags(),
14653 0,
14654 0,
14655 corr,
14656 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14657 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14658 capabilities: Some(capabilities),
14659 ready: None,
14660 })
14661 .expect("capability catalog.update serializes"),
14662 )
14663 .expect("capability catalog.update frame builds")
14664 }
14665
14666 async fn register_capability_manifest(
14667 handler: &ControlHandler,
14668 ctx: &RouteCtx,
14669 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14670 manifest: ModuleManifest,
14671 corr: u64,
14672 ) {
14673 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14674 }
14675
14676 async fn open_route_for_capability_test(
14677 handler: &ControlHandler,
14678 target_ctx: &RouteCtx,
14679 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14680 client_connection_id: u64,
14681 corr: u64,
14682 target_module_id: &str,
14683 consumer_identity: Option<ConsumerIdentity>,
14684 ) -> (
14685 mpsc::Receiver<crate::router::OutboundFrame>,
14686 ModuleControlRequest,
14687 ) {
14688 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14689 let route_handler = handler.clone();
14690 let target_module_id = target_module_id.to_string();
14691 let route_task = tokio::spawn(async move {
14692 route_handler
14693 .handle_control_frame(
14694 &client_ctx,
14695 route_open_frame_with_admission_facts(
14696 corr,
14697 &target_module_id,
14698 unique_project_root("admission-facts"),
14699 consumer_identity,
14700 None,
14701 ),
14702 )
14703 .await
14704 .expect("capability test route.open succeeds")
14705 });
14706 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14707 .await
14708 .expect("capability test route.open must reach route.bind")
14709 .expect("target control receiver stays open");
14710 let bind_request: ModuleControlRequest =
14711 serde_json::from_slice(&bind.body).expect("route.bind decodes");
14712 handler
14713 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14714 .await
14715 .expect("capability test route.bind ACK succeeds");
14716 assert!(route_task.await.expect("route.open task joins").is_empty());
14717 let opened = client_rx
14718 .recv()
14719 .await
14720 .expect("successful route.open publishes a response");
14721 assert!(matches!(
14722 serde_json::from_slice::<ClientControlResponse>(&opened.body),
14723 Ok(ClientControlResponse::RouteOpen { .. })
14724 ));
14725 (client_rx, bind_request)
14726 }
14727
14728 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
14729 assert_eq!(frame.header.ty, FrameType::Push);
14730 assert_eq!(frame.header.channel, 0);
14731 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14732 .expect("route.closed control push decodes");
14733 let ClientControlPush::RouteClosed { channels, .. } = &push else {
14734 panic!("expected route.closed");
14735 };
14736 assert_eq!(channels.len(), 1, "exactly one violating route closed");
14737 let channels = channels.clone();
14738 assert_eq!(
14739 push,
14740 ClientControlPush::RouteClosed {
14741 module_id: target_module_id.to_string(),
14742 channels,
14743 reason: RouteCloseReason::CapabilityDenied,
14744 drained: false,
14745 abandoned: 0,
14746 excluded_subscriptions: 0,
14747 terminal: Some(false),
14748 }
14749 );
14750 }
14751
14752 #[tokio::test]
14753 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14754 let registry = Arc::new(Registry::default());
14755 let forwarding = Arc::new(ForwardingTable::default());
14756 let supervisor = SupervisorHandle::new();
14757 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14758 let handler =
14759 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14760 .with_supervisor(supervisor);
14761 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14762 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14763 register_capability_manifest(
14764 &handler,
14765 &target_ctx,
14766 &mut target_rx,
14767 capability_manifest("target", &["credentials-provider/v1"], &[]),
14768 1,
14769 )
14770 .await;
14771 register_capability_manifest(
14772 &handler,
14773 &opener_ctx,
14774 &mut opener_rx,
14775 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14776 2,
14777 )
14778 .await;
14779
14780 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14781 let replies = handler
14782 .handle_control_frame(
14783 &client_ctx,
14784 route_open_frame_with_admission_facts(
14785 3,
14786 "target",
14787 unique_project_root("admission-facts"),
14788 Some(ConsumerIdentity {
14789 module_id: "opener".to_string(),
14790 launch_nonce: "opener-nonce".to_string(),
14791 }),
14792 None,
14793 ),
14794 )
14795 .await
14796 .expect("denied route.open returns a typed frame");
14797 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14798 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14799 assert!(
14800 target_rx.try_recv().is_err(),
14801 "forbidden route.open must not relay route.bind"
14802 );
14803 }
14804
14805 #[tokio::test]
14806 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
14807 let registry = Arc::new(Registry::default());
14808 let forwarding = Arc::new(ForwardingTable::default());
14809 let supervisor = SupervisorHandle::new();
14810 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14811 let handler =
14812 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14813 .with_supervisor(supervisor);
14814 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
14815 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
14816 register_capability_manifest(
14817 &handler,
14818 &target_ctx,
14819 &mut target_rx,
14820 capability_manifest("target", &["credentials-provider/v1"], &[]),
14821 1,
14822 )
14823 .await;
14824 register_capability_manifest(
14825 &handler,
14826 &old_opener_ctx,
14827 &mut old_opener_rx,
14828 capability_manifest("opener", &[], &[]),
14829 2,
14830 )
14831 .await;
14832 let (mut client_rx, _) = open_route_for_capability_test(
14833 &handler,
14834 &target_ctx,
14835 &mut target_rx,
14836 712,
14837 3,
14838 "target",
14839 Some(ConsumerIdentity {
14840 module_id: "opener".to_string(),
14841 launch_nonce: "opener-nonce".to_string(),
14842 }),
14843 )
14844 .await;
14845 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14846
14847 handler
14848 .cleanup_connection(old_opener_ctx.connection_id)
14849 .expect("old opener registration cleans up");
14850 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
14851 register_capability_manifest(
14852 &handler,
14853 &new_opener_ctx,
14854 &mut new_opener_rx,
14855 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14856 4,
14857 )
14858 .await;
14859
14860 assert_capability_denied_push(
14861 client_rx
14862 .try_recv()
14863 .expect("HELLO deny addition must emit route.closed")
14864 .frame,
14865 "target",
14866 );
14867 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14868 assert!(matches!(
14869 target_rx.try_recv(),
14870 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14871 ));
14872 }
14873
14874 #[tokio::test]
14875 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
14876 let registry = Arc::new(Registry::default());
14877 let forwarding = Arc::new(ForwardingTable::default());
14878 let supervisor = SupervisorHandle::new();
14879 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14880 let handler =
14881 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14882 .with_supervisor(supervisor);
14883 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
14884 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
14885 register_capability_manifest(
14886 &handler,
14887 &target_ctx,
14888 &mut target_rx,
14889 capability_manifest("target", &[], &[]),
14890 1,
14891 )
14892 .await;
14893 register_capability_manifest(
14894 &handler,
14895 &opener_ctx,
14896 &mut opener_rx,
14897 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14898 2,
14899 )
14900 .await;
14901 let (mut client_rx, _) = open_route_for_capability_test(
14902 &handler,
14903 &target_ctx,
14904 &mut target_rx,
14905 722,
14906 3,
14907 "target",
14908 Some(ConsumerIdentity {
14909 module_id: "opener".to_string(),
14910 launch_nonce: "opener-nonce".to_string(),
14911 }),
14912 )
14913 .await;
14914 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14915
14916 let replies = handler
14917 .handle_control_frame(
14918 &target_ctx,
14919 catalog_update_with_capabilities_frame(
14920 4,
14921 CapabilityDeclarations {
14922 provides: vec!["credentials-provider/v1".to_string()],
14923 requires: Vec::new(),
14924 must_never_reach: Vec::new(),
14925 },
14926 ),
14927 )
14928 .await
14929 .expect("claim catalog.update succeeds");
14930 assert!(matches!(
14931 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
14932 Ok(ModuleControlResponseToModule::CatalogUpdate {})
14933 ));
14934 assert_capability_denied_push(
14935 client_rx
14936 .try_recv()
14937 .expect("claim addition must emit route.closed")
14938 .frame,
14939 "target",
14940 );
14941 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14942 assert!(matches!(
14943 target_rx.try_recv(),
14944 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14945 ));
14946 }
14947
14948 #[tokio::test]
14949 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
14950 let registry = Arc::new(Registry::default());
14951 let forwarding = Arc::new(ForwardingTable::default());
14952 let supervisor = SupervisorHandle::new();
14953 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14954 let handler =
14955 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14956 .with_supervisor(supervisor);
14957 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
14958 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
14959 register_capability_manifest(
14960 &handler,
14961 &target_ctx,
14962 &mut target_rx,
14963 capability_manifest("target", &["credentials-provider/v1"], &[]),
14964 1,
14965 )
14966 .await;
14967 register_capability_manifest(
14968 &handler,
14969 &opener_ctx,
14970 &mut opener_rx,
14971 capability_manifest("opener", &[], &[]),
14972 2,
14973 )
14974 .await;
14975 let (mut client_rx, _) = open_route_for_capability_test(
14976 &handler,
14977 &target_ctx,
14978 &mut target_rx,
14979 732,
14980 3,
14981 "target",
14982 Some(ConsumerIdentity {
14983 module_id: "opener".to_string(),
14984 launch_nonce: "opener-nonce".to_string(),
14985 }),
14986 )
14987 .await;
14988 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14989
14990 handler
14991 .handle_control_frame(
14992 &target_ctx,
14993 catalog_update_with_capabilities_frame(
14994 4,
14995 CapabilityDeclarations {
14996 provides: Vec::new(),
14997 requires: Vec::new(),
14998 must_never_reach: Vec::new(),
14999 },
15000 ),
15001 )
15002 .await
15003 .expect("claim removal catalog.update succeeds");
15004 assert_eq!(
15005 forwarding.active_binding_count().unwrap(),
15006 1,
15007 "removing an attested target claim must leave the route census unchanged"
15008 );
15009 assert!(
15010 client_rx.try_recv().is_err(),
15011 "claim removal must not emit route.closed capability_denied"
15012 );
15013 assert!(
15014 target_rx.try_recv().is_err(),
15015 "claim removal must not send the target a route GOODBYE"
15016 );
15017 }
15018
15019 #[tokio::test]
15022 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15023 let registry = Arc::new(Registry::default());
15024 let forwarding = Arc::new(ForwardingTable::default());
15025 let supervisor = SupervisorHandle::new();
15026 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15027 let handler =
15028 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15029 .with_supervisor(supervisor);
15030 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15031 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15032 register_capability_manifest(
15033 &handler,
15034 &target_ctx,
15035 &mut target_rx,
15036 capability_manifest("target", &["credentials-provider/v1"], &[]),
15037 1,
15038 )
15039 .await;
15040 register_capability_manifest(
15041 &handler,
15042 &opener_ctx,
15043 &mut opener_rx,
15044 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15045 2,
15046 )
15047 .await;
15048
15049 let (_client_rx, bind) = open_route_for_capability_test(
15050 &handler,
15051 &target_ctx,
15052 &mut target_rx,
15053 742,
15054 3,
15055 "target",
15056 None,
15057 )
15058 .await;
15059 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15060 panic!("direct scope-honesty route must bind");
15061 };
15062 assert_eq!(principal, Some(Principal::Direct));
15063 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15064 }
15065
15066 #[tokio::test]
15069 async fn must_never_reach_self_route_is_capability_forbidden() {
15070 let registry = Arc::new(Registry::default());
15071 let forwarding = Arc::new(ForwardingTable::default());
15072 let supervisor = SupervisorHandle::new();
15073 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15074 let handler =
15075 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15076 .with_supervisor(supervisor);
15077 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15078 register_capability_manifest(
15079 &handler,
15080 &self_ctx,
15081 &mut self_rx,
15082 capability_manifest(
15083 "self-provider",
15084 &["credentials-provider/v1"],
15085 &["credentials-provider/v1"],
15086 ),
15087 1,
15088 )
15089 .await;
15090
15091 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15092 let replies = handler
15093 .handle_control_frame(
15094 &client_ctx,
15095 route_open_frame_with_admission_facts(
15096 2,
15097 "self-provider",
15098 unique_project_root("admission-facts"),
15099 Some(ConsumerIdentity {
15100 module_id: "self-provider".to_string(),
15101 launch_nonce: "self-nonce".to_string(),
15102 }),
15103 None,
15104 ),
15105 )
15106 .await
15107 .expect("self-route refusal returns a typed frame");
15108 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15109 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15110 assert!(
15111 self_rx.try_recv().is_err(),
15112 "self denial must not relay route.bind"
15113 );
15114 }
15115
15116 #[test]
15117 fn unsupported_channel_zero_frame_returns_error() {
15118 let handler = ControlHandler::default();
15119 let request = Frame::build(
15120 FrameType::Request,
15121 control_flags(),
15122 0,
15123 0,
15124 21,
15125 b"opaque".to_vec(),
15126 )
15127 .unwrap();
15128
15129 let response = handler
15130 .handle_control(ConnectionId::new(1), request)
15131 .unwrap();
15132
15133 assert_eq!(response[0].header.ty, FrameType::Error);
15134 assert_eq!(
15135 parse_error(&response[0])["code"],
15136 "unsupported_control_frame"
15137 );
15138 }
15139
15140 mod swap {
15145 use super::*;
15146
15147 const INCUMBENT: ConnectionId = ConnectionId::new(30);
15148 const CANDIDATE: ConnectionId = ConnectionId::new(40);
15149
15150 struct Swap {
15151 registry: Arc<Registry>,
15152 forwarding: Arc<ForwardingTable>,
15153 handler: ControlHandler,
15154 incumbent_ctx: RouteCtx,
15155 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15156 candidate_ctx: RouteCtx,
15157 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15158 }
15159
15160 async fn swap_with_incumbent() -> Swap {
15161 let registry = Arc::new(Registry::default());
15162 let forwarding = Arc::new(ForwardingTable::default());
15163 let handler =
15164 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15165 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15166 hello_via_sink(
15167 &handler,
15168 &incumbent_ctx,
15169 &mut incumbent_rx,
15170 hello_frame("aft", PROTOCOL_VERSION, 7),
15171 )
15172 .await;
15173 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15174 Swap {
15175 registry,
15176 forwarding,
15177 handler,
15178 incumbent_ctx,
15179 incumbent_rx,
15180 candidate_ctx,
15181 candidate_rx,
15182 }
15183 }
15184
15185 fn register_candidate(swap: &Swap, ready: Option<bool>) {
15186 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15187 candidate_manifest.ready = ready;
15188 let registration = swap
15189 .registry
15190 .register_candidate_with_control_ops(
15191 candidate_manifest,
15192 PROTOCOL_VERSION,
15193 CANDIDATE,
15194 module_baseline_control_ops(),
15195 )
15196 .unwrap();
15197 swap.forwarding
15198 .register_candidate_module_connection(
15199 CANDIDATE,
15200 "aft".to_string(),
15201 PROTOCOL_VERSION,
15202 manifest_concurrency(®istration.manifest),
15203 swap.candidate_ctx.egress.clone(),
15204 )
15205 .unwrap();
15206 }
15207
15208 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15209 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15210 swap.registry.promote_candidate("aft").unwrap().unwrap();
15211 cutover.incumbent.unwrap()
15212 }
15213
15214 #[tokio::test]
15223 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15224 let mut swap = swap_with_incumbent().await;
15225 let hold = Arc::new(std::sync::Barrier::new(2));
15226 let handler = swap
15227 .handler
15228 .clone()
15229 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15230 handler
15231 .supervisor
15232 .open_swap("aft", "candidate-nonce".to_string());
15233 let hello = {
15234 let handler = handler.clone();
15235 let candidate_ctx = swap.candidate_ctx.clone();
15236 std::thread::spawn(move || {
15237 handler
15238 .handle_hello(
15239 candidate_ctx.connection_id,
15240 Some(candidate_ctx.egress.clone()),
15241 hello_frame_with_nonce(
15242 "aft",
15243 PROTOCOL_VERSION,
15244 8,
15245 Some("candidate-nonce"),
15246 ),
15247 )
15248 .unwrap()
15249 })
15250 };
15251
15252 hold.wait();
15253 assert!(swap
15256 .registry
15257 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15258 .unwrap()
15259 .is_some());
15260 assert!(swap
15261 .registry
15262 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15263 .unwrap()
15264 .is_none());
15265 assert!(swap
15266 .forwarding
15267 .module_endpoint_for_connection(CANDIDATE)
15268 .unwrap()
15269 .is_none());
15270
15271 hold.wait();
15272 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15273 assert_eq!(
15274 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15275 FrameType::HelloAck
15276 );
15277 let published = swap
15278 .registry
15279 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15280 .unwrap()
15281 .expect("candidate is published once its HELLO completes");
15282 assert_eq!(published.connection_id, CANDIDATE);
15283 assert!(swap
15284 .forwarding
15285 .module_endpoint_for_connection(CANDIDATE)
15286 .unwrap()
15287 .is_some());
15288 }
15289
15290 fn keyed_total(counters: &Value, key: &str) -> u64 {
15291 counters[key]
15292 .as_object()
15293 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15294 .unwrap_or(0)
15295 }
15296
15297 #[tokio::test]
15298 async fn replacement_logs_old_end_and_new_admission_once() {
15299 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15300 let mut swap = swap_with_incumbent().await;
15301 swap.handler
15302 .supervisor
15303 .open_swap("aft", "candidate-nonce".to_string());
15304 hello_via_sink(
15305 &swap.handler,
15306 &swap.candidate_ctx,
15307 &mut swap.candidate_rx,
15308 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15309 )
15310 .await;
15311 cutover(&swap);
15312 swap.handler.cleanup_connection(INCUMBENT).unwrap();
15313
15314 let captured = crate::router::test_log::captured_logs(&logs);
15315 println!("captured replacement registry logs:\n{captured}");
15316 let new_admissions: Vec<_> = captured
15317 .lines()
15318 .filter(|line| {
15319 line.contains("connection_id=40")
15320 && line.contains("swap candidate registered; not routable until cutover")
15321 })
15322 .collect();
15323 assert_eq!(
15324 new_admissions.len(),
15325 1,
15326 "unexpected admission logs: {captured}"
15327 );
15328 assert!(new_admissions[0].contains("module_id=aft"));
15329 assert!(new_admissions[0].contains("connection_id=40"));
15330 assert!(
15331 new_admissions[0].contains("swap candidate registered; not routable until cutover")
15332 );
15333 assert!(new_admissions[0].contains("ready=true"));
15334
15335 let old_admissions: Vec<_> = captured
15336 .lines()
15337 .filter(|line| {
15338 line.contains("module registered module_id=aft ")
15339 && line.contains("connection_id=30")
15340 })
15341 .collect();
15342 assert_eq!(
15343 old_admissions.len(),
15344 1,
15345 "unexpected incumbent admission: {captured}"
15346 );
15347
15348 let old_ends: Vec<_> = captured
15349 .lines()
15350 .filter(|line| {
15351 line.contains("connection_id=30") && line.contains("module registration ended")
15352 })
15353 .collect();
15354 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15355 assert!(old_ends[0].contains("module_id=aft"));
15356 assert!(old_ends[0].contains("connection_id=30"));
15357 assert!(old_ends[0].contains("reason=replaced"));
15358 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15359 assert_eq!(
15360 captured
15361 .lines()
15362 .filter(|line| line.contains("module registration promoted"))
15363 .count(),
15364 1,
15365 "unexpected promotion logs: {captured}"
15366 );
15367 }
15368
15369 #[tokio::test]
15375 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15376 let mut swap = swap_with_incumbent().await;
15377 let handler = swap.handler.clone();
15378
15379 let cotenant = ConnectionId::new(31);
15381 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15382 let (cotenant_task, cotenant_bind) = relay_route_open(
15383 &handler,
15384 cotenant,
15385 &cotenant_ctx.egress,
15386 &mut swap.incumbent_rx,
15387 100,
15388 "aft",
15389 "swap-cotenant",
15390 )
15391 .await;
15392 handler
15393 .handle_control_frame(
15394 &swap.incumbent_ctx,
15395 route_bind_ack(cotenant_bind.header.corr),
15396 )
15397 .await
15398 .unwrap();
15399 assert!(cotenant_task.await.unwrap().is_empty());
15400 let (cotenant_channel, cotenant_epoch) =
15401 published_route(&cotenant_rx.recv().await.unwrap());
15402
15403 let caller = ConnectionId::new(32);
15405 let (caller_ctx, mut caller_rx) = route_ctx(caller);
15406 let (caller_task, caller_bind) = relay_route_open(
15407 &handler,
15408 caller,
15409 &caller_ctx.egress,
15410 &mut swap.incumbent_rx,
15411 101,
15412 "aft",
15413 "swap-caller",
15414 )
15415 .await;
15416 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15417
15418 register_candidate(&swap, None);
15419 cutover(&swap);
15420
15421 let ack = handler
15423 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15424 .await;
15425 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15426 if module_loop_error.is_some() {
15427 handler.cleanup_connection(INCUMBENT).unwrap();
15430 }
15431
15432 assert!(
15434 cotenant_rx.try_recv().is_err(),
15435 "the co-tenant route on the incumbent was torn down by one late ack: \
15436 {module_loop_error:?}"
15437 );
15438 assert!(matches!(
15439 swap.forwarding
15440 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15441 .unwrap(),
15442 DataRoute::Client(DataRouteState::Bound(_))
15443 ));
15444 assert_eq!(module_loop_error, None);
15445 assert!(swap
15446 .registry
15447 .get_module_by_connection(INCUMBENT)
15448 .unwrap()
15449 .is_some());
15450
15451 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15453 .await
15454 .expect("the incumbent is told to drop the abandoned binding")
15455 .unwrap()
15456 .frame;
15457 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15458 assert_eq!(goodbye.header.channel, abandoned_channel);
15459 assert_eq!(goodbye.header.epoch, abandoned_epoch);
15460 assert!(swap.incumbent_rx.try_recv().is_err());
15461
15462 let response = caller_task.await.unwrap();
15464 assert_eq!(response.len(), 1);
15465 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15466 assert!(caller_rx.try_recv().is_err());
15467
15468 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15472 let counters = handler.counters().snapshot();
15473 assert_eq!(
15474 keyed_total(&counters, "route_open_accepted_by_principal"),
15475 1
15476 );
15477 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15478 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15479 }
15480
15481 #[tokio::test]
15485 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15486 let mut swap = swap_with_incumbent().await;
15487 register_candidate(&swap, None);
15488 let incumbent = cutover(&swap);
15489 swap.forwarding
15490 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15491 .unwrap()
15492 .expect("the incumbent is still registered");
15493
15494 let client = ConnectionId::new(33);
15495 let (client_ctx, mut client_rx) = route_ctx(client);
15496 let route_handler = swap.handler.clone();
15497 let open_ctx = RouteCtx {
15498 connection_id: client,
15499 egress: client_ctx.egress.clone(),
15500 };
15501 let mut route_task = tokio::spawn(async move {
15502 route_handler
15503 .handle_control_frame(
15504 &open_ctx,
15505 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15506 )
15507 .await
15508 .unwrap()
15509 });
15510 let bind = tokio::select! {
15511 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15512 response = &mut route_task => {
15513 let response = response.unwrap();
15514 panic!(
15515 "post-cutover route.open was refused instead of relayed to the candidate: {}",
15516 parse_error(&response[0])["code"]
15517 );
15518 }
15519 };
15520 swap.handler
15521 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15522 .await
15523 .unwrap();
15524 assert!(route_task.await.unwrap().is_empty());
15525 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15526 match swap
15527 .forwarding
15528 .lookup_data_route(client, channel, epoch)
15529 .unwrap()
15530 {
15531 DataRoute::Client(DataRouteState::Bound(route)) => {
15532 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15533 }
15534 other => panic!("expected a bound route on the candidate, got {other:?}"),
15535 }
15536 assert!(swap.incumbent_rx.try_recv().is_err());
15537 }
15538
15539 #[tokio::test]
15544 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15545 let swap = swap_with_incumbent().await;
15546 register_candidate(&swap, Some(false));
15547 let update = Frame::build(
15548 FrameType::Request,
15549 control_flags(),
15550 0,
15551 0,
15552 55,
15553 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15554 provides: manifest("aft", PROTOCOL_VERSION).provides,
15555 capabilities: None,
15556 ready: Some(true),
15557 })
15558 .unwrap(),
15559 )
15560 .unwrap();
15561
15562 let replies = swap
15563 .handler
15564 .handle_control_frame(&swap.candidate_ctx, update)
15565 .await
15566 .unwrap();
15567
15568 assert_eq!(replies.len(), 1);
15569 assert_eq!(
15570 replies[0].header.ty,
15571 FrameType::Response,
15572 "candidate catalog.update was refused: {:?}",
15573 serde_json::from_slice::<Value>(&replies[0].body).ok()
15574 );
15575 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15576 assert_eq!(
15577 swap.registry
15578 .get_module("aft")
15579 .unwrap()
15580 .unwrap()
15581 .connection_id,
15582 INCUMBENT
15583 );
15584 }
15585 }
15586
15587 mod swap_admission {
15591 use super::*;
15592
15593 const INCUMBENT_NONCE: &str = "incumbent-nonce";
15594 const CANDIDATE_NONCE: &str = "candidate-nonce";
15595
15596 fn handler_with_incumbent(
15597 module_id: &str,
15598 reserved: bool,
15599 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15600 let registry = Arc::new(Registry::default());
15601 let supervisor = SupervisorHandle::new();
15602 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15603 if reserved {
15604 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15605 }
15606 let handler =
15607 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
15608 let incumbent = handler
15609 .handle_control(
15610 ConnectionId::new(1),
15611 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15612 )
15613 .unwrap();
15614 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15615 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15616 (registry, supervisor, handler)
15617 }
15618
15619 #[test]
15625 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15626 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15627
15628 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15629 let replies = handler
15630 .handle_control(
15631 ConnectionId::new(connection),
15632 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15633 )
15634 .unwrap();
15635 assert_eq!(replies[0].header.ty, FrameType::Error);
15636 assert_eq!(
15637 parse_error(&replies[0])["code"],
15638 "swap_token_invalid",
15639 "nonce {nonce:?}"
15640 );
15641 }
15642 assert!(registry.get_candidate("aft").unwrap().is_none());
15643 assert_eq!(
15644 registry.get_module("aft").unwrap().unwrap().connection_id,
15645 ConnectionId::new(1)
15646 );
15647
15648 let admitted = handler
15651 .handle_control(
15652 ConnectionId::new(4),
15653 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15654 )
15655 .unwrap();
15656 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15657 assert_eq!(
15658 registry
15659 .get_candidate("aft")
15660 .unwrap()
15661 .unwrap()
15662 .connection_id,
15663 ConnectionId::new(4)
15664 );
15665 assert_eq!(
15666 registry.get_module("aft").unwrap().unwrap().connection_id,
15667 ConnectionId::new(1),
15668 "the candidate must not take the active slot"
15669 );
15670 let replayed = handler
15671 .handle_control(
15672 ConnectionId::new(5),
15673 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15674 )
15675 .unwrap();
15676 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15677
15678 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15682 let squatter = handler
15683 .handle_control(
15684 ConnectionId::new(6),
15685 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15686 )
15687 .unwrap();
15688 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15689 assert!(
15690 registry.get_module("aft").unwrap().is_none(),
15691 "a squatter took the active slot of an id being swapped"
15692 );
15693 }
15694
15695 #[test]
15700 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15701 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15702
15703 let replies = handler
15704 .handle_control(
15705 ConnectionId::new(2),
15706 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15707 )
15708 .unwrap();
15709
15710 assert_eq!(
15711 replies[0].header.ty,
15712 FrameType::HelloAck,
15713 "reserved candidate refused: {:?}",
15714 serde_json::from_slice::<Value>(&replies[0].body).ok()
15715 );
15716 assert_eq!(
15717 registry
15718 .get_candidate("vault")
15719 .unwrap()
15720 .unwrap()
15721 .connection_id,
15722 ConnectionId::new(2)
15723 );
15724 }
15725
15726 #[test]
15729 fn without_an_open_swap_the_ordinary_gates_decide() {
15730 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15731 supervisor.close_swap("vault");
15732
15733 let candidate = handler
15734 .handle_control(
15735 ConnectionId::new(2),
15736 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15737 )
15738 .unwrap();
15739 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15740 let duplicate = handler
15741 .handle_control(
15742 ConnectionId::new(3),
15743 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15744 )
15745 .unwrap();
15746 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15747 assert!(registry.get_candidate("vault").unwrap().is_none());
15748 }
15749 }
15750
15751 mod scopes {
15755 mod idle_tests {
15756 use super::*;
15757 include!("scope_idle_tests.rs");
15758 }
15759 mod route_open_timings {
15760 use super::*;
15761 include!("route_open_timing_tests.rs");
15762 }
15763 mod route_open_refusal_log {
15764 use super::*;
15765 include!("route_open_refusal_log_tests.rs");
15766 }
15767 mod activity {
15768 include!("activity_scope_tests.rs");
15769 }
15770 mod delta {
15771 include!("scope_handler_delta_tests.rs");
15772 }
15773 mod call_modes {
15774 include!("scope_call_mode_tests.rs");
15775 }
15776 #[tokio::test]
15777 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15778 let rig = rig().await;
15779 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15780 "s".into(),
15781 1,
15782 "tool".into(),
15783 ))
15784 .unwrap();
15785 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
15786 let (direct, _rx) = wide_ctx(999);
15787 for (ctx, expected) in [
15788 (&rig.owner, FrameType::Response),
15789 (&direct, FrameType::Error),
15790 ] {
15791 let frame =
15792 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
15793 .unwrap();
15794 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
15795 assert_eq!(replies[0].header.ty, expected);
15796 if expected == FrameType::Response {
15797 let reply: subc_control::ScopeServedByReply =
15798 serde_json::from_slice(&replies[0].body).unwrap();
15799 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
15800 } else {
15801 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
15802 }
15803 }
15804 let reply = call(
15805 &rig.handler,
15806 &rig.owner,
15807 &ModuleControlRequestFromModule::ScopeDescribe {
15808 owner: Principal::Reserved {
15809 module_id: OWNER.into(),
15810 },
15811 scope_ref: "s".into(),
15812 },
15813 )
15814 .await;
15815 assert!(matches!(
15816 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
15817 ModuleControlResponseToModule::ScopeDescribe { .. }
15818 ));
15819 }
15820
15821 use subc_protocol::scope::{
15822 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
15823 ScopeStatus,
15824 };
15825
15826 use super::*;
15827
15828 const OWNER: &str = "prefrontal-core";
15829
15830 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
15831 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
15832 }
15833
15834 async fn call(
15835 handler: &ControlHandler,
15836 ctx: &RouteCtx,
15837 request: &ModuleControlRequestFromModule,
15838 ) -> Frame {
15839 let body = serde_json::to_vec(request).unwrap();
15840 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
15841 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
15842 assert_eq!(replies.len(), 1, "{replies:?}");
15843 replies.pop().unwrap()
15844 }
15845
15846 async fn sync(
15847 handler: &ControlHandler,
15848 ctx: &RouteCtx,
15849 generation: u64,
15850 scopes: Vec<ScopeRecord>,
15851 ) -> Result<ModuleControlResponseToModule, String> {
15852 let reply = call(
15853 handler,
15854 ctx,
15855 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
15856 )
15857 .await;
15858 match reply.header.ty {
15859 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
15860 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
15861 }
15862 }
15863
15864 async fn describe(
15865 handler: &ControlHandler,
15866 ctx: &RouteCtx,
15867 owner: &str,
15868 scope_ref: &str,
15869 ) -> ModuleControlResponseToModule {
15870 let reply = call(
15871 handler,
15872 ctx,
15873 &ModuleControlRequestFromModule::ScopeDescribe {
15874 owner: Principal::Reserved {
15875 module_id: owner.to_string(),
15876 },
15877 scope_ref: scope_ref.to_string(),
15878 },
15879 )
15880 .await;
15881 assert_eq!(
15882 reply.header.ty,
15883 FrameType::Response,
15884 "{:?}",
15885 parse_error(&reply)
15886 );
15887 serde_json::from_slice(&reply.body).unwrap()
15888 }
15889
15890 #[tokio::test]
15891 async fn scope_describe_never_echoes_bind_proof_or_hash() {
15892 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15893 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15894 let mut rig = rig().await;
15895 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15896 .await;
15897 let _bound = rig
15898 .bound(
15899 Some(AFT),
15900 PLEXUS,
15901 Some(
15902 rig.selector("s", Some(1))
15903 .with_bind_proof(Some(PROOF.into())),
15904 ),
15905 )
15906 .await;
15907 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
15908 let output = serde_json::to_string(&reply).unwrap();
15909 assert!(output.contains("\"status\":\"live\""));
15910 assert!(!output.contains(PROOF));
15911 assert!(!output.contains(HASH));
15912 }
15913
15914 async fn module(
15916 handler: &ControlHandler,
15917 connection: u64,
15918 module_id: &str,
15919 nonce: Option<&str>,
15920 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15921 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
15922 hello_via_sink(
15923 handler,
15924 &ctx,
15925 &mut rx,
15926 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
15927 )
15928 .await;
15929 (ctx, rx)
15930 }
15931
15932 #[tokio::test]
15935 async fn a_client_connection_cannot_sync_or_describe() {
15936 let handler = ControlHandler::new(Arc::new(Registry::default()));
15937 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
15938 for request in [
15939 ModuleControlRequestFromModule::ScopeSync {
15940 generation: 1,
15941 scopes: vec![head("s", 1)],
15942 },
15943 ModuleControlRequestFromModule::ScopeDescribe {
15944 owner: Principal::Direct,
15945 scope_ref: "s".to_string(),
15946 },
15947 ] {
15948 let reply = call(&handler, &ctx, &request).await;
15949 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
15950 }
15951 assert!(
15952 !handler
15953 .scopes
15954 .read()
15955 .unwrap()
15956 .describe(
15957 &Principal::Reserved {
15958 module_id: OWNER.to_string()
15959 },
15960 "s"
15961 )
15962 .owner_synced
15963 );
15964 }
15965
15966 #[tokio::test]
15969 async fn a_module_without_a_supervised_launch_cannot_sync() {
15970 let handler = ControlHandler::new(Arc::new(Registry::default()));
15971 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
15972 assert_eq!(
15973 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
15974 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15975 );
15976 }
15977
15978 #[tokio::test]
15979 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
15980 let supervisor = SupervisorHandle::new();
15981 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15982 let handler = ControlHandler::new(Arc::new(Registry::default()))
15983 .with_supervisor(supervisor.clone());
15984 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15985 sync(&handler, &incumbent, 1, vec![head("s", 1)])
15986 .await
15987 .expect("the current launch syncs");
15988
15989 supervisor.open_swap(OWNER, "n2".to_string());
15992 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
15993 assert_eq!(
15994 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15995 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15996 );
15997 sync(&handler, &incumbent, 2, vec![head("s", 1)])
15998 .await
15999 .expect("the serving owner syncs during the swap");
16000
16001 supervisor.close_swap(OWNER);
16004 assert_eq!(
16005 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16006 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16007 );
16008 sync(&handler, &incumbent, 3, vec![head("s", 1)])
16009 .await
16010 .expect("the serving owner syncs after the rollback");
16011 handler.cleanup_connection(candidate.connection_id).unwrap();
16012
16013 supervisor.open_swap(OWNER, "n3".to_string());
16018 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16019 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16020 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16021 .await
16022 .expect("the promoted launch takes authority");
16023 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16024 panic!("unexpected reply {reply:?}");
16025 };
16026 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16027 assert_eq!(
16028 sync(&handler, &incumbent, 4, Vec::new()).await,
16029 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16030 );
16031 }
16032
16033 #[tokio::test]
16036 async fn closing_the_authority_connection_frees_sync_authority() {
16037 let supervisor = SupervisorHandle::new();
16038 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16039 let handler = ControlHandler::new(Arc::new(Registry::default()))
16040 .with_supervisor(supervisor.clone());
16041 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16042 sync(&handler, &first, 10, vec![head("s", 1)])
16043 .await
16044 .unwrap();
16045 handler.cleanup_connection(first.connection_id).unwrap();
16046
16047 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16048 sync(&handler, &second, 1, vec![head("s", 1)])
16049 .await
16050 .expect("the next connection takes the released authority");
16051 }
16052
16053 #[tokio::test]
16054 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16055 let supervisor = SupervisorHandle::new();
16056 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16057 let handler =
16058 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16059 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16060 sync(&handler, &first, 10, vec![head("s", 1)])
16061 .await
16062 .unwrap();
16063 handler
16064 .handle_control_frame(
16065 &first,
16066 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16067 )
16068 .await
16069 .unwrap();
16070 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16071 sync(&handler, &second, 1, vec![head("s", 1)])
16072 .await
16073 .expect("GOODBYE releases authority even if the old socket remains open");
16074 }
16075
16076 #[tokio::test]
16077 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16078 let registry = Arc::new(Registry::default());
16079 let supervisor_handle = SupervisorHandle::new();
16080 let supervisor =
16081 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16082 .with_handle(supervisor_handle.clone())
16083 .with_daemon_incarnation("incarnation-7".to_string());
16084 supervisor
16087 .supervise_configured(
16088 ModuleSpec {
16089 module_id: OWNER.to_string(),
16090 program: PathBuf::from("/nonexistent/prefrontal-core"),
16091 args: Vec::new(),
16092 env: Vec::new(),
16093 reserved: false,
16094 reserved_prefixes: Vec::new(),
16095 protocol: ModuleProtocol::Subc,
16096 overlap: Default::default(),
16097 },
16098 false,
16099 )
16100 .unwrap();
16101 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16102 let handler =
16103 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16104 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16105
16106 let ModuleControlResponseToModule::ScopeDescribe {
16108 status,
16109 daemon_incarnation,
16110 owner_synced,
16111 owner_configured,
16112 scope,
16113 ..
16114 } = describe(&handler, &reader, OWNER, "s").await
16115 else {
16116 panic!("not a describe reply");
16117 };
16118 assert_eq!(status, ScopeStatus::NotLive);
16119 assert_eq!(daemon_incarnation, "incarnation-7");
16120 assert!(!owner_synced);
16121 assert!(owner_configured);
16122 assert!(scope.is_none());
16123
16124 let ModuleControlResponseToModule::ScopeDescribe {
16127 status,
16128 owner_configured,
16129 ..
16130 } = describe(&handler, &reader, "ghost", "s").await
16131 else {
16132 panic!("not a describe reply");
16133 };
16134 assert_eq!(status, ScopeStatus::NotLive);
16135 assert!(!owner_configured);
16136
16137 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16139 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16140 let ModuleControlResponseToModule::ScopeDescribe {
16141 status,
16142 scope_epoch,
16143 owner_synced,
16144 scope,
16145 ..
16146 } = describe(&handler, &reader, OWNER, "s").await
16147 else {
16148 panic!("not a describe reply");
16149 };
16150 assert_eq!(status, ScopeStatus::Live);
16151 assert_eq!(scope_epoch, Some(4));
16152 assert!(owner_synced);
16153 let stamp = scope.expect("a live scope carries its stamp");
16154 assert!(
16155 stamp.owner_authorized,
16156 "prefrontal-core is the default authority"
16157 );
16158 assert_eq!(stamp.kind, ScopeKind::Head);
16159 }
16160
16161 #[tokio::test]
16162 async fn scope_authority_owners_decides_owner_authorized() {
16163 let supervisor = SupervisorHandle::new();
16164 supervisor.set_spawn_nonce("broca", "b1".to_string());
16165 let handler = ControlHandler::new(Arc::new(Registry::default()))
16166 .with_supervisor(supervisor)
16167 .with_scope_authority_owners(vec!["broca".to_string()]);
16168 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16169 let mut gated = head("s", 1);
16170 gated.attributes.agent_id = Some("agent".to_string());
16171 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16172 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16173 describe(&handler, &broca, "broca", "s").await
16174 else {
16175 panic!("not a describe reply");
16176 };
16177 assert!(scope.unwrap().owner_authorized);
16178 }
16179
16180 #[tokio::test]
16184 async fn scope_ops_and_the_scopes_capability_are_advertised() {
16185 let handler = ControlHandler::new(Arc::new(Registry::default()));
16186 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16187 let ack = hello_via_sink(
16188 &handler,
16189 &ctx,
16190 &mut rx,
16191 hello_frame("m", PROTOCOL_VERSION, 1),
16192 )
16193 .await;
16194 let ack = parse_ack(&ack);
16195 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16196 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16197 }
16198 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16199
16200 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16201 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16202 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16203 let reply = handler
16204 .handle_control_frame(&client, frame)
16205 .await
16206 .unwrap()
16207 .pop()
16208 .unwrap();
16209 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16210 serde_json::from_slice(&reply.body).unwrap()
16211 else {
16212 panic!("not a server.describe reply");
16213 };
16214 assert!(
16215 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16216 "{capabilities:?}"
16217 );
16218 }
16219
16220 const PLEXUS: &str = "plexus";
16223 const OTHER: &str = "other";
16224 const AFT: &str = "aft";
16225 const BROCA: &str = "broca";
16226 const MAGIC: &str = "magic-context";
16227
16228 fn nonce(module_id: &str) -> String {
16229 format!("nonce-{module_id}")
16230 }
16231
16232 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16233 let (tx, rx) = mpsc::channel(64);
16234 (
16235 RouteCtx {
16236 connection_id: ConnectionId::new(connection),
16237 egress: FrameSink::new(tx),
16238 },
16239 rx,
16240 )
16241 }
16242
16243 struct Rig {
16247 handler: ControlHandler,
16248 forwarding: Arc<ForwardingTable>,
16249 owner: RouteCtx,
16250 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16251 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16252 generation: u64,
16253 next_connection: u64,
16254 _supervisor: Supervisor,
16255 }
16256
16257 async fn rig() -> Rig {
16258 rig_with_flow_support(true).await
16259 }
16260
16261 async fn rig_with_flow_support(flow_support: bool) -> Rig {
16262 rig_with_scope_capabilities(flow_support, false).await
16263 }
16264
16265 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16266 let mut provides = Vec::new();
16267 if flow_support {
16268 provides.push("flow-scopes/v1");
16269 }
16270 if run_support {
16271 provides.push("agent-run-scopes/v1");
16272 }
16273 rig_with_target_capabilities(&provides).await
16274 }
16275
16276 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16277 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16278 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16279 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16282 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16283 Frame::build(
16284 FrameType::Hello,
16285 control_flags(),
16286 0,
16287 0,
16288 connection,
16289 serde_json::to_vec(&body).unwrap(),
16290 )
16291 .unwrap()
16292 }
16293
16294 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16295 let registry = Arc::new(Registry::default());
16296 let forwarding = Arc::new(ForwardingTable::default());
16297 let supervisor_handle = SupervisorHandle::new();
16298 let supervisor =
16299 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16300 .with_handle(supervisor_handle.clone());
16301 supervisor
16302 .supervise_configured(
16303 ModuleSpec {
16304 module_id: OWNER.to_string(),
16305 program: PathBuf::from("/nonexistent/prefrontal-core"),
16306 args: Vec::new(),
16307 env: Vec::new(),
16308 reserved: false,
16309 reserved_prefixes: Vec::new(),
16310 protocol: ModuleProtocol::Subc,
16311 overlap: Default::default(),
16312 },
16313 false,
16314 )
16315 .unwrap();
16316 for module_id in [OWNER, AFT, BROCA, MAGIC] {
16317 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16318 }
16319 let handler =
16320 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16321 .with_supervisor(supervisor_handle);
16322 let (owner, mut owner_rx) = wide_ctx(1);
16323 hello_via_sink(
16324 &handler,
16325 &owner,
16326 &mut owner_rx,
16327 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16328 )
16329 .await;
16330 let mut modules = BTreeMap::new();
16331 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16332 let (ctx, mut rx) = wide_ctx(connection);
16333 let hello = hello_with_capabilities(module_id, connection, provides);
16334 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16335 modules.insert(module_id.to_string(), (ctx, rx));
16336 }
16337 Rig {
16338 handler,
16339 forwarding,
16340 owner,
16341 _owner_rx: owner_rx,
16342 modules,
16343 generation: 0,
16344 next_connection: 100,
16345 _supervisor: supervisor,
16346 }
16347 }
16348
16349 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16350 ScopeCarrier::new(Principal::Reserved {
16351 module_id: module_id.to_string(),
16352 })
16353 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16354 }
16355
16356 fn session(scope_epoch: u64) -> ScopeRecord {
16359 let mut record = head("s", scope_epoch);
16360 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16361 record.attributes.agent_id = Some("agent-1".to_string());
16362 record.attributes.delegates = true;
16363 record
16364 }
16365
16366 impl Rig {
16367 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16368 self.generation += 1;
16369 sync(&self.handler, &self.owner, self.generation, scopes)
16370 .await
16371 .expect("the owner's sync is accepted");
16372 }
16373
16374 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16375 ScopeSelector::new(
16376 Principal::Reserved {
16377 module_id: OWNER.to_string(),
16378 },
16379 scope_ref.to_string(),
16380 scope_epoch,
16381 )
16382 }
16383
16384 fn open_frame(
16385 &mut self,
16386 opener: Option<&str>,
16387 target: &str,
16388 scope: Option<ScopeSelector>,
16389 ) -> (
16390 RouteCtx,
16391 mpsc::Receiver<crate::router::OutboundFrame>,
16392 Frame,
16393 ) {
16394 self.next_connection += 1;
16395 let (ctx, rx) = wide_ctx(self.next_connection);
16396 let root = unique_project_root("scoped-open");
16397 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16398 target: RouteTarget::ToolProvider {
16399 module_id: target.to_string(),
16400 },
16401 identity: BindIdentity::new(
16402 root.path().to_path_buf(),
16403 "unit".to_string(),
16404 "session".to_string(),
16405 ),
16406 consumer_identity: opener.map(|module_id| ConsumerIdentity {
16407 module_id: module_id.to_string(),
16408 launch_nonce: nonce(module_id),
16409 }),
16410 consumer_capabilities: None,
16411 role_versions: None,
16412 admission_facts: None,
16413 scope,
16414 })
16415 .unwrap();
16416 let frame = Frame::build(
16417 FrameType::Request,
16418 control_flags(),
16419 0,
16420 0,
16421 self.next_connection,
16422 body,
16423 )
16424 .unwrap();
16425 (ctx, rx, frame)
16426 }
16427
16428 async fn refused(
16430 &mut self,
16431 opener: Option<&str>,
16432 target: &str,
16433 scope: Option<ScopeSelector>,
16434 ) -> String {
16435 self.refusal_body(opener, target, scope).await["code"]
16436 .as_str()
16437 .unwrap()
16438 .to_string()
16439 }
16440
16441 async fn refusal_body(
16442 &mut self,
16443 opener: Option<&str>,
16444 target: &str,
16445 scope: Option<ScopeSelector>,
16446 ) -> Value {
16447 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16448 let replies = tokio::time::timeout(
16449 Duration::from_secs(2),
16450 self.handler.handle_control_frame(&ctx, frame),
16451 )
16452 .await
16453 .expect("the open must be refused before waiting for a bind ack")
16454 .unwrap();
16455 assert_eq!(replies.len(), 1, "{replies:?}");
16456 assert_eq!(replies[0].header.ty, FrameType::Error);
16457 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16458 assert!(
16459 module_rx.try_recv().is_err(),
16460 "a refused open relays nothing"
16461 );
16462 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16463 parse_error(&replies[0])
16464 }
16465
16466 async fn relayed(
16468 &mut self,
16469 opener: Option<&str>,
16470 target: &str,
16471 scope: Option<ScopeSelector>,
16472 ) -> Relayed {
16473 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16474 let handler = self.handler.clone();
16475 let task_ctx = ctx.clone();
16476 let task = tokio::spawn(async move {
16477 handler
16478 .handle_control_frame(&task_ctx, frame)
16479 .await
16480 .unwrap()
16481 });
16482 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16483 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16484 .await
16485 .expect("the target receives the relayed route.bind")
16486 .unwrap()
16487 .frame;
16488 Relayed {
16489 target: target.to_string(),
16490 client: ctx,
16491 client_rx: rx,
16492 task,
16493 bind,
16494 }
16495 }
16496
16497 async fn ack(&self, relayed: &Relayed) {
16498 let (module, _) = &self.modules[&relayed.target];
16499 self.handler
16500 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16501 .await
16502 .unwrap();
16503 }
16504
16505 async fn bound(
16507 &mut self,
16508 opener: Option<&str>,
16509 target: &str,
16510 scope: Option<ScopeSelector>,
16511 ) -> Bound {
16512 let relayed = self.relayed(opener, target, scope).await;
16513 self.ack(&relayed).await;
16514 let Relayed {
16515 target,
16516 client,
16517 mut client_rx,
16518 task,
16519 bind,
16520 } = relayed;
16521 assert!(
16522 task.await.unwrap().is_empty(),
16523 "the open is answered by commit"
16524 );
16525 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16526 Bound {
16527 target,
16528 client,
16529 client_rx,
16530 channel,
16531 epoch,
16532 bind,
16533 }
16534 }
16535
16536 fn live(&self, route: &Bound) -> bool {
16537 matches!(
16538 self.forwarding
16539 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16540 .unwrap(),
16541 DataRoute::Client(DataRouteState::Bound(_))
16542 )
16543 }
16544 }
16545
16546 struct Relayed {
16547 target: String,
16548 client: RouteCtx,
16549 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16550 task: tokio::task::JoinHandle<Vec<Frame>>,
16551 bind: Frame,
16552 }
16553
16554 struct Bound {
16555 target: String,
16556 client: RouteCtx,
16557 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16558 channel: u16,
16559 epoch: u32,
16560 bind: Frame,
16561 }
16562
16563 impl Bound {
16564 fn closed_reason(&mut self) -> RouteCloseReason {
16567 let mut reason = None;
16568 let mut goodbye = false;
16569 while let Ok(outbound) = self.client_rx.try_recv() {
16570 let frame = outbound.frame;
16571 match frame.header.ty {
16572 FrameType::Goodbye => {
16573 assert_eq!(
16574 (frame.header.channel, frame.header.epoch),
16575 (self.channel, self.epoch)
16576 );
16577 goodbye = true;
16578 }
16579 FrameType::Push => {
16580 let ClientControlPush::RouteClosed {
16581 reason: r,
16582 module_id,
16583 ..
16584 } = serde_json::from_slice(&frame.body).unwrap()
16585 else {
16586 panic!("unexpected push");
16587 };
16588 assert_eq!(module_id, self.target);
16589 reason = Some(r);
16590 }
16591 other => panic!("unexpected frame {other:?}"),
16592 }
16593 }
16594 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16595 reason.expect("the client is told why the route closed")
16596 }
16597
16598 fn untouched(&mut self) -> bool {
16599 self.client_rx.try_recv().is_err()
16600 }
16601
16602 fn stamp(&self) -> Option<ScopeStamp> {
16603 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16604 ModuleControlRequest::RouteBind { scope, .. } => scope,
16605 other => panic!("expected a route.bind, got {other:?}"),
16606 }
16607 }
16608 }
16609
16610 #[tokio::test]
16611 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16612 ) {
16613 let mut rig = rig().await;
16614 let mut record = session(1);
16615 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16616 record.child_owners = vec![Principal::Reserved {
16617 module_id: MAGIC.to_string(),
16618 }];
16619 rig.sync(vec![record]).await;
16620 let scope = || Some(rig_selector("s", Some(1)));
16621
16622 rig.bound(Some(OWNER), PLEXUS, scope()).await;
16625 rig.bound(Some(AFT), OTHER, scope()).await;
16626 rig.bound(Some(BROCA), PLEXUS, scope()).await;
16627
16628 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16632 assert_eq!(
16633 rig.refused(opener, target, scope()).await,
16634 error_codes::SCOPE_NOT_CARRIER,
16635 "{opener:?} -> {target}"
16636 );
16637 }
16638 }
16639
16640 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16641 ScopeSelector::new(
16642 Principal::Reserved {
16643 module_id: OWNER.to_string(),
16644 },
16645 scope_ref.to_string(),
16646 scope_epoch,
16647 )
16648 }
16649
16650 #[tokio::test]
16651 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16652 let mut rig = rig().await;
16653 rig.sync(vec![session(1)]).await;
16654 for opener in [OWNER, AFT] {
16655 assert_eq!(
16656 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16657 .await,
16658 error_codes::SCOPE_EPOCH_REQUIRED,
16659 "{opener}"
16660 );
16661 }
16662 }
16663
16664 #[tokio::test]
16665 async fn admission_separates_not_synced_not_live_and_ended() {
16666 let mut rig = rig().await;
16667 let code = rig
16669 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16670 .await;
16671 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16672 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16673
16674 let ghost = ScopeSelector::new(
16676 Principal::Reserved {
16677 module_id: "ghost".to_string(),
16678 },
16679 "s".to_string(),
16680 Some(1),
16681 );
16682 assert_eq!(
16683 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16684 error_codes::SCOPE_NOT_LIVE
16685 );
16686
16687 rig.sync(vec![session(2)]).await;
16688 assert_eq!(
16689 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16690 .await,
16691 error_codes::SCOPE_NOT_LIVE
16692 );
16693 for epoch in [1, 3] {
16694 assert_eq!(
16695 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16696 .await,
16697 error_codes::SCOPE_ENDED,
16698 "epoch {epoch}"
16699 );
16700 }
16701 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16703 .await;
16704 }
16705
16706 #[tokio::test]
16707 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16708 let mut rig = rig().await;
16709 rig.sync(vec![session(1)]).await;
16710 let route = rig
16711 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16712 .await;
16713 let stamp = route.stamp().expect("a scoped bind carries the stamp");
16714 assert_eq!(stamp.scope_ref, "s");
16715 assert_eq!(stamp.scope_epoch, 1);
16716 assert_eq!(stamp.kind, ScopeKind::Head);
16717 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16718 assert!(stamp.attributes.delegates);
16719 assert!(stamp.owner_authorized);
16720 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16721 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16722
16723 let (broca, mut broca_rx) = wide_ctx(50);
16726 hello_via_sink(
16727 &rig.handler,
16728 &broca,
16729 &mut broca_rx,
16730 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16731 )
16732 .await;
16733 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16734 .await
16735 .unwrap();
16736 let own = ScopeSelector::new(
16737 Principal::Reserved {
16738 module_id: BROCA.to_string(),
16739 },
16740 "b".to_string(),
16741 Some(1),
16742 );
16743 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16744 assert!(!route.stamp().unwrap().owner_authorized);
16745 }
16746
16747 #[tokio::test]
16748 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16749 let mut rig = rig().await;
16750 let mut record = head("s", 1);
16751 record.carriers = vec![carrier(AFT, None)];
16752 let flow_id = "Flow:run-7/step_2!~";
16753 record.attributes.flow_id = Some(flow_id.to_string());
16754 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16755 .await
16756 .unwrap();
16757 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16758 panic!("not a sync reply");
16759 };
16760 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16761 let route = rig
16762 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16763 .await;
16764 assert!(rig.live(&route), "the stamped bind committed");
16765 let stamp = route.stamp().expect("a flow scope carries a stamp");
16766 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16767 assert_eq!(stamp.attributes.agent_id, None);
16768 assert!(!stamp.attributes.delegates);
16769 assert!(stamp.owner_authorized);
16770 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16771 assert_eq!(unscoped.stamp(), None);
16772 }
16773
16774 #[tokio::test]
16775 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16776 let mut rig = rig_with_flow_support(false).await;
16777 let mut record = session(1);
16778 record.attributes.flow_id = Some("flow:7".to_string());
16779 rig.sync(vec![record]).await;
16780 for opener in [OWNER, AFT] {
16781 let body = rig
16782 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16783 .await;
16784 assert_eq!(body["code"], "target_flow_unsupported");
16785 let message = body["message"].as_str().unwrap();
16786 for required in [PLEXUS, "flow-scopes/v1"] {
16787 assert!(message.contains(required), "{message}");
16788 }
16789 }
16790 }
16791
16792 #[tokio::test]
16793 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
16794 let mut rig = rig_with_flow_support(true).await;
16795 let mut record = session(1);
16796 record.attributes.flow_id = Some("flow:7".to_string());
16797 rig.sync(vec![record]).await;
16798 for opener in [OWNER, AFT] {
16799 let route = rig
16800 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16801 .await;
16802 assert!(rig.live(&route));
16803 assert_eq!(
16804 route.stamp().unwrap().attributes.flow_id.as_deref(),
16805 Some("flow:7")
16806 );
16807 }
16808 }
16809
16810 #[tokio::test]
16811 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
16812 use std::future::Future;
16813
16814 let mut rig = rig_with_flow_support(true).await;
16815 let mut record = session(1);
16816 record.attributes.flow_id = Some("flow:7".to_string());
16817 rig.sync(vec![record]).await;
16818 let (client, mut client_rx, frame) =
16819 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
16820 for _ in 0..64 {
16823 client.egress.try_send(route_bind_ack(1)).unwrap();
16824 }
16825 let handler = rig.handler.clone();
16826 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16827 std::future::poll_fn(|cx| {
16828 assert!(open.as_mut().poll(cx).is_pending());
16829 std::task::Poll::Ready(())
16830 })
16831 .await;
16832 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16833
16834 let old_connection = rig.modules[PLEXUS].0.connection_id;
16835 rig.handler.cleanup_connection(old_connection).unwrap();
16836 let (replacement, mut replacement_rx) = wide_ctx(200);
16837 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
16838 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16839 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16840 let hello = Frame::build(
16841 FrameType::Hello,
16842 control_flags(),
16843 0,
16844 0,
16845 200,
16846 serde_json::to_vec(&body).unwrap(),
16847 )
16848 .unwrap();
16849 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
16850
16851 client_rx.try_recv().unwrap();
16852 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16853 .await
16854 .expect("the replacement is refused without waiting for a bind ack")
16855 .unwrap();
16856 assert_eq!(replies.len(), 1);
16857 let body = parse_error(&replies[0]);
16858 assert_eq!(body["code"], "target_flow_unsupported");
16859 for required in [PLEXUS, "flow-scopes/v1"] {
16860 assert!(body["message"].as_str().unwrap().contains(required));
16861 }
16862 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
16863 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16864 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16865 assert!(rig
16866 .handler
16867 .registry
16868 .get_module_by_connection(replacement.connection_id)
16869 .unwrap()
16870 .is_some());
16871 }
16872
16873 #[tokio::test]
16874 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
16875 use std::future::Future;
16876 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16877 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16878 const NEXT_HASH: &str =
16879 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16880 let mut rig = rig().await;
16881 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
16882 record.carriers.push(ScopeCarrier::new(Principal::Direct));
16883 rig.sync(vec![record.clone()]).await;
16884 let (client, mut rx, frame) = rig.open_frame(
16885 None,
16886 PLEXUS,
16887 Some(
16888 rig.selector("s", Some(1))
16889 .with_bind_proof(Some(PROOF.into())),
16890 ),
16891 );
16892 for _ in 0..64 {
16895 client.egress.try_send(route_bind_ack(1)).unwrap();
16896 }
16897 let handler = rig.handler.clone();
16898 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16899 std::future::poll_fn(|cx| {
16900 assert!(open.as_mut().poll(cx).is_pending());
16901 std::task::Poll::Ready(())
16902 })
16903 .await;
16904 record.bind_proof_sha256 = Some(NEXT_HASH.into());
16905 rig.sync(vec![record]).await;
16906 rx.try_recv().unwrap();
16907 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16908 .await
16909 .unwrap()
16910 .unwrap();
16911 assert_eq!(
16912 parse_error(&replies[0])["code"],
16913 error_codes::SCOPE_BIND_PROOF_MISMATCH
16914 );
16915 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
16916 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16917 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16918 }
16919
16920 #[tokio::test]
16921 async fn scope_bind_proof_reserved_openers_ignore_proof() {
16922 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16923 let mut rig = rig().await;
16924 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16925 .await;
16926 for opener in [OWNER, AFT] {
16927 for proof in [None, Some("malformed+proof=")] {
16928 let route = rig
16929 .bound(
16930 Some(opener),
16931 PLEXUS,
16932 Some(
16933 rig.selector("s", Some(1))
16934 .with_bind_proof(proof.map(str::to_string)),
16935 ),
16936 )
16937 .await;
16938 assert!(rig.live(&route));
16939 }
16940 }
16941 }
16942
16943 #[tokio::test]
16944 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
16945 use std::future::Future;
16946 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16947 const NEXT_HASH: &str =
16948 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16949 let mut rig = rig().await;
16950 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16951 .await;
16952 let (client, mut rx, frame) =
16953 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
16954 for _ in 0..64 {
16955 client.egress.try_send(route_bind_ack(1)).unwrap();
16956 }
16957 let handler = rig.handler.clone();
16958 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16959 std::future::poll_fn(|cx| {
16960 assert!(open.as_mut().poll(cx).is_pending());
16961 std::task::Poll::Ready(())
16962 })
16963 .await;
16964 rig.sync(vec![
16965 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
16966 ])
16967 .await;
16968 rx.try_recv().unwrap();
16969 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16970 let bind = std::future::poll_fn(|cx| {
16971 assert!(open.as_mut().poll(cx).is_pending());
16972 module_rx.poll_recv(cx)
16973 })
16974 .await
16975 .unwrap()
16976 .frame;
16977 handler
16978 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
16979 .await
16980 .unwrap();
16981 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16982 .await
16983 .unwrap()
16984 .unwrap();
16985 assert_eq!(replies.len(), 1);
16986 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
16989 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16990 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
16991 let route = rig
16992 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
16993 .await;
16994 assert!(rig.live(&route));
16995 }
16996
16997 #[tokio::test]
16998 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
16999 {
17000 let mut rig = rig_with_flow_support(false).await;
17001 rig.sync(vec![session(1)]).await;
17002 let route = rig
17003 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17004 .await;
17005 assert!(rig.live(&route));
17006 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17007 let mut record = session(1);
17008 record.attributes.flow_id = Some("flow:7".to_string());
17009 rig.sync(vec![record]).await;
17010 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17011 assert!(rig.live(&unscoped));
17012 assert_eq!(unscoped.stamp(), None);
17013 }
17014
17015 #[tokio::test]
17016 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17017 let mut rig = rig().await;
17018 let mut record = session(1);
17019 record.attributes.flow_id = Some("flow:7".to_string());
17020 rig.sync(vec![record.clone()]).await;
17021 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17022 let mut owner_route = rig
17023 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17024 .await;
17025 let mut carrier_route = rig
17026 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17027 .await;
17028 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17029 rig.sync(vec![record.clone()]).await;
17030 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17031 assert!(rig.live(&owner_route) && owner_route.untouched());
17032 assert!(rig.live(&carrier_route) && carrier_route.untouched());
17033
17034 record.attributes.flow_id = Some("flow:8".to_string());
17035 rig.sync(vec![record]).await;
17036 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17037 let before = before.unwrap();
17038 assert_eq!(after.scope_epoch, before.scope_epoch);
17039 assert!(after.version > before.version);
17040 for route in [&mut owner_route, &mut carrier_route] {
17041 assert!(!rig.live(route));
17042 assert_eq!(
17043 route.closed_reason(),
17044 RouteCloseReason::ScopeDelegationChanged
17045 );
17046 }
17047 assert!(rig.live(&unscoped) && unscoped.untouched());
17048 for target in [PLEXUS, OTHER] {
17051 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17052 let goodbye = module_rx
17053 .try_recv()
17054 .expect("the provider sees the drain")
17055 .frame;
17056 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17057 assert!(module_rx.try_recv().is_err());
17058 }
17059 let rebound = rig
17060 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17061 .await;
17062 assert_eq!(
17063 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17064 Some("flow:8")
17065 );
17066 }
17067
17068 #[tokio::test]
17073 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17074 let mut rig = rig().await;
17075 rig.sync(vec![session(1)]).await;
17076 let mut cotenant = rig
17077 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17078 .await;
17079
17080 let mut changed = session(1);
17081 changed.child_owners.push(Principal::Reserved {
17082 module_id: MAGIC.to_string(),
17083 });
17084 let mut ended = None;
17085 for (code, next) in [
17086 (error_codes::SCOPE_CHANGED, vec![changed]),
17087 (error_codes::SCOPE_ENDED, Vec::new()),
17088 ] {
17089 let relayed = rig
17090 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17091 .await;
17092 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17093 ended = Some(next.is_empty());
17094 rig.sync(next).await;
17095 rig.ack(&relayed).await;
17096 let replies = relayed.task.await.unwrap();
17097 assert_eq!(replies.len(), 1, "{replies:?}");
17098 assert_eq!(parse_error(&replies[0])["code"], code);
17099 assert_eq!(
17100 subc_protocol::error_codes::is_retryable_route_open(code),
17101 code == error_codes::SCOPE_CHANGED
17102 );
17103 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17104 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17106 let mut goodbyes = Vec::new();
17109 while let Ok(outbound) = plexus_rx.try_recv() {
17110 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17111 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17112 }
17113 assert!(
17114 goodbyes.contains(&(bind_channel, bind_epoch)),
17115 "{goodbyes:?}"
17116 );
17117 assert!(rig
17118 .handler
17119 .registry
17120 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17121 .unwrap()
17122 .is_some());
17123 }
17124 assert_eq!(ended, Some(true));
17125 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17128 }
17129
17130 #[tokio::test]
17133 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17134 struct Case {
17135 name: &'static str,
17136 change: fn(&mut ScopeRecord),
17137 closed: [Option<RouteCloseReason>; 4],
17140 }
17141 use RouteCloseReason::*;
17142 let cases = [
17143 Case {
17144 name: "a carrier entry removed",
17145 change: |r| {
17146 r.carriers.retain(|c| {
17147 c.principal
17148 != Principal::Reserved {
17149 module_id: AFT.to_string(),
17150 }
17151 })
17152 },
17153 closed: [None, Some(ScopeCarrierRemoved), None, None],
17154 },
17155 Case {
17156 name: "a target removed from a carrier",
17157 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17158 closed: [None, None, None, Some(ScopeCarrierRemoved)],
17159 },
17160 Case {
17161 name: "a bare carrier narrowed to targets",
17162 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17163 closed: [None, Some(ScopeCarrierRemoved), None, None],
17164 },
17165 Case {
17166 name: "delegates turned off",
17167 change: |r| r.attributes.delegates = false,
17168 closed: [Some(ScopeDelegationChanged); 4],
17169 },
17170 Case {
17171 name: "agent_id changed",
17172 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17173 closed: [Some(ScopeDelegationChanged); 4],
17174 },
17175 Case {
17176 name: "a carrier added, child owners changed, the record re-sent",
17177 change: |r| {
17178 r.carriers.push(carrier(MAGIC, None));
17179 r.child_owners.push(Principal::Reserved {
17180 module_id: MAGIC.to_string(),
17181 });
17182 },
17183 closed: [None; 4],
17184 },
17185 Case {
17186 name: "a target added",
17187 change: |r| {
17188 r.carriers[1]
17189 .targets
17190 .as_mut()
17191 .unwrap()
17192 .push("third".to_string())
17193 },
17194 closed: [None; 4],
17195 },
17196 Case {
17197 name: "delegates turned on",
17198 change: |r| r.attributes.delegates = true,
17199 closed: [None; 4],
17200 },
17201 ];
17202 for case in cases {
17203 let mut rig = rig().await;
17204 rig.sync(vec![session(1)]).await;
17205 let scope = || Some(rig_selector("s", Some(1)));
17206 let mut routes = [
17207 rig.bound(Some(OWNER), PLEXUS, scope()).await,
17208 rig.bound(Some(AFT), PLEXUS, scope()).await,
17209 rig.bound(Some(BROCA), PLEXUS, scope()).await,
17210 rig.bound(Some(BROCA), OTHER, scope()).await,
17211 ];
17212 let mut record = session(1);
17213 (case.change)(&mut record);
17214 rig.sync(vec![record]).await;
17215 for (index, expected) in case.closed.iter().enumerate() {
17216 let route = &mut routes[index];
17217 match expected {
17218 Some(reason) => {
17219 assert!(!rig.live(route), "{}: route {index} still live", case.name);
17220 assert_eq!(
17221 route.closed_reason(),
17222 *reason,
17223 "{}: route {index}",
17224 case.name
17225 );
17226 }
17227 None => {
17228 assert!(rig.live(route), "{}: route {index} closed", case.name);
17229 assert!(
17230 route.untouched(),
17231 "{}: route {index} was told something",
17232 case.name
17233 );
17234 }
17235 }
17236 }
17237 }
17238 }
17239
17240 #[tokio::test]
17241 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17242 for next in [Vec::new(), vec![session(2)]] {
17244 let mut rig = rig().await;
17245 rig.sync(vec![session(1)]).await;
17246 let mut route = rig
17247 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17248 .await;
17249 rig.sync(next).await;
17250 assert!(!rig.live(&route));
17251 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17252 }
17253
17254 let mut rig = rig().await;
17257 let mut child = session(1);
17258 child.scope_ref = "child".to_string();
17259 child.kind = ScopeKind::Worker;
17260 child.parent = Some(ScopeParent::new(
17261 Principal::Reserved {
17262 module_id: OWNER.to_string(),
17263 },
17264 "s".to_string(),
17265 1,
17266 ));
17267 rig.sync(vec![session(1), child.clone()]).await;
17268 let mut child_route = rig
17269 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17270 .await;
17271 assert_eq!(
17272 child_route.stamp().unwrap().parent_state,
17273 Some(ParentState::Linked)
17274 );
17275 rig.sync(vec![child]).await;
17276 assert!(!rig.live(&child_route));
17277 assert_eq!(
17278 child_route.closed_reason(),
17279 RouteCloseReason::ScopeParentEnded
17280 );
17281 }
17282
17283 #[tokio::test]
17284 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17285 ) {
17286 let mut rig = rig().await;
17287 rig.sync(vec![session(1)]).await;
17288 let mut route = rig
17289 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17290 .await;
17291 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17292 rig.sync(vec![session(1)]).await;
17293 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17294 assert!(rig.live(&route) && route.untouched());
17295
17296 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17300 .forwarding
17301 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17302 .unwrap()
17303 else {
17304 panic!("the route is bound");
17305 };
17306 binding.flow.acquire_tagged(9, false).await.unwrap();
17307 let mut widened = session(1);
17308 widened.carriers.push(carrier(MAGIC, None));
17309 rig.sync(vec![widened]).await;
17310 assert!(rig.live(&route) && route.untouched());
17311 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17312 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17313 assert_eq!(binding.flow.in_flight(), 1);
17316 binding
17317 .flow
17318 .acquire_tagged(10, false)
17319 .await
17320 .expect("the flow is still open");
17321 }
17322
17323 #[tokio::test]
17326 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17327 let mut rig = rig().await;
17328 rig.sync(vec![session(1)]).await;
17329 let mut on_incumbent = rig
17330 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17331 .await;
17332
17333 let (candidate, _candidate_rx) = wide_ctx(9);
17336 let registration = rig
17337 .handler
17338 .registry
17339 .register_candidate_with_control_ops(
17340 manifest(PLEXUS, PROTOCOL_VERSION),
17341 PROTOCOL_VERSION,
17342 candidate.connection_id,
17343 module_baseline_control_ops(),
17344 )
17345 .unwrap();
17346 rig.forwarding
17347 .register_candidate_module_connection(
17348 candidate.connection_id,
17349 PLEXUS.to_string(),
17350 PROTOCOL_VERSION,
17351 manifest_concurrency(®istration.manifest),
17352 candidate.egress.clone(),
17353 )
17354 .unwrap();
17355 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17356 rig.handler
17357 .registry
17358 .promote_candidate(PLEXUS)
17359 .unwrap()
17360 .unwrap();
17361 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17362
17363 rig.sync(Vec::new()).await;
17364 assert!(!rig.live(&on_incumbent));
17365 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17366 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17367 let goodbye = incumbent_rx
17368 .try_recv()
17369 .expect("the superseded endpoint is told")
17370 .frame;
17371 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17372 }
17373 }
17374}
17375
17376#[cfg(test)]
17377mod concurrency_default_exposure_tests {
17378 use super::*;
17379
17380 fn hello_body(role_json: &str) -> Vec<u8> {
17381 format!(
17382 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17383 )
17384 .into_bytes()
17385 }
17386
17387 fn manifest_from(body: &[u8]) -> ModuleManifest {
17388 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17389 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17390 .expect("manifest parses")
17391 }
17392
17393 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17394
17395 #[test]
17396 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17397 let body = hello_body(&format!(
17398 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17399 ));
17400 let manifest = manifest_from(&body);
17401 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17404 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17405 }
17406
17407 #[test]
17408 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17409 let body = hello_body(&format!(
17410 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17411 ));
17412 let manifest = manifest_from(&body);
17413 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17414 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17415 }
17416
17417 #[test]
17418 fn non_management_roles_are_never_reported() {
17419 let body = hello_body(
17420 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17421 );
17422 let manifest = manifest_from(&body);
17423 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17424 }
17425}