1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_rich_ui_descriptor, validate_hello_self_signal_declarations,
26 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
27 ProviderRole,
28 },
29 scope::{
30 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32 SCOPE_SYNC_OP,
33 },
34 session::{
35 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38 ROLE_VERSIONS_FIELD,
39 },
40 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41 Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47 caller_identity::CallerIdentity,
48 capability_requirements::{
49 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51 RuntimeModule,
52 },
53 child_roster::DaemonShutdownFlag,
54 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55 forwarding::{
56 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59 },
60 observability::{
61 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62 },
63 privileged_prompt::{prompt_text, PrivilegedOperation},
64 provenance::{
65 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66 },
67 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68 router::{RouteCtx, RouterError},
69 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71 stderr_tail::{CaptureState, TailEntry},
72 supervise::{
73 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74 SupervisorHandle, SwapHelloAdmission,
75 },
76 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93 ops::ACTIVITY_RECORD,
94 ops::ACTIVITY_QUERY,
95 ops::ACTIVITY_SUBSCRIBE,
96 ops::SERVER_DESCRIBE,
97 ops::CATALOG_LIST,
98 ops::ROUTE_OPEN,
99 ops::ROUTE_POLL,
100 ops::ROUTE_CLOSING,
101 ops::ROUTE_CLOSED,
102 ops::SUPERVISOR_LIST,
103 ops::SUPERVISOR_RESTART,
104 ops::SUPERVISOR_SWAP,
105 ops::SUPERVISOR_RELOAD,
106 ops::SUPERVISOR_RESCAN,
107 ops::SUPERVISOR_RELEASE_RESERVED,
108 ops::SUPERVISOR_SET_ENABLED,
109 ops::SUPERVISOR_HEALTH_PROBE,
110 ops::SUPERVISOR_HEALTH,
111 ops::SUPERVISOR_STDERR_TAIL,
112 ops::SUPERVISOR_TERMINALS,
113 ops::SUPERVISOR_ROUTES,
114 ops::SUPERVISOR_PROVENANCE,
115 ops::SUPERVISOR_SPAWN_SNAPSHOT,
116 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132 "supervisor.live_roots",
133 SCOPE_SYNC_OP,
134 SCOPE_APPLY_OP,
135 SCOPE_DESCRIBE_OP,
136 "operator.confirm",
137];
138
139const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194 configured: &Path,
195 spawned_from: Option<&Path>,
196 image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198 let path = match spawned_from {
199 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200 ReloadPathAgreement::Match
201 }
202 Some(spawned_from) => ReloadPathAgreement::Mismatch {
203 configured: configured.to_path_buf(),
204 spawned_from: spawned_from.to_path_buf(),
205 },
206 None => ReloadPathAgreement::Unavailable {
207 reason: if matches!(
208 image,
209 subc_control::RunningImageAgreement::Unavailable {
210 reason: subc_control::RunningImageUnavailableReason::NotRunning
211 }
212 ) {
213 ReloadPathUnavailableReason::NotRunning
214 } else {
215 ReloadPathUnavailableReason::SpawnedPathUnavailable
216 },
217 },
218 };
219 PendingReloadVerdict {
220 path,
221 image,
222 spec: None,
223 }
224}
225
226fn launch_spec_agreement(
227 configured: &crate::supervise::ModuleSpec,
228 running: bool,
229 spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232 if !running {
233 return Agreement::Unavailable {
234 reason: Reason::NotRunning,
235 };
236 }
237 match spawned {
238 None => Agreement::Unavailable {
239 reason: Reason::SpawnedSpecUnavailable,
240 },
241 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242 Agreement::Match
243 }
244 Some(_) => Agreement::Mismatch,
245 }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249 if configured == spawned_from {
250 return true;
251 }
252 #[cfg(windows)]
253 {
254 match (
260 std::fs::canonicalize(configured),
261 std::fs::canonicalize(spawned_from),
262 ) {
263 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264 _ => false,
265 }
266 }
267 #[cfg(not(windows))]
268 {
269 false
270 }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275 build: DaemonBuildProvenance,
276 pid: Option<u32>,
277 started_at_ms: Option<u64>,
278 start_clock: Option<crate::clock::StartClock>,
279 executable_path: Option<PathBuf>,
280 executable_identity: Option<SpawnedFileIdentity>,
281 process_start_time: Option<u64>,
282 probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286 fn default() -> Self {
287 Self {
288 build: DaemonBuildProvenance {
289 build_git_sha: None,
290 build_lock_digest: None,
291 },
292 pid: None,
293 started_at_ms: None,
294 start_clock: None,
295 executable_path: None,
296 executable_identity: None,
297 process_start_time: None,
298 probe: ExecutableIdentityProbe::default(),
299 }
300 }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305 supervisor: Supervisor,
306 config_path: PathBuf,
307 configured_port: Option<u16>,
308 storage_config: Option<crate::daemon_config::StorageConfig>,
309 admission_facts_carrier_module_id: Option<String>,
310 admission_facts_targets: Option<Vec<String>>,
311 scope_authority_owners: Vec<String>,
312 privileged_operator_prompt: PrivilegedOperatorPrompt,
313 generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342 "reloading",
343 "supervisor_not_live",
344 "registration_not_active",
345 "no_forwarding_connection",
346 "relay_send_failed",
347];
348
349#[derive(Clone)]
351pub struct ControlHandler {
352 registry: Arc<Registry>,
353 activity: Option<crate::activity::ActivityLedger>,
354 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355 forwarding: Arc<ForwardingTable>,
356 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357 supervisor: SupervisorHandle,
358 subc_capabilities: Arc<[String]>,
359 route_bind_relay_timeout: Duration,
363 route_bind_relay_timeouts: BTreeMap<String, Duration>,
367 route_bind_breakers: RouteBindBreakers,
370 route_bind_concurrency: RouteBindConcurrency,
373 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378 route_bind_breaker_threshold: u32,
380 route_bind_breaker_cooldown: Duration,
382 health_probe_timeout: Duration,
383 storage_config: Option<crate::daemon_config::StorageConfig>,
386 machine_id: Option<crate::machine_id::MachineId>,
390 admission_facts_carrier_module_id: Option<String>,
391 admission_facts_targets: Option<Vec<String>>,
392 scopes: Arc<RwLock<ScopeTable>>,
396 scope_deadline_added: Arc<tokio::sync::Notify>,
400 scope_authority_owners: Vec<String>,
403 privileged_operator_prompt: PrivilegedOperatorPrompt,
405 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408 #[cfg(test)]
409 caller_lookup: Option<TestCallerLookup>,
410 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413 rescan: Option<SupervisorRescanContext>,
414 connected_clients: ConnectedClients,
415 counters: DaemonCounters,
416 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417 daemon_shutdown: DaemonShutdownFlag,
422 daemon_provenance: DaemonProvenanceFacts,
423 #[cfg(test)]
424 control_dispatch_delay: Option<Duration>,
425 #[cfg(test)]
426 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427 #[cfg(test)]
431 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432 #[cfg(test)]
438 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443 f.debug_struct("ControlHandler")
444 .field("registry", &self.registry)
445 .field("forwarding", &self.forwarding)
446 .field("process_liveness", &self.process_liveness.is_some())
447 .field("supervisor", &self.supervisor)
448 .field("subc_capabilities", &self.subc_capabilities)
449 .finish()
450 }
451}
452
453struct RouteOpenRequest {
454 target: RouteTarget,
455 identity: BindIdentity,
456 consumer_identity: Option<ConsumerIdentity>,
457 consumer_capabilities: Option<Vec<String>>,
458 role_versions: Option<BTreeMap<String, String>>,
459 admission_facts: Option<serde_json::Value>,
460 scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465 Admission,
466 ProjectRoot,
467 EgressReservation,
468 BindSend,
469 BindAckWait,
470}
471
472struct RouteOpenTimings {
476 module_id: String,
477 principal: String,
478 admission: Duration,
479 project_root: Duration,
480 egress_reservation: Duration,
481 bind_send: Duration,
482 bind_ack_wait: Duration,
483 phase: Option<RouteOpenPhase>,
484 phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488 fn new() -> Self {
489 Self {
490 module_id: String::new(),
491 principal: "unresolved".to_string(),
493 admission: Duration::ZERO,
494 project_root: Duration::ZERO,
495 egress_reservation: Duration::ZERO,
496 bind_send: Duration::ZERO,
497 bind_ack_wait: Duration::ZERO,
498 phase: Some(RouteOpenPhase::Admission),
499 phase_started_at: Instant::now(),
500 }
501 }
502
503 fn finish(&mut self) {
504 let elapsed = self.phase_started_at.elapsed();
505 match self.phase.take() {
506 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511 None => {}
512 }
513 }
514
515 fn enter(&mut self, phase: RouteOpenPhase) {
516 self.finish();
517 self.phase = Some(phase);
518 self.phase_started_at = Instant::now();
519 }
520}
521
522struct RouteBindReservationGuard {
523 forwarding: Arc<ForwardingTable>,
524 endpoint: ModuleEndpointId,
525 relay_corr: u64,
526 armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530 forwarding: Arc<ForwardingTable>,
531 endpoint: ModuleEndpointId,
532 corr: u64,
533 armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538 Self {
539 forwarding,
540 endpoint,
541 corr,
542 armed: true,
543 }
544 }
545
546 fn disarm(&mut self) {
547 self.armed = false;
548 }
549}
550
551impl Drop for ModuleControlRpcGuard {
552 fn drop(&mut self) {
553 if self.armed {
554 let _ = self
555 .forwarding
556 .cancel_module_control_rpc(self.endpoint, self.corr);
557 }
558 }
559}
560
561impl RouteBindReservationGuard {
562 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563 Self {
564 forwarding,
565 endpoint,
566 relay_corr,
567 armed: true,
568 }
569 }
570
571 fn release_and_disarm(&mut self) {
572 if !self.armed {
573 return;
574 }
575 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576 self.endpoint,
577 self.relay_corr,
578 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579 ) {
580 send_goodbye_target_best_effort(
581 &self.forwarding.counters(),
582 &target,
583 "canceled route.bind",
584 );
585 }
586 self.armed = false;
587 }
588
589 fn disarm(&mut self) {
590 self.armed = false;
591 }
592}
593
594impl Drop for RouteBindReservationGuard {
595 fn drop(&mut self) {
596 self.release_and_disarm();
597 }
598}
599
600#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636 modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640 concurrency: RouteBindConcurrency,
641 module_id: String,
642}
643
644impl RouteBindConcurrency {
645 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648 let mut modules = self
649 .modules
650 .lock()
651 .expect("route.bind concurrency mutex poisoned");
652 let in_flight = modules.entry(module_id.to_string()).or_default();
653 if *in_flight >= limit {
654 return Err(*in_flight);
655 }
656 *in_flight += 1;
657 Ok(RouteBindConcurrencyGuard {
658 concurrency: self.clone(),
659 module_id: module_id.to_string(),
660 })
661 }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665 fn drop(&mut self) {
666 let mut modules = self
667 .concurrency
668 .modules
669 .lock()
670 .expect("route.bind concurrency mutex poisoned");
671 let remove = {
672 let in_flight = modules
673 .get_mut(&self.module_id)
674 .expect("admitted route.bind has a concurrency entry");
675 *in_flight -= 1;
676 *in_flight == 0
677 };
678 if remove {
679 modules.remove(&self.module_id);
680 }
681 }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686 consecutive_timeouts: u32,
688 cooldown_until: Option<Instant>,
691 probe_in_flight: Option<Arc<()>>,
696}
697
698enum RouteBindAdmission<'a> {
700 Admitted {
701 guard: RouteBindBreakerGuard<'a>,
702 probe: bool,
705 },
706 Refused {
707 consecutive_timeouts: u32,
708 retry_in: Duration,
712 probe_in_flight: bool,
713 },
714}
715
716struct RouteBindBreakerGuard<'a> {
723 breakers: RouteBindBreakers,
724 module_id: &'a str,
725 probe_token: Option<Arc<()>>,
726 settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730 fn record_accepted(&mut self) -> bool {
734 self.settled = true;
735 self.breakers.record_accepted(self.module_id)
736 }
737
738 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741 self.settled = true;
742 self.breakers.record_timeout(
743 self.module_id,
744 self.probe_token.as_ref(),
745 threshold,
746 cooldown,
747 )
748 }
749
750 fn record_inconclusive(&mut self) {
759 self.settled = true;
760 self.breakers
761 .record_inconclusive(self.module_id, self.probe_token.as_ref());
762 }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766 fn drop(&mut self) {
767 if !self.settled {
768 self.breakers
769 .record_inconclusive(self.module_id, self.probe_token.as_ref());
770 }
771 }
772}
773
774struct BreakerOpened {
778 consecutive_timeouts: u32,
779 reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786 self.modules
787 .lock()
788 .expect("route.bind breaker mutex poisoned")
789 }
790
791 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795 probe: probe_token.is_some(),
796 guard: RouteBindBreakerGuard {
797 breakers: self.clone(),
798 module_id,
799 probe_token,
800 settled: false,
801 },
802 };
803
804 let mut modules = self.lock();
805 let Some(state) = modules.get_mut(module_id) else {
806 return admitted(None);
807 };
808 let Some(cooldown_until) = state.cooldown_until else {
809 return admitted(None);
810 };
811 if state.probe_in_flight.is_some() {
812 return RouteBindAdmission::Refused {
813 consecutive_timeouts: state.consecutive_timeouts,
814 retry_in: Duration::ZERO,
815 probe_in_flight: true,
816 };
817 }
818 let now = Instant::now();
819 if now < cooldown_until {
820 return RouteBindAdmission::Refused {
821 consecutive_timeouts: state.consecutive_timeouts,
822 retry_in: cooldown_until - now,
823 probe_in_flight: false,
824 };
825 }
826 let token = Arc::new(());
827 state.probe_in_flight = Some(Arc::clone(&token));
828 admitted(Some(token))
829 }
830
831 fn record_accepted(&self, module_id: &str) -> bool {
832 self.lock()
833 .remove(module_id)
834 .is_some_and(|state| state.cooldown_until.is_some())
835 }
836
837 fn record_timeout(
838 &self,
839 module_id: &str,
840 probe_token: Option<&Arc<()>>,
841 threshold: u32,
842 cooldown: Duration,
843 ) -> Option<BreakerOpened> {
844 let mut modules = self.lock();
845 let state = modules.entry(module_id.to_string()).or_default();
846 let was_open = state.cooldown_until.is_some();
847 let was_probe = Self::owns_probe(state, probe_token);
848 if was_probe {
849 state.probe_in_flight = None;
850 }
851 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852 if state.consecutive_timeouts < threshold {
853 return None;
854 }
855 state.cooldown_until = Some(Instant::now() + cooldown);
856 Some(BreakerOpened {
857 consecutive_timeouts: state.consecutive_timeouts,
858 reopened_after_probe: was_open && was_probe,
859 })
860 }
861
862 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863 state
867 .probe_in_flight
868 .as_ref()
869 .zip(token)
870 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871 }
872
873 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874 if let Some(state) = self.lock().get_mut(module_id) {
875 if Self::owns_probe(state, probe_token) {
876 state.probe_in_flight = None;
877 }
878 }
879 }
880
881 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896 self.lock()
897 .remove(module_id)
898 .map(|state| state.consecutive_timeouts)
899 .filter(|discarded| *discarded > 0)
900 }
901
902 fn open_snapshot(&self) -> Option<serde_json::Value> {
909 let now = Instant::now();
910 let modules = self.lock();
911 let open = modules
912 .iter()
913 .filter_map(|(module_id, state)| {
914 let cooldown_until = state.cooldown_until?;
915 Some((
916 module_id.clone(),
917 serde_json::json!({
918 "consecutive_timeouts": state.consecutive_timeouts,
919 "cooldown_remaining_ms":
920 cooldown_until.saturating_duration_since(now).as_millis() as u64,
921 "probe_in_flight": state.probe_in_flight.is_some(),
922 }),
923 ))
924 })
925 .collect::<serde_json::Map<String, serde_json::Value>>();
926 (!open.is_empty()).then_some(serde_json::Value::Object(open))
927 }
928}
929
930impl ControlHandler {
931 pub fn new(registry: Arc<Registry>) -> Self {
932 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933 }
934
935 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936 let counters = forwarding.counters();
937 let route_bind_breakers = forwarding.route_bind_breakers();
941 let route_bind_concurrency = forwarding.route_bind_concurrency();
942 let route_outages = forwarding.route_outages();
943 Self {
944 registry,
945 activity: None,
946 activity_streams: Arc::new(Mutex::new(HashMap::new())),
947 forwarding,
948 process_liveness: None,
949 supervisor: SupervisorHandle::new(),
950 subc_capabilities: Arc::from([
951 CAP_MANIFEST_REGISTRATION.to_string(),
952 CAP_CHANNEL_LIFECYCLE.to_string(),
953 CAP_PING_PONG.to_string(),
954 CAP_SESSION_ATTACH.to_string(),
955 CAP_ADMISSION_FACTS_RELAY.to_string(),
956 CAP_SCOPES_V1.to_string(),
957 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
964 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
965 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
966 ]),
967 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
968 route_bind_relay_timeouts: BTreeMap::new(),
969 route_bind_breakers,
970 route_bind_concurrency,
971 route_outages,
972 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
973 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
974 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
975 storage_config: None,
976 machine_id: None,
977 admission_facts_carrier_module_id: None,
978 admission_facts_targets: None,
979 scopes: Arc::new(RwLock::new(ScopeTable::new(
980 crate::daemon_config::default_scope_authority_owners(),
981 ))),
982 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
983 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
984 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
985 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
986 #[cfg(test)]
987 caller_lookup: None,
988 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
989 rescan: None,
990 connected_clients: ConnectedClients::new(),
991 counters,
992 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
993 daemon_shutdown: DaemonShutdownFlag::default(),
994 daemon_provenance: DaemonProvenanceFacts::default(),
995 #[cfg(test)]
996 control_dispatch_delay: None,
997 #[cfg(test)]
998 provenance_probe_override: None,
999 #[cfg(test)]
1000 route_bind_relay_sent_hold: None,
1001 #[cfg(test)]
1002 hello_registry_to_forwarding_hold: None,
1003 }
1004 }
1005
1006 pub fn with_storage_config(
1009 mut self,
1010 storage_config: Option<crate::daemon_config::StorageConfig>,
1011 ) -> Self {
1012 self.storage_config = storage_config;
1013 self
1014 }
1015
1016 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1019 self.activity = Some(crate::activity::ActivityLedger::open(
1020 run_dir.join("activity.jsonl"),
1021 incarnation,
1022 ));
1023 self.supervisor.set_activity(self.activity.clone());
1024 self
1025 }
1026
1027 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1030 self.machine_id = machine_id;
1031 self
1032 }
1033
1034 pub fn with_admission_facts_config(
1038 mut self,
1039 carrier_module_id: Option<String>,
1040 targets: Option<Vec<String>>,
1041 ) -> Self {
1042 self.admission_facts_carrier_module_id = carrier_module_id;
1043 self.admission_facts_targets = targets;
1044 self
1045 }
1046
1047 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1051 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1052 self.scope_authority_owners = owners;
1053 self
1054 }
1055
1056 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1058 self.privileged_operator_prompt = mode;
1059 self
1060 }
1061
1062 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1065 self.route_bind_relay_timeout = timeout;
1066 self
1067 }
1068
1069 pub fn with_route_bind_relay_timeouts(
1075 mut self,
1076 timeouts: impl IntoIterator<Item = (String, Duration)>,
1077 ) -> Self {
1078 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1079 self
1080 }
1081
1082 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1088 self.route_bind_relay_timeouts
1089 .get(module_id)
1090 .copied()
1091 .unwrap_or(self.route_bind_relay_timeout)
1092 }
1093
1094 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1102 self.route_bind_breaker_threshold = threshold.max(1);
1103 self.route_bind_breaker_cooldown = cooldown;
1104 self
1105 }
1106
1107 #[cfg(test)]
1108 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1109 self.health_probe_timeout = timeout;
1110 self
1111 }
1112
1113 #[cfg(test)]
1114 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1115 self.control_dispatch_delay = Some(delay);
1116 self
1117 }
1118
1119 #[cfg(test)]
1123 pub(crate) fn with_route_bind_relay_sent_hold(
1124 mut self,
1125 hold: Arc<tokio::sync::Barrier>,
1126 ) -> Self {
1127 self.route_bind_relay_sent_hold = Some(hold);
1128 self
1129 }
1130
1131 #[cfg(test)]
1136 pub(crate) fn with_hello_registry_to_forwarding_hold(
1137 mut self,
1138 hold: Arc<std::sync::Barrier>,
1139 ) -> Self {
1140 self.hello_registry_to_forwarding_hold = Some(hold);
1141 self
1142 }
1143
1144 #[cfg(test)]
1145 fn hold_between_registry_and_forwarding(&self) {
1146 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1147 hold.wait();
1148 hold.wait();
1149 }
1150 }
1151
1152 pub fn with_process_liveness(
1153 mut self,
1154 process_liveness: Arc<dyn ModuleProcessLiveness>,
1155 ) -> Self {
1156 self.process_liveness = Some(process_liveness);
1157 self
1158 }
1159
1160 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1161 self.supervisor = supervisor;
1162 self.supervisor.set_activity(self.activity.clone());
1163 self
1164 }
1165
1166 pub fn with_daemon_provenance(
1167 mut self,
1168 pid: u32,
1169 started_at_ms: u64,
1170 executable_path: Option<PathBuf>,
1171 build_git_sha: Option<String>,
1172 build_lock_digest: Option<String>,
1173 ) -> Self {
1174 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1175 let process_start_time = process_start_time(pid);
1176 self.daemon_provenance = DaemonProvenanceFacts {
1177 build: DaemonBuildProvenance {
1178 build_git_sha,
1179 build_lock_digest,
1180 },
1181 pid: Some(pid),
1182 started_at_ms: Some(started_at_ms),
1183 start_clock: None,
1184 executable_path,
1185 executable_identity,
1186 process_start_time,
1187 probe: ExecutableIdentityProbe::default(),
1188 };
1189 self
1190 }
1191
1192 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1193 self.daemon_provenance.start_clock = Some(clock);
1194 self
1195 }
1196
1197 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1199 self.scopes
1200 .write()
1201 .unwrap_or_else(|poisoned| poisoned.into_inner())
1202 .set_wall_clock(Arc::new(clock));
1203 self
1204 }
1205
1206 #[cfg(test)]
1207 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1208 self.provenance_probe_override = Some(result);
1209 self
1210 }
1211
1212 pub fn with_capability_config(
1216 self,
1217 modules: impl IntoIterator<Item = (String, bool)>,
1218 reserved_capabilities: BTreeMap<String, String>,
1219 ) -> Self {
1220 self.capability_evaluator
1221 .configure(modules, reserved_capabilities);
1222 self
1223 }
1224
1225 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1229 self.daemon_shutdown = flag;
1230 self
1231 }
1232
1233 pub fn with_supervisor_rescan(
1234 mut self,
1235 supervisor: Supervisor,
1236 config_path: impl Into<PathBuf>,
1237 configured_port: Option<u16>,
1238 ) -> Self {
1239 self.rescan = Some(SupervisorRescanContext {
1240 supervisor,
1241 config_path: config_path.into(),
1242 configured_port,
1243 storage_config: self.storage_config.clone(),
1244 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1245 admission_facts_targets: self.admission_facts_targets.clone(),
1246 scope_authority_owners: self.scope_authority_owners.clone(),
1247 privileged_operator_prompt: self.privileged_operator_prompt,
1248 generation: Arc::default(),
1249 });
1250 self
1251 }
1252
1253 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1254 self.connected_clients = connected_clients;
1255 self
1256 }
1257
1258 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1259 Arc::clone(&self.forwarding)
1260 }
1261
1262 pub(crate) fn counters(&self) -> DaemonCounters {
1263 self.counters.clone()
1264 }
1265
1266 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1269 tokio::spawn(async move {
1270 loop {
1271 self.capability_evaluator
1272 .wait_for_change_or_deadline()
1273 .await;
1274 self.refresh_capability_requirements();
1275 }
1276 });
1277 }
1278
1279 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1285 tokio::spawn(async move {
1286 loop {
1287 let has_deadlines = self
1288 .scopes
1289 .read()
1290 .unwrap_or_else(|poisoned| poisoned.into_inner())
1291 .has_deadlines();
1292 if !has_deadlines {
1293 self.scope_deadline_added.notified().await;
1294 continue;
1295 }
1296 tokio::time::sleep(Duration::from_secs(1)).await;
1297 if let Err(error) = self.sweep_expired_scopes() {
1298 warn!(%error, "scope expiry sweep failed");
1299 }
1300 }
1301 });
1302 }
1303
1304 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1305 let mut table = self
1306 .scopes
1307 .write()
1308 .unwrap_or_else(|poisoned| poisoned.into_inner());
1309 let swept = table.sweep_expired();
1310 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1311 drop(table);
1312 self.close_scope_drained_routes(drained);
1313 Ok(())
1314 }
1315
1316 fn publish_scope_changes(
1317 &self,
1318 changes: &[crate::scopes::ScopeTagChange],
1319 expired: &[crate::scopes::ScopeExpired],
1320 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1321 if changes.is_empty() {
1322 return Ok(Vec::new());
1323 }
1324 let drained = self
1325 .forwarding
1326 .publish_scope_changes(changes)
1327 .map_err(RouterError::Forwarding)?;
1328 let mut counts = HashMap::new();
1329 for route in &drained {
1330 *counts
1331 .entry((
1332 route.scope.owner.as_str(),
1333 route.scope.scope_ref.as_str(),
1334 route.scope.tag.scope_epoch,
1335 ))
1336 .or_insert(0usize) += 1;
1337 }
1338 for scope in expired {
1339 let routes_closed = counts
1340 .get(&(
1341 scope.owner.as_str(),
1342 scope.scope_ref.as_str(),
1343 scope.scope_epoch,
1344 ))
1345 .copied()
1346 .unwrap_or(0);
1347 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1348 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1349 routes_closed, "scope expired");
1350 }
1351 Ok(drained)
1352 }
1353
1354 fn runtime_capability_snapshot(
1355 &self,
1356 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1357 let runtime = self
1358 .supervisor
1359 .list()
1360 .into_iter()
1361 .map(|module| {
1362 let status = module.status().map_err(|err| {
1363 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1364 })?;
1365 Ok(RuntimeModule {
1366 module_id: status.module_id,
1367 state: status.state,
1368 enabled: status.enabled,
1369 })
1370 })
1371 .collect::<Result<Vec<_>, RouterError>>()?;
1372 let registrations = self.registry.list_registered_modules().map_err(|err| {
1378 RouterError::backend(
1379 0,
1380 0,
1381 format!("failed to list capability registrations: {err}"),
1382 )
1383 })?;
1384 let registrations = registrations
1385 .into_iter()
1386 .map(|registration| RegisteredModule {
1387 module_id: registration.manifest.module_id,
1388 module_version: registration.manifest.module_version,
1389 capabilities: registration.manifest.capabilities,
1390 })
1391 .collect();
1392 Ok((runtime, registrations))
1393 }
1394
1395 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1402 let cached_registration = RegisteredModule {
1403 module_id: registration.manifest.module_id.clone(),
1404 module_version: registration.manifest.module_version.clone(),
1405 capabilities: registration.manifest.capabilities.clone(),
1406 };
1407 if self.capability_evaluator.record_hello(&cached_registration) {
1408 warn!(
1409 module_id = %cached_registration.module_id,
1410 "capability claims drifted from the cached manifest"
1411 );
1412 }
1413 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1414 self.enforce_capability_denies();
1415 }
1416 self.refresh_capability_requirements();
1417 }
1418
1419 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1423 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1424 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1425 self.supervisor.set_swap_promotion_observer(observer);
1426 }
1427
1428 pub fn refresh_capability_requirements(&self) {
1429 match self.runtime_capability_snapshot() {
1430 Ok((runtime, registrations)) => {
1431 log_requirement_events(
1432 self.capability_evaluator
1433 .evaluate_now(&runtime, ®istrations),
1434 &self.daemon_shutdown,
1435 );
1436 }
1437 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1438 }
1439 }
1440
1441 fn enforce_capability_denies(&self) {
1445 let registrations = match self.registry.list_registered_modules() {
1446 Ok(snapshot) => snapshot,
1447 Err(err) => {
1448 warn!(error = %err, "failed to read registrations for capability deny census");
1449 return;
1450 }
1451 };
1452 let manifests = registrations
1453 .into_iter()
1454 .map(|registration| {
1455 (
1456 registration.manifest.module_id.clone(),
1457 registration.manifest,
1458 )
1459 })
1460 .collect::<BTreeMap<_, _>>();
1461 let census = match self.forwarding.route_census(None) {
1462 Ok(census) => census,
1463 Err(err) => {
1464 warn!(error = %err, "failed to read route census for capability deny enforcement");
1465 return;
1466 }
1467 };
1468
1469 for (target_module_id, routes) in census {
1470 let Some(target_manifest) = manifests.get(&target_module_id) else {
1471 continue;
1472 };
1473 let mut closed_routes = Vec::new();
1474 let mut module_goodbyes = Vec::new();
1475 for route in routes {
1476 let Principal::Reserved {
1477 module_id: opening_module_id,
1478 } = &route.principal
1479 else {
1480 continue;
1481 };
1482 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1483 continue;
1484 };
1485 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1486 continue;
1487 };
1488
1489 match self.forwarding.release_client_route(
1490 route.goodbye_target.connection_id,
1491 route.goodbye_target.channel,
1492 route.goodbye_target.epoch,
1493 ) {
1494 Ok(RouteRelease::Removed(module_goodbye)) => {
1495 warn!(
1496 opening_module_id,
1497 target_module_id,
1498 capability,
1499 "force-closing route because an attested capability deny edge now matches"
1500 );
1501 closed_routes.push(route);
1502 module_goodbyes.push(module_goodbye);
1503 }
1504 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1505 Err(err) => warn!(
1506 opening_module_id,
1507 target_module_id,
1508 capability,
1509 error = %err,
1510 "failed to force-close capability-denied route"
1511 ),
1512 }
1513 }
1514
1515 if closed_routes.is_empty() {
1516 continue;
1517 }
1518 send_route_control_pushes(
1519 &self.forwarding,
1520 closed_routes,
1521 ClientControlPush::RouteClosed {
1522 module_id: target_module_id,
1523 channels: Vec::new(),
1524 reason: RouteCloseReason::CapabilityDenied,
1525 drained: false,
1526 abandoned: 0,
1527 excluded_subscriptions: 0,
1528 terminal: Some(false),
1529 },
1530 );
1531 self.emit_route_goodbyes(module_goodbyes);
1532 }
1533 }
1534
1535 fn not_ready_reason(
1541 &self,
1542 registration: &crate::registry::ModuleRegistration,
1543 ) -> Option<NotReadyReason> {
1544 if !registration.ready {
1545 return Some(NotReadyReason {
1546 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1547 capability: None,
1548 });
1549 }
1550 self.first_unprovided_required_capability(registration)
1551 .map(|capability| NotReadyReason {
1552 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1553 capability: Some(capability),
1554 })
1555 }
1556
1557 fn first_unprovided_required_capability(
1575 &self,
1576 registration: &crate::registry::ModuleRegistration,
1577 ) -> Option<String> {
1578 let required = registration
1579 .manifest
1580 .capabilities
1581 .iter()
1582 .flat_map(|declarations| declarations.requires.iter())
1583 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1584 .map(|requirement| requirement.capability.as_str())
1585 .collect::<BTreeSet<_>>();
1586 if required.is_empty() {
1587 return None;
1588 }
1589 let module_id = registration.manifest.module_id.as_str();
1590 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1591 if required
1592 .iter()
1593 .any(|capability| verdict(capability).is_none())
1594 {
1595 self.refresh_capability_requirements();
1596 }
1597 required
1598 .into_iter()
1599 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1600 .map(str::to_string)
1601 }
1602
1603 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1604 self.capability_evaluator
1605 .statuses()
1606 .into_iter()
1607 .map(capability_requirement_status)
1608 .collect()
1609 }
1610
1611 fn deregister_connection(
1618 &self,
1619 connection_id: ConnectionId,
1620 reason: RegistrationEndReason,
1621 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1622 self.registry
1623 .deregister_connection_with_reason(connection_id, reason)
1624 }
1625
1626 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1627 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1628 return None;
1629 }
1630 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1631 parse_client_control_request(&frame.body)
1632 else {
1633 return None;
1634 };
1635 Some(target_module_id(&target).to_string())
1636 }
1637
1638 pub(crate) fn route_open_capacity_refusal(
1639 &self,
1640 ctx: &RouteCtx,
1641 frame: &Frame,
1642 target_module_id: &str,
1643 in_flight: usize,
1644 limit: usize,
1645 ) -> Result<Frame, RouterError> {
1646 self.route_open_admission_refusal_frame(
1647 ctx,
1648 frame,
1649 target_module_id,
1650 "open_admission_full",
1651 (in_flight, limit),
1652 format!(
1653 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1654 ),
1655 )
1656 }
1657
1658 fn route_open_target_capacity_refusal(
1659 &self,
1660 ctx: &RouteCtx,
1661 frame: &Frame,
1662 target_module_id: &str,
1663 in_flight: usize,
1664 ) -> Result<Frame, RouterError> {
1665 self.route_open_admission_refusal_frame(
1666 ctx,
1667 frame,
1668 target_module_id,
1669 "target_binds_full",
1670 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1671 format!(
1672 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1673 ),
1674 )
1675 }
1676
1677 fn route_open_admission_refusal_frame(
1685 &self,
1686 ctx: &RouteCtx,
1687 frame: &Frame,
1688 target_module_id: &str,
1689 reason: &'static str,
1690 (in_flight, limit): (usize, usize),
1691 message: impl Into<String>,
1692 ) -> Result<Frame, RouterError> {
1693 let code = error_codes::TARGET_UNAVAILABLE;
1694 self.counters.increment_route_open_refused(code);
1695 info!(
1696 target: "control",
1697 code,
1698 reason,
1699 module_id = ?target_module_id,
1700 connection_id = ctx.connection_id.get(),
1701 in_flight,
1702 limit,
1703 "route.open refused"
1704 );
1705 control_error_frame(frame, code, message.into())
1706 }
1707
1708 #[cfg(test)]
1714 pub fn handle_control(
1715 &self,
1716 connection_id: ConnectionId,
1717 frame: Frame,
1718 ) -> Result<Vec<Frame>, RouterError> {
1719 match frame.header.ty {
1720 FrameType::Ping => Ok(vec![pong(&frame)?]),
1721 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1722 FrameType::Goodbye => self.handle_goodbye(connection_id),
1723 ty => Ok(vec![control_error_frame(
1724 &frame,
1725 "unsupported_control_frame",
1726 format!("unsupported channel-0 frame {ty:?}"),
1727 )?]),
1728 }
1729 }
1730
1731 pub async fn handle_control_frame(
1732 &self,
1733 ctx: &RouteCtx,
1734 frame: Frame,
1735 ) -> Result<Vec<Frame>, RouterError> {
1736 self.handle_control_frame_timed(ctx, frame, None).await
1737 }
1738
1739 pub(crate) fn record_caller_endpoints(
1740 &self,
1741 connection: ConnectionId,
1742 endpoints: Option<(SocketAddr, SocketAddr)>,
1743 ) {
1744 let mut callers = self
1745 .caller_endpoints
1746 .lock()
1747 .unwrap_or_else(|p| p.into_inner());
1748 callers.remove(&connection);
1749 if let Some(endpoints) = endpoints {
1750 callers.insert(connection, endpoints);
1751 }
1752 }
1753
1754 #[cfg(test)]
1755 pub(crate) fn with_caller_lookup_for_test(
1756 mut self,
1757 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1758 ) -> Self {
1759 self.caller_lookup = Some(Arc::new(lookup));
1760 self
1761 }
1762
1763 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1764 let endpoints = self
1765 .caller_endpoints
1766 .lock()
1767 .unwrap_or_else(|p| p.into_inner())
1768 .get(&connection)
1769 .copied();
1770 #[cfg(test)]
1771 let lookup = self.caller_lookup.clone();
1772 tokio::task::spawn_blocking(move || {
1773 #[cfg(test)]
1774 if let Some(lookup) = lookup {
1775 return lookup(endpoints);
1776 }
1777 endpoints
1778 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1779 .unwrap_or_default()
1780 })
1781 .await
1782 .unwrap_or_default()
1783 }
1784
1785 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1786 frame.header.channel == 0
1787 && frame.header.ty == FrameType::Request
1788 && self
1789 .forwarding
1790 .module_endpoint_for_connection(ctx.connection_id)
1791 .is_ok_and(|endpoint| endpoint.is_none())
1792 && parse_client_control_request(&frame.body)
1793 .ok()
1794 .is_some_and(|request| privileged_operation(&request).is_some())
1795 }
1796
1797 pub(crate) async fn handle_control_frame_timed(
1798 &self,
1799 ctx: &RouteCtx,
1800 frame: Frame,
1801 dispatch_started_at: Option<StdInstant>,
1802 ) -> Result<Vec<Frame>, RouterError> {
1803 match frame.header.ty {
1804 FrameType::Ping => Ok(vec![pong(&frame)?]),
1805 FrameType::Hello => {
1806 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1807 }
1808 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1809 FrameType::Cancel => {
1810 if self
1813 .forwarding
1814 .operator_confirms()
1815 .cancel(ctx.connection_id, frame.header.corr)
1816 || self
1817 .supervisor
1818 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1819 || self.cancel_activity_subscription(ctx, &frame)?
1820 {
1821 Ok(Vec::new())
1822 } else {
1823 Ok(vec![control_error_frame(
1824 &frame,
1825 "unknown_subscription",
1826 "no subscription has this correlation id",
1827 )?])
1828 }
1829 }
1830 FrameType::Request => {
1831 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1834 if op
1835 .as_ref()
1836 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1837 {
1838 return self.handle_activity_record(ctx, frame).await;
1839 }
1840 if op.as_ref().is_some_and(|probe| {
1841 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1842 }) {
1843 return self.handle_activity_read(ctx, frame).await;
1844 }
1845 if op
1846 .as_ref()
1847 .is_some_and(|probe| probe.op == "operator.confirm")
1848 {
1849 return self.handle_operator_confirm(ctx, frame);
1850 }
1851 if op
1852 .as_ref()
1853 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1854 {
1855 return self.handle_scope_served_by(ctx.connection_id, frame);
1856 }
1857 if self
1858 .forwarding
1859 .module_endpoint_for_connection(ctx.connection_id)
1860 .map_err(RouterError::Forwarding)?
1861 .is_some()
1862 {
1863 if !is_known_module_request_op(&frame.body) {
1864 return Ok(vec![control_error_frame(
1865 &frame,
1866 "unsupported_control_frame",
1867 "module-originated channel-0 REQUEST is not supported",
1868 )?]);
1869 }
1870 let request = match parse_module_control_request_from_module(&frame.body) {
1871 Ok(request) => request,
1872 Err((err, ControlRequestBodyError::UnknownOp)) => {
1873 return Ok(vec![control_error_frame(
1874 &frame,
1875 "unsupported_control_frame",
1876 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1877 )?])
1878 }
1879 Err((err, ControlRequestBodyError::InvalidBody)) => {
1880 return Ok(vec![control_error_frame(
1881 &frame,
1882 "invalid_control_body",
1883 format!("malformed module control body: {err}"),
1884 )?])
1885 }
1886 };
1887 let op = module_control_request_op(&request);
1888 let corr = frame.header.corr;
1889 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1890 let result =
1891 self.handle_module_control_request(ctx.connection_id, frame, request);
1892 log_slow_control_dispatch(
1893 dispatch_started_at,
1894 op,
1895 ctx.connection_id,
1896 corr,
1897 None,
1898 );
1899 return result;
1900 }
1901
1902 if is_known_module_request_op(&frame.body) {
1903 return Ok(vec![control_error_frame(
1904 &frame,
1905 "not_registered",
1906 "catalog.update requires an active module registration owned by this connection",
1907 )?]);
1908 }
1909
1910 let request = match parse_client_control_request(&frame.body) {
1911 Ok(request) => request,
1912 Err((err, ControlRequestBodyError::UnknownOp)) => {
1913 return Ok(vec![control_error_frame(
1914 &frame,
1915 "unknown_control_op",
1916 format!("unknown client control op: {err}"),
1917 )?])
1918 }
1919 Err((err, ControlRequestBodyError::InvalidBody)) => {
1920 return Ok(vec![control_error_frame(
1921 &frame,
1922 "invalid_control_body",
1923 format!("malformed client control body: {err}"),
1924 )?])
1925 }
1926 };
1927 let op = client_control_request_op(&request);
1928 let corr = frame.header.corr;
1929 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1930 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1931 #[cfg(test)]
1932 if let Some(delay) = self.control_dispatch_delay {
1933 tokio::time::sleep(delay).await;
1934 }
1935 let result = self
1936 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1937 .await;
1938 if let Some(timings) = route_open_timings.as_mut() {
1939 timings.finish();
1940 }
1941 log_slow_control_dispatch(
1942 dispatch_started_at,
1943 op,
1944 ctx.connection_id,
1945 corr,
1946 route_open_timings.as_ref(),
1947 );
1948 result
1949 }
1950 FrameType::Push => {
1951 let Some(endpoint) = self
1952 .forwarding
1953 .module_endpoint_for_connection(ctx.connection_id)
1954 .map_err(RouterError::Forwarding)?
1955 else {
1956 return Ok(vec![control_error_frame(
1957 &frame,
1958 "unsupported_control_frame",
1959 "client-originated channel-0 PUSH is not supported",
1960 )?]);
1961 };
1962 self.handle_status_update(endpoint, frame)
1963 }
1964 FrameType::Response | FrameType::Error
1965 if self
1966 .forwarding
1967 .module_endpoint_for_connection(ctx.connection_id)
1968 .map_err(RouterError::Forwarding)?
1969 .is_some() =>
1970 {
1971 self.handle_module_relay_response(ctx.connection_id, frame)
1972 }
1973 ty => Ok(vec![control_error_frame(
1974 &frame,
1975 "unsupported_control_frame",
1976 format!("unsupported channel-0 frame {ty:?}"),
1977 )?]),
1978 }
1979 }
1980
1981 pub fn cleanup_connection(
1982 &self,
1983 connection_id: ConnectionId,
1984 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1985 self.caller_endpoints
1986 .lock()
1987 .unwrap_or_else(|p| p.into_inner())
1988 .remove(&connection_id);
1989 self.forwarding
1990 .operator_confirms()
1991 .connection_closed(connection_id);
1992 self.cleanup_connection_with_end_reason(
1993 connection_id,
1994 RegistrationEndReason::ConnectionClosed,
1995 )
1996 }
1997
1998 fn cleanup_connection_with_end_reason(
1999 &self,
2000 connection_id: ConnectionId,
2001 requested_reason: RegistrationEndReason,
2002 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2003 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2004 self.registry
2005 .get_module_by_connection(connection_id)?
2006 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2007 .and_then(|module| module.registration_end_reason().ok().flatten())
2008 .unwrap_or(requested_reason)
2009 } else {
2010 requested_reason
2011 };
2012 let crash_closed = self
2013 .registry
2014 .get_module_by_connection(connection_id)?
2015 .and_then(|registration| {
2016 self.forwarding
2017 .module_endpoint_for_connection(connection_id)
2018 .ok()
2019 .flatten()
2020 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2021 .map(|routes| (registration.manifest.module_id, routes))
2022 });
2023 let crash_closed = crash_closed.map(|(module_id, routes)| {
2024 let terminal = match self.supervisor.get(&module_id) {
2025 None => false,
2026 Some(module) => match module.will_recover_after_connection_loss() {
2027 Ok(will_recover) => !will_recover,
2028 Err(err) => {
2029 warn!(
2030 %module_id,
2031 error = %err,
2032 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2033 );
2034 false
2035 }
2036 },
2037 };
2038 let (reason, drained) = match self
2042 .forwarding
2043 .daemon_drain_result_for_connection(connection_id)
2044 {
2045 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2046 Ok(None) => (RouteCloseReason::Crash, false),
2047 Err(err) => {
2048 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2049 (RouteCloseReason::Crash, false)
2050 }
2051 };
2052 (module_id, routes, reason, drained, terminal)
2053 });
2054 let registrations = self.deregister_connection(connection_id, end_reason);
2055 let cleanup = if crash_closed.is_some() {
2056 self.forwarding.cleanup_connection_counted(connection_id)
2057 } else {
2058 self.forwarding
2064 .cleanup_connection(connection_id)
2065 .map(|released| crate::forwarding::ConnectionCleanup {
2066 released,
2067 abandoned_relays: 0,
2068 })
2069 };
2070 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2075 let abandoned = cleanup
2076 .as_ref()
2077 .map_or(0, |cleanup| cleanup.abandoned_relays);
2078 send_route_control_pushes(
2079 &self.forwarding,
2080 routes,
2081 ClientControlPush::RouteClosed {
2082 module_id,
2083 channels: Vec::new(),
2084 reason,
2085 drained,
2086 abandoned,
2087 excluded_subscriptions: 0,
2088 terminal: Some(terminal),
2089 },
2090 );
2091 }
2092 if let Ok(cleanup) = cleanup {
2093 self.emit_route_goodbyes(cleanup.released);
2094 }
2095 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2099 crate::supervise::notify_registration_release();
2100 self.capability_evaluator.wake_deadline_loop();
2101 self.refresh_capability_requirements();
2102 }
2103 self.supervisor.remove_spawn_subscribers(connection_id);
2104 self.activity_streams
2105 .lock()
2106 .unwrap_or_else(|p| p.into_inner())
2107 .retain(|(id, _), task| {
2108 if *id == connection_id {
2109 task.abort();
2110 false
2111 } else {
2112 true
2113 }
2114 });
2115 self.hello_launch_nonces
2118 .lock()
2119 .unwrap_or_else(|poisoned| poisoned.into_inner())
2120 .forget(connection_id);
2121 self.scopes
2122 .write()
2123 .unwrap_or_else(|poisoned| poisoned.into_inner())
2124 .release_connection(connection_id);
2125 registrations
2126 }
2127
2128 pub(crate) fn handle_route_goodbye(
2129 &self,
2130 connection_id: ConnectionId,
2131 route_channel: u16,
2132 route_epoch: u32,
2133 ) -> Result<bool, RouterError> {
2134 debug!(
2135 connection_id = connection_id.get(),
2136 route_channel, route_epoch, "handling route GOODBYE"
2137 );
2138 let RouteRelease::Removed(released_route) = self
2139 .forwarding
2140 .release_client_route(connection_id, route_channel, route_epoch)
2141 .map_err(RouterError::Forwarding)?
2142 else {
2143 return Ok(false);
2144 };
2145 self.emit_route_goodbyes(vec![released_route]);
2146 Ok(true)
2147 }
2148
2149 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2150 for released in released_routes {
2151 let frame = match Frame::build_with_version(
2152 released.negotiated_ver,
2153 FrameType::Goodbye,
2154 control_flags(),
2155 released.channel,
2156 released.epoch,
2157 0,
2158 Vec::new(),
2159 ) {
2160 Ok(frame) => frame,
2161 Err(err) => {
2162 warn!(
2163 route_channel = released.channel,
2164 error = %err,
2165 "failed to build route GOODBYE frame"
2166 );
2167 continue;
2168 }
2169 };
2170 if !released.close_on_delivery_failure() {
2171 crate::forwarding::send_module_route_goodbye(
2172 &self.counters,
2173 &released.sink,
2174 frame,
2175 released.module_id.as_deref(),
2176 "client route released",
2177 );
2178 continue;
2179 }
2180 if let Err(err) = released.sink.try_send(frame) {
2181 warn!(
2182 target_connection_id = released.connection_id.get(),
2183 route_channel = released.channel,
2184 error = %err,
2185 "route GOODBYE was not delivered to client; closing target connection"
2186 );
2187 if self
2188 .forwarding
2189 .escalate_client_delivery_failure(
2190 released.connection_id,
2191 released.channel,
2192 released.epoch,
2193 CloseReason::new(
2194 "route_goodbye_delivery_failed",
2195 format!(
2196 "failed to enqueue route GOODBYE for channel {}: {err}",
2197 released.channel
2198 ),
2199 ),
2200 crate::forwarding::UndeliveredFrame {
2201 module_id: released.module_id.as_deref(),
2202 sink: &released.sink,
2203 },
2204 )
2205 .unwrap_or(false)
2206 {
2207 self.counters.increment_goodbye_relay_client_failed();
2208 }
2209 }
2210 }
2211 }
2212
2213 fn send_abandoned_route_bind_goodbye(
2225 &self,
2226 module_sink: &crate::FrameSink,
2227 negotiated_ver: u8,
2228 module_channel: u16,
2229 module_epoch: u32,
2230 ) {
2231 let frame = match Frame::build_with_version(
2232 negotiated_ver,
2233 FrameType::Goodbye,
2234 control_flags(),
2235 module_channel,
2236 module_epoch,
2237 0,
2238 Vec::new(),
2239 ) {
2240 Ok(frame) => frame,
2241 Err(err) => {
2242 warn!(
2243 route_channel = module_channel,
2244 error = %err,
2245 "failed to build GOODBYE for abandoned route.bind"
2246 );
2247 return;
2248 }
2249 };
2250 crate::forwarding::send_module_route_goodbye(
2251 &self.counters,
2252 module_sink,
2253 frame,
2254 None,
2255 "abandoned route.bind",
2256 );
2257 }
2258
2259 fn handle_hello(
2260 &self,
2261 connection_id: ConnectionId,
2262 sink: Option<crate::FrameSink>,
2263 frame: Frame,
2264 ) -> Result<Vec<Frame>, RouterError> {
2265 debug!(
2266 connection_id = connection_id.get(),
2267 corr = frame.header.corr,
2268 "handling HELLO"
2269 );
2270 if self
2274 .registry
2275 .get_module_by_connection(connection_id)
2276 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2277 .is_some()
2278 {
2279 return Ok(vec![control_error_frame(
2280 &frame,
2281 "invalid_hello",
2282 "connection is already registered as a module",
2283 )?]);
2284 }
2285 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2286 Ok(value) => value,
2287 Err(err) => {
2288 return Ok(vec![control_error_frame(
2289 &frame,
2290 "invalid_hello",
2291 format!("malformed HELLO body: {err}"),
2292 )?])
2293 }
2294 };
2295 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2296 return Ok(vec![control_error_frame(
2297 &frame,
2298 "invalid_capability_grammar",
2299 err.to_string(),
2300 )?]);
2301 }
2302 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2303 return Ok(vec![control_error_frame(
2304 &frame,
2305 "invalid_manifest",
2306 err.to_string(),
2307 )?]);
2308 }
2309 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2310 return Ok(vec![control_error_frame(
2311 &frame,
2312 "invalid_event_declaration",
2313 err.to_string(),
2314 )?]);
2315 }
2316 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2317 return Ok(vec![control_error_frame(
2318 &frame,
2319 "invalid_manifest",
2320 err.to_string(),
2321 )?]);
2322 }
2323 if let Some(provenance) = hello_value
2324 .get("manifest")
2325 .and_then(|manifest| manifest.get("provenance"))
2326 {
2327 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2328 return Ok(vec![control_error_frame(
2329 &frame,
2330 "invalid_manifest",
2331 format!("malformed manifest provenance: {err}"),
2332 )?]);
2333 }
2334 }
2335 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2336 Ok(hello) => hello,
2337 Err(err) => {
2338 return Ok(vec![control_error_frame(
2339 &frame,
2340 "invalid_hello",
2341 format!("malformed HELLO body: {err}"),
2342 )?])
2343 }
2344 };
2345
2346 if hello.protocol_ver != hello.manifest.protocol_ver {
2347 return Ok(vec![control_error_frame(
2348 &frame,
2349 "invalid_manifest",
2350 format!(
2351 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2352 hello.protocol_ver, hello.manifest.protocol_ver
2353 ),
2354 )?]);
2355 }
2356
2357 if hello.manifest.module_id.trim().is_empty() {
2358 return Ok(vec![control_error_frame(
2359 &frame,
2360 "invalid_manifest",
2361 "manifest module_id must not be empty",
2362 )?]);
2363 }
2364
2365 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2366 Ok(negotiated_ver) => negotiated_ver,
2367 Err(message) => {
2368 return Ok(vec![control_error_frame(
2369 &frame,
2370 "version_unsupported",
2371 message,
2372 )?])
2373 }
2374 };
2375
2376 let swap_admission = self
2384 .supervisor
2385 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2386 if swap_admission == SwapHelloAdmission::Refused {
2387 warn!(
2388 module_id = %hello.manifest.module_id,
2389 connection_id = connection_id.get(),
2390 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2391 );
2392 return Ok(vec![control_error_frame(
2393 &frame,
2394 "swap_token_invalid",
2395 format!(
2396 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2397 hello.manifest.module_id
2398 ),
2399 )?]);
2400 }
2401 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2402
2403 let nonce_verified = if swap_candidate {
2408 true
2409 } else {
2410 match self.supervisor.reserved_hello_verification(
2411 &hello.manifest.module_id,
2412 hello.launch_nonce.as_deref(),
2413 ) {
2414 Ok(verified) => verified,
2415 Err(rejection) => {
2416 let message = match rejection {
2417 ReservedHelloRejection::Exact { module_id } => format!(
2418 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2419 ),
2420 ReservedHelloRejection::Prefix {
2421 prefix,
2422 owner_module_id,
2423 } => format!(
2424 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2425 hello.manifest.module_id
2426 ),
2427 };
2428 return Ok(vec![control_error_frame(
2429 &frame,
2430 "reserved_module",
2431 message,
2432 )?]);
2433 }
2434 }
2435 };
2436
2437 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2438 &hello.manifest.module_id,
2439 hello.manifest.capabilities.as_ref(),
2440 );
2441 if let Some(refusal) = reserved_capability_refusals.first() {
2442 let capability = refusal.capability.clone();
2443 let bound_module = refusal.claimants[0].clone();
2444 log_duplicate_claim_events(reserved_capability_refusals);
2445 return Ok(vec![control_error_frame(
2446 &frame,
2447 "reserved_capability",
2448 format!(
2449 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2450 capability, bound_module, hello.manifest.module_id
2451 ),
2452 )?]);
2453 }
2454
2455 if self
2458 .forwarding
2459 .connection_has_client_routes(connection_id)
2460 .map_err(RouterError::Forwarding)?
2461 {
2462 return Ok(vec![control_error_frame(
2463 &frame,
2464 "invalid_hello",
2465 "connection has open client routes and cannot also register as a module",
2466 )?]);
2467 }
2468
2469 self.hello_launch_nonces
2474 .lock()
2475 .unwrap_or_else(|poisoned| poisoned.into_inner())
2476 .record(connection_id, hello.launch_nonce.as_deref());
2477 let control_ops = effective_module_control_ops(hello.control_ops);
2478 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2481 if swap_candidate {
2482 return self.register_swap_candidate(
2483 connection_id,
2484 sink,
2485 &frame,
2486 hello.manifest,
2487 negotiated_ver,
2488 control_ops,
2489 hello_ack,
2490 );
2491 }
2492 let registration = match self
2499 .registry
2500 .register_unpublished_verified_with_control_ops(
2501 hello.manifest,
2502 negotiated_ver,
2503 connection_id,
2504 control_ops,
2505 nonce_verified,
2506 ) {
2507 Ok(registration) => registration,
2508 Err(RegistryError::DuplicateModuleId { module_id }) => {
2509 return Ok(vec![control_error_frame(
2510 &frame,
2511 "duplicate_module_id",
2512 format!(
2513 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2514 ),
2515 )?])
2516 }
2517 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2518 return Ok(vec![control_error_frame(
2519 &frame,
2520 "invalid_module_id",
2521 err.to_string(),
2522 )?])
2523 }
2524 Err(err) => {
2525 return Ok(vec![control_error_frame(
2526 &frame,
2527 "registry_error",
2528 err.to_string(),
2529 )?])
2530 }
2531 };
2532 #[cfg(test)]
2533 self.hold_between_registry_and_forwarding();
2534
2535 let reply = if let Some(sink) = sink {
2536 let concurrency = manifest_concurrency(®istration.manifest);
2552 if let Err(err) = self.forwarding.register_module_connection_acked(
2553 connection_id,
2554 registration.manifest.module_id.clone(),
2555 negotiated_ver,
2556 concurrency,
2557 sink,
2558 hello_ack,
2559 ) {
2560 if matches!(
2565 self.deregister_connection(
2566 connection_id,
2567 RegistrationEndReason::RegistrationFailed,
2568 ),
2569 Ok(r) if !r.is_empty()
2570 ) {
2571 crate::supervise::notify_registration_release();
2572 }
2573 return Ok(vec![control_error_frame(
2574 &frame,
2575 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2576 "invalid_hello"
2577 } else {
2578 forwarding_error_code(&err)
2579 },
2580 err.to_string(),
2581 )?]);
2582 }
2583 Vec::new()
2584 } else {
2585 vec![hello_ack]
2588 };
2589
2590 if !self
2595 .registry
2596 .publish_registration(connection_id)
2597 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2598 {
2599 debug!(
2600 module_id = %registration.manifest.module_id,
2601 connection_id = connection_id.get(),
2602 "registration was removed before it could be published"
2603 );
2604 }
2605
2606 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2615 info!(
2616 module_id = %registration.manifest.module_id,
2617 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2618 );
2619 }
2620
2621 self.apply_registration_capabilities(®istration);
2622
2623 self.supervisor
2624 .record_activity_registration(®istration.manifest.module_id);
2625
2626 info!(
2627 module_id = %registration.manifest.module_id,
2628 module_version = %registration.manifest.module_version,
2629 negotiated_ver,
2630 routable_provider = manifest_provides_routable_role(®istration.manifest),
2631 connection_id = connection_id.get(),
2632 "module registered"
2633 );
2634
2635 Ok(reply)
2636 }
2637
2638 #[allow(clippy::too_many_arguments)]
2655 fn register_swap_candidate(
2656 &self,
2657 connection_id: ConnectionId,
2658 sink: Option<crate::FrameSink>,
2659 frame: &Frame,
2660 manifest: ModuleManifest,
2661 negotiated_ver: u8,
2662 control_ops: Vec<String>,
2663 hello_ack: Frame,
2664 ) -> Result<Vec<Frame>, RouterError> {
2665 let module_id = manifest.module_id.clone();
2666 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2667 manifest,
2668 negotiated_ver,
2669 connection_id,
2670 control_ops,
2671 ) {
2672 Ok(registration) => registration,
2673 Err(RegistryError::DuplicateModuleId { module_id }) => {
2674 return Ok(vec![control_error_frame(
2675 frame,
2676 "duplicate_module_id",
2677 format!(
2678 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2679 ),
2680 )?])
2681 }
2682 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2683 return Ok(vec![control_error_frame(
2684 frame,
2685 "invalid_module_id",
2686 err.to_string(),
2687 )?])
2688 }
2689 Err(err) => {
2690 return Ok(vec![control_error_frame(
2691 frame,
2692 "registry_error",
2693 err.to_string(),
2694 )?])
2695 }
2696 };
2697 #[cfg(test)]
2698 self.hold_between_registry_and_forwarding();
2699 let reply = if let Some(sink) = sink {
2700 let concurrency = manifest_concurrency(®istration.manifest);
2704 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2705 connection_id,
2706 module_id.clone(),
2707 negotiated_ver,
2708 concurrency,
2709 sink,
2710 hello_ack,
2711 ) {
2712 if matches!(
2713 self.deregister_connection(
2714 connection_id,
2715 RegistrationEndReason::RegistrationFailed,
2716 ),
2717 Ok(r) if !r.is_empty()
2718 ) {
2719 crate::supervise::notify_registration_release();
2720 }
2721 return Ok(vec![control_error_frame(
2722 frame,
2723 forwarding_error_code(&err),
2724 err.to_string(),
2725 )?]);
2726 }
2727 Vec::new()
2728 } else {
2729 vec![hello_ack]
2730 };
2731 if !self
2732 .registry
2733 .publish_registration(connection_id)
2734 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2735 {
2736 debug!(
2737 module_id = %module_id,
2738 connection_id = connection_id.get(),
2739 "swap candidate registration was removed before it could be published"
2740 );
2741 }
2742 self.supervisor.mark_swap_candidate_admitted(&module_id);
2743 info!(
2744 module_id = %module_id,
2745 module_version = %registration.manifest.module_version,
2746 negotiated_ver,
2747 ready = registration.ready,
2748 connection_id = connection_id.get(),
2749 "swap candidate registered; not routable until cutover"
2750 );
2751 Ok(reply)
2752 }
2753
2754 fn build_hello_ack(
2755 &self,
2756 frame: &Frame,
2757 negotiated_ver: u8,
2758 module_id: &str,
2759 ) -> Result<Frame, RouterError> {
2760 let ack = ModuleHelloAckBody {
2761 negotiated_ver,
2762 subc_ops: module_subc_ops(),
2763 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2764 storage: self
2765 .storage_config
2766 .as_ref()
2767 .map(|cfg| cfg.descriptor_for(module_id)),
2768 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2769 };
2770 let body = serde_json::to_vec(&ack).map_err(|err| {
2771 RouterError::backend(
2772 0,
2773 frame.header.corr,
2774 format!("failed to encode HELLO_ACK: {err}"),
2775 )
2776 })?;
2777
2778 Frame::build_with_version(
2779 negotiated_ver,
2780 FrameType::HelloAck,
2781 control_flags(),
2782 0,
2783 0,
2784 frame.header.corr,
2785 body,
2786 )
2787 .map_err(RouterError::FrameBuild)
2788 }
2789
2790 async fn handle_client_control_request(
2791 &self,
2792 ctx: &RouteCtx,
2793 frame: Frame,
2794 request: ClientControlRequest,
2795 route_open_timings: &mut Option<RouteOpenTimings>,
2796 ) -> Result<Vec<Frame>, RouterError> {
2797 if let Some(operation) = privileged_operation(&request) {
2798 let caller = self.resolve_caller(ctx.connection_id).await;
2799 let op = client_control_request_op(&request);
2800 let text = prompt_text(operation, &caller);
2801 match self.privileged_operator_prompt {
2802 PrivilegedOperatorPrompt::Log => {
2803 info!(target: "subc_daemon::privileged_prompt", operation = op,
2804 target = ?operation, caller = %caller.description(), summary = %text,
2805 "privileged request would prompt the operator");
2806 }
2807 PrivilegedOperatorPrompt::Enforce => {
2808 let key =
2809 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2810 let outcome = self
2811 .forwarding
2812 .operator_confirms()
2813 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2814 .await;
2815 if outcome.code != "confirmed" {
2816 return Ok(vec![outcome.frame(&frame)]);
2817 }
2818 }
2819 }
2820 }
2821 match request {
2822 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2823 ClientControlRequest::CatalogList { module_id } => {
2824 self.handle_catalog_list(frame, module_id)
2825 }
2826 ClientControlRequest::RouteOpen {
2827 target,
2828 identity,
2829 consumer_identity,
2830 consumer_capabilities,
2831 role_versions,
2832 admission_facts,
2833 scope,
2834 } => {
2835 self.handle_route_open(
2836 ctx,
2837 frame,
2838 RouteOpenRequest {
2839 target,
2840 identity,
2841 consumer_identity,
2842 consumer_capabilities,
2843 role_versions,
2844 admission_facts,
2845 scope,
2846 },
2847 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2848 )
2849 .await
2850 }
2851 ClientControlRequest::RoutePoll {
2852 route_channel,
2853 route_epoch,
2854 kind,
2855 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2856 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2857 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2858 self.handle_supervisor_spawn_snapshot(frame)
2859 }
2860 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2861 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2862 }
2863 ClientControlRequest::SupervisorRestart {
2864 module_id,
2865 drain_timeout_ms,
2866 } => {
2867 self.log_supervisor_request_received(
2868 ctx,
2869 frame.header.corr,
2870 ops::SUPERVISOR_RESTART,
2871 Some(&module_id),
2872 None,
2873 )?;
2874 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2875 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2876 .await
2877 }
2878 ClientControlRequest::SupervisorSwap {
2879 module_id,
2880 ready_timeout_ms,
2881 } => {
2882 self.log_supervisor_request_received(
2883 ctx,
2884 frame.header.corr,
2885 ops::SUPERVISOR_SWAP,
2886 Some(&module_id),
2887 None,
2888 )?;
2889 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2890 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2891 .await
2892 }
2893 ClientControlRequest::SupervisorReload { module_id } => {
2894 self.log_supervisor_request_received(
2895 ctx,
2896 frame.header.corr,
2897 ops::SUPERVISOR_RELOAD,
2898 Some(&module_id),
2899 None,
2900 )?;
2901 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2902 self.handle_supervisor_reload(frame, module_id, requester)
2903 .await
2904 }
2905 ClientControlRequest::SupervisorRescan { preview } => {
2906 if !preview {
2907 self.log_supervisor_request_received(
2908 ctx,
2909 frame.header.corr,
2910 ops::SUPERVISOR_RESCAN,
2911 None,
2912 None,
2913 )?;
2914 }
2915 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2916 self.handle_supervisor_rescan(
2917 frame,
2918 if preview {
2919 RescanMode::Preview
2920 } else {
2921 RescanMode::Execute
2922 },
2923 requester,
2924 )
2925 .await
2926 }
2927 ClientControlRequest::SupervisorRescanChecked {
2928 preview,
2929 candidate,
2930 expect,
2931 } => {
2932 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2933 .await
2934 }
2935 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2936 self.log_supervisor_request_received(
2937 ctx,
2938 frame.header.corr,
2939 ops::SUPERVISOR_RELEASE_RESERVED,
2940 Some(&module_id),
2941 None,
2942 )?;
2943 self.handle_supervisor_release_reserved(frame, module_id)
2944 .await
2945 }
2946 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2947 self.log_supervisor_request_received(
2948 ctx,
2949 frame.header.corr,
2950 ops::SUPERVISOR_SET_ENABLED,
2951 Some(&module_id),
2952 Some(enabled),
2953 )?;
2954 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2955 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2956 .await
2957 }
2958 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2959 self.handle_supervisor_health_probe(frame, module_id).await
2960 }
2961 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2962 ClientControlRequest::SupervisorRoutes { module_id } => {
2963 self.handle_supervisor_routes(frame, module_id)
2964 }
2965 ClientControlRequest::SupervisorProvenance { module_id } => {
2966 self.handle_supervisor_provenance(frame, module_id).await
2967 }
2968 ClientControlRequest::SupervisorStderrTail {
2969 module_id,
2970 max_lines,
2971 max_bytes,
2972 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2973 ClientControlRequest::SupervisorTerminals { module_id } => {
2974 self.handle_supervisor_terminals(frame, module_id).await
2975 }
2976 }
2977 }
2978
2979 fn log_supervisor_request_received(
2980 &self,
2981 ctx: &RouteCtx,
2982 corr: u64,
2983 op: &'static str,
2984 module_id: Option<&str>,
2985 enabled: Option<bool>,
2986 ) -> Result<(), RouterError> {
2987 let caller = self.supervisor_requester(ctx, corr)?;
2988 let caller = principal_label(&caller);
2989 info!(
2992 target: "control",
2993 op,
2994 module_id,
2995 enabled,
2996 connection_id = ctx.connection_id.get(),
2997 caller = %caller,
2998 "supervisor request received"
2999 );
3000 Ok(())
3001 }
3002
3003 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3004 Ok(self
3005 .registry
3006 .get_module_by_connection(ctx.connection_id)
3007 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3008 .map(|registration| Principal::Reserved {
3009 module_id: registration.manifest.module_id,
3010 })
3011 .unwrap_or(Principal::Direct))
3012 }
3013
3014 fn handle_module_control_request(
3015 &self,
3016 connection_id: ConnectionId,
3017 frame: Frame,
3018 request: ModuleControlRequestFromModule,
3019 ) -> Result<Vec<Frame>, RouterError> {
3020 match request {
3021 ModuleControlRequestFromModule::CatalogUpdate {
3022 provides,
3023 capabilities,
3024 ready,
3025 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3026 ModuleControlRequestFromModule::LiveRoots {} => {
3027 let registered = self
3028 .registry
3029 .get_module_by_connection(connection_id)
3030 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3031 let Some(registration) = registered else {
3032 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3033 };
3034 let response = self
3035 .forwarding
3036 .live_roots(®istration.manifest.module_id)
3037 .map_err(RouterError::Forwarding)?;
3038 Ok(vec![control_response_body_frame(
3039 &frame,
3040 &response,
3041 "ModuleControlResponseToModule::LiveRoots",
3042 )?])
3043 }
3044 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3045 self.handle_scope_sync(connection_id, frame, generation, scopes)
3046 }
3047 ModuleControlRequestFromModule::ScopeApply {
3048 generation,
3049 upsert,
3050 end,
3051 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3052 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3053 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3054 }
3055 }
3056 }
3057
3058 fn handle_scope_served_by(
3059 &self,
3060 connection_id: ConnectionId,
3061 frame: Frame,
3062 ) -> Result<Vec<Frame>, RouterError> {
3063 let registered = self
3064 .registry
3065 .get_module_by_connection(connection_id)
3066 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3067 let Some(registered) = registered else {
3068 return Ok(vec![control_error_frame(
3069 &frame,
3070 "not_registered",
3071 "scope.served_by requires an active module registration owned by this connection",
3072 )?]);
3073 };
3074 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3075 {
3076 Ok(request) => request,
3077 Err(err) => {
3078 return Ok(vec![control_error_frame(
3079 &frame,
3080 "invalid_control_body",
3081 format!("malformed scope.served_by body: {err}"),
3082 )?])
3083 }
3084 };
3085 let answer = self
3086 .scopes
3087 .read()
3088 .unwrap_or_else(|poisoned| poisoned.into_inner())
3089 .served_by(
3090 ®istered.manifest.module_id,
3091 &request.scope_ref,
3092 request.scope_epoch,
3093 &request.module_id,
3094 );
3095 let reply = subc_control::ScopeServedByReply::new(
3096 answer,
3097 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3098 );
3099 Ok(vec![control_response_body_frame(
3100 &frame,
3101 &reply,
3102 "ScopeServedByReply",
3103 )?])
3104 }
3105
3106 fn handle_operator_confirm(
3107 &self,
3108 ctx: &RouteCtx,
3109 frame: Frame,
3110 ) -> Result<Vec<Frame>, RouterError> {
3111 use crate::operator_confirm::{audit, Outcome};
3112 let registration = self
3113 .registry
3114 .get_module_by_connection(ctx.connection_id)
3115 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3116 let Some(registration) = registration else {
3117 let outcome = Outcome::refusal("not_registered");
3118 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3119 return Ok(vec![outcome.frame(&frame)]);
3120 };
3121 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3122 Ok(request) => request,
3123 Err(_) => {
3124 let outcome = Outcome::refusal("invalid_control_body");
3125 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3126 return Ok(vec![outcome.frame(&frame)]);
3127 }
3128 };
3129 let module_id = registration.manifest.module_id;
3130 let nonce = self
3134 .hello_launch_nonces
3135 .lock()
3136 .unwrap_or_else(|p| p.into_inner())
3137 .nonce(ctx.connection_id)
3138 .map(str::to_owned);
3139 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3140 self.supervisor
3141 .spawned_consumer_authorized(&module_id, nonce)
3142 });
3143 let confirms = self.forwarding.operator_confirms();
3144 self.forwarding
3145 .with_operator_route(
3146 ctx.connection_id,
3147 request.route_channel,
3148 request.route_epoch,
3149 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3150 )
3151 .map_err(RouterError::Forwarding)
3152 }
3153
3154 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3157 let mut scopes = ScopeRecords::from(scopes);
3158 for (index, record) in scopes.records.iter_mut().enumerate() {
3159 if let Some(refusal) = Self::canonicalize_workdir(record) {
3160 scopes.refusals.insert(index, refusal);
3161 }
3162 }
3163 scopes
3164 }
3165
3166 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3167 let workdir = record.workdir.as_mut()?;
3168 let path = std::path::Path::new(workdir);
3169 if !path.is_absolute() {
3170 return Some((
3171 error_codes::INVALID_WORKDIR,
3172 "workdir must be absolute".to_string(),
3173 ));
3174 }
3175 match ProjectRootId::from_path_allowing_missing(path) {
3176 Ok(root) => match root.as_path().to_str() {
3177 Some(canonical) => {
3178 *workdir = canonical.to_string();
3179 None
3180 }
3181 None => Some((
3182 error_codes::INVALID_WORKDIR,
3183 "canonical workdir is not UTF-8".to_string(),
3184 )),
3185 },
3186 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3187 }
3188 }
3189
3190 fn handle_scope_sync(
3194 &self,
3195 connection_id: ConnectionId,
3196 frame: Frame,
3197 generation: u64,
3198 scopes: Vec<ScopeRecord>,
3199 ) -> Result<Vec<Frame>, RouterError> {
3200 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3201 }
3202
3203 fn handle_scope_change(
3204 &self,
3205 connection_id: ConnectionId,
3206 frame: Frame,
3207 generation: u64,
3208 scopes: Vec<ScopeRecord>,
3209 end: Option<Vec<ScopeEnd>>,
3210 ) -> Result<Vec<Frame>, RouterError> {
3211 let is_apply = end.is_some();
3212 let Some(registration) = self
3213 .registry
3214 .get_module_by_connection(connection_id)
3215 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3216 else {
3217 return Ok(vec![control_error_frame(
3218 &frame,
3219 "not_registered",
3220 "scope.sync and scope.apply require an active module registration owned by this connection",
3221 )?]);
3222 };
3223 let owner = registration.manifest.module_id;
3224 let scopes = Self::prepare_scope_records(scopes);
3225 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3226 let is_current_launch = |connection: ConnectionId| {
3227 self.hello_launch_nonces
3228 .lock()
3229 .unwrap_or_else(|poisoned| poisoned.into_inner())
3230 .presented(connection, current_nonce.as_deref())
3231 };
3232 let mut table = self
3237 .scopes
3238 .write()
3239 .unwrap_or_else(|poisoned| poisoned.into_inner());
3240 let outcome = match end {
3241 Some(end) => table.apply(
3242 &owner,
3243 connection_id,
3244 is_current_launch,
3245 generation,
3246 scopes,
3247 end,
3248 ),
3249 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3250 };
3251 let drained = match &outcome {
3252 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3253 Err(_) => Vec::new(),
3254 };
3255 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3256 drop(table);
3257 if has_deadlines {
3258 self.scope_deadline_added.notify_one();
3259 }
3260 match outcome {
3261 Ok(applied) => {
3262 let counts = ScopeOutcomeCounts::of(&applied.results);
3263 info!(
3264 owner = %owner,
3265 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3266 generation,
3267 records = applied.results.len(),
3268 created = counts.created,
3269 replaced = counts.replaced,
3270 updated = counts.updated,
3271 unchanged = counts.unchanged,
3272 refused = counts.refused,
3273 ended = applied.ended.len(),
3274 tag_changes = applied.tag_changes.len(),
3275 routes_closed = drained.len(),
3276 "scope change accepted"
3277 );
3278 for refused in applied
3284 .results
3285 .iter()
3286 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3287 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3288 {
3289 warn!(
3290 owner = %owner,
3291 generation,
3292 scope_ref = %refused.scope_ref,
3293 scope_epoch = refused.scope_epoch,
3294 code = refused.code.as_deref().unwrap_or(""),
3295 "scope record refused"
3296 );
3297 }
3298 self.close_scope_drained_routes(drained);
3299 let response = if is_apply {
3300 ModuleControlResponseToModule::ScopeApply {
3301 generation,
3302 results: applied.results,
3303 end_results: applied.end_results,
3304 ended: applied.ended,
3305 }
3306 } else {
3307 ModuleControlResponseToModule::ScopeSync {
3308 generation,
3309 results: applied.results,
3310 ended: applied.ended,
3311 }
3312 };
3313 Ok(vec![control_response_body_frame(
3314 &frame,
3315 &response,
3316 "scope change response",
3317 )?])
3318 }
3319 Err(refusal) => {
3320 info!(
3321 owner = %owner,
3322 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3323 generation,
3324 code = refusal.code,
3325 "scope change refused"
3326 );
3327 Ok(vec![control_error_frame(
3328 &frame,
3329 refusal.code,
3330 refusal.message,
3331 )?])
3332 }
3333 }
3334 }
3335
3336 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3342 if drained.is_empty() {
3343 return;
3344 }
3345 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3346 BTreeMap::new();
3347 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3348 for route in drained {
3349 warn!(
3350 module_id = %route.module_id,
3351 reason = ?route.reason,
3352 client_connection_id = route.client.connection_id.get(),
3353 route_channel = route.client.channel,
3354 "closing route because its scope changed"
3355 );
3356 pushes
3357 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3358 .or_insert_with(|| (route.reason, Vec::new()))
3359 .1
3360 .push(EndpointRoute {
3361 goodbye_target: route.client.clone(),
3362 principal: Principal::Unverified,
3363 bound_at: Instant::now(),
3364 draining: false,
3365 drain_reason: None,
3366 });
3367 goodbyes.push(route.module);
3368 goodbyes.push(route.client);
3369 }
3370 for ((module_id, _), (reason, routes)) in pushes {
3371 send_route_control_pushes(
3372 &self.forwarding,
3373 routes,
3374 ClientControlPush::RouteClosed {
3375 module_id,
3376 channels: Vec::new(),
3377 reason,
3378 drained: false,
3379 abandoned: 0,
3380 excluded_subscriptions: 0,
3381 terminal: Some(false),
3382 },
3383 );
3384 }
3385 self.emit_route_goodbyes(goodbyes);
3386 }
3387
3388 fn handle_scope_describe(
3391 &self,
3392 connection_id: ConnectionId,
3393 frame: Frame,
3394 owner: Principal,
3395 scope_ref: String,
3396 ) -> Result<Vec<Frame>, RouterError> {
3397 let registered = self
3398 .registry
3399 .get_module_by_connection(connection_id)
3400 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3401 let Some(registered) = registered else {
3402 return Ok(vec![control_error_frame(
3403 &frame,
3404 "not_registered",
3405 "scope.describe requires an active module registration owned by this connection",
3406 )?]);
3407 };
3408 let mut description = self
3409 .scopes
3410 .read()
3411 .unwrap_or_else(|poisoned| poisoned.into_inner())
3412 .describe(&owner, &scope_ref);
3413 if !registered
3416 .manifest
3417 .capabilities
3418 .as_ref()
3419 .is_some_and(|caps| {
3420 caps.provides
3421 .iter()
3422 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3423 })
3424 {
3425 if let Some(stamp) = description.stamp.as_mut() {
3426 stamp.attributes.on_behalf_of_agent = None;
3427 }
3428 }
3429 let owner_configured = match &owner {
3430 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3435 _ => false,
3436 };
3437 let response = ModuleControlResponseToModule::ScopeDescribe {
3438 status: description.status,
3439 scope_epoch: description.scope_epoch,
3440 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3441 owner_synced: description.owner_synced,
3442 owner_configured,
3443 scope: description.stamp,
3444 };
3445 Ok(vec![control_response_body_frame(
3446 &frame,
3447 &response,
3448 "ModuleControlResponseToModule::ScopeDescribe",
3449 )?])
3450 }
3451
3452 fn handle_catalog_update(
3453 &self,
3454 connection_id: ConnectionId,
3455 frame: Frame,
3456 provides: Vec<ProviderRole>,
3457 capabilities: Option<CapabilityDeclarations>,
3458 ready: Option<bool>,
3459 ) -> Result<Vec<Frame>, RouterError> {
3460 self.refresh_capability_requirements();
3461 let Some(registration) = self
3462 .registry
3463 .get_module_by_connection(connection_id)
3464 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3465 else {
3466 return Ok(vec![control_error_frame(
3467 &frame,
3468 "not_registered",
3469 "catalog.update requires an active module registration owned by this connection",
3470 )?]);
3471 };
3472
3473 if let Some(message) =
3474 catalog_update_frozen_field_message(®istration.manifest, &provides)
3475 {
3476 return Ok(vec![control_error_frame(
3477 &frame,
3478 "catalog_update_frozen_field",
3479 message,
3480 )?]);
3481 }
3482
3483 let mut candidate = registration.manifest.clone();
3484 candidate.provides = provides.clone();
3485 candidate.capabilities = capabilities
3486 .clone()
3487 .or_else(|| registration.manifest.capabilities.clone());
3488 if let Some(rich_ui) = &candidate.rich_ui {
3489 if let Err(err) = rich_ui.validate() {
3490 return Ok(vec![control_error_frame(
3491 &frame,
3492 "invalid_manifest",
3493 err.to_string(),
3494 )?]);
3495 }
3496 }
3497 if let Err(err) = candidate.validate_capability_grammar() {
3498 return Ok(vec![control_error_frame(
3499 &frame,
3500 "invalid_capability_grammar",
3501 err.to_string(),
3502 )?]);
3503 }
3504
3505 let mut conflicts = self
3508 .capability_evaluator
3509 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3510 if let Some(conflict) = conflicts.first() {
3511 let message = format!(
3512 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3513 conflict.capability, conflict.claimants[0], candidate.module_id
3514 );
3515 for conflict in &mut conflicts {
3516 conflict.source = DuplicateClaimSource::CatalogUpdate;
3517 }
3518 log_duplicate_claim_events(conflicts);
3519 return Ok(vec![control_error_frame(
3520 &frame,
3521 "reserved_capability",
3522 message,
3523 )?]);
3524 }
3525
3526 let updated = self
3527 .registry
3528 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3529 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3530 if updated.is_none() {
3531 return Ok(vec![control_error_frame(
3532 &frame,
3533 "not_registered",
3534 "catalog.update requires an active module registration owned by this connection",
3535 )?]);
3536 }
3537 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3538 log_duplicate_claim_events(
3539 self.capability_evaluator
3540 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3541 );
3542 }
3543 if capability_census_trigger(
3544 registration.manifest.capabilities.as_ref(),
3545 updated
3546 .as_ref()
3547 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3548 ) {
3549 self.enforce_capability_denies();
3550 }
3551 self.refresh_capability_requirements();
3552
3553 let response = ModuleControlResponseToModule::CatalogUpdate {};
3554 control_response_body_frame(
3555 &frame,
3556 &response,
3557 "ModuleControlResponseToModule::CatalogUpdate",
3558 )
3559 .map(|frame| vec![frame])
3560 }
3561
3562 pub(crate) fn with_activity_ledger_config(
3563 mut self,
3564 run_dir: &Path,
3565 incarnation: String,
3566 config: crate::activity::ActivityConfig,
3567 ) -> Self {
3568 self.activity = Some(crate::activity::ActivityLedger::open_config(
3569 run_dir.join("activity.jsonl"),
3570 incarnation,
3571 config,
3572 ));
3573 self.supervisor.set_activity(self.activity.clone());
3574 self
3575 }
3576
3577 pub(crate) async fn configure_activity(
3578 &self,
3579 config: crate::activity::ActivityConfig,
3580 ) -> Result<(), &'static str> {
3581 match &self.activity {
3582 Some(ledger) => ledger.configure(config).await,
3583 None => Ok(()),
3584 }
3585 }
3586
3587 fn cancel_activity_subscription(
3588 &self,
3589 ctx: &RouteCtx,
3590 frame: &Frame,
3591 ) -> Result<bool, RouterError> {
3592 let task = self
3593 .activity_streams
3594 .lock()
3595 .unwrap_or_else(|p| p.into_inner())
3596 .remove(&(ctx.connection_id, frame.header.corr));
3597 let Some(task) = task else {
3598 return Ok(false);
3599 };
3600 task.abort();
3601 let end = Frame::build_with_version(
3602 response_version(frame),
3603 FrameType::StreamEnd,
3604 control_flags(),
3605 0,
3606 0,
3607 frame.header.corr,
3608 Vec::new(),
3609 )
3610 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3611 let sink = ctx.egress.clone();
3612 tokio::spawn(async move {
3613 let _ = sink.send(end).await;
3614 });
3615 Ok(true)
3616 }
3617
3618 async fn handle_activity_read(
3619 &self,
3620 ctx: &RouteCtx,
3621 frame: Frame,
3622 ) -> Result<Vec<Frame>, RouterError> {
3623 use subc_control::{
3624 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3625 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3626 };
3627 #[derive(Deserialize)]
3628 struct Identity {
3629 #[serde(default)]
3630 consumer_identity: Option<ConsumerIdentity>,
3631 }
3632 let principal = match self
3636 .forwarding
3637 .module_id_for_connection(ctx.connection_id)
3638 .map_err(RouterError::Forwarding)?
3639 {
3640 Some(module_id) => Ok(Principal::Reserved { module_id }),
3641 None => {
3642 let identity: Identity = match serde_json::from_slice(&frame.body) {
3643 Ok(identity) => identity,
3644 Err(_) => {
3645 return Ok(vec![control_error_frame(
3646 &frame,
3647 ACTIVITY_MALFORMED,
3648 "malformed activity read",
3649 )?])
3650 }
3651 };
3652 self.route_open_principal(&frame, identity.consumer_identity)?
3653 }
3654 };
3655 let admitted = matches!(principal, Ok(Principal::Direct))
3656 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3657 if !admitted {
3658 return Ok(vec![control_error_frame(
3659 &frame,
3660 ACTIVITY_READ_FORBIDDEN,
3661 "activity reads require an operator or trusted mirror principal",
3662 )?]);
3663 }
3664 let Some(ledger) = &self.activity else {
3665 return Ok(vec![control_error_frame(
3666 &frame,
3667 ACTIVITY_UNAVAILABLE,
3668 "activity ledger unavailable",
3669 )?]);
3670 };
3671 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3672 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3673 {
3674 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3675 Ok(request) => ledger.query(request).await,
3676 Err(_) => Err(ACTIVITY_MALFORMED),
3677 };
3678 return match result {
3679 Ok(reply) => Ok(vec![control_response_body_frame(
3680 &frame,
3681 &reply,
3682 ops::ACTIVITY_QUERY,
3683 )?]),
3684 Err(code) => Ok(vec![control_error_frame(
3685 &frame,
3686 code,
3687 "activity query refused",
3688 )?]),
3689 };
3690 }
3691 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3692 Ok(request) => request,
3693 Err(_) => {
3694 return Ok(vec![control_error_frame(
3695 &frame,
3696 ACTIVITY_MALFORMED,
3697 "malformed activity subscription",
3698 )?])
3699 }
3700 };
3701 let mut subscription = match ledger.subscribe(request.since).await {
3702 Ok(subscription) => subscription,
3703 Err(code) => {
3704 return Ok(vec![control_error_frame(
3705 &frame,
3706 code,
3707 "activity subscription refused",
3708 )?])
3709 }
3710 };
3711 let key = (ctx.connection_id, frame.header.corr);
3712 let sink = ctx.egress.clone();
3713 let mut streams = self
3714 .activity_streams
3715 .lock()
3716 .unwrap_or_else(|p| p.into_inner());
3717 streams.retain(|_, task| !task.is_finished());
3718 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3719 return Ok(vec![control_error_frame(
3720 &frame,
3721 "duplicate_subscription",
3722 "activity subscription correlation id already in use",
3723 )?]);
3724 }
3725 let task = tokio::spawn(async move {
3726 loop {
3727 let entry = if let Some(entry) = subscription.replay.pop_front() {
3728 Some(entry)
3729 } else {
3730 subscription.receiver.recv().await
3731 };
3732 let Some(entry) = entry else {
3733 break;
3734 };
3735 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3736 return;
3737 };
3738 let Ok(next) = Frame::build_with_version(
3739 response_version(&frame),
3740 FrameType::StreamData,
3741 control_flags(),
3742 0,
3743 0,
3744 frame.header.corr,
3745 body,
3746 ) else {
3747 return;
3748 };
3749 if sink.send(next).await.is_err() {
3750 return;
3751 }
3752 }
3753 if let Some(code) = subscription.terminal_code() {
3756 if let Ok(end) =
3757 control_error_frame(&frame, code, "activity subscriber fell behind")
3758 {
3759 let _ = sink.send(end).await;
3760 }
3761 }
3762 });
3763 streams.insert(key, task.abort_handle());
3764 Ok(Vec::new())
3765 }
3766
3767 async fn handle_activity_record(
3768 &self,
3769 ctx: &RouteCtx,
3770 frame: Frame,
3771 ) -> Result<Vec<Frame>, RouterError> {
3772 use subc_control::{
3773 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3774 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3775 };
3776 let result = async {
3777 let module = self
3778 .forwarding
3779 .module_id_for_connection(ctx.connection_id)
3780 .ok()
3781 .flatten()
3782 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3783 let request: ActivityRecordRequest =
3784 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3785 crate::activity::validate(&module, &request)?;
3786 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3787 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3788 let at_ms = std::time::SystemTime::now()
3789 .duration_since(std::time::UNIX_EPOCH)
3790 .unwrap_or_default()
3791 .as_millis() as u64;
3792 ledger.record(module, request, actor, at_ms).await
3793 }
3794 .await;
3795 match result {
3796 Ok(entry) => Ok(vec![control_response_body_frame(
3797 &frame,
3798 &ActivityRecordResponse { seq: entry.seq },
3799 "activity.record",
3800 )?]),
3801 Err(code) => Ok(vec![control_error_frame(
3802 &frame,
3803 code,
3804 "activity recording refused",
3805 )?]),
3806 }
3807 }
3808
3809 fn activity_actor(
3810 &self,
3811 connection: ConnectionId,
3812 module: &str,
3813 on: Option<&subc_control::ActivityOn>,
3814 ) -> Result<subc_control::ActivityActor, &'static str> {
3815 use crate::forwarding::{DataRoute, DataRouteState};
3816 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3817 let principal = |id: &str| Principal::Reserved {
3818 module_id: id.to_owned(),
3819 };
3820 let mut actor = ActivityActor {
3821 principal: principal(module),
3822 scope_owner: None,
3823 agent_id: None,
3824 run_id: None,
3825 flow_id: None,
3826 project_root: None,
3827 workdir: None,
3828 project_id: None,
3829 };
3830 let scope = match on {
3831 None => return Ok(actor),
3832 Some(ActivityOn::Scope(scope)) => {
3833 let selector = subc_protocol::scope::ScopeSelector::new(
3834 principal(module),
3835 scope.scope_ref.clone(),
3836 Some(scope.epoch),
3837 );
3838 self.scopes
3839 .read()
3840 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3841 .attribute(&principal(module), module, &selector, true)
3842 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3843 }
3844 Some(ActivityOn::Route(route)) => {
3845 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3846 .forwarding
3847 .lookup_data_route(connection, route.channel, route.epoch)
3848 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3849 else {
3850 return Err(ACTIVITY_ACTOR_UNKNOWN);
3851 };
3852 if binding.module_id != module {
3853 return Err(ACTIVITY_ACTOR_UNKNOWN);
3854 }
3855 actor.principal = binding.principal.clone();
3856 actor.project_root = binding
3857 .project_root
3858 .as_ref()
3859 .map(|root| root.as_path().to_path_buf());
3860 let Some(bound) = &binding.scope else {
3861 return Ok(actor);
3862 };
3863 let owner = principal(&bound.owner);
3864 let selector = subc_protocol::scope::ScopeSelector::new(
3865 owner.clone(),
3866 bound.scope_ref.clone(),
3867 Some(bound.tag.scope_epoch),
3868 );
3869 let admitted = self
3870 .scopes
3871 .read()
3872 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3873 .attribute(&owner, module, &selector, true)
3874 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3875 if admitted.tag != bound.tag {
3876 return Err(ACTIVITY_ACTOR_UNKNOWN);
3877 }
3878 admitted
3879 }
3880 };
3881 actor.scope_owner = Some(scope.stamp.owner);
3882 actor.agent_id = scope.stamp.attributes.agent_id;
3883 actor.run_id = scope.stamp.attributes.run_id;
3884 actor.flow_id = scope.stamp.attributes.flow_id;
3885 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3886 actor.project_id = scope.stamp.project_id;
3887 Ok(actor)
3888 }
3889
3890 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3891 self.refresh_capability_requirements();
3892 let mut counters = self.counters.snapshot();
3898 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3899 obj.insert(
3900 "activity_unparseable_lines".into(),
3901 ledger.read_errors().into(),
3902 );
3903 obj.insert(
3904 "activity_lifecycle_append_failures".into(),
3905 ledger.lifecycle_failures().into(),
3906 );
3907 obj.insert(
3908 "activity_started_digest_skips".into(),
3909 ledger.lifecycle_digest_skips().into(),
3910 );
3911 }
3912 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3913 self.forwarding.client_route_concentration(),
3914 counters.as_object_mut(),
3915 ) {
3916 obj.insert(
3917 "client_connections_with_routes".into(),
3918 connections_with_routes.into(),
3919 );
3920 obj.insert("max_routes_on_one_connection".into(), max.into());
3921 }
3922 if let (Some(open_breakers), Some(obj)) = (
3928 self.route_bind_breakers.open_snapshot(),
3929 counters.as_object_mut(),
3930 ) {
3931 obj.insert("route_bind_breakers_open".into(), open_breakers);
3932 }
3933 let response = ClientControlResponse::ServerDescribe {
3934 protocol_ver: PROTOCOL_VERSION,
3935 subc_ops: subc_ops(),
3936 capabilities: self.describe_capabilities(),
3937 connected_clients: self.connected_clients.count(),
3938 counters: Some(counters),
3939 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3940 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3941 capability_requirements: self.capability_requirement_statuses(),
3942 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3943 config_path: self
3944 .rescan
3945 .as_ref()
3946 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3947 };
3948 Ok(vec![control_response_body_frame(
3949 &frame,
3950 &response,
3951 "ClientControlResponse::ServerDescribe",
3952 )?])
3953 }
3954
3955 fn handle_catalog_list(
3956 &self,
3957 frame: Frame,
3958 module_id: Option<String>,
3959 ) -> Result<Vec<Frame>, RouterError> {
3960 let (generation, modules) = self
3961 .registry
3962 .list_modules_with_nonce_verification()
3963 .map_err(|err| {
3964 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3965 })?;
3966 let entries = modules
3967 .into_iter()
3968 .filter(|(registration, _)| {
3969 module_id
3970 .as_deref()
3971 .map(|wanted| registration.manifest.module_id == wanted)
3972 .unwrap_or(true)
3973 })
3974 .map(|(registration, nonce_verified)| {
3975 let not_ready = self.not_ready_reason(®istration);
3976 let roles = registration.manifest.provides;
3977 CatalogEntry::new(
3978 registration.manifest.module_id,
3979 roles,
3980 registration.control_ops,
3981 )
3982 .with_ready(not_ready.is_none())
3983 .with_not_ready(not_ready)
3984 .with_module_version(Some(registration.manifest.module_version))
3985 .with_capabilities(registration.manifest.capabilities)
3986 .with_self_signals(registration.manifest.self_signals)
3987 .with_events(registration.manifest.events)
3988 .with_operation_delivery(registration.manifest.operation_delivery)
3989 .with_rich_ui(registration.manifest.rich_ui)
3990 .with_nonce_verified(nonce_verified)
3991 })
3992 .collect();
3993 let response = ClientControlResponse::CatalogList {
3994 generation,
3995 modules: entries,
3996 subc_ops: subc_ops(),
3997 };
3998 Ok(vec![control_response_body_frame(
3999 &frame,
4000 &response,
4001 "ClientControlResponse::CatalogList",
4002 )?])
4003 }
4004
4005 fn route_open_principal(
4006 &self,
4007 frame: &Frame,
4008 consumer_identity: Option<ConsumerIdentity>,
4009 ) -> Result<Result<Principal, Frame>, RouterError> {
4010 let Some(consumer_identity) = consumer_identity else {
4011 return Ok(Ok(Principal::Direct));
4012 };
4013
4014 if self.supervisor.spawned_consumer_authorized(
4015 &consumer_identity.module_id,
4016 &consumer_identity.launch_nonce,
4017 ) {
4018 return Ok(Ok(Principal::Reserved {
4019 module_id: consumer_identity.module_id,
4020 }));
4021 }
4022
4023 Ok(Err(control_error_frame(
4024 frame,
4025 "bad_consumer_identity",
4026 format!(
4027 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4028 consumer_identity.module_id
4029 ),
4030 )?))
4031 }
4032
4033 fn route_open_refusal_frame(
4040 &self,
4041 ctx: &RouteCtx,
4042 frame: &Frame,
4043 module_id: &str,
4044 reason: &'static str,
4045 code: &'static str,
4046 message: impl Into<String>,
4047 ) -> Result<Frame, RouterError> {
4048 self.observe_route_open_refusal(ctx, module_id, reason, code);
4049 control_error_frame(frame, code, message.into())
4050 }
4051
4052 fn route_open_breaker_refusal_frame(
4070 &self,
4071 ctx: &RouteCtx,
4072 frame: &Frame,
4073 module_id: &str,
4074 consecutive_timeouts: u32,
4075 retry_in: Duration,
4076 probe_in_flight: bool,
4077 ) -> Result<Frame, RouterError> {
4078 self.counters
4079 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4080 debug!(
4081 target: "control",
4082 code = "module_timeout",
4083 module_id = ?module_id,
4084 connection_id = ctx.connection_id.get(),
4085 consecutive_timeouts,
4086 retry_in_ms = retry_in.as_millis() as u64,
4087 probe_in_flight,
4088 "route.open refused by open bind-relay breaker"
4089 );
4090 let detail = if probe_in_flight {
4097 "one test route is already being tried; retry once it settles".to_string()
4098 } else {
4099 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4100 };
4101 control_error_frame(
4102 frame,
4103 "module_timeout",
4104 format!(
4105 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4106 timed out in a row); {detail}; its established routes are unaffected"
4107 ),
4108 )
4109 }
4110
4111 fn observe_route_open_refusal(
4124 &self,
4125 ctx: &RouteCtx,
4126 module_id: &str,
4127 reason: &'static str,
4128 code: &'static str,
4129 ) {
4130 self.counters.increment_route_open_refused(code);
4131 info!(
4132 target: "control",
4133 code,
4134 reason,
4135 module_id = ?module_id,
4136 connection_id = ctx.connection_id.get(),
4137 "route.open refused"
4138 );
4139 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4140 self.route_outages.record_not_serving(module_id, reason);
4141 }
4142 }
4143
4144 fn observe_route_open_accept(
4189 &self,
4190 ctx: &RouteCtx,
4191 module_id: &str,
4192 principal: &str,
4193 scope: &str,
4194 ) {
4195 self.route_outages.record_accepted(module_id);
4196 self.counters.increment_route_open_accepted(principal);
4197 info!(
4198 target: "control",
4199 principal,
4200 module_id,
4201 connection_id = ctx.connection_id.get(),
4202 scope = %scope,
4203 "route.open accepted"
4204 );
4205 }
4206
4207 fn supervised_absent_route_open_refusal_frame(
4208 &self,
4209 ctx: &RouteCtx,
4210 frame: &Frame,
4211 module_id: &str,
4212 code: &'static str,
4213 status: &crate::supervise::ModuleStatus,
4214 ) -> Result<Frame, RouterError> {
4215 self.counters.increment_route_open_refused(code);
4216 info!(
4217 target: "control",
4218 code,
4219 reason = "supervised_not_registered",
4220 module_id = ?module_id,
4221 connection_id = ctx.connection_id.get(),
4222 state = %status.state,
4223 enabled = status.enabled,
4224 live = status.live,
4225 "route.open refused"
4226 );
4227 self.route_outages
4231 .record_not_serving(module_id, "supervised_not_registered");
4232 let message = if let Some(pid) = status.waiting_for_previous_pid {
4233 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4234 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4235 })
4236 } else {
4237 format!(
4238 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4239 status.state, status.enabled, status.live
4240 )
4241 };
4242 control_error_frame(frame, code, message)
4243 }
4244
4245 async fn handle_route_open(
4246 &self,
4247 ctx: &RouteCtx,
4248 frame: Frame,
4249 request: RouteOpenRequest,
4250 timings: &mut RouteOpenTimings,
4251 ) -> Result<Vec<Frame>, RouterError> {
4252 let RouteOpenRequest {
4253 target,
4254 mut identity,
4255 consumer_identity,
4256 consumer_capabilities,
4257 role_versions,
4258 admission_facts,
4259 scope,
4260 } = request;
4261 let target_module_id = target_module_id(&target).to_string();
4262 timings.module_id.clone_from(&target_module_id);
4263 if self
4264 .registry
4265 .get_module_by_connection(ctx.connection_id)
4266 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4267 .is_some()
4268 {
4269 return Ok(vec![control_error_frame(
4270 &frame,
4271 "invalid_request",
4272 "module connections cannot open client routes",
4273 )?]);
4274 }
4275 debug!(
4276 connection_id = ctx.connection_id.get(),
4277 corr = frame.header.corr,
4278 module_id = %target_module_id,
4279 "handling route.open"
4280 );
4281
4282 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4288 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4289 self.observe_route_open_refusal(
4290 ctx,
4291 &target_module_id,
4292 "invalid_role_versions",
4293 error_codes::INVALID_REQUEST,
4294 );
4295 return Ok(vec![control_error_body_frame(
4296 &frame,
4297 ErrorBody {
4298 code: error_codes::INVALID_REQUEST.to_string(),
4299 message: error.to_string(),
4300 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4301 },
4302 )?]);
4303 }
4304
4305 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4306 if status.waiting_for_previous_pid.is_some() {
4307 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4308 ctx,
4309 &frame,
4310 &target_module_id,
4311 "module_warming",
4312 &status,
4313 )?]);
4314 }
4315 }
4316
4317 let Some(registration) = self
4336 .registry
4337 .get_module(&target_module_id)
4338 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4339 else {
4340 if let Some((status, warming)) =
4341 self.supervisor_status(&target_module_id, frame.header.corr)?
4342 {
4343 if status.protocol == ModuleProtocol::None {
4350 return Ok(vec![self.route_open_refusal_frame(
4351 ctx,
4352 &frame,
4353 &target_module_id,
4354 "protocol_none",
4355 error_codes::MODULE_NO_PROTOCOL,
4356 format!(
4357 "module_id '{target_module_id}' is declared protocol: none; \
4358 it speaks no subc wire and serves no routes"
4359 ),
4360 )?]);
4361 }
4362 let code = if warming {
4363 "module_warming"
4364 } else {
4365 "target_unavailable"
4366 };
4367 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4368 ctx,
4369 &frame,
4370 &target_module_id,
4371 code,
4372 &status,
4373 )?]);
4374 }
4375 if let Some(removed_ago_ms) =
4376 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4377 {
4378 return Ok(vec![self.route_open_refusal_frame(
4379 ctx,
4380 &frame,
4381 &target_module_id,
4382 "removed",
4383 error_codes::MODULE_REMOVED,
4384 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4385 )?]);
4386 }
4387 return Ok(vec![self.route_open_refusal_frame(
4388 ctx,
4389 &frame,
4390 &target_module_id,
4391 "not_registered",
4392 error_codes::UNKNOWN_MODULE,
4393 format!("module_id '{target_module_id}' is not registered"),
4394 )?]);
4395 };
4396
4397 if !registration.ready {
4401 self.counters
4402 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4403 info!(
4404 target: "control",
4405 code = error_codes::MODULE_WARMING,
4406 module_id = ?target_module_id,
4407 connection_id = ctx.connection_id.get(),
4408 reason = "declared_not_ready",
4409 "route.open refused"
4410 );
4411 self.route_outages
4414 .record_not_serving(&target_module_id, "declared_not_ready");
4415 return Ok(vec![control_error_body_frame(
4416 &frame,
4417 ErrorBody {
4418 code: error_codes::MODULE_WARMING.to_string(),
4419 message: format!(
4420 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4421 ),
4422 detail: Some(serde_json::json!({
4423 "reason": "declared_not_ready"
4424 })),
4425 },
4426 )?]);
4427 }
4428
4429 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4448 self.counters
4449 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4450 info!(
4451 target: "control",
4452 code = error_codes::MODULE_WARMING,
4453 module_id = ?target_module_id,
4454 connection_id = ctx.connection_id.get(),
4455 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4456 capability = %capability,
4457 "route.open refused"
4458 );
4459 return Ok(vec![control_error_body_frame(
4460 &frame,
4461 ErrorBody {
4462 code: error_codes::MODULE_WARMING.to_string(),
4463 message: format!(
4464 "module_id '{target_module_id}' requires capability '{capability}', \
4465 which no registered module provides; retry"
4466 ),
4467 detail: Some(serde_json::json!({
4468 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4469 "capability": capability,
4470 })),
4471 },
4472 )?]);
4473 }
4474
4475 if !target_has_required_role(&target, ®istration.manifest.provides) {
4476 return Ok(vec![self.route_open_refusal_frame(
4477 ctx,
4478 &frame,
4479 &target_module_id,
4480 "role_not_provided",
4481 "target_unavailable",
4482 format!("module_id '{target_module_id}' does not provide the requested target"),
4483 )?]);
4484 }
4485
4486 if registration.state != ChannelState::Active {
4487 return Ok(vec![self.route_open_refusal_frame(
4488 ctx,
4489 &frame,
4490 &target_module_id,
4491 "registration_not_active",
4492 "target_unavailable",
4493 format!("module_id '{target_module_id}' is not active"),
4494 )?]);
4495 }
4496
4497 if self
4498 .forwarding
4499 .module_is_draining(&target_module_id)
4500 .map_err(RouterError::Forwarding)?
4501 {
4502 return Ok(vec![self.route_open_refusal_frame(
4503 ctx,
4504 &frame,
4505 &target_module_id,
4506 "reloading",
4507 "module_reloading",
4508 format!("module_id '{target_module_id}' is reloading"),
4509 )?]);
4510 }
4511
4512 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4513 process_liveness.process_live(&target_module_id) == Some(false)
4514 }) {
4515 if process_liveness.process_replacing(&target_module_id) {
4522 return Ok(vec![self.route_open_refusal_frame(
4523 ctx,
4524 &frame,
4525 &target_module_id,
4526 "reloading",
4527 "module_reloading",
4528 format!("module_id '{target_module_id}' is reloading"),
4529 )?]);
4530 }
4531 return Ok(vec![self.route_open_refusal_frame(
4532 ctx,
4533 &frame,
4534 &target_module_id,
4535 "supervisor_not_live",
4536 "target_unavailable",
4537 format!("module_id '{target_module_id}' is not live"),
4538 )?]);
4539 }
4540
4541 if !self
4542 .forwarding
4543 .has_live_module_connection(&target_module_id)
4544 .map_err(RouterError::Forwarding)?
4545 {
4546 return Ok(vec![self.route_open_refusal_frame(
4547 ctx,
4548 &frame,
4549 &target_module_id,
4550 "no_forwarding_connection",
4551 "target_unavailable",
4552 format!("module_id '{target_module_id}' has no live forwarding connection"),
4553 )?]);
4554 }
4555
4556 if let Some(error) =
4557 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4558 {
4559 self.observe_route_open_refusal(
4560 ctx,
4561 &target_module_id,
4562 "op_not_allowed",
4563 "op_not_allowed",
4564 );
4565 return Ok(vec![error]);
4566 }
4567
4568 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4569 Ok(principal) => principal,
4570 Err(error) => {
4571 self.observe_route_open_refusal(
4572 ctx,
4573 &target_module_id,
4574 "bad_consumer_identity",
4575 "bad_consumer_identity",
4576 );
4577 return Ok(vec![error]);
4578 }
4579 };
4580 timings.principal = principal_label(&principal);
4581
4582 if let Principal::Reserved {
4586 module_id: opening_module_id,
4587 } = &principal
4588 {
4589 if let Some(opening_registration) = self
4590 .registry
4591 .get_module(opening_module_id)
4592 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4593 {
4594 if let Some(capability) =
4595 denied_capability(&opening_registration.manifest, ®istration.manifest)
4596 {
4597 warn!(
4598 opening_module_id,
4599 target_module_id,
4600 capability,
4601 "refusing route.open because an attested capability deny edge matches"
4602 );
4603 return Ok(vec![self.route_open_refusal_frame(
4604 ctx,
4605 &frame,
4606 &target_module_id,
4607 "capability_deny_edge",
4608 "capability_forbidden",
4609 format!(
4610 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4611 ),
4612 )?]);
4613 }
4614 }
4615 }
4616
4617 if admission_facts.is_some() {
4618 let carrier_matches = matches!(
4619 &principal,
4620 Principal::Reserved { module_id }
4621 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4622 );
4623 if !carrier_matches {
4624 return Ok(vec![self.route_open_refusal_frame(
4625 ctx,
4626 &frame,
4627 &target_module_id,
4628 "admission_facts_carrier_not_permitted",
4629 "admission_facts_not_permitted",
4630 "admission facts may only be carried by the configured reserved module",
4631 )?]);
4632 }
4633
4634 let target_allowed = self
4635 .admission_facts_targets
4636 .as_ref()
4637 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4638 if !target_allowed {
4639 return Ok(vec![self.route_open_refusal_frame(
4640 ctx,
4641 &frame,
4642 &target_module_id,
4643 "admission_facts_target_not_listed",
4644 "admission_facts_target_not_allowed",
4645 format!(
4646 "admission facts are not permitted for target module_id '{target_module_id}'"
4647 ),
4648 )?]);
4649 }
4650
4651 }
4655
4656 let (bound_scope, mut scope_stamp) = match &scope {
4662 None => (None, None),
4663 Some(selector) => {
4664 let owner_configured = match &selector.owner {
4665 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4671 _ => false,
4672 };
4673 let mut scopes = self
4674 .scopes
4675 .write()
4676 .unwrap_or_else(|poisoned| poisoned.into_inner());
4677 let admitted = scopes
4678 .admit(&principal, &target_module_id, selector, owner_configured)
4679 .and_then(|admission| {
4680 crate::scopes::check_target_flow_support(
4681 &admission.stamp,
4682 &target_module_id,
4683 registration.manifest.capabilities.as_ref(),
4684 )?;
4685 crate::scopes::check_target_agent_run_support(
4686 &admission.stamp,
4687 &target_module_id,
4688 registration.manifest.capabilities.as_ref(),
4689 )?;
4690 let bound = scopes.reserve_bind(
4691 &admission,
4692 &principal,
4693 ctx.connection_id,
4694 selector.hold_presence,
4695 );
4696 Ok((admission, bound))
4697 });
4698 match admitted {
4699 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4700 Err(refusal) => {
4701 return Ok(vec![self.route_open_refusal_frame(
4702 ctx,
4703 &frame,
4704 &target_module_id,
4705 refusal.code,
4706 refusal.code,
4707 refusal.message,
4708 )?]);
4709 }
4710 }
4711 }
4712 };
4713
4714 timings.enter(RouteOpenPhase::ProjectRoot);
4736 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4737 Ok(project_root) => project_root,
4738 Err(err) => {
4739 return Ok(vec![control_error_frame(
4740 &frame,
4741 "invalid_project_root",
4742 err.to_string(),
4743 )?])
4744 }
4745 };
4746 timings.enter(RouteOpenPhase::Admission);
4747 identity.project_root = project_root.as_path().to_path_buf();
4748
4749 let _concurrency_guard = match self
4760 .route_bind_concurrency
4761 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4762 {
4763 Ok(guard) => guard,
4764 Err(in_flight) => {
4765 return Ok(vec![self.route_open_target_capacity_refusal(
4766 ctx,
4767 &frame,
4768 &target_module_id,
4769 in_flight,
4770 )?]);
4771 }
4772 };
4773
4774 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4777 RouteBindAdmission::Admitted { guard, probe } => {
4778 if probe {
4779 info!(
4780 module_id = %target_module_id,
4781 connection_id = ctx.connection_id.get(),
4782 "route.bind breaker half-open: admitting one probe"
4783 );
4784 }
4785 guard
4786 }
4787 RouteBindAdmission::Refused {
4788 consecutive_timeouts,
4789 retry_in,
4790 probe_in_flight,
4791 } => {
4792 return Ok(vec![self.route_open_breaker_refusal_frame(
4793 ctx,
4794 &frame,
4795 &target_module_id,
4796 consecutive_timeouts,
4797 retry_in,
4798 probe_in_flight,
4799 )?]);
4800 }
4801 };
4802
4803 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4808 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4809 timings.enter(RouteOpenPhase::EgressReservation);
4810 let pending = match self
4811 .forwarding
4812 .begin_route_bind_relay_for(
4813 ctx.connection_id,
4814 ctx.egress.clone(),
4815 response_version(&frame),
4816 frame.header.corr,
4817 &target_module_id,
4818 principal.clone(),
4819 bound_scope,
4820 Some(project_root),
4821 relay_deadline,
4822 )
4823 .await
4824 {
4825 Ok(pending) => pending,
4826 Err(err) => {
4827 return Ok(vec![self.route_open_refusal_frame(
4828 ctx,
4829 &frame,
4830 &target_module_id,
4831 "relay_reservation_failed",
4832 forwarding_error_code(&err),
4833 err.to_string(),
4834 )?])
4835 }
4836 };
4837 timings.enter(RouteOpenPhase::Admission);
4838 let crate::forwarding::PendingRouteBindRelay {
4839 endpoint,
4840 module_sink,
4841 negotiated_ver,
4842 client_channel,
4843 client_epoch,
4844 module_channel,
4845 module_epoch,
4846 corr: relay_corr,
4847 receiver,
4848 } = pending;
4849 let mut reservation =
4850 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4851
4852 let relay_registration = self
4857 .registry
4858 .get_module_by_connection(endpoint.connection_id)
4859 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4860 let relay_capabilities = relay_registration
4861 .as_ref()
4862 .and_then(|registration| registration.manifest.capabilities.as_ref());
4863 if let Some(selector) = &scope {
4864 if let Err(refusal) = self
4867 .scopes
4868 .read()
4869 .unwrap_or_else(|poisoned| poisoned.into_inner())
4870 .check_bind_proof(&principal, selector)
4871 {
4872 reservation.release_and_disarm();
4873 return Ok(vec![self.route_open_refusal_frame(
4874 ctx,
4875 &frame,
4876 &target_module_id,
4877 refusal.code,
4878 refusal.code,
4879 refusal.message,
4880 )?]);
4881 }
4882 }
4883 if let Some(stamp) = scope_stamp
4884 .as_ref()
4885 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4886 {
4887 if let Err(refusal) = crate::scopes::check_target_flow_support(
4888 stamp,
4889 &target_module_id,
4890 relay_capabilities,
4891 )
4892 .and_then(|()| {
4893 crate::scopes::check_target_agent_run_support(
4894 stamp,
4895 &target_module_id,
4896 relay_capabilities,
4897 )
4898 }) {
4899 reservation.release_and_disarm();
4900 return Ok(vec![self.route_open_refusal_frame(
4901 ctx,
4902 &frame,
4903 &target_module_id,
4904 refusal.code,
4905 refusal.code,
4906 refusal.message,
4907 )?]);
4908 }
4909 }
4910
4911 if !relay_capabilities.is_some_and(|caps| {
4914 caps.provides
4915 .iter()
4916 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
4917 }) {
4918 if let Some(stamp) = scope_stamp.as_mut() {
4919 stamp.workdir = None;
4920 stamp.project_id = None;
4921 }
4922 }
4923 if !relay_capabilities.is_some_and(|caps| {
4924 caps.provides
4925 .iter()
4926 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4927 }) {
4928 if let Some(stamp) = scope_stamp.as_mut() {
4929 stamp.attributes.on_behalf_of_agent = None;
4930 }
4931 }
4932 let call_mode = if relay_capabilities
4933 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4934 {
4935 Some(derive_call_mode(scope_stamp.as_ref()))
4936 } else {
4937 if let Some(stamp) = scope_stamp.as_mut() {
4938 stamp.expires_at_ms = None;
4939 }
4940 None
4941 };
4942
4943 debug!(
4944 connection_id = ctx.connection_id.get(),
4945 client_channel,
4946 client_epoch,
4947 module_channel,
4948 module_epoch,
4949 "reserved route handle pair"
4950 );
4951 let route_principal_label = principal_label(&principal);
4954 let scope_label = scope_stamp
4955 .as_ref()
4956 .map_or_else(|| "none".to_string(), route_open_scope_label);
4957 let relay = ModuleControlRequest::RouteBind {
4958 route_channel: module_channel,
4959 epoch: module_epoch,
4960 target,
4961 identity,
4962 principal: Some(principal),
4963 consumer_capabilities,
4964 role_versions,
4965 admission_facts,
4966 scope: scope_stamp,
4967 call_mode,
4968 };
4969 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4970 RouterError::backend(
4971 0,
4972 frame.header.corr,
4973 format!("failed to encode route.bind request: {err}"),
4974 )
4975 })?;
4976 let relay_frame = Frame::build_with_version(
4977 negotiated_ver,
4978 FrameType::Request,
4979 control_flags(),
4980 0,
4981 0,
4982 relay_corr,
4983 relay_body,
4984 )
4985 .map_err(RouterError::FrameBuild)?;
4986
4987 timings.enter(RouteOpenPhase::BindSend);
4988 let sent = module_sink.send(relay_frame).await;
4989 timings.enter(RouteOpenPhase::Admission);
4990 if let Err(err) = sent {
4991 reservation.release_and_disarm();
4992 return Ok(vec![self.route_open_refusal_frame(
4993 ctx,
4994 &frame,
4995 &target_module_id,
4996 "relay_send_failed",
4997 "target_unavailable",
4998 err.to_string(),
4999 )?]);
5000 }
5001
5002 #[cfg(test)]
5003 if let Some(hold) = &self.route_bind_relay_sent_hold {
5004 hold.wait().await;
5008 hold.wait().await;
5009 }
5010
5011 if self
5016 .forwarding
5017 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5018 .map_err(RouterError::Forwarding)?
5019 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5020 {
5021 self.send_abandoned_route_bind_goodbye(
5022 &module_sink,
5023 negotiated_ver,
5024 module_channel,
5025 module_epoch,
5026 );
5027 }
5028
5029 timings.enter(RouteOpenPhase::BindAckWait);
5030 let outcome = timeout_at(relay_deadline, receiver).await;
5031 timings.enter(RouteOpenPhase::Admission);
5032 match outcome {
5033 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5034 reservation.disarm();
5035 if breaker.record_accepted() {
5036 info!(
5037 module_id = %target_module_id,
5038 "route.bind breaker closed: the probe was accepted"
5039 );
5040 }
5041 self.observe_route_open_accept(
5042 ctx,
5043 &target_module_id,
5044 &route_principal_label,
5045 &scope_label,
5046 );
5047 Ok(Vec::new())
5048 }
5049 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5050 reservation.release_and_disarm();
5051 breaker.record_inconclusive();
5055 let scope_code = match body.code.as_str() {
5059 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5060 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5061 _ => None,
5062 };
5063 if let Some(code) = scope_code {
5064 self.observe_route_open_refusal(
5065 ctx,
5066 &target_module_id,
5067 "scope_changed_before_commit",
5068 code,
5069 );
5070 return Ok(vec![control_error_body_frame(&frame, body)?]);
5071 }
5072 self.counters
5073 .increment_route_open_refused("module_rejected");
5074 info!(
5075 target: "control",
5076 code = "module_rejected",
5077 module_code = ?body.code,
5078 module_id = ?target_module_id,
5079 connection_id = ctx.connection_id.get(),
5080 "route.open refused"
5081 );
5082 Ok(vec![control_error_body_frame(&frame, body)?])
5083 }
5084 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5085 reservation.release_and_disarm();
5086 breaker.record_inconclusive();
5087 tracing::warn!(
5094 module_id = %target_module_id,
5095 "route.bind relay abandoned: {message}"
5096 );
5097 Ok(vec![self.route_open_refusal_frame(
5098 ctx,
5099 &frame,
5100 &target_module_id,
5101 "relay_abandoned",
5102 "target_unavailable",
5103 message,
5104 )?])
5105 }
5106 Ok(Err(_)) => {
5107 reservation.release_and_disarm();
5108 breaker.record_inconclusive();
5109 Ok(vec![self.route_open_refusal_frame(
5110 ctx,
5111 &frame,
5112 &target_module_id,
5113 "relay_waiter_canceled",
5114 "target_unavailable",
5115 "route.bind relay waiter was canceled before the module responded",
5116 )?])
5117 }
5118 Err(_) => {
5119 reservation.release_and_disarm();
5120 if let Some(opened) = breaker.record_timeout(
5124 self.route_bind_breaker_threshold,
5125 self.route_bind_breaker_cooldown,
5126 ) {
5127 warn!(
5128 module_id = %target_module_id,
5129 consecutive_timeouts = opened.consecutive_timeouts,
5130 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5131 reopened_after_probe = opened.reopened_after_probe,
5132 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5133 );
5134 }
5135 tracing::warn!(
5143 module_id = %target_module_id,
5144 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5145 "route.bind relay timed out: module did not ack within budget"
5146 );
5147 Ok(vec![self.route_open_refusal_frame(
5148 ctx,
5149 &frame,
5150 &target_module_id,
5151 "relay_timed_out",
5152 "module_timeout",
5153 format!(
5154 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5155 route_bind_relay_timeout
5156 ),
5157 )?])
5158 }
5159 }
5160 }
5161
5162 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5163 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5164 snapshot: self.supervisor.spawn_snapshot(),
5165 };
5166 Ok(vec![control_response_body_frame(
5167 &frame,
5168 &response,
5169 "ClientControlResponse::SupervisorSpawnSnapshot",
5170 )?])
5171 }
5172
5173 fn handle_supervisor_spawn_subscribe(
5174 &self,
5175 ctx: &RouteCtx,
5176 frame: Frame,
5177 since: Option<SpawnCursor>,
5178 ) -> Result<Vec<Frame>, RouterError> {
5179 match self.supervisor.subscribe_spawns(
5180 ctx.connection_id,
5181 frame.header.corr,
5182 response_version(&frame),
5183 since,
5184 ctx.egress.clone(),
5185 ) {
5186 Ok(()) => Ok(Vec::new()),
5187 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5188 Ok(vec![control_error_body_frame(
5189 &frame,
5190 ErrorBody {
5191 code: "spawn_cursor_incarnation_mismatch".to_string(),
5192 message: "spawn cursor belongs to a different daemon incarnation"
5193 .to_string(),
5194 detail: Some(serde_json::json!({
5195 "current_daemon_incarnation": current
5196 })),
5197 },
5198 )?])
5199 }
5200 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5201 &frame,
5202 ErrorBody {
5203 code: "spawn_cursor_too_old".to_string(),
5204 message: "spawn cursor predates the retained event ring".to_string(),
5205 detail: Some(serde_json::json!({
5206 "oldest_retained_cursor": oldest
5207 })),
5208 },
5209 )?]),
5210 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5211 0,
5212 frame.header.corr,
5213 format!("failed to open supervisor spawn subscription: {error}"),
5214 )),
5215 }
5216 }
5217
5218 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5219 let generation = self
5220 .registry
5221 .generation()
5222 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5223 let mut modules = Vec::new();
5224 for module in self.supervisor.list() {
5225 let status = module.status_for_control("list").map_err(|err| {
5226 RouterError::backend(
5227 0,
5228 frame.header.corr,
5229 format!("failed to read supervisor status: {err}"),
5230 )
5231 })?;
5232 let (configured, _) = module.configuration().map_err(|err| {
5233 RouterError::backend(
5234 0,
5235 frame.header.corr,
5236 format!("failed to read module configuration: {err}"),
5237 )
5238 })?;
5239 let image = module.running_image_agreement().await;
5241 let resources = Some(module.child_resource_usage());
5244 let mut verdict =
5245 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5246 verdict.spec = Some(launch_spec_agreement(
5247 &configured,
5248 status.process_alive,
5249 status.launch_spec_digest.as_ref(),
5250 ));
5251 let pending_reload = Some(verdict);
5252 let configured_enabled = self
5253 .capability_evaluator
5254 .configured_enabled(&status.module_id);
5255 modules.push(
5258 SupervisorEntry::new(
5259 status.module_id,
5260 status.state.to_string(),
5261 status.enabled,
5262 status.live,
5263 status.health.status,
5264 )
5265 .with_configured_enabled(configured_enabled)
5266 .with_launch_nonce_env(Some(!cfg!(unix)))
5267 .with_launch_nonce_source(status.launch_nonce_source)
5268 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5269 .with_protocol(status.protocol)
5270 .with_pending_reload(pending_reload)
5271 .with_last_probe_ms(status.health.last_probe_ms)
5272 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5273 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5274 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5275 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5276 .with_restart_count(Some(status.restart_count))
5277 .with_max_restarts(Some(status.max_restarts))
5278 .with_lifetime_restarts(Some(status.lifetime_restarts))
5279 .with_spawn_generation(Some(status.spawn_generation))
5280 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5281 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5282 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5283 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5284 .with_resources(resources),
5285 );
5286 }
5287 let response = ClientControlResponse::SupervisorList {
5288 generation,
5289 modules,
5290 };
5291 Ok(vec![control_response_body_frame(
5292 &frame,
5293 &response,
5294 "ClientControlResponse::SupervisorList",
5295 )?])
5296 }
5297
5298 fn handle_supervisor_stderr_tail(
5299 &self,
5300 frame: Frame,
5301 module_id: String,
5302 max_lines: Option<u32>,
5303 max_bytes: Option<u32>,
5304 ) -> Result<Vec<Frame>, RouterError> {
5305 let Some(module) = self.supervisor.get(&module_id) else {
5306 return Ok(vec![control_error_frame(
5307 &frame,
5308 "unknown_module",
5309 format!("module_id '{module_id}' is not supervised"),
5310 )?]);
5311 };
5312
5313 let snapshot = module.stderr_tail(
5314 max_lines.map(|value| value as usize),
5315 max_bytes.map(|value| value as usize),
5316 );
5317
5318 let response = ClientControlResponse::SupervisorStderrTail {
5319 module_id,
5320 tail: StderrTail {
5321 capture: match snapshot.capture {
5322 CaptureState::Captured => StderrCaptureState::Captured,
5323 CaptureState::Incomplete { reason } => {
5324 StderrCaptureState::Incomplete { reason }
5325 }
5326 CaptureState::NotCaptured { reason } => {
5327 StderrCaptureState::NotCaptured { reason }
5328 }
5329 },
5330 entries: snapshot
5331 .entries
5332 .into_iter()
5333 .map(|entry| match entry {
5334 TailEntry::Line {
5335 text,
5336 truncated,
5337 at_ms,
5338 } => StderrTailEntry::Line {
5339 text,
5340 truncated,
5341 at_ms,
5342 },
5343 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5344 })
5345 .collect(),
5346 dropped_lines: snapshot.dropped_lines,
5347 },
5348 };
5349 Ok(vec![control_response_body_frame(
5350 &frame,
5351 &response,
5352 "ClientControlResponse::SupervisorStderrTail",
5353 )?])
5354 }
5355
5356 async fn handle_supervisor_terminals(
5357 &self,
5358 frame: Frame,
5359 module_id: String,
5360 ) -> Result<Vec<Frame>, RouterError> {
5361 let Some(module) = self.supervisor.get(&module_id) else {
5362 return Ok(vec![control_error_frame(
5363 &frame,
5364 "unknown_module",
5365 format!("module_id '{module_id}' is not supervised"),
5366 )?]);
5367 };
5368
5369 let terminals = module
5372 .read_durable_terminal_history()
5373 .await
5374 .map_err(|error| {
5375 RouterError::backend(
5376 0,
5377 frame.header.corr,
5378 format!("failed to read terminal history: {error}"),
5379 )
5380 })?;
5381 let response = ClientControlResponse::SupervisorTerminals {
5382 module_id,
5383 terminals,
5384 };
5385 Ok(vec![control_response_body_frame(
5386 &frame,
5387 &response,
5388 "ClientControlResponse::SupervisorTerminals",
5389 )?])
5390 }
5391
5392 fn handle_supervisor_routes(
5393 &self,
5394 frame: Frame,
5395 module_id: Option<String>,
5396 ) -> Result<Vec<Frame>, RouterError> {
5397 let modules = self
5398 .forwarding
5399 .route_census(module_id.as_deref())
5400 .map_err(RouterError::Forwarding)?
5401 .into_iter()
5402 .map(|(module_id, routes)| SupervisorRouteModule {
5403 module_id,
5404 routes: routes
5405 .into_iter()
5406 .map(|route| SupervisorRoute {
5407 consumer: match route.principal {
5408 Principal::Reserved { module_id } => {
5409 SupervisorRouteConsumer::Reserved { module_id }
5410 }
5411 Principal::Direct | Principal::Unverified => {
5412 SupervisorRouteConsumer::Direct {
5413 connection_id: route.goodbye_target.connection_id.get(),
5414 }
5415 }
5416 },
5417 age_ms: Instant::now()
5418 .saturating_duration_since(route.bound_at)
5419 .as_millis()
5420 .try_into()
5421 .unwrap_or(u64::MAX),
5422 draining: route.draining,
5423 drain_reason: route.drain_reason,
5424 })
5425 .collect(),
5426 })
5427 .collect();
5428 let response = ClientControlResponse::SupervisorRoutes { modules };
5429 Ok(vec![control_response_body_frame(
5430 &frame,
5431 &response,
5432 "ClientControlResponse::SupervisorRoutes",
5433 )?])
5434 }
5435
5436 async fn handle_supervisor_provenance(
5437 &self,
5438 frame: Frame,
5439 module_id: Option<String>,
5440 ) -> Result<Vec<Frame>, RouterError> {
5441 let mut selected = if let Some(module_id) = module_id {
5442 let Some(module) = self.supervisor.get(&module_id) else {
5443 return Ok(vec![control_error_frame(
5444 &frame,
5445 "unknown_module",
5446 format!("module_id '{module_id}' is not supervised"),
5447 )?]);
5448 };
5449 vec![module]
5450 } else {
5451 self.supervisor.list()
5452 };
5453
5454 let mut modules = Vec::with_capacity(selected.len());
5455 for module in selected.drain(..) {
5456 let (status, observed_image) = module
5457 .status_and_running_image_agreement()
5458 .await
5459 .map_err(|err| {
5460 RouterError::backend(
5461 0,
5462 frame.header.corr,
5463 format!("failed to read supervisor status: {err}"),
5464 )
5465 })?;
5466 let module_declared = self
5467 .registry
5468 .get_module(&status.module_id)
5469 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5470 .and_then(|registration| registration.manifest.provenance)
5471 .map(|build| ModuleDeclaredProvenance::Reported { build })
5472 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5473 #[cfg(test)]
5474 let running_image = match &self.provenance_probe_override {
5475 Some(result) => result.clone(),
5476 None => observed_image,
5477 };
5478 #[cfg(not(test))]
5479 let running_image = observed_image;
5480 modules.push(SupervisorModuleProvenance {
5481 module_id: status.module_id,
5482 module_declared,
5483 daemon_observed: SupervisorObservedProcess {
5484 launch_nonce_source: status.launch_nonce_source,
5485 pid: status.pid,
5486 spawned_at_ms: status.spawned_at_ms,
5487 spawned_from: status.spawned_from,
5488 running_image,
5489 },
5490 });
5491 }
5492 let daemon = SupervisorDaemonProvenance {
5493 daemon_build: self.daemon_provenance.build.clone(),
5494 daemon_observed: DaemonObservedProcess {
5495 pid: self.daemon_provenance.pid,
5496 started_at_ms: self
5497 .daemon_provenance
5498 .start_clock
5499 .map(|clock| clock.started_at_ms())
5500 .or(self.daemon_provenance.started_at_ms),
5501 running_image: self
5502 .daemon_provenance
5503 .probe
5504 .observe(
5505 self.daemon_provenance.pid,
5506 self.daemon_provenance.executable_path.as_deref(),
5507 self.daemon_provenance.executable_identity.clone(),
5508 self.daemon_provenance.process_start_time,
5509 )
5510 .await,
5511 },
5512 };
5513 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5514 Ok(vec![control_response_body_frame(
5515 &frame,
5516 &response,
5517 "ClientControlResponse::SupervisorProvenance",
5518 )?])
5519 }
5520
5521 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5522 self.refresh_capability_requirements();
5523 let generation = self
5524 .registry
5525 .generation()
5526 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5527 let modules = self
5528 .supervisor
5529 .list()
5530 .into_iter()
5531 .map(|module| {
5532 let status = module.status_for_control("health").map_err(|err| {
5533 RouterError::backend(
5534 0,
5535 frame.header.corr,
5536 format!("failed to read supervisor health: {err}"),
5537 )
5538 })?;
5539 let module_id = status.module_id;
5540 let capability_detail = self
5541 .capability_evaluator
5542 .required_problem_detail(&module_id);
5543 Ok(SupervisorHealthEntry {
5544 module_id,
5545 status: status.health.status,
5546 detail: append_capability_problem_detail(
5547 status.health.detail,
5548 capability_detail,
5549 ),
5550 metrics: status.health.metrics,
5551 consecutive_failures: status.health.consecutive_failures,
5552 late_answer_count: status.health.late_answer_count,
5553 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5554 last_action: status.health.last_action,
5555 last_action_ms: status.health.last_action_ms,
5556 last_probe_ms: status.health.last_probe_ms,
5557 })
5558 })
5559 .collect::<Result<Vec<_>, RouterError>>()?;
5560 let response = ClientControlResponse::SupervisorHealth {
5561 generation,
5562 modules,
5563 };
5564 Ok(vec![control_response_body_frame(
5565 &frame,
5566 &response,
5567 "ClientControlResponse::SupervisorHealth",
5568 )?])
5569 }
5570
5571 async fn handle_supervisor_restart(
5572 &self,
5573 frame: Frame,
5574 module_id: String,
5575 drain_timeout_ms: Option<u64>,
5576 requester: Principal,
5577 ) -> Result<Vec<Frame>, RouterError> {
5578 let operation_lock = self.supervisor.operation_lock();
5579 let _operation_guard = operation_lock.lock().await;
5580 let Some(module) = self.supervisor.get(&module_id) else {
5581 return Ok(vec![control_error_frame(
5582 &frame,
5583 "unknown_module",
5584 format!("module_id '{module_id}' is not supervised"),
5585 )?]);
5586 };
5587
5588 self.route_outages.mark_operator_action(&module_id);
5589 if let Err(err) = module
5590 .restart_requested(drain_timeout_ms, Some(requester))
5591 .await
5592 {
5593 self.route_outages
5594 .operator_action_ended_unrefused(&module_id);
5595 let (code, message) = match err {
5596 crate::supervise::SuperviseError::Disabled { .. } => {
5597 ("module_disabled", err.to_string())
5598 }
5599 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5600 ("swap_in_progress", err.to_string())
5601 }
5602 _ => (
5603 "target_unavailable",
5604 format!("failed to restart module_id '{module_id}': {err}"),
5605 ),
5606 };
5607 return Ok(vec![control_error_frame(&frame, code, message)?]);
5608 }
5609
5610 let response = ClientControlResponse::SupervisorAck {
5611 module_id,
5612 applied: true,
5613 };
5614 Ok(vec![control_response_body_frame(
5615 &frame,
5616 &response,
5617 "ClientControlResponse::SupervisorAck",
5618 )?])
5619 }
5620
5621 async fn handle_supervisor_swap(
5625 &self,
5626 frame: Frame,
5627 module_id: String,
5628 ready_timeout_ms: Option<u64>,
5629 requester: Principal,
5630 ) -> Result<Vec<Frame>, RouterError> {
5631 let module = {
5638 let operation_lock = self.supervisor.operation_lock();
5639 let _operation_guard = operation_lock.lock().await;
5640 self.supervisor.get(&module_id)
5641 };
5642 let Some(module) = module else {
5643 return Ok(vec![control_error_frame(
5644 &frame,
5645 "unknown_module",
5646 format!("module_id '{module_id}' is not supervised"),
5647 )?]);
5648 };
5649
5650 self.route_outages.mark_operator_action(&module_id);
5651 if let Err(err) = module
5652 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5653 .await
5654 {
5655 self.route_outages
5656 .operator_action_ended_unrefused(&module_id);
5657 use crate::supervise::SuperviseError;
5658 let message = err.to_string();
5659 let error = match err {
5660 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5661 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5662 code: "swap_refused".to_string(),
5663 message,
5664 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5665 },
5666 SuperviseError::SwapFailed {
5667 arm,
5668 candidate_exit,
5669 ..
5670 } => ErrorBody {
5671 code: "swap_failed".to_string(),
5672 message,
5673 detail: Some(serde_json::json!({
5674 "arm": arm.as_str(),
5675 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5676 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5677 })),
5678 },
5679 _ => ErrorBody::new(
5680 "target_unavailable",
5681 format!("failed to swap module_id '{module_id}': {message}"),
5682 ),
5683 };
5684 return Ok(vec![control_error_body_frame(&frame, error)?]);
5685 }
5686 self.route_outages
5690 .operator_action_ended_unrefused(&module_id);
5691
5692 let response = ClientControlResponse::SupervisorAck {
5693 module_id,
5694 applied: true,
5695 };
5696 Ok(vec![control_response_body_frame(
5697 &frame,
5698 &response,
5699 "ClientControlResponse::SupervisorAck",
5700 )?])
5701 }
5702
5703 async fn handle_supervisor_reload(
5704 &self,
5705 frame: Frame,
5706 module_id: String,
5707 requester: Principal,
5708 ) -> Result<Vec<Frame>, RouterError> {
5709 let operation_lock = self.supervisor.operation_lock();
5710 let _operation_guard = operation_lock.lock().await;
5711 let Some(module) = self.supervisor.get(&module_id) else {
5712 return Ok(vec![control_error_frame(
5713 &frame,
5714 "unknown_module",
5715 format!("module_id '{module_id}' is not supervised"),
5716 )?]);
5717 };
5718
5719 self.route_outages.mark_operator_action(&module_id);
5720 if let Err(err) = module.reload_requested(Some(requester)).await {
5721 self.route_outages
5722 .operator_action_ended_unrefused(&module_id);
5723 let (code, message) = match err {
5724 crate::supervise::SuperviseError::Disabled { .. } => {
5725 ("module_disabled", err.to_string())
5726 }
5727 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5728 ("swap_in_progress", err.to_string())
5729 }
5730 _ => (
5731 "reload_failed",
5732 format!("failed to reload module_id '{module_id}': {err}"),
5733 ),
5734 };
5735 return Ok(vec![control_error_frame(&frame, code, message)?]);
5736 }
5737
5738 let response = ClientControlResponse::SupervisorAck {
5739 module_id,
5740 applied: true,
5741 };
5742 Ok(vec![control_response_body_frame(
5743 &frame,
5744 &response,
5745 "ClientControlResponse::SupervisorAck",
5746 )?])
5747 }
5748
5749 async fn handle_supervisor_rescan(
5753 &self,
5754 frame: Frame,
5755 mode: RescanMode,
5756 requester: Principal,
5757 ) -> Result<Vec<Frame>, RouterError> {
5758 let preview = mode.previews();
5759 let Some(context) = self.rescan.clone() else {
5760 return Ok(vec![control_error_frame(
5761 &frame,
5762 "rescan_unavailable",
5763 "the daemon was not started with a reloadable config path".to_string(),
5764 )?]);
5765 };
5766
5767 let operation_lock = self.supervisor.operation_lock();
5768 let _operation_guard = operation_lock.lock().await;
5769 let loaded = match self.load_rescan_config(&mode, &context) {
5773 Ok(config) => config,
5774 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5775 };
5776 let Some((config, activity_config)) = loaded else {
5784 return Ok(vec![control_error_frame(
5785 &frame,
5786 "invalid_daemon_config",
5787 format!(
5788 "daemon config not found at {}; refusing to rescan (an absent config would \
5789 retire every supervised module)",
5790 context.config_path.display()
5791 ),
5792 )?]);
5793 };
5794 let (
5795 configured_port,
5796 storage_config,
5797 admission_facts_carrier_module_id,
5798 admission_facts_targets,
5799 scope_authority_owners,
5800 privileged_operator_prompt,
5801 modules,
5802 reserved_capabilities,
5803 ) = (
5804 config.port,
5805 config.storage,
5806 config.admission_facts_carrier_module_id,
5807 config.admission_facts_targets,
5808 config.scope_authority_owners,
5809 config.privileged_operator_prompt,
5810 config.modules,
5811 config.reserved_capabilities,
5812 );
5813
5814 let mut restart_required = Vec::new();
5823 for section in RestartRequiredSection::ALL {
5824 let changed = match section {
5825 RestartRequiredSection::Port => configured_port != context.configured_port,
5826 RestartRequiredSection::Storage => storage_config != context.storage_config,
5827 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5828 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5829 }
5830 RestartRequiredSection::AdmissionFactsTargets => {
5831 admission_facts_targets != context.admission_facts_targets
5832 }
5833 RestartRequiredSection::ScopeAuthorityOwners => {
5834 scope_authority_owners != context.scope_authority_owners
5835 }
5836 RestartRequiredSection::PrivilegedOperatorPrompt => {
5837 privileged_operator_prompt != context.privileged_operator_prompt
5838 }
5839 };
5840 if changed {
5841 restart_required.push(section.label().to_string());
5842 }
5843 }
5844 if !restart_required.is_empty() {
5845 warn!(
5846 config_path = %context.config_path.display(),
5847 sections = %restart_required.join(", "),
5848 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5849 );
5850 }
5851
5852 for configured in &modules {
5853 if let Err(err) = validate_spec(&configured.module_spec()) {
5854 return Ok(vec![control_error_frame(
5855 &frame,
5856 "invalid_daemon_config",
5857 format!("supervisor rescan rejected daemon config: {err}"),
5858 )?]);
5859 }
5860 }
5861
5862 let configured_capabilities = modules
5863 .iter()
5864 .map(|module| (module.module_id.clone(), module.enabled))
5865 .collect::<Vec<_>>();
5866 let preview_capability_warnings = if preview {
5867 let (_, registrations) = self.runtime_capability_snapshot()?;
5868 let current_modules = self
5869 .supervisor
5870 .list()
5871 .into_iter()
5872 .map(|module| module.module_id().to_string())
5873 .collect::<BTreeSet<_>>();
5874 let resulting_modules = configured_capabilities.clone();
5875 let removed = current_modules
5876 .into_iter()
5877 .filter(|module_id| {
5878 !resulting_modules
5879 .iter()
5880 .any(|(configured_id, _)| configured_id == module_id)
5881 })
5882 .collect::<Vec<_>>();
5883 let disabled = match &mode {
5886 RescanMode::Candidate(_) => resulting_modules
5887 .iter()
5888 .filter(|(_, enabled)| !enabled)
5889 .map(|(module_id, _)| module_id.clone())
5890 .collect::<Vec<_>>(),
5891 _ => Vec::new(),
5892 };
5893 self.capability_evaluator.preview_warnings(
5894 resulting_modules,
5895 &removed,
5896 &disabled,
5897 ®istrations,
5898 )
5899 } else {
5900 Vec::new()
5901 };
5902 let reconciled = match &mode {
5903 RescanMode::Candidate(candidate) => self
5904 .preview_candidate(&context, modules, candidate)
5905 .map(|result| (result, Vec::new())),
5906 _ => {
5907 self.reconcile_supervised_modules(
5908 &context.supervisor,
5909 modules,
5910 preview,
5911 requester.clone(),
5912 )
5913 .await
5914 }
5915 };
5916 let (result, changed) = match reconciled {
5917 Ok(result) => result,
5918 Err(message) => {
5919 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5920 }
5921 };
5922 if !preview {
5923 if let Some(ledger) = &self.activity {
5924 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
5925 ledger.config_changed(&result.added, &result.removed, &changed, requester);
5926 }
5927 if let Err(code) = self.configure_activity(activity_config).await {
5928 warn!(
5929 code,
5930 "activity retention could not be applied during rescan"
5931 );
5932 }
5933 self.capability_evaluator
5934 .configure(configured_capabilities, reserved_capabilities);
5935 self.capability_evaluator.wake_deadline_loop();
5936 self.refresh_capability_requirements();
5937 }
5938 let mut result = result;
5939 result.restart_required = restart_required;
5940 result.capability_warnings = preview_capability_warnings;
5941 let response = ClientControlResponse::SupervisorRescan { result };
5942 Ok(vec![control_response_body_frame(
5943 &frame,
5944 &response,
5945 "ClientControlResponse::SupervisorRescan",
5946 )?])
5947 }
5948
5949 async fn handle_supervisor_release_reserved(
5950 &self,
5951 frame: Frame,
5952 module_id: String,
5953 ) -> Result<Vec<Frame>, RouterError> {
5954 let Some(context) = self.rescan.clone() else {
5955 return Ok(vec![control_error_frame(
5956 &frame,
5957 "release_unavailable",
5958 "reserved-id release requires a daemon started with a reloadable config path",
5959 )?]);
5960 };
5961 let operation_lock = self.supervisor.operation_lock();
5962 let _operation_guard = operation_lock.lock().await;
5963 let loaded = match crate::daemon_config::load(&context.config_path) {
5964 Ok(Some(config)) => config,
5965 Ok(None) => {
5966 return Ok(vec![control_error_frame(
5967 &frame,
5968 "invalid_daemon_config",
5969 format!(
5970 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5971 context.config_path.display()
5972 ),
5973 )?])
5974 }
5975 Err(err) => {
5976 return Ok(vec![control_error_frame(
5977 &frame,
5978 "invalid_daemon_config",
5979 format!("unable to verify reserved-id release against daemon config: {err}"),
5980 )?])
5981 }
5982 };
5983 if loaded
5984 .modules
5985 .iter()
5986 .any(|configured| configured.module_id == module_id)
5987 {
5988 return Ok(vec![control_error_frame(
5989 &frame,
5990 "reserved_module_configured",
5991 format!(
5992 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5993 ),
5994 )?]);
5995 }
5996 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5997 return Ok(vec![control_error_frame(
5998 &frame,
5999 "reserved_gate_not_retained",
6000 format!(
6001 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6002 ),
6003 )?]);
6004 }
6005
6006 let response = ClientControlResponse::SupervisorAck {
6007 module_id,
6008 applied: true,
6009 };
6010 Ok(vec![control_response_body_frame(
6011 &frame,
6012 &response,
6013 "ClientControlResponse::SupervisorAck",
6014 )?])
6015 }
6016
6017 fn plan_rescan(
6021 &self,
6022 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6023 ) -> Result<RescanPlan, String> {
6024 let mut current = BTreeMap::new();
6025 for module in self.supervisor.list() {
6026 let (spec, health) = module.configuration().map_err(|err| {
6027 format!(
6028 "failed to read configuration for module_id '{}': {err}",
6029 module.module_id()
6030 )
6031 })?;
6032 let enabled = module
6033 .status()
6034 .map_err(|err| {
6035 format!(
6036 "failed to read status for module_id '{}': {err}",
6037 module.module_id()
6038 )
6039 })?
6040 .enabled;
6041 current.insert(
6042 module.module_id().to_string(),
6043 (module, spec, health, enabled),
6044 );
6045 }
6046 let configured = configured_modules
6047 .into_iter()
6048 .map(|module| (module.module_id.clone(), module))
6049 .collect::<BTreeMap<_, _>>();
6050
6051 let added = configured
6052 .keys()
6053 .filter(|module_id| !current.contains_key(*module_id))
6054 .cloned()
6055 .collect::<Vec<_>>();
6056 let removed = current
6057 .keys()
6058 .filter(|module_id| !configured.contains_key(*module_id))
6059 .cloned()
6060 .collect::<Vec<_>>();
6061 let mut changed_pending_reload = Vec::new();
6062 let mut configuration_changes = BTreeSet::new();
6063 let mut enabled_changes = BTreeSet::new();
6064 let mut unchanged = 0_u32;
6065
6066 for (module_id, configured_module) in &configured {
6067 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6068 else {
6069 continue;
6070 };
6071 let launch_changed = *current_spec != configured_module.module_spec();
6075 let configuration_changed =
6076 launch_changed || *current_health != configured_module.health;
6077 let enabled_changed = *current_enabled != configured_module.enabled;
6078 if configuration_changed {
6079 configuration_changes.insert(module_id.clone());
6080 }
6081 if launch_changed {
6082 changed_pending_reload.push(module_id.clone());
6083 }
6084 if enabled_changed {
6085 enabled_changes.insert(module_id.clone());
6086 }
6087 if !configuration_changed && !enabled_changed {
6088 unchanged = unchanged.saturating_add(1);
6089 }
6090 }
6091 Ok(RescanPlan {
6092 current,
6093 configured,
6094 added,
6095 removed,
6096 changed_pending_reload,
6097 configuration_changes,
6098 enabled_changes,
6099 unchanged,
6100 })
6101 }
6102
6103 async fn reconcile_supervised_modules(
6112 &self,
6113 supervisor: &Supervisor,
6114 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6115 preview: bool,
6116 requester: Principal,
6117 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6118 let plan = self.plan_rescan(configured_modules)?;
6119
6120 if preview {
6126 return Ok((plan.preview_result(), Vec::new()));
6127 }
6128 let changed = plan
6129 .configuration_changes
6130 .union(&plan.enabled_changes)
6131 .cloned()
6132 .collect::<Vec<_>>();
6133 let RescanPlan {
6134 current,
6135 configured,
6136 added,
6137 removed,
6138 changed_pending_reload,
6139 configuration_changes,
6140 enabled_changes,
6141 unchanged,
6142 } = plan;
6143 if !(added.is_empty()
6148 && removed.is_empty()
6149 && configuration_changes.is_empty()
6150 && enabled_changes.is_empty())
6151 {
6152 if let Some(context) = &self.rescan {
6153 context
6154 .generation
6155 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6156 }
6157 }
6158
6159 for module_id in &removed {
6160 let module = ¤t
6161 .get(module_id)
6162 .expect("removed module came from current supervisor state")
6163 .0;
6164 module
6165 .retire_requested(Some(requester.clone()))
6166 .await
6167 .map_err(|err| {
6168 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6169 })?;
6170 self.supervisor.record_rescan_removal(module_id);
6204 self.supervisor.retire(module_id);
6205 self.route_outages.forget(module_id);
6206 }
6207
6208 for module_id in configured.keys() {
6209 let Some((module, _, _, _)) = current.get(module_id) else {
6210 continue;
6211 };
6212 let configured_module = configured
6213 .get(module_id)
6214 .expect("configured module id came from configured map");
6215 if configuration_changes.contains(module_id) {
6216 module
6217 .update_configuration(
6218 configured_module.module_spec(),
6219 configured_module.health.clone(),
6220 configured_module.drain_timeout_ms,
6221 )
6222 .await
6223 .map_err(|err| {
6224 format!(
6225 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6226 )
6227 })?;
6228 }
6229 if enabled_changes.contains(module_id) {
6230 self.route_outages.mark_operator_action(module_id);
6233 module
6234 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6235 .await
6236 .map_err(|err| {
6237 self.route_outages.operator_action_ended_unrefused(module_id);
6238 format!(
6239 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6240 configured_module.enabled
6241 )
6242 })?;
6243 }
6244 }
6245
6246 for module_id in &added {
6247 let configured_module = configured
6248 .get(module_id)
6249 .expect("added module id came from configured map");
6250 supervisor
6251 .supervise_configured_requested(
6252 configured_module.module_spec(),
6253 configured_module.enabled,
6254 configured_module.health.clone(),
6255 configured_module.drain_timeout_ms,
6256 configured_module.restart,
6257 Some(requester.clone()),
6258 )
6259 .map_err(|err| {
6260 format!("failed to add module_id '{module_id}' during rescan: {err}")
6261 })?;
6262 }
6263
6264 Ok((
6265 SupervisorRescanResult {
6266 added,
6267 removed,
6268 changed_pending_reload,
6269 enabled_changes: enabled_changes.iter().cloned().collect(),
6270 unchanged,
6271 preview: false,
6272 restart_required: Vec::new(),
6275 capability_warnings: Vec::new(),
6276 module_actions: Vec::new(),
6277 earlier_pending: Vec::new(),
6278 expect: None,
6279 },
6280 changed,
6281 ))
6282 }
6283
6284 async fn handle_supervisor_set_enabled(
6285 &self,
6286 frame: Frame,
6287 module_id: String,
6288 enabled: bool,
6289 requester: Principal,
6290 ) -> Result<Vec<Frame>, RouterError> {
6291 let operation_lock = self.supervisor.operation_lock();
6292 let _operation_guard = operation_lock.lock().await;
6293 let Some(module) = self.supervisor.get(&module_id) else {
6294 return Ok(vec![control_error_frame(
6295 &frame,
6296 "unknown_module",
6297 format!("module_id '{module_id}' is not supervised"),
6298 )?]);
6299 };
6300
6301 self.route_outages.mark_operator_action(&module_id);
6304 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6305 Ok(applied) => applied,
6306 Err(err) => {
6307 self.route_outages
6308 .operator_action_ended_unrefused(&module_id);
6309 return Ok(vec![control_error_frame(
6310 &frame,
6311 "target_unavailable",
6312 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6313 )?]);
6314 }
6315 };
6316 if !applied {
6317 self.route_outages
6320 .operator_action_ended_unrefused(&module_id);
6321 }
6322
6323 self.capability_evaluator.wake_deadline_loop();
6324 self.refresh_capability_requirements();
6325 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6326 Ok(vec![control_response_body_frame(
6327 &frame,
6328 &response,
6329 "ClientControlResponse::SupervisorAck",
6330 )?])
6331 }
6332
6333 async fn handle_supervisor_health_probe(
6334 &self,
6335 frame: Frame,
6336 module_id: String,
6337 ) -> Result<Vec<Frame>, RouterError> {
6338 self.refresh_capability_requirements();
6339 let Some(registration) = self
6340 .registry
6341 .get_module(&module_id)
6342 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6343 else {
6344 return Ok(vec![control_error_frame(
6345 &frame,
6346 "unknown_module",
6347 format!("module_id '{module_id}' is not registered"),
6348 )?]);
6349 };
6350
6351 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6364 {
6365 return Ok(vec![control_error_frame(
6366 &frame,
6367 "health_not_advertised",
6368 format!("module_id '{module_id}' did not advertise health.check"),
6369 )?]);
6370 }
6371
6372 let deadline = Instant::now() + self.health_probe_timeout;
6373 let pending = match self.forwarding.begin_module_control_rpc_for(
6374 &module_id,
6375 MODULE_CONTROL_OP_HEALTH_CHECK,
6376 deadline,
6377 ) {
6378 Ok(pending) => pending,
6379 Err(err) => {
6380 return Ok(vec![control_error_frame(
6381 &frame,
6382 forwarding_error_code(&err),
6383 err.to_string(),
6384 )?])
6385 }
6386 };
6387
6388 let PendingModuleControlRpc {
6389 endpoint,
6390 module_sink,
6391 negotiated_ver,
6392 corr: probe_corr,
6393 receiver,
6394 } = pending;
6395 let mut guard =
6396 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6397 let probe_body =
6398 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6399 RouterError::backend(
6400 0,
6401 frame.header.corr,
6402 format!("failed to encode health.check request: {err}"),
6403 )
6404 })?;
6405 let probe_frame = Frame::build_with_version(
6406 negotiated_ver,
6407 FrameType::Request,
6408 control_flags(),
6409 0,
6410 0,
6411 probe_corr,
6412 probe_body,
6413 )
6414 .map_err(RouterError::FrameBuild)?;
6415
6416 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6419 Ok(Ok(())) => {}
6420 Ok(Err(err)) => {
6421 return Ok(vec![control_error_frame(
6422 &frame,
6423 "target_unavailable",
6424 err.to_string(),
6425 )?]);
6426 }
6427 Err(_) => {
6428 return Ok(vec![control_error_frame(
6429 &frame,
6430 "module_timeout",
6431 format!(
6432 "module_id '{module_id}' did not answer health.check within {:?}",
6433 self.health_probe_timeout
6434 ),
6435 )?]);
6436 }
6437 }
6438
6439 match timeout_at(deadline, receiver).await {
6440 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6441 guard.disarm();
6442 let Some(report) = response.health_report() else {
6443 return Ok(vec![control_error_frame(
6444 &frame,
6445 "invalid_control_body",
6446 "health.check RPC returned a non-health response",
6447 )?]);
6448 };
6449 let HealthReport {
6454 status,
6455 detail,
6456 metrics,
6457 } = report;
6458 let capability_detail = self
6459 .capability_evaluator
6460 .required_problem_detail(&module_id);
6461 let response = ClientControlResponse::SupervisorHealthProbe {
6462 module_id,
6463 status,
6464 detail: append_capability_problem_detail(detail, capability_detail),
6465 metrics,
6466 };
6467 Ok(vec![control_response_body_frame(
6468 &frame,
6469 &response,
6470 "ClientControlResponse::SupervisorHealthProbe",
6471 )?])
6472 }
6473 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6474 guard.disarm();
6475 Ok(vec![control_error_body_frame(&frame, body)?])
6476 }
6477 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6478 guard.disarm();
6479 Ok(vec![control_error_frame(
6480 &frame,
6481 "target_unavailable",
6482 message,
6483 )?])
6484 }
6485 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6486 guard.disarm();
6487 Ok(vec![control_error_frame(
6488 &frame,
6489 "invalid_control_body",
6490 message,
6491 )?])
6492 }
6493 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6494 guard.disarm();
6495 Ok(vec![control_error_frame(
6496 &frame,
6497 "invalid_control_body",
6498 format!("expected module-control op '{expected}', got '{actual}'"),
6499 )?])
6500 }
6501 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6502 guard.disarm();
6503 Ok(vec![control_error_frame(
6504 &frame,
6505 "module_timeout",
6506 format!(
6507 "module_id '{module_id}' answered health.check after {:?}",
6508 self.health_probe_timeout
6509 ),
6510 )?])
6511 }
6512 Ok(Err(_)) => Ok(vec![control_error_frame(
6513 &frame,
6514 "target_unavailable",
6515 "health.check waiter was canceled before the module responded",
6516 )?]),
6517 Err(_) => Ok(vec![control_error_frame(
6518 &frame,
6519 "module_timeout",
6520 format!(
6521 "module_id '{module_id}' did not answer health.check within {:?}",
6522 self.health_probe_timeout
6523 ),
6524 )?]),
6525 }
6526 }
6527
6528 fn supervisor_status(
6529 &self,
6530 module_id: &str,
6531 corr: u64,
6532 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6533 self.supervisor
6534 .get(module_id)
6535 .map(|module| {
6536 let warming = module.is_warming_for_control("status").map_err(|err| {
6537 RouterError::backend(
6538 0,
6539 corr,
6540 format!(
6541 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6542 ),
6543 )
6544 })?;
6545 module.status_for_control("status").map_err(|err| {
6546 RouterError::backend(
6547 0,
6548 corr,
6549 format!(
6550 "failed to read supervisor status for module_id '{module_id}': {err}"
6551 ),
6552 )
6553 }).map(|status| (status, warming))
6554 })
6555 .transpose()
6556 }
6557
6558 fn guard_module_control_op(
6559 &self,
6560 frame: &Frame,
6561 module_id: &str,
6562 op: &str,
6563 ) -> Result<Option<Frame>, RouterError> {
6564 if self.module_grants_op(module_id, op, frame.header.corr)? {
6565 return Ok(None);
6566 }
6567
6568 Ok(Some(control_error_frame(
6569 frame,
6570 "op_not_allowed",
6571 format!("module_id '{module_id}' did not grant control op '{op}'"),
6572 )?))
6573 }
6574
6575 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6576 let Some(registration) = self
6577 .registry
6578 .get_module(module_id)
6579 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6580 else {
6581 return Ok(false);
6582 };
6583 Ok(module_registration_grants_op(®istration.control_ops, op))
6584 }
6585
6586 fn handle_status_update(
6587 &self,
6588 endpoint: ModuleEndpointId,
6589 frame: Frame,
6590 ) -> Result<Vec<Frame>, RouterError> {
6591 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6592 Ok(update) => update,
6593 Err(err) => {
6594 if is_known_module_push_op(&frame.body) {
6599 return Ok(vec![control_error_frame(
6600 &frame,
6601 "invalid_control_body",
6602 format!("malformed module control push body: {err}"),
6603 )?]);
6604 }
6605 return Ok(Vec::new());
6606 }
6607 };
6608
6609 match update {
6610 ModuleControlPush::RouteStatus {
6611 route_channel,
6612 route_epoch,
6613 status,
6614 } => {
6615 self.forwarding
6616 .cache_status(endpoint, route_channel, route_epoch, status)
6617 .map_err(RouterError::Forwarding)?;
6618 }
6619 }
6620 Ok(Vec::new())
6621 }
6622
6623 fn handle_route_poll(
6624 &self,
6625 ctx: &RouteCtx,
6626 frame: Frame,
6627 route_channel: u16,
6628 route_epoch: u32,
6629 kind: PollKind,
6630 ) -> Result<Vec<Frame>, RouterError> {
6631 let snapshot = self
6632 .forwarding
6633 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6634 .map_err(RouterError::Forwarding)?;
6635 let response = match (kind, snapshot) {
6636 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6637 ClientControlResponse::RoutePoll {
6638 route_channel,
6639 route_epoch,
6640 status,
6641 live: None,
6642 }
6643 }
6644 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6645 route_channel,
6646 route_epoch,
6647 status: None,
6648 live: None,
6649 },
6650 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6651 let live = self
6677 .process_liveness
6678 .as_ref()
6679 .and_then(|source| source.process_live(&module_id))
6680 .unwrap_or(true);
6681 ClientControlResponse::RoutePoll {
6682 route_channel,
6683 route_epoch,
6684 status: None,
6685 live: Some(live),
6686 }
6687 }
6688 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6689 route_channel,
6690 route_epoch,
6691 status: None,
6692 live: Some(false),
6693 },
6694 };
6695
6696 Ok(vec![control_response_body_frame(
6697 &frame,
6698 &response,
6699 "ClientControlResponse::RoutePoll",
6700 )?])
6701 }
6702
6703 pub(crate) fn observe_module_control_completion(
6704 &self,
6705 completion: ModuleControlRpcCompletion,
6706 ) -> bool {
6707 match completion {
6708 ModuleControlRpcCompletion::Unknown => false,
6709 ModuleControlRpcCompletion::Settled => true,
6710 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6711 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6712 info!(
6713 module_id = %module_id,
6714 latency_ms,
6715 "late health.check answer proves the module is alive"
6716 );
6717 match self
6718 .supervisor
6719 .record_late_health_answer(&module_id, latency_ms)
6720 {
6721 Ok(true) => {}
6722 Ok(false) => debug!(
6723 module_id = %module_id,
6724 latency_ms,
6725 "late health.check answer has no active supervisor snapshot"
6726 ),
6727 Err(err) => warn!(
6728 module_id = %module_id,
6729 latency_ms,
6730 error = %err,
6731 "failed to record late health.check answer"
6732 ),
6733 }
6734 true
6735 }
6736 }
6737 }
6738
6739 fn refuse_to_end_module_connection_for_a_client(
6760 &self,
6761 module_connection_id: ConnectionId,
6762 corr: u64,
6763 err: ForwardingError,
6764 ) -> Result<(), RouterError> {
6765 if let ForwardingError::ConnectionClosing { connection_id } = err {
6766 if connection_id != module_connection_id {
6767 warn!(
6768 module_connection_id = module_connection_id.get(),
6769 client_connection_id = connection_id.get(),
6770 corr,
6771 "dropping a route.bind response for a closing client; the module connection keeps serving"
6772 );
6773 return Ok(());
6774 }
6775 }
6776 Err(RouterError::Forwarding(err))
6777 }
6778
6779 fn handle_module_relay_response(
6780 &self,
6781 connection_id: ConnectionId,
6782 frame: Frame,
6783 ) -> Result<Vec<Frame>, RouterError> {
6784 let mut secondary_error = None;
6785 let outcome = match frame.header.ty {
6786 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6787 Ok(probe) if probe.op == "route.bind" => {
6788 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6789 Ok(ModuleControlResponse::RouteBindAck {}) => {
6790 RouteBindRelayOutcome::Accepted
6791 }
6792 Ok(other) => {
6793 let message =
6794 format!("route.bind response carried unexpected body: {other:?}");
6795 secondary_error = Some(control_error_frame(
6796 &frame,
6797 "invalid_control_body",
6798 message.clone(),
6799 )?);
6800 RouteBindRelayOutcome::ModuleGone(message)
6801 }
6802 Err(err) => {
6803 let message = format!("malformed route.bind response body: {err}");
6804 secondary_error = Some(control_error_frame(
6805 &frame,
6806 "invalid_control_body",
6807 message.clone(),
6808 )?);
6809 RouteBindRelayOutcome::ModuleGone(message)
6810 }
6811 }
6812 }
6813 Ok(probe) => {
6814 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6815 {
6816 Ok(response) => ModuleControlRpcOutcome::Response(response),
6817 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6818 "malformed {} response body: {err}",
6819 probe.op
6820 )),
6821 };
6822 let completion = self
6823 .forwarding
6824 .complete_module_control_rpc(
6825 connection_id,
6826 frame.header.corr,
6827 Some(&probe.op),
6828 outcome,
6829 )
6830 .map_err(RouterError::Forwarding)?;
6831 if !self.observe_module_control_completion(completion) {
6832 debug!(
6833 connection_id = connection_id.get(),
6834 corr = frame.header.corr,
6835 op = %probe.op,
6836 "dropping late or unknown module-control RPC response"
6837 );
6838 }
6839 return Ok(Vec::new());
6840 }
6841 Err(err) => {
6842 if let Some(expected_op) = self
6843 .forwarding
6844 .pending_module_control_op(connection_id, frame.header.corr)
6845 .map_err(RouterError::Forwarding)?
6846 {
6847 let completion = self
6848 .forwarding
6849 .complete_module_control_rpc(
6850 connection_id,
6851 frame.header.corr,
6852 None,
6853 ModuleControlRpcOutcome::MalformedResponse(format!(
6854 "malformed {expected_op} response body: {err}"
6855 )),
6856 )
6857 .map_err(RouterError::Forwarding)?;
6858 if !self.observe_module_control_completion(completion) {
6859 debug!(
6860 connection_id = connection_id.get(),
6861 corr = frame.header.corr,
6862 "dropping late malformed module-control RPC response"
6863 );
6864 }
6865 return Ok(Vec::new());
6866 }
6867 let message = format!("malformed route.bind response body: {err}");
6868 secondary_error = Some(control_error_frame(
6869 &frame,
6870 "invalid_control_body",
6871 message.clone(),
6872 )?);
6873 RouteBindRelayOutcome::ModuleGone(message)
6874 }
6875 },
6876 FrameType::Error => {
6877 if self
6878 .forwarding
6879 .pending_module_control_op(connection_id, frame.header.corr)
6880 .map_err(RouterError::Forwarding)?
6881 .is_some()
6882 {
6883 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6884 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6885 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6886 "malformed module-control ERROR body: {err}"
6887 )),
6888 };
6889 let completion = self
6890 .forwarding
6891 .complete_module_control_rpc(
6892 connection_id,
6893 frame.header.corr,
6894 None,
6895 outcome,
6896 )
6897 .map_err(RouterError::Forwarding)?;
6898 if !self.observe_module_control_completion(completion) {
6899 debug!(
6900 connection_id = connection_id.get(),
6901 corr = frame.header.corr,
6902 "dropping late or unknown module-control RPC error"
6903 );
6904 }
6905 return Ok(Vec::new());
6906 }
6907 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6908 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6909 Err(err) => {
6910 let message = format!("malformed route.bind ERROR body: {err}");
6911 secondary_error = Some(control_error_frame(
6912 &frame,
6913 "invalid_control_body",
6914 message.clone(),
6915 )?);
6916 RouteBindRelayOutcome::ModuleGone(message)
6917 }
6918 }
6919 }
6920 ty => {
6921 return Ok(vec![control_error_frame(
6922 &frame,
6923 "unsupported_control_frame",
6924 format!("unsupported module channel-0 frame {ty:?}"),
6925 )?])
6926 }
6927 };
6928
6929 let settled =
6930 self.forwarding
6931 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6932 let completion = match settled {
6933 Ok(completion) => completion,
6934 Err(err) => {
6935 self.refuse_to_end_module_connection_for_a_client(
6936 connection_id,
6937 frame.header.corr,
6938 err,
6939 )?;
6940 return Ok(secondary_error.into_iter().collect());
6941 }
6942 };
6943 if let Some(target) = completion.abandoned.as_ref() {
6944 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6945 }
6946 if !completion.settled {
6947 debug!(
6948 connection_id = connection_id.get(),
6949 corr = frame.header.corr,
6950 frame_type = ?frame.header.ty,
6951 "dropping late or unknown route.bind relay response"
6952 );
6953 }
6954 Ok(secondary_error.into_iter().collect())
6955 }
6956
6957 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6958 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6959 self.cleanup_connection_with_end_reason(
6963 connection_id,
6964 RegistrationEndReason::ExplicitGoodbye,
6965 )
6966 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6967 Ok(Vec::new())
6968 }
6969}
6970
6971impl Default for ControlHandler {
6972 fn default() -> Self {
6973 Self::new(Arc::new(Registry::default()))
6974 }
6975}
6976
6977impl crate::supervise::SwapPromotionObserver for ControlHandler {
6978 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6979 self.apply_registration_capabilities(registration);
6980 }
6981}
6982
6983fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6984 CapabilityRequirementStatus {
6985 consumer: status.consumer,
6986 capability: status.capability,
6987 need: match status.need {
6988 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6989 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6990 },
6991 verdict: status.verdict.as_str().to_string(),
6992 episode_seq: status.episode_seq,
6993 config_satisfiable: status.config_satisfiable,
6994 runtime_available: status.runtime_available,
6995 detail: status.detail,
6996 }
6997}
6998
6999fn append_capability_problem_detail(
7000 detail: Option<String>,
7001 capability_detail: Option<String>,
7002) -> Option<String> {
7003 match (detail, capability_detail) {
7004 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7005 (Some(detail), None) => Some(detail),
7006 (None, Some(capability_detail)) => Some(capability_detail),
7007 (None, None) => None,
7008 }
7009}
7010
7011fn subc_ops() -> Vec<String> {
7012 SUBC_CONTROL_OPS
7013 .iter()
7014 .map(|op| (*op).to_string())
7015 .collect()
7016}
7017
7018fn module_subc_ops() -> Vec<String> {
7019 SUBC_CONTROL_OPS
7020 .iter()
7021 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7022 .map(|op| (*op).to_string())
7023 .collect()
7024}
7025
7026#[cfg(test)]
7027fn module_baseline_control_ops() -> Vec<String> {
7028 MODULE_BASELINE_CONTROL_OPS
7029 .iter()
7030 .map(|op| (*op).to_string())
7031 .collect()
7032}
7033
7034fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7035 let mut seen = HashSet::new();
7036 let mut effective = Vec::new();
7037 for op in MODULE_BASELINE_CONTROL_OPS {
7038 if seen.insert((*op).to_string()) {
7039 effective.push((*op).to_string());
7040 }
7041 }
7042 for op in declared.unwrap_or_default() {
7043 if seen.insert(op.clone()) {
7044 effective.push(op);
7045 }
7046 }
7047 effective
7048}
7049
7050fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7051 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7052}
7053
7054fn target_module_id(target: &RouteTarget) -> &str {
7055 match target {
7056 RouteTarget::ToolProvider { module_id }
7057 | RouteTarget::ManagementSurface { module_id }
7058 | RouteTarget::InternalService { module_id, .. } => module_id,
7059 }
7060}
7061
7062fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7063 roles.iter().any(|role| match (target, role) {
7064 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7065 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7066 (
7067 RouteTarget::InternalService { service_id, .. },
7068 ProviderRole::InternalService {
7069 service_id: provided,
7070 ..
7071 },
7072 ) => service_id == provided,
7073 _ => false,
7074 })
7075}
7076
7077fn is_routable_role(role: &ProviderRole) -> bool {
7078 matches!(
7079 role,
7080 ProviderRole::ToolProvider { .. }
7081 | ProviderRole::ManagementSurface { .. }
7082 | ProviderRole::InternalService { .. }
7083 )
7084}
7085
7086#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7087enum ControlRequestBodyError {
7088 UnknownOp,
7089 InvalidBody,
7090}
7091
7092#[derive(Debug, Deserialize)]
7093struct ControlOpProbe {
7094 op: String,
7095}
7096
7097const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7100
7101fn is_known_module_push_op(body: &[u8]) -> bool {
7102 serde_json::from_slice::<ControlOpProbe>(body)
7103 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7104 .unwrap_or(false)
7105}
7106
7107fn is_known_module_request_op(body: &[u8]) -> bool {
7108 serde_json::from_slice::<ControlOpProbe>(body)
7109 .map(|probe| is_module_to_subc_op(&probe.op))
7110 .unwrap_or(false)
7111}
7112
7113fn is_module_to_subc_op(op: &str) -> bool {
7114 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7115}
7116
7117fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7118 debug!(
7119 op = %op,
7120 connection_id = connection_id.get(),
7121 corr,
7122 "control dispatch"
7123 );
7124}
7125
7126fn log_slow_control_dispatch(
7127 dispatch_started_at: Option<StdInstant>,
7128 op: &'static str,
7129 connection_id: ConnectionId,
7130 corr: u64,
7131 route_open_timings: Option<&RouteOpenTimings>,
7132) {
7133 let Some(dispatch_started_at) = dispatch_started_at else {
7134 return;
7135 };
7136 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7138 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7139 if let Some(timings) = route_open_timings {
7140 warn!(
7141 op = %op,
7142 connection_id = connection_id.get(),
7143 corr,
7144 elapsed_ms = elapsed.as_millis() as u64,
7145 module_id = ?timings.module_id,
7146 principal = %timings.principal,
7147 admission_ms = timings.admission.as_millis() as u64,
7148 project_root_ms = timings.project_root.as_millis() as u64,
7149 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7150 bind_send_ms = timings.bind_send.as_millis() as u64,
7151 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7152 "slow control dispatch"
7153 );
7154 return;
7155 }
7156 warn!(
7157 op = %op,
7158 connection_id = connection_id.get(),
7159 corr,
7160 elapsed_ms = elapsed.as_millis() as u64,
7161 "slow control dispatch"
7162 );
7163 }
7164}
7165
7166#[cfg(test)]
7168type TestCallerLookup =
7169 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7170
7171fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7173 match request {
7174 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7175 Some(PrivilegedOperation::Restart { module_id })
7176 }
7177 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7178 Some(PrivilegedOperation::Swap { module_id })
7179 }
7180 ClientControlRequest::SupervisorReload { module_id } => {
7181 Some(PrivilegedOperation::Reload { module_id })
7182 }
7183 ClientControlRequest::SupervisorRescan { preview } => {
7184 if *preview {
7185 None
7186 } else {
7187 Some(PrivilegedOperation::Rescan)
7188 }
7189 }
7190 ClientControlRequest::SupervisorRescanChecked {
7195 preview,
7196 candidate,
7197 expect,
7198 } => (!*preview && candidate.is_none() && expect.is_some())
7199 .then_some(PrivilegedOperation::Rescan),
7200 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7201 Some(PrivilegedOperation::SetEnabled {
7202 module_id,
7203 enabled: *enabled,
7204 })
7205 }
7206 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7207 Some(PrivilegedOperation::ReleaseReserved { module_id })
7208 }
7209 ClientControlRequest::ServerDescribe {}
7210 | ClientControlRequest::CatalogList { .. }
7211 | ClientControlRequest::RouteOpen { .. }
7212 | ClientControlRequest::RoutePoll { .. }
7213 | ClientControlRequest::SupervisorList {}
7214 | ClientControlRequest::SupervisorHealth {}
7215 | ClientControlRequest::SupervisorHealthProbe { .. }
7216 | ClientControlRequest::SupervisorRoutes { .. }
7217 | ClientControlRequest::SupervisorProvenance { .. }
7218 | ClientControlRequest::SupervisorStderrTail { .. }
7219 | ClientControlRequest::SupervisorTerminals { .. }
7220 | ClientControlRequest::SupervisorSpawnSnapshot {}
7221 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7222 }
7223}
7224
7225fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7226 match request {
7227 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7228 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7229 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7230 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7231 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7232 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7233 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7234 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7235 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7236 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7237 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7238 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7239 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7240 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7241 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7242 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7243 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7244 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7245 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7246 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7247 }
7248}
7249
7250fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7256 use subc_protocol::scope::ScopeKind;
7257 const MAX_REF_CHARS: usize = 64;
7258 let kind = match stamp.kind {
7259 ScopeKind::Head => "head",
7260 ScopeKind::Worker => "worker",
7261 ScopeKind::Ephemeral => "ephemeral",
7262 };
7263 let mut scope_ref: String = stamp.scope_ref.chars().take(MAX_REF_CHARS).collect();
7264 if stamp.scope_ref.chars().nth(MAX_REF_CHARS).is_some() {
7265 scope_ref.push('…');
7266 }
7267 format!(
7268 "{kind}/{}/{scope_ref:?}@{}",
7269 principal_label(&stamp.owner),
7270 stamp.scope_epoch
7271 )
7272}
7273
7274fn principal_label(principal: &Principal) -> String {
7275 match principal {
7276 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7277 Principal::Direct => "direct".to_string(),
7278 other => format!("{other:?}"),
7279 }
7280}
7281
7282fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7283 match request {
7284 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7285 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7286 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7287 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7288 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7289 }
7290}
7291
7292fn parse_client_control_request(
7293 body: &[u8],
7294) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7295 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7296 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7297 Ok(probe)
7298 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7299 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7300 {
7301 ControlRequestBodyError::InvalidBody
7302 }
7303 Ok(_) => ControlRequestBodyError::UnknownOp,
7304 Err(_) => ControlRequestBodyError::InvalidBody,
7305 };
7306 (err, classification)
7307 })
7308}
7309
7310fn parse_module_control_request_from_module(
7311 body: &[u8],
7312) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7313 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7314 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7315 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7316 Ok(_) => ControlRequestBodyError::UnknownOp,
7317 Err(_) => ControlRequestBodyError::InvalidBody,
7318 };
7319 (err, classification)
7320 })
7321}
7322
7323#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7324enum ProviderRoleKind {
7325 ToolProvider,
7326 PipelineStage,
7327 ManagementSurface,
7328 InternalService,
7329}
7330
7331fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7332 match role {
7333 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7334 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7335 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7336 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7337 }
7338}
7339
7340fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7341 roles.iter().map(provider_role_kind).collect()
7342}
7343
7344const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7347
7348#[derive(Debug, Default, PartialEq, Eq)]
7350struct ScopeOutcomeCounts {
7351 created: usize,
7352 replaced: usize,
7353 updated: usize,
7354 unchanged: usize,
7355 refused: usize,
7356}
7357
7358impl ScopeOutcomeCounts {
7359 fn of(results: &[ScopeRecordResult]) -> Self {
7360 let mut counts = Self::default();
7361 for result in results {
7362 let slot = match result.outcome {
7363 ScopeRecordOutcome::Created => &mut counts.created,
7364 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7365 ScopeRecordOutcome::Updated => &mut counts.updated,
7366 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7367 ScopeRecordOutcome::Refused => &mut counts.refused,
7368 };
7369 *slot += 1;
7370 }
7371 counts
7372 }
7373}
7374
7375#[cfg(test)]
7376mod scope_outcome_count_tests {
7377 use super::*;
7378
7379 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7380 ScopeRecordResult {
7381 scope_ref: "r".to_string(),
7382 scope_epoch: 1,
7383 outcome,
7384 code: None,
7385 message: None,
7386 version: None,
7387 parent_state: None,
7388 }
7389 }
7390
7391 #[test]
7394 fn every_outcome_is_counted_in_its_own_field() {
7395 let results = [
7396 result(ScopeRecordOutcome::Created),
7397 result(ScopeRecordOutcome::Created),
7398 result(ScopeRecordOutcome::Replaced),
7399 result(ScopeRecordOutcome::Updated),
7400 result(ScopeRecordOutcome::Unchanged),
7401 result(ScopeRecordOutcome::Refused),
7402 result(ScopeRecordOutcome::Refused),
7403 result(ScopeRecordOutcome::Refused),
7404 ];
7405 assert_eq!(
7406 ScopeOutcomeCounts::of(&results),
7407 ScopeOutcomeCounts {
7408 created: 2,
7409 replaced: 1,
7410 updated: 1,
7411 unchanged: 1,
7412 refused: 3,
7413 }
7414 );
7415 }
7416}
7417
7418fn capability_census_trigger(
7422 old: Option<&CapabilityDeclarations>,
7423 new: Option<&CapabilityDeclarations>,
7424) -> bool {
7425 let old_provides = old
7426 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7427 .unwrap_or_default();
7428 let old_denies = old
7429 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7430 .unwrap_or_default();
7431 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7432 provides: Vec::new(),
7433 requires: Vec::new(),
7434 must_never_reach: Vec::new(),
7435 });
7436
7437 new.provides
7438 .iter()
7439 .any(|capability| !old_provides.contains(capability))
7440 || new
7441 .must_never_reach
7442 .iter()
7443 .any(|capability| !old_denies.contains(capability))
7444}
7445
7446fn denied_capability<'a>(
7449 opening_manifest: &'a ModuleManifest,
7450 target_manifest: &ModuleManifest,
7451) -> Option<&'a str> {
7452 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7453 let target_capabilities = target_manifest.capabilities.as_ref()?;
7454 opening_capabilities
7455 .must_never_reach
7456 .iter()
7457 .find(|denied| {
7458 target_capabilities
7459 .provides
7460 .iter()
7461 .any(|provided| provided == *denied)
7462 })
7463 .map(String::as_str)
7464}
7465
7466fn catalog_update_frozen_field_message(
7467 registered: &ModuleManifest,
7468 provides: &[ProviderRole],
7469) -> Option<String> {
7470 let old_has_provides = !registered.provides.is_empty();
7471 let new_has_provides = !provides.is_empty();
7472 if old_has_provides != new_has_provides {
7473 return Some(format!(
7474 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7475 registered.module_id
7476 ));
7477 }
7478
7479 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7480 return Some(format!(
7481 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7482 registered.module_id
7483 ));
7484 }
7485
7486 let registered_concurrency = manifest_concurrency(registered);
7487 let mut candidate = registered.clone();
7488 candidate.provides = provides.to_vec();
7489 let candidate_concurrency = manifest_concurrency(&candidate);
7490 if candidate_concurrency != registered_concurrency {
7491 return Some(format!(
7492 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7493 registered.module_id, registered_concurrency, candidate_concurrency
7494 ));
7495 }
7496
7497 None
7500}
7501
7502fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7503 manifest.provides.iter().any(is_routable_role)
7504}
7505
7506fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7512 manifest
7513 .provides
7514 .iter()
7515 .find_map(|provider| match provider {
7516 ProviderRole::ToolProvider { concurrency, .. }
7517 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7518 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7519 })
7520 .unwrap_or(Concurrency::ModuleManaged)
7521}
7522
7523fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7530 let has_management_surface = manifest
7531 .provides
7532 .iter()
7533 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7534 if !has_management_surface {
7535 return false;
7536 }
7537 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7538 return false;
7539 };
7540 let Some(provides) = raw
7541 .get("manifest")
7542 .and_then(|manifest| manifest.get("provides"))
7543 .and_then(serde_json::Value::as_array)
7544 else {
7545 return false;
7546 };
7547 provides.iter().any(|role| {
7552 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7553 && role.get("concurrency").is_none()
7554 })
7555}
7556
7557fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7558 if peer_version != PROTOCOL_VERSION {
7559 return Err(format!(
7560 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7561 ));
7562 }
7563 Ok(PROTOCOL_VERSION)
7564}
7565
7566fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7567 Frame::build_with_version(
7568 response_version(frame),
7569 FrameType::Pong,
7570 frame.header.flags,
7571 0,
7572 0,
7573 frame.header.corr,
7574 Vec::new(),
7575 )
7576 .map_err(RouterError::FrameBuild)
7577}
7578
7579pub(crate) fn control_error_frame(
7580 frame: &Frame,
7581 code: &'static str,
7582 message: impl Into<String>,
7583) -> Result<Frame, RouterError> {
7584 control_error_body_frame(
7585 frame,
7586 ErrorBody {
7587 code: code.to_string(),
7588 message: message.into(),
7589 detail: None,
7590 },
7591 )
7592}
7593
7594fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7595 let body = serde_json::to_vec(&error).map_err(|err| {
7596 RouterError::backend(
7597 0,
7598 frame.header.corr,
7599 format!("failed to encode control ERROR: {err}"),
7600 )
7601 })?;
7602
7603 Frame::build_with_version(
7604 response_version(frame),
7605 FrameType::Error,
7606 control_flags(),
7607 0,
7608 0,
7609 frame.header.corr,
7610 body,
7611 )
7612 .map_err(RouterError::FrameBuild)
7613}
7614
7615fn control_response_body_frame<T: Serialize>(
7616 frame: &Frame,
7617 reply: &T,
7618 label: &'static str,
7619) -> Result<Frame, RouterError> {
7620 let body = serde_json::to_vec(reply).map_err(|err| {
7621 RouterError::backend(
7622 0,
7623 frame.header.corr,
7624 format!("failed to encode {label}: {err}"),
7625 )
7626 })?;
7627
7628 Frame::build_with_version(
7629 response_version(frame),
7630 FrameType::Response,
7631 control_flags(),
7632 0,
7633 0,
7634 frame.header.corr,
7635 body,
7636 )
7637 .map_err(RouterError::FrameBuild)
7638}
7639
7640fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7670 match err {
7671 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7672 ForwardingError::NoModuleConnection => "target_unavailable",
7673 ForwardingError::ModuleReloading { .. } => "module_reloading",
7674 ForwardingError::ClientRouteChannelExhausted { .. }
7675 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7676 ForwardingError::StaleModuleEndpoint
7677 | ForwardingError::UnknownReservation { .. }
7678 | ForwardingError::ConnectionClosing { .. }
7679 | ForwardingError::ClientEgressClosed { .. }
7680 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7681 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7684 ForwardingError::RelayCorrelationExhausted
7685 | ForwardingError::RouteOpenBuild(_)
7686 | ForwardingError::Poisoned => "forwarding_error",
7687 }
7688}
7689
7690fn response_version(frame: &Frame) -> u8 {
7691 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7692 frame.header.ver
7693 } else {
7694 PROTOCOL_VERSION
7695 }
7696}
7697
7698fn control_flags() -> Flags {
7699 Flags::new(false, Priority::Passive, false)
7700}
7701
7702fn send_goodbye_target_best_effort(
7707 counters: &DaemonCounters,
7708 target: &GoodbyeTarget,
7709 context: &'static str,
7710) {
7711 let Ok(frame) = Frame::build_with_version(
7712 target.negotiated_ver,
7713 FrameType::Goodbye,
7714 control_flags(),
7715 target.channel,
7716 target.epoch,
7717 0,
7718 Vec::new(),
7719 ) else {
7720 return;
7721 };
7722 crate::forwarding::send_module_route_goodbye(
7723 counters,
7724 &target.sink,
7725 frame,
7726 target.module_id.as_deref(),
7727 context,
7728 );
7729}
7730
7731pub(crate) fn send_route_control_pushes(
7732 forwarding: &ForwardingTable,
7733 routes: Vec<EndpointRoute>,
7734 push: ClientControlPush,
7735) {
7736 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7737 for route in routes {
7738 let target = route.goodbye_target;
7739 if let Some((existing, channels)) = targets
7740 .iter_mut()
7741 .find(|(existing, _)| existing.connection_id == target.connection_id)
7742 {
7743 debug_assert_eq!(
7744 existing.negotiated_ver, target.negotiated_ver,
7745 "one connection cannot negotiate multiple frame versions"
7746 );
7747 if !channels.contains(&target.channel) {
7748 channels.push(target.channel);
7749 }
7750 continue;
7751 }
7752 let channel = target.channel;
7753 targets.push((target, vec![channel]));
7754 }
7755 for (target, mut channels) in targets {
7756 channels.sort_unstable();
7757 let mut push = push.clone();
7758 match &mut push {
7759 ClientControlPush::RouteClosing {
7760 channels: covered, ..
7761 }
7762 | ClientControlPush::RouteClosed {
7763 channels: covered, ..
7764 } => *covered = channels,
7765 }
7766 let body = match serde_json::to_vec(&push) {
7767 Ok(body) => body,
7768 Err(err) => {
7769 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7770 continue;
7771 }
7772 };
7773 let frame = match Frame::build_with_version(
7774 target.negotiated_ver,
7775 FrameType::Push,
7776 control_flags(),
7777 0,
7778 0,
7779 0,
7780 body.clone(),
7781 ) {
7782 Ok(frame) => frame,
7783 Err(err) => {
7784 warn!(
7785 route_channel = target.channel,
7786 error = %err,
7787 "failed to build route lifecycle control PUSH frame"
7788 );
7789 continue;
7790 }
7791 };
7792 if let Err(err) = target.sink.try_send(frame) {
7793 if target.close_on_delivery_failure() {
7794 warn!(
7795 target_connection_id = target.connection_id.get(),
7796 route_channel = target.channel,
7797 error = %err,
7798 "route lifecycle control PUSH was not delivered to client; closing target connection"
7799 );
7800 let _ = forwarding.escalate_client_delivery_failure(
7801 target.connection_id,
7802 target.channel,
7803 target.epoch,
7804 CloseReason::new(
7805 "route_lifecycle_push_delivery_failed",
7806 format!(
7807 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7808 target.channel
7809 ),
7810 ),
7811 crate::forwarding::UndeliveredFrame {
7812 module_id: target.module_id.as_deref(),
7813 sink: &target.sink,
7814 },
7815 );
7816 }
7817 }
7818 }
7819}
7820
7821#[cfg(test)]
7822mod tests {
7823 mod activity_read {
7824 include!("activity_read_tests.rs");
7825 }
7826
7827 mod lifecycle_activity {
7828 include!("control_lifecycle_tests.rs");
7829 }
7830 #[cfg(unix)]
7831 #[tokio::test]
7832 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7833 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7834 let path = dir.join("subc.jsonc");
7835 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7836 "program":"/bin/sleep","args":["60"],"protocol":"none",
7837 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7838 }}}).to_string()).unwrap();
7839 let mut configured = crate::daemon_config::load(&path)
7840 .unwrap()
7841 .unwrap()
7842 .modules
7843 .pop()
7844 .unwrap();
7845 let registry = std::sync::Arc::new(crate::Registry::default());
7846 let handle = crate::SupervisorHandle::new();
7847 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7848 .with_handle(handle.clone());
7849 let module = supervisor
7850 .supervise_configured_with_health(
7851 configured.module_spec(),
7852 true,
7853 configured.health.clone(),
7854 None,
7855 configured.restart,
7856 )
7857 .unwrap();
7858 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7859 let before = module.status().unwrap().pid;
7860 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7861 configured.health.cadence = std::time::Duration::from_secs(3600);
7862 let health_only = handler
7863 .reconcile_supervised_modules(
7864 &supervisor,
7865 vec![configured.clone()],
7866 false,
7867 Principal::Direct,
7868 )
7869 .await
7870 .unwrap();
7871 assert!(
7872 health_only.0.changed_pending_reload.is_empty(),
7873 "health policy is already applied live"
7874 );
7875 assert_eq!(module.status().unwrap().pid, before);
7876 assert_eq!(
7877 module.configuration().unwrap().1.http,
7878 configured.health.http
7879 );
7880 configured.args = vec!["61".into()];
7881 let launch = handler
7882 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7883 .await
7884 .unwrap();
7885 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
7886 assert_eq!(
7887 module.status().unwrap().pid,
7888 before,
7889 "a launch edit is stored until reload"
7890 );
7891 module.drain().await.unwrap();
7892 }
7893 use cortexkit_test_support::ScratchDir;
7894 use std::{
7895 collections::BTreeMap,
7896 fmt,
7897 path::PathBuf,
7898 sync::{Arc, Mutex},
7899 time::Duration,
7900 };
7901
7902 use serde_json::{json, Value};
7903 use subc_protocol::{
7904 manifest::{
7905 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7906 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7907 },
7908 session::HealthStatus,
7909 FrameType,
7910 };
7911
7912 use super::*;
7913 use crate::{
7914 forwarding::{DataRoute, DataRouteState},
7915 registry::ChannelState,
7916 router::FrameSink,
7917 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7918 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7919 RouteCtx, Router,
7920 };
7921 use tokio::{
7922 sync::mpsc,
7923 time::{sleep, Instant},
7924 };
7925 use tracing::{
7926 field::{Field, Visit},
7927 Event, Subscriber,
7928 };
7929 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7930
7931 fn fake_aft_stub_path() -> PathBuf {
7950 let mut path = std::env::current_exe().expect("current_exe available in tests");
7951 path.pop(); path.pop(); path.push(if cfg!(windows) {
7954 "fake-aft-stub.exe"
7955 } else {
7956 "fake-aft-stub"
7957 });
7958 assert!(
7959 path.exists(),
7960 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7961 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7962 path.display()
7963 );
7964 path
7965 }
7966
7967 fn client_retries(code: &str) -> bool {
7972 subc_protocol::error_codes::is_retryable_route_open(code)
7973 }
7974
7975 #[test]
7986 fn retryability_of_forwarding_codes_matches_the_failure() {
7987 let transient = [
7990 ForwardingError::NoModuleConnection,
7991 ForwardingError::ModuleReloading {
7992 module_id: "m".into(),
7993 },
7994 ForwardingError::StaleModuleEndpoint,
7995 ForwardingError::UnknownReservation {
7996 client_channel: 1,
7997 module_channel: 1,
7998 },
7999 ForwardingError::ConnectionClosing {
8000 connection_id: ConnectionId::new(1),
8001 },
8002 ForwardingError::ClientEgressClosed {
8003 connection_id: ConnectionId::new(1),
8004 },
8005 ForwardingError::ModuleEgressUnavailable {
8006 connection_id: ConnectionId::new(1),
8007 },
8008 ];
8009 for err in transient {
8010 let code = forwarding_error_code(&err);
8011 assert!(
8012 client_retries(code),
8013 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8014 );
8015 }
8016
8017 let permanent = [
8022 ForwardingError::ConnectionRoleConflict {
8023 connection_id: ConnectionId::new(1),
8024 },
8025 ForwardingError::ClientRouteChannelExhausted {
8026 connection_id: ConnectionId::new(1),
8027 },
8028 ForwardingError::ModuleRouteChannelExhausted {
8029 endpoint: ModuleEndpointId {
8030 connection_id: ConnectionId::new(1),
8031 generation: 1,
8032 },
8033 },
8034 ForwardingError::RelayCorrelationExhausted,
8035 ForwardingError::RouteOpenBuild("x".into()),
8036 ForwardingError::Poisoned,
8037 ];
8038 for err in permanent {
8039 let code = forwarding_error_code(&err);
8040 assert!(
8041 !client_retries(code),
8042 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8043 );
8044 }
8045 }
8046
8047 #[tokio::test]
8075 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8076 let handler = ControlHandler::default();
8077 let frame =
8078 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8079
8080 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8084 assert_eq!(
8085 stamped,
8086 Principal::Direct,
8087 "a caller that proved nothing must not be stamped as a supervised module"
8088 );
8089
8090 let forged = handler
8095 .route_open_principal(
8096 &frame,
8097 Some(ConsumerIdentity {
8098 module_id: "aft".to_string(),
8099 launch_nonce: "not-a-real-nonce".to_string(),
8100 }),
8101 )
8102 .unwrap();
8103 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8104 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8105 }
8106
8107 #[test]
8124 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8125 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8126 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8127 let ClientControlRequest::RouteOpen {
8128 consumer_identity, ..
8129 } = parsed
8130 else {
8131 panic!("route.open body must parse as RouteOpen");
8132 };
8133 assert_eq!(
8134 consumer_identity,
8135 Some(ConsumerIdentity {
8136 module_id: "aft".to_string(),
8137 launch_nonce: "n".to_string(),
8138 }),
8139 "the wire field name must reach the value route_open_principal reads"
8140 );
8141 }
8142
8143 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8144 ModuleManifest::builder(module_id, "0.1.0")
8145 .protocol_ver(protocol_ver)
8146 .provides(vec![ProviderRole::ToolProvider {
8147 tools: vec![Tool {
8148 name: "read".to_string(),
8149 description: None,
8150 execution_mode: ExecutionMode::Pure,
8151 schema: json!({"type": "object"}),
8152 }],
8153 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8154 concurrency: Concurrency::ModuleManaged,
8155 emits_push: true,
8156 sub_supervises: true,
8157 }])
8158 .build()
8159 }
8160
8161 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8162 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8163 }
8164
8165 fn hello_frame_with_control_ops(
8166 module_id: &str,
8167 protocol_ver: u8,
8168 corr: u64,
8169 control_ops: Option<Vec<String>>,
8170 ) -> Frame {
8171 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8172 }
8173
8174 fn hello_frame_with_nonce(
8175 module_id: &str,
8176 protocol_ver: u8,
8177 corr: u64,
8178 launch_nonce: Option<&str>,
8179 ) -> Frame {
8180 hello_frame_full(
8181 module_id,
8182 protocol_ver,
8183 corr,
8184 None,
8185 launch_nonce.map(ToOwned::to_owned),
8186 )
8187 }
8188
8189 fn hello_frame_full(
8190 module_id: &str,
8191 protocol_ver: u8,
8192 corr: u64,
8193 control_ops: Option<Vec<String>>,
8194 launch_nonce: Option<String>,
8195 ) -> Frame {
8196 let body = serde_json::to_vec(&ModuleHelloBody {
8197 manifest: manifest(module_id, protocol_ver),
8198 protocol_ver,
8199 control_ops,
8200 launch_nonce,
8201 })
8202 .unwrap();
8203 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8204 }
8205
8206 fn hello_frame_with_rich_ui(
8207 module_id: &str,
8208 protocol_ver: u8,
8209 corr: u64,
8210 launch_nonce: Option<&str>,
8211 rich_ui: Value,
8212 ) -> Frame {
8213 let mut body = serde_json::to_value(ModuleHelloBody {
8214 manifest: manifest(module_id, protocol_ver),
8215 protocol_ver,
8216 control_ops: None,
8217 launch_nonce: launch_nonce.map(str::to_string),
8218 })
8219 .unwrap();
8220 body["manifest"]["rich_ui"] = rich_ui;
8221 Frame::build(
8222 FrameType::Hello,
8223 control_flags(),
8224 0,
8225 0,
8226 corr,
8227 serde_json::to_vec(&body).unwrap(),
8228 )
8229 .unwrap()
8230 }
8231
8232 fn non_routable_hello_frame_with_control_ops(
8233 module_id: &str,
8234 corr: u64,
8235 control_ops: Option<Vec<String>>,
8236 ) -> Frame {
8237 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8238 manifest.provides.clear();
8239 let body = serde_json::to_vec(&ModuleHelloBody {
8240 manifest,
8241 protocol_ver: PROTOCOL_VERSION,
8242 control_ops,
8243 launch_nonce: None,
8244 })
8245 .unwrap();
8246 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8247 }
8248
8249 fn capability_grammar_hello_frame(
8250 capabilities: Value,
8251 runtime_computed: Option<Value>,
8252 corr: u64,
8253 ) -> Frame {
8254 let mut body = serde_json::to_value(ModuleHelloBody {
8255 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8256 protocol_ver: PROTOCOL_VERSION,
8257 control_ops: None,
8258 launch_nonce: None,
8259 })
8260 .expect("HELLO body serializes");
8261 body["manifest"]["capabilities"] = capabilities;
8262 if let Some(runtime_computed) = runtime_computed {
8263 body["runtime_computed"] = runtime_computed;
8264 }
8265 Frame::build(
8266 FrameType::Hello,
8267 control_flags(),
8268 0,
8269 0,
8270 corr,
8271 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8272 )
8273 .expect("HELLO frame builds")
8274 }
8275
8276 fn channel_request(channel: u16, corr: u64) -> Frame {
8277 Frame::build(
8278 FrameType::Request,
8279 Flags::new(true, Priority::Interactive, false),
8280 channel,
8281 0,
8282 corr,
8283 b"opaque".to_vec(),
8284 )
8285 .unwrap()
8286 }
8287
8288 fn route_ctx(
8289 connection_id: ConnectionId,
8290 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8291 let (tx, rx) = mpsc::channel(8);
8292 (
8293 RouteCtx {
8294 connection_id,
8295 egress: FrameSink::new(tx),
8296 },
8297 rx,
8298 )
8299 }
8300
8301 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8302 serde_json::from_slice(&frame.body).unwrap()
8303 }
8304
8305 async fn hello_via_sink(
8310 handler: &ControlHandler,
8311 ctx: &RouteCtx,
8312 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8313 hello: Frame,
8314 ) -> Frame {
8315 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8316 assert!(
8317 replies.is_empty(),
8318 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8319 );
8320 let ack = rx
8321 .try_recv()
8322 .expect("HELLO_ACK is queued on the module sink")
8323 .frame;
8324 assert_eq!(ack.header.ty, FrameType::HelloAck);
8325 ack
8326 }
8327
8328 fn parse_error(frame: &Frame) -> Value {
8329 serde_json::from_slice(&frame.body).unwrap()
8330 }
8331
8332 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8333 serde_json::from_slice(&frame.body).unwrap()
8334 }
8335
8336 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8337 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8338 route_channel,
8339 route_epoch: 0,
8340 kind,
8341 })
8342 .unwrap();
8343 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8344 }
8345
8346 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8347 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8348 module_id: module_id.to_string(),
8349 })
8350 .unwrap();
8351 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8352 }
8353
8354 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8355 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8356 }
8357
8358 fn route_open_frame_with_consumer_capabilities(
8359 corr: u64,
8360 module_id: &str,
8361 project_root: ScratchDir,
8362 consumer_capabilities: Option<Vec<String>>,
8363 ) -> Frame {
8364 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8365 target: RouteTarget::ToolProvider {
8366 module_id: module_id.to_string(),
8367 },
8368 identity: BindIdentity::new(
8369 project_root.path().to_path_buf(),
8370 "unit".to_string(),
8371 "session".to_string(),
8372 ),
8373 consumer_identity: None,
8374 consumer_capabilities,
8375 role_versions: None,
8376 admission_facts: None,
8377 scope: None,
8378 })
8379 .unwrap();
8380 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8381 }
8382
8383 fn route_open_frame_with_role_versions(
8384 corr: u64,
8385 module_id: &str,
8386 project_root: ScratchDir,
8387 role_versions: Option<BTreeMap<String, String>>,
8388 ) -> Frame {
8389 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8390 target: RouteTarget::ToolProvider {
8391 module_id: module_id.to_string(),
8392 },
8393 identity: BindIdentity::new(
8394 project_root.path().to_path_buf(),
8395 "unit".to_string(),
8396 format!("session-{corr}"),
8397 ),
8398 consumer_identity: None,
8399 consumer_capabilities: None,
8400 role_versions,
8401 admission_facts: None,
8402 scope: None,
8403 })
8404 .unwrap();
8405 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8406 }
8407
8408 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8409 entries
8410 .iter()
8411 .map(|(role, version)| (role.to_string(), version.to_string()))
8412 .collect()
8413 }
8414
8415 fn route_open_frame_with_admission_facts(
8416 corr: u64,
8417 module_id: &str,
8418 project_root: ScratchDir,
8419 consumer_identity: Option<subc_control::ConsumerIdentity>,
8420 facts: Option<Value>,
8421 ) -> Frame {
8422 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8423 target: RouteTarget::ToolProvider {
8424 module_id: module_id.to_string(),
8425 },
8426 identity: BindIdentity::new(
8427 project_root.path().to_path_buf(),
8428 "unit".to_string(),
8429 format!("session-{corr}"),
8430 ),
8431 consumer_identity,
8432 consumer_capabilities: None,
8433 role_versions: None,
8434 admission_facts: facts,
8435 scope: None,
8436 })
8437 .unwrap();
8438 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8439 }
8440
8441 #[derive(Clone, Default)]
8442 struct EventCapture {
8443 events: Arc<Mutex<Vec<CapturedEvent>>>,
8444 }
8445
8446 #[derive(Clone, Debug)]
8447 struct CapturedEvent {
8448 target: String,
8449 level: tracing::Level,
8450 fields: BTreeMap<String, String>,
8451 }
8452
8453 impl EventCapture {
8454 fn events(&self) -> Vec<CapturedEvent> {
8455 self.events.lock().unwrap().clone()
8456 }
8457 }
8458
8459 impl<S> Layer<S> for EventCapture
8460 where
8461 S: Subscriber,
8462 {
8463 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8464 let mut visitor = EventFieldVisitor::default();
8465 event.record(&mut visitor);
8466 self.events.lock().unwrap().push(CapturedEvent {
8467 target: event.metadata().target().to_string(),
8468 level: *event.metadata().level(),
8469 fields: visitor.fields,
8470 });
8471 }
8472 }
8473
8474 #[derive(Default)]
8475 struct EventFieldVisitor {
8476 fields: BTreeMap<String, String>,
8477 }
8478
8479 impl Visit for EventFieldVisitor {
8480 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8481 self.fields
8482 .insert(field.name().to_string(), format!("{value:?}"));
8483 }
8484 }
8485
8486 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8487 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8488 status,
8489 detail: Some("warming".to_string()),
8490 metrics: Some(json!({"queue_depth": 3})),
8491 })
8492 .unwrap();
8493 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8494 }
8495
8496 fn route_bind_ack(corr: u64) -> Frame {
8497 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8498 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8499 }
8500
8501 fn unique_project_root(label: &str) -> ScratchDir {
8502 ScratchDir::new(label)
8503 }
8504
8505 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8506 match parse_route_poll(frame) {
8507 ClientControlResponse::RoutePoll {
8508 status: None,
8509 live: Some(live),
8510 ..
8511 } => assert_eq!(live, expected_live),
8512 other => panic!("unexpected route.poll response: {other:?}"),
8513 }
8514 }
8515
8516 fn bind_liveness_route(
8517 registry: &Registry,
8518 forwarding: &ForwardingTable,
8519 module_id: &str,
8520 ) -> (RouteCtx, u16, u32) {
8521 let module_connection = ConnectionId::new(101);
8522 let client_connection = ConnectionId::new(202);
8523 let registration = registry
8524 .register_with_control_ops(
8525 manifest(module_id, PROTOCOL_VERSION),
8526 PROTOCOL_VERSION,
8527 module_connection,
8528 module_baseline_control_ops(),
8529 )
8530 .unwrap();
8531 let (module_tx, _module_rx) = mpsc::channel(8);
8532 let endpoint = forwarding
8533 .register_module_connection(
8534 module_connection,
8535 module_id.to_string(),
8536 PROTOCOL_VERSION,
8537 manifest_concurrency(®istration.manifest),
8538 FrameSink::new(module_tx),
8539 )
8540 .unwrap();
8541 let (client_ctx, _client_rx) = route_ctx(client_connection);
8542 let pending = forwarding
8543 .begin_route_bind_relay_for_test(
8544 client_connection,
8545 client_ctx.egress.clone(),
8546 1,
8547 module_id,
8548 )
8549 .unwrap();
8550 assert_eq!(pending.endpoint, endpoint);
8551 let route_channel = pending.client_channel;
8552 let route_epoch = pending.client_epoch;
8553 forwarding
8554 .complete_pending_relay(
8555 module_connection,
8556 pending.corr,
8557 RouteBindRelayOutcome::Accepted,
8558 )
8559 .unwrap();
8560 (client_ctx, route_channel, route_epoch)
8561 }
8562
8563 struct FakeProcessLiveness {
8564 live: Option<bool>,
8565 }
8566
8567 impl ModuleProcessLiveness for FakeProcessLiveness {
8568 fn process_live(&self, _module_id: &str) -> Option<bool> {
8569 self.live
8570 }
8571 }
8572
8573 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8574 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8575 {
8576 let registry = Arc::new(Registry::default());
8577 let supervisor_handle = SupervisorHandle::new();
8578 let supervisor = Supervisor::new_for_test(
8579 Arc::clone(®istry),
8580 RestartPolicy::new(1, Duration::from_millis(10)),
8581 )
8582 .with_handle(supervisor_handle.clone());
8583 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8584 let module = supervisor
8585 .spawn(ModuleSpec {
8586 module_id: "stderr-tail-wire".to_string(),
8587 program: fake_aft_stub_path(),
8588 args: Vec::new(),
8589 env: vec![
8590 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8591 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8592 ],
8593 reserved: false,
8594 reserved_prefixes: Vec::new(),
8595 protocol: ModuleProtocol::Subc,
8596 overlap: Default::default(),
8597 })
8598 .unwrap();
8599
8600 let deadline = Instant::now() + Duration::from_secs(5);
8601 loop {
8602 let tail = module.stderr_tail(None, None);
8603 if tail
8604 .entries
8605 .iter()
8606 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8607 && tail.entries.iter().any(|entry| {
8608 matches!(
8609 entry,
8610 TailEntry::Line {
8611 truncated: true,
8612 ..
8613 }
8614 )
8615 })
8616 {
8617 break;
8618 }
8619 assert!(
8620 Instant::now() < deadline,
8621 "module did not produce a truncated line and restart boundary: {tail:?}"
8622 );
8623 sleep(Duration::from_millis(10)).await;
8624 }
8625
8626 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8627 let request = ClientControlRequest::SupervisorStderrTail {
8628 module_id: "stderr-tail-wire".to_string(),
8629 max_lines: None,
8630 max_bytes: None,
8631 };
8632 let frame = Frame::build(
8633 FrameType::Request,
8634 control_flags(),
8635 0,
8636 0,
8637 1,
8638 serde_json::to_vec(&request).unwrap(),
8639 )
8640 .unwrap();
8641 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8642 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8643 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8644 serde_json::from_slice(&responses[0].body).unwrap()
8645 else {
8646 panic!("expected supervisor.stderr_tail response");
8647 };
8648
8649 assert!(
8650 tail.entries
8651 .iter()
8652 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8653 "the control response lost the restart boundary"
8654 );
8655 let Some(StderrTailEntry::Line {
8656 text,
8657 truncated,
8658 at_ms,
8659 }) = tail.entries.iter().find(|entry| {
8660 matches!(
8661 entry,
8662 StderrTailEntry::Line {
8663 truncated: true,
8664 ..
8665 }
8666 )
8667 })
8668 else {
8669 panic!("the control response lost the truncated line");
8670 };
8671 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8672 assert!(*truncated);
8673 assert!(
8674 at_ms.is_some(),
8675 "the control response lost the line's capture time"
8676 );
8677 }
8678
8679 #[tokio::test(flavor = "current_thread")]
8684 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8685 let dir = ScratchDir::new("terminals-off-worker");
8686 let journal_path = dir.join("terminals.jsonl");
8687 let registry = Arc::new(Registry::default());
8688 let supervisor_handle = SupervisorHandle::new();
8689 let supervisor =
8690 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8691 .with_handle(supervisor_handle.clone())
8692 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8693 let module = supervisor
8694 .spawn(ModuleSpec {
8695 module_id: "terminal-off-worker".to_string(),
8696 program: fake_aft_stub_path(),
8697 args: Vec::new(),
8698 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8699 reserved: false,
8700 reserved_prefixes: Vec::new(),
8701 protocol: ModuleProtocol::Subc,
8702 overlap: Default::default(),
8703 })
8704 .unwrap();
8705 let deadline = Instant::now() + Duration::from_secs(5);
8706 while module.terminal_history().entries.len() != 2 {
8707 assert!(Instant::now() < deadline, "module did not record two exits");
8708 sleep(Duration::from_millis(10)).await;
8709 }
8710
8711 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8712 let handler =
8713 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8714 let frame = Frame::build(
8715 FrameType::Request,
8716 control_flags(),
8717 0,
8718 0,
8719 1,
8720 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8721 module_id: "terminal-off-worker".to_string(),
8722 })
8723 .unwrap(),
8724 )
8725 .unwrap();
8726 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8727 let spawned_at = std::time::Instant::now();
8728 let read = tokio::spawn({
8729 let handler = Arc::clone(&handler);
8730 async move { handler.handle_control_frame(&ctx, frame).await }
8731 });
8732 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8735 .await
8736 .unwrap()
8737 .expect("the history read reached its pause");
8738 let elapsed = spawned_at.elapsed();
8739 assert!(
8740 elapsed < Duration::from_secs(2) && !read.is_finished(),
8741 "this task could not run while the history read was paused \
8742 (resumed after {elapsed:?}, read finished: {})",
8743 read.is_finished()
8744 );
8745
8746 drop(release);
8747 let responses = read.await.unwrap().unwrap();
8748 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8749 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8750 panic!("expected supervisor.terminals response");
8751 };
8752 assert_eq!(terminals.entries.len(), 2);
8753 assert_eq!(terminals.journal_skipped_lines, 0);
8754 assert_eq!(terminals.journal_read_errors, 0);
8755 }
8756
8757 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8758 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8759 let registry = Arc::new(Registry::default());
8760 let supervisor_handle = SupervisorHandle::new();
8761 let supervisor =
8762 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8763 .with_handle(supervisor_handle.clone());
8764 let module = supervisor
8765 .spawn(ModuleSpec {
8766 module_id: "terminal-golden".to_string(),
8767 program: fake_aft_stub_path(),
8768 args: Vec::new(),
8769 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8770 reserved: false,
8771 reserved_prefixes: Vec::new(),
8772 protocol: ModuleProtocol::Subc,
8773 overlap: Default::default(),
8774 })
8775 .unwrap();
8776
8777 let deadline = Instant::now() + Duration::from_secs(5);
8778 while module.terminal_history().entries.len() != 2 {
8779 assert!(
8780 Instant::now() < deadline,
8781 "module did not retain two terminal exits: {:?}",
8782 module.terminal_history()
8783 );
8784 sleep(Duration::from_millis(10)).await;
8785 }
8786
8787 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8788 let request = ClientControlRequest::SupervisorTerminals {
8789 module_id: "terminal-golden".to_string(),
8790 };
8791 let frame = Frame::build(
8792 FrameType::Request,
8793 control_flags(),
8794 0,
8795 0,
8796 1,
8797 serde_json::to_vec(&request).unwrap(),
8798 )
8799 .unwrap();
8800 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8801 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8802 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8803 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8804 panic!("expected supervisor.terminals response");
8805 };
8806 assert_eq!(terminals.entries.len(), 2);
8807 assert_eq!(terminals.dropped, 0);
8808
8809 let mut rendered = serde_json::to_value(response).unwrap();
8810 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8813 for (index, entry) in rendered["entries"]
8814 .as_array_mut()
8815 .expect("terminal response entries array")
8816 .iter_mut()
8817 .enumerate()
8818 {
8819 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8820 }
8821
8822 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8823 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8824 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8825 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8826 std::fs::write(&golden_path, &serialized).unwrap();
8827 }
8828 let expected: Value =
8829 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8830 assert_eq!(rendered, expected);
8831 }
8832
8833 #[test]
8834 fn hello_registers_manifest_and_returns_ack() {
8835 let registry = Arc::new(Registry::default());
8836 let handler = ControlHandler::new(Arc::clone(®istry));
8837 let conn = ConnectionId::new(1);
8838
8839 let responses = handler
8840 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8841 .unwrap();
8842
8843 assert_eq!(responses.len(), 1);
8844 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8845 assert_eq!(responses[0].header.channel, 0);
8846 assert_eq!(responses[0].header.corr, 7);
8847 let ack = parse_ack(&responses[0]);
8848 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8849 assert!(ack
8850 .subc_capabilities
8851 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8852 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8853 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8854 assert!(ack
8855 .subc_ops
8856 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8857 assert!(ack
8858 .subc_ops
8859 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8860
8861 let registration = registry.get_module("aft").unwrap().unwrap();
8862 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8863 assert_eq!(registration.state, ChannelState::Active);
8864 assert_eq!(registration.connection_id, conn);
8865 assert_eq!(registration.control_ops, module_baseline_control_ops());
8866 }
8867
8868 #[test]
8869 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8870 let invalid_identifiers = [
8871 ("case_change", "credentials-Provider/v1"),
8872 ("leading_zero", "credentials-provider/v01"),
8873 ("trailing_hyphen", "credentials-provider-/v1"),
8874 ("consecutive_hyphens", "credentials--provider/v1"),
8875 ("uppercase", "Credentials-provider/v1"),
8876 ("missing_v", "credentials-provider/1"),
8877 ("whitespace", "credentials provider/v1"),
8878 ("zero_version", "credentials-provider/v0"),
8879 ("out_of_range_version", "credentials-provider/v4294967296"),
8880 (
8881 "overlength_name",
8882 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8883 ),
8884 ];
8885 let mut cases = invalid_identifiers
8886 .into_iter()
8887 .map(|(name, identifier)| {
8888 (
8889 format!("identifier_{name}"),
8890 "capabilities.provides[0]".to_string(),
8891 identifier.to_string(),
8892 json!({ "provides": [identifier] }),
8893 None,
8894 )
8895 })
8896 .collect::<Vec<_>>();
8897 cases.extend([
8898 (
8899 "unknown_need".to_string(),
8900 "capabilities.requires[0].need".to_string(),
8901 "deferred".to_string(),
8902 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8903 None,
8904 ),
8905 (
8906 "duplicate_provides".to_string(),
8907 "capabilities.provides[1]".to_string(),
8908 "credentials-provider/v1".to_string(),
8909 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8910 None,
8911 ),
8912 (
8913 "duplicate_must_never_reach".to_string(),
8914 "capabilities.must_never_reach[1]".to_string(),
8915 "credentials-provider/v1".to_string(),
8916 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8917 None,
8918 ),
8919 (
8920 "duplicate_requires_same_need".to_string(),
8921 "capabilities.requires[1]".to_string(),
8922 "credentials-provider/v1".to_string(),
8923 json!({ "requires": [
8924 { "capability": "credentials-provider/v1", "need": "required" },
8925 { "capability": "credentials-provider/v1", "need": "required" }
8926 ] }),
8927 None,
8928 ),
8929 (
8930 "duplicate_requires_conflicting_need".to_string(),
8931 "capabilities.requires[1]".to_string(),
8932 "credentials-provider/v1".to_string(),
8933 json!({ "requires": [
8934 { "capability": "credentials-provider/v1", "need": "required" },
8935 { "capability": "credentials-provider/v1", "need": "optional" }
8936 ] }),
8937 None,
8938 ),
8939 (
8940 "capabilities_root_pointer".to_string(),
8941 "runtime_computed[0]".to_string(),
8942 "/capabilities".to_string(),
8943 json!({}),
8944 Some(json!(["/capabilities"])),
8945 ),
8946 (
8947 "capabilities_descendant_pointer".to_string(),
8948 "runtime_computed[0]".to_string(),
8949 "/capabilities/provides".to_string(),
8950 json!({}),
8951 Some(json!(["/capabilities/provides"])),
8952 ),
8953 (
8954 "malformed_pointer_without_leading_slash".to_string(),
8955 "runtime_computed[0]".to_string(),
8956 "capabilities".to_string(),
8957 json!({}),
8958 Some(json!(["capabilities"])),
8959 ),
8960 (
8961 "malformed_pointer_escape".to_string(),
8962 "runtime_computed[0]".to_string(),
8963 "/roles/~2/tools".to_string(),
8964 json!({}),
8965 Some(json!(["/roles/~2/tools"])),
8966 ),
8967 (
8968 "unknown_capabilities_field".to_string(),
8969 "capabilities.future".to_string(),
8970 "<array>".to_string(),
8971 json!({ "future": [] }),
8972 None,
8973 ),
8974 ]);
8975
8976 for (index, (name, field, value, capabilities, runtime_computed)) in
8977 cases.into_iter().enumerate()
8978 {
8979 let registry = Arc::new(Registry::default());
8980 let handler = ControlHandler::new(Arc::clone(®istry));
8981 let response = handler
8982 .handle_control(
8983 ConnectionId::new((index + 1) as u64),
8984 capability_grammar_hello_frame(
8985 capabilities,
8986 runtime_computed,
8987 index as u64 + 1,
8988 ),
8989 )
8990 .expect("invalid HELLO returns a refusal");
8991
8992 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8993 let error = parse_error(&response[0]);
8994 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8995 let message = error["message"]
8996 .as_str()
8997 .expect("error message is a string");
8998 assert!(
8999 message.contains(&field),
9000 "{name}: field missing from {message}"
9001 );
9002 assert!(
9003 message.contains(&value),
9004 "{name}: value missing from {message}"
9005 );
9006 assert_eq!(
9007 registry
9008 .active_registration_count()
9009 .expect("registry reads"),
9010 0,
9011 "{name}: refused HELLO must not create a catalog entry"
9012 );
9013 }
9014 }
9015
9016 #[test]
9017 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9018 let registry = Arc::new(Registry::default());
9019 let handler = ControlHandler::new(Arc::clone(®istry));
9020 let capabilities = json!({
9021 "provides": ["credentials-provider/v1"],
9022 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9023 "must_never_reach": ["federation-transport/v1"]
9024 });
9025 let response = handler
9026 .handle_control(
9027 ConnectionId::new(99),
9028 capability_grammar_hello_frame(
9029 capabilities.clone(),
9030 Some(json!(["/roles/0/tools"])),
9031 99,
9032 ),
9033 )
9034 .expect("valid HELLO registers");
9035 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9036
9037 let request = Frame::build(
9038 FrameType::Request,
9039 control_flags(),
9040 0,
9041 0,
9042 100,
9043 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9044 .expect("catalog request serializes"),
9045 )
9046 .expect("catalog request frame builds");
9047 let response = handler
9048 .handle_catalog_list(request, None)
9049 .expect("catalog list succeeds");
9050 let ClientControlResponse::CatalogList { modules, .. } =
9051 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9052 else {
9053 panic!("catalog request must return catalog.list");
9054 };
9055 assert_eq!(modules.len(), 1);
9056 assert_eq!(
9057 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9058 capabilities
9059 );
9060 }
9061
9062 #[test]
9063 fn catalog_list_mirrors_management_operation_description() {
9064 let registry = Arc::new(Registry::default());
9065 let handler = ControlHandler::new(Arc::clone(®istry));
9066 let description = "List managed records and return their identifiers and metadata.";
9067 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9068 manifest.provides = vec![ProviderRole::ManagementSurface {
9069 operations: vec![ManagementOperation {
9070 name: "records.list".to_string(),
9071 kind: ManagementOperationKind::Query,
9072 description: Some(description.to_string()),
9073 }],
9074 config_schema: json!({"type": "object"}),
9075 observability: vec![ObservabilitySurface {
9076 name: "records.stats".to_string(),
9077 kind: ObservabilityKind::Snapshot,
9078 }],
9079 identity_scope: vec![IdentityScope::Project],
9080 concurrency: Concurrency::ModuleManaged,
9081 }];
9082 registry
9083 .register_with_control_ops(
9084 manifest,
9085 PROTOCOL_VERSION,
9086 ConnectionId::new(99),
9087 Vec::new(),
9088 )
9089 .expect("described management manifest registers");
9090
9091 let request = Frame::build(
9092 FrameType::Request,
9093 control_flags(),
9094 0,
9095 0,
9096 100,
9097 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9098 .expect("catalog request serializes"),
9099 )
9100 .expect("catalog request frame builds");
9101 let response = handler
9102 .handle_catalog_list(request, None)
9103 .expect("catalog list succeeds");
9104 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9105 assert_eq!(
9106 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9107 "catalog.list must preserve the declared operation description verbatim"
9108 );
9109 }
9110
9111 #[test]
9112 fn catalog_list_mirrors_registered_operation_delivery() {
9113 let registry = Arc::new(Registry::default());
9114 let handler = ControlHandler::new(Arc::clone(®istry));
9115 let delivery =
9116 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9117 let mut body = serde_json::to_value(ModuleHelloBody {
9118 manifest: manifest("watch-management", PROTOCOL_VERSION),
9119 protocol_ver: PROTOCOL_VERSION,
9120 control_ops: None,
9121 launch_nonce: None,
9122 })
9123 .unwrap();
9124 body["manifest"]["provides"] = json!([{
9125 "role": "management_surface",
9126 "operations": [
9127 {"name": "status", "kind": "query"},
9128 {"name": "future", "kind": "query"},
9129 {"name": "reset", "kind": "mutate"}
9130 ],
9131 "config_schema": {}, "observability": [], "identity_scope": []
9132 }]);
9133 body["manifest"]["operation_delivery"] = delivery.clone();
9134 let hello = Frame::build(
9135 FrameType::Hello,
9136 control_flags(),
9137 0,
9138 0,
9139 99,
9140 serde_json::to_vec(&body).unwrap(),
9141 )
9142 .unwrap();
9143 let response = handler
9144 .handle_control(ConnectionId::new(99), hello)
9145 .unwrap();
9146 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9147
9148 let (_, registrations) = registry.list_modules().unwrap();
9149 let registered = ®istrations[0].manifest;
9150 assert!(registered.is_watch_operation("status"));
9151 assert!(!registered.is_watch_operation("future"));
9152 assert!(!registered.is_watch_operation("reset"));
9153 assert!(!registered.is_watch_operation("missing"));
9154
9155 let request = Frame::build(
9156 FrameType::Request,
9157 control_flags(),
9158 0,
9159 0,
9160 100,
9161 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9162 )
9163 .unwrap();
9164 let response = handler.handle_catalog_list(request, None).unwrap();
9165 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9166 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9167 let ClientControlResponse::CatalogList { modules, .. } =
9168 serde_json::from_slice(&response[0].body).unwrap()
9169 else {
9170 panic!("expected catalog.list");
9171 };
9172 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9173 }
9174
9175 #[test]
9176 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9177 let registry = Arc::new(Registry::default());
9178 let supervisor = SupervisorHandle::new();
9179 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9180 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9181 let descriptor = json!({
9182 "schema": "ckios.richui.manifest/1",
9183 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9184 "bytes": 4096,
9185 "chunks": 4
9186 });
9187 let connection = ConnectionId::new(201);
9188 let hello = handler
9189 .handle_control(
9190 connection,
9191 hello_frame_with_rich_ui(
9192 "vault",
9193 PROTOCOL_VERSION,
9194 201,
9195 Some("the-real-nonce"),
9196 descriptor.clone(),
9197 ),
9198 )
9199 .unwrap();
9200 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9201
9202 let request =
9203 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9204 let provides = registry
9205 .get_module_by_connection(connection)
9206 .unwrap()
9207 .unwrap()
9208 .manifest
9209 .provides;
9210 handler
9211 .handle_catalog_update(connection, request, provides, None, Some(false))
9212 .unwrap();
9213
9214 let request = Frame::build(
9215 FrameType::Request,
9216 control_flags(),
9217 0,
9218 0,
9219 203,
9220 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9221 )
9222 .unwrap();
9223 let response = handler.handle_catalog_list(request, None).unwrap();
9224 let ClientControlResponse::CatalogList { modules, .. } =
9225 serde_json::from_slice(&response[0].body).unwrap()
9226 else {
9227 panic!("expected catalog.list");
9228 };
9229 assert_eq!(
9230 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9231 descriptor
9232 );
9233 assert!(modules[0].nonce_verified);
9234 }
9235
9236 #[test]
9237 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9238 let registry = Arc::new(Registry::default());
9239 let handler = ControlHandler::new(registry.clone());
9240 let descriptor = json!({
9241 "schema": "ckios.richui.manifest/1",
9242 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9243 "bytes": 128,
9244 "chunks": 2
9245 });
9246 let malformed = [
9247 ("schema", json!("")),
9248 ("schema", json!("a".repeat(65))),
9249 ("schema", json!("Bad schema")),
9250 (
9251 "manifest_digest",
9252 json!(format!("sha256:{}", "A".repeat(64))),
9253 ),
9254 ("manifest_digest", json!("sha256:abc")),
9255 ("bytes", json!(0)),
9256 ("bytes", json!(16_777_217)),
9257 ("bytes", json!("128")),
9258 ("chunks", json!(0)),
9259 ("chunks", json!(1025)),
9260 ("chunks", json!("2")),
9261 ];
9262 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9263 let mut invalid = descriptor.clone();
9264 invalid[field] = invalid_value;
9265 let response = handler
9266 .handle_control(
9267 ConnectionId::new(210 + index as u64),
9268 hello_frame_with_rich_ui(
9269 &format!("bad-rich-ui-{index}"),
9270 PROTOCOL_VERSION,
9271 210 + index as u64,
9272 None,
9273 invalid,
9274 ),
9275 )
9276 .unwrap();
9277 assert_eq!(response[0].header.ty, FrameType::Error);
9278 let error = parse_error(&response[0]);
9279 assert_eq!(error["code"], "invalid_manifest");
9280 assert!(error["message"]
9281 .as_str()
9282 .unwrap()
9283 .contains(&format!("rich_ui.{field}")));
9284 assert!(registry
9285 .get_module(&format!("bad-rich-ui-{index}"))
9286 .unwrap()
9287 .is_none());
9288 }
9289 }
9290
9291 #[test]
9292 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9293 let registry = Arc::new(Registry::default());
9294 let handler = ControlHandler::new(Arc::clone(®istry));
9295 let connection = ConnectionId::new(219);
9296 registry
9297 .register_with_control_ops(
9298 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9299 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9300 "bad schema",
9301 format!("sha256:{}", "a".repeat(64)),
9302 128,
9303 2,
9304 )))
9305 .build(),
9306 PROTOCOL_VERSION,
9307 connection,
9308 Vec::new(),
9309 )
9310 .unwrap();
9311 let request =
9312 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9313 let response = handler
9314 .handle_catalog_update(connection, request, Vec::new(), None, None)
9315 .unwrap();
9316 assert_eq!(response[0].header.ty, FrameType::Error);
9317 let error = parse_error(&response[0]);
9318 assert_eq!(error["code"], "invalid_manifest");
9319 assert!(error["message"]
9320 .as_str()
9321 .unwrap()
9322 .contains("rich_ui.schema"));
9323 }
9324
9325 #[test]
9326 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9327 let registry = Arc::new(Registry::default());
9328 let supervisor = SupervisorHandle::new();
9329 supervisor.open_swap("provider", "candidate-nonce".to_string());
9330 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9331 let connection = ConnectionId::new(218);
9332 let hello = handler
9333 .handle_control(
9334 connection,
9335 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9336 )
9337 .unwrap();
9338 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9339 assert!(registry.list_modules().unwrap().1.is_empty());
9340
9341 registry.promote_candidate("provider").unwrap().unwrap();
9342 let request = Frame::build(
9343 FrameType::Request,
9344 control_flags(),
9345 0,
9346 0,
9347 219,
9348 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9349 )
9350 .unwrap();
9351 let response = handler.handle_catalog_list(request, None).unwrap();
9352 let ClientControlResponse::CatalogList { modules, .. } =
9353 serde_json::from_slice(&response[0].body).unwrap()
9354 else {
9355 panic!("expected catalog.list");
9356 };
9357 assert_eq!(modules.len(), 1);
9358 assert!(modules[0].nonce_verified);
9359 }
9360
9361 #[test]
9362 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9363 let registry = Arc::new(Registry::default());
9364 let handler = ControlHandler::new(Arc::clone(®istry));
9365 let hello = Frame::build(
9366 FrameType::Hello,
9367 control_flags(),
9368 0,
9369 0,
9370 220,
9371 serde_json::to_vec(&ModuleHelloBody {
9372 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9373 protocol_ver: PROTOCOL_VERSION,
9374 control_ops: None,
9375 launch_nonce: Some("unverified".to_string()),
9376 })
9377 .unwrap(),
9378 )
9379 .unwrap();
9380 assert_eq!(
9381 handler
9382 .handle_control(ConnectionId::new(220), hello)
9383 .unwrap()[0]
9384 .header
9385 .ty,
9386 FrameType::HelloAck
9387 );
9388 let request = Frame::build(
9389 FrameType::Request,
9390 control_flags(),
9391 0,
9392 0,
9393 221,
9394 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9395 )
9396 .unwrap();
9397 let response = handler.handle_catalog_list(request, None).unwrap();
9398 let ClientControlResponse::CatalogList { modules, .. } =
9399 serde_json::from_slice(&response[0].body).unwrap()
9400 else {
9401 panic!("expected catalog.list");
9402 };
9403 assert_eq!(
9404 serde_json::to_vec(&modules[0]).unwrap(),
9405 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9406 );
9407 assert!(!modules[0].nonce_verified);
9408 }
9409
9410 #[test]
9411 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9412 let registry = Arc::new(Registry::default());
9413 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9414 [("vault".to_string(), true), ("squatter".to_string(), true)],
9415 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9416 );
9417 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9418 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9419 provides: vec!["credentials-provider/v1".to_string()],
9420 requires: Vec::new(),
9421 must_never_reach: Vec::new(),
9422 });
9423 let frame = Frame::build(
9424 FrameType::Hello,
9425 control_flags(),
9426 0,
9427 0,
9428 77,
9429 serde_json::to_vec(&ModuleHelloBody {
9430 manifest: squatter,
9431 protocol_ver: PROTOCOL_VERSION,
9432 control_ops: None,
9433 launch_nonce: None,
9434 })
9435 .expect("HELLO serializes"),
9436 )
9437 .expect("HELLO frame builds");
9438 let response = handler
9439 .handle_control(ConnectionId::new(77), frame)
9440 .expect("reserved claim receives a typed refusal");
9441 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9442 assert_eq!(
9443 registry
9444 .active_registration_count()
9445 .expect("registry reads"),
9446 0,
9447 "a reserved capability refusal must not leave a catalog entry"
9448 );
9449 }
9450
9451 #[test]
9452 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9453 for settlement in ["timeout", "inconclusive", "drop"] {
9454 let breakers = RouteBindBreakers::default();
9455 let RouteBindAdmission::Admitted {
9456 guard: mut old,
9457 probe: false,
9458 } = breakers.admit("prov")
9459 else {
9460 panic!("ordinary relay admitted")
9461 };
9462 let RouteBindAdmission::Admitted {
9463 guard: mut opener, ..
9464 } = breakers.admit("prov")
9465 else {
9466 panic!("second relay admitted")
9467 };
9468 assert!(
9469 !opener
9470 .record_timeout(1, Duration::ZERO)
9471 .unwrap()
9472 .reopened_after_probe
9473 );
9474 let RouteBindAdmission::Admitted {
9475 guard: mut probe,
9476 probe: true,
9477 } = breakers.admit("prov")
9478 else {
9479 panic!("one cooldown probe admitted")
9480 };
9481 match settlement {
9482 "timeout" => assert!(
9483 !old.record_timeout(1, Duration::ZERO)
9484 .unwrap()
9485 .reopened_after_probe
9486 ),
9487 "inconclusive" => old.record_inconclusive(),
9488 "drop" => drop(old),
9489 _ => unreachable!(),
9490 }
9491 assert!(
9492 matches!(
9493 breakers.admit("prov"),
9494 RouteBindAdmission::Refused {
9495 probe_in_flight: true,
9496 ..
9497 }
9498 ),
9499 "{settlement} of a pre-open relay cannot release the real probe"
9500 );
9501 assert!(
9502 probe
9503 .record_timeout(1, Duration::ZERO)
9504 .unwrap()
9505 .reopened_after_probe
9506 );
9507 assert!(matches!(
9508 breakers.admit("prov"),
9509 RouteBindAdmission::Admitted { probe: true, .. }
9510 ));
9511 }
9512 let breakers = RouteBindBreakers::default();
9513 let admit = || match breakers.admit("prov") {
9514 RouteBindAdmission::Admitted { guard, .. } => guard,
9515 _ => panic!("relay admitted"),
9516 };
9517 admit().record_timeout(1, Duration::ZERO);
9518 let mut old_probe = admit();
9519 breakers.reset_for_new_module_connection("prov");
9520 admit().record_timeout(1, Duration::ZERO);
9521 let _new_probe = admit();
9522 old_probe.record_inconclusive();
9523 assert!(matches!(
9524 breakers.admit("prov"),
9525 RouteBindAdmission::Refused {
9526 probe_in_flight: true,
9527 ..
9528 }
9529 ));
9530 }
9531
9532 #[tokio::test]
9533 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9534 for candidate in [false, true] {
9535 let registry = Arc::new(Registry::default());
9536 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9537 [("vault".to_string(), true), ("squatter".to_string(), true)],
9538 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9539 );
9540 let conn = ConnectionId::new(77);
9541 let (ctx, mut rx) = route_ctx(conn);
9542 let initial = capability_manifest("squatter", &[], &[]);
9543 if candidate {
9544 registry
9545 .register_candidate_with_control_ops(
9546 initial.clone(),
9547 PROTOCOL_VERSION,
9548 conn,
9549 module_baseline_control_ops(),
9550 )
9551 .unwrap();
9552 handler
9553 .forwarding
9554 .register_candidate_module_connection(
9555 conn,
9556 "squatter".to_string(),
9557 PROTOCOL_VERSION,
9558 manifest_concurrency(&initial),
9559 ctx.egress.clone(),
9560 )
9561 .unwrap();
9562 } else {
9563 hello_via_sink(
9564 &handler,
9565 &ctx,
9566 &mut rx,
9567 hello_frame_with_manifest(initial.clone(), 1),
9568 )
9569 .await;
9570 }
9571 let response = handler
9572 .handle_control_frame(
9573 &ctx,
9574 catalog_update_with_capabilities_frame(
9575 2,
9576 capability_manifest("squatter", &["credentials-provider/v1"], &[])
9577 .capabilities
9578 .unwrap(),
9579 ),
9580 )
9581 .await
9582 .unwrap();
9583 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9584 assert_eq!(
9585 registry
9586 .get_module_by_connection(conn)
9587 .unwrap()
9588 .unwrap()
9589 .manifest,
9590 initial
9591 );
9592 }
9593 }
9594
9595 #[test]
9596 fn server_describe_surfaces_required_capability_verdict_fields() {
9597 let registry = Arc::new(Registry::default());
9598 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9599 [
9600 ("consumer".to_string(), true),
9601 ("provider".to_string(), false),
9602 ],
9603 BTreeMap::new(),
9604 );
9605 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9606 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9607 provides: Vec::new(),
9608 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9609 capability: "credentials-provider/v1".to_string(),
9610 need: subc_protocol::manifest::CapabilityNeed::Required,
9611 }],
9612 must_never_reach: Vec::new(),
9613 });
9614 let hello = Frame::build(
9615 FrameType::Hello,
9616 control_flags(),
9617 0,
9618 0,
9619 78,
9620 serde_json::to_vec(&ModuleHelloBody {
9621 manifest: consumer,
9622 protocol_ver: PROTOCOL_VERSION,
9623 control_ops: None,
9624 launch_nonce: None,
9625 })
9626 .expect("HELLO serializes"),
9627 )
9628 .expect("HELLO frame builds");
9629 handler
9630 .handle_control(ConnectionId::new(78), hello)
9631 .expect("consumer registers");
9632 let describe = Frame::build(
9633 FrameType::Request,
9634 control_flags(),
9635 0,
9636 0,
9637 79,
9638 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9639 .expect("request serializes"),
9640 )
9641 .expect("describe frame builds");
9642 let response = handler
9643 .handle_server_describe(describe)
9644 .expect("server.describe succeeds");
9645 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9646 let requirement = &rendered["capability_requirements"][0];
9647 assert_eq!(requirement["consumer"], "consumer");
9648 assert_eq!(requirement["verdict"], "never_provided");
9649 assert_eq!(requirement["episode_seq"], 1);
9650 assert_eq!(requirement["config_satisfiable"], false);
9651 assert_eq!(requirement["runtime_available"], false);
9652 assert!(requirement["detail"]
9653 .as_str()
9654 .expect("detail string")
9655 .contains("credentials-provider/v1"));
9656 }
9657
9658 fn deregister_required_provider(
9664 daemon_shutdown_begun: bool,
9665 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
9666 const CAPABILITY: &str = "project-identity/v1";
9667 let registry = Arc::new(Registry::default());
9668 let daemon_shutdown = DaemonShutdownFlag::default();
9669 let handler = ControlHandler::new(Arc::clone(®istry))
9670 .with_capability_config(
9671 [
9672 ("consumer".to_string(), true),
9673 ("provider".to_string(), true),
9674 ],
9675 BTreeMap::new(),
9676 )
9677 .with_daemon_shutdown_flag(daemon_shutdown.clone());
9678 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9679 consumer.capabilities = Some(CapabilityDeclarations {
9680 provides: Vec::new(),
9681 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9682 capability: CAPABILITY.to_string(),
9683 need: subc_protocol::manifest::CapabilityNeed::Required,
9684 }],
9685 must_never_reach: Vec::new(),
9686 });
9687 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
9688 let provider_conn = ConnectionId::new(182);
9689 for (conn, manifest, corr) in [
9690 (ConnectionId::new(181), consumer, 181),
9691 (provider_conn, provider, 182),
9692 ] {
9693 let hello = handler
9694 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
9695 .expect("module registers");
9696 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9697 }
9698 assert_eq!(
9699 handler.capability_evaluator.verdict("consumer", CAPABILITY),
9700 Some(CapabilityVerdict::Provided),
9701 "the registered provider must satisfy the requirement first"
9702 );
9703 if daemon_shutdown_begun {
9704 daemon_shutdown.set_for_test();
9705 }
9706
9707 let capture = EventCapture::default();
9708 let _subscriber =
9709 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9710 handler
9711 .cleanup_connection(provider_conn)
9712 .expect("provider deregisters");
9713 let events = capture
9714 .events()
9715 .into_iter()
9716 .filter(|event| {
9717 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
9718 })
9719 .collect();
9720 (
9721 events,
9722 handler.capability_evaluator.verdict("consumer", CAPABILITY),
9723 )
9724 }
9725
9726 #[test]
9727 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
9728 let (events, verdict) = deregister_required_provider(true);
9729 assert_eq!(
9730 verdict,
9731 Some(CapabilityVerdict::NeverProvided),
9732 "shutdown must not change the verdict route.open refusals read"
9733 );
9734 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9735 let event = &events[0];
9736 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
9737 assert_eq!(
9738 event.fields.get("cause").map(String::as_str),
9739 Some("\"daemon_shutdown\""),
9740 "{event:?}"
9741 );
9742 assert_eq!(
9743 event.fields.get("verdict").map(String::as_str),
9744 Some("\"never_provided\""),
9745 "{event:?}"
9746 );
9747 }
9748
9749 #[test]
9750 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
9751 let (events, verdict) = deregister_required_provider(false);
9752 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
9753 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9754 let event = &events[0];
9755 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
9756 assert_eq!(event.fields.get("cause"), None, "{event:?}");
9757 assert_eq!(
9758 event.fields.get("verdict").map(String::as_str),
9759 Some("\"never_provided\""),
9760 "{event:?}"
9761 );
9762 }
9763
9764 #[test]
9765 fn catalog_list_omits_capabilities_for_legacy_manifest() {
9766 let registry = Arc::new(Registry::default());
9767 let handler = ControlHandler::new(Arc::clone(®istry));
9768 let hello = handler
9769 .handle_control(
9770 ConnectionId::new(101),
9771 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9772 )
9773 .expect("legacy HELLO registers");
9774 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9775
9776 let request = Frame::build(
9777 FrameType::Request,
9778 control_flags(),
9779 0,
9780 0,
9781 102,
9782 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9783 .expect("catalog request serializes"),
9784 )
9785 .expect("catalog request frame builds");
9786 let response = handler
9787 .handle_catalog_list(request, None)
9788 .expect("catalog list succeeds");
9789 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9790 assert!(
9791 body["modules"][0].get("capabilities").is_none(),
9792 "legacy manifest must retain an absent capabilities field on catalog.list"
9793 );
9794 }
9795
9796 #[test]
9797 fn hello_ack_omits_storage_when_no_storage_config() {
9798 let registry = Arc::new(Registry::default());
9799 let handler = ControlHandler::new(Arc::clone(®istry));
9800 let responses = handler
9801 .handle_control(
9802 ConnectionId::new(1),
9803 hello_frame("aft", PROTOCOL_VERSION, 7),
9804 )
9805 .unwrap();
9806 let ack = parse_ack(&responses[0]);
9807 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9808 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9809 }
9810
9811 #[tokio::test]
9812 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9813 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9814 let registry = Arc::new(Registry::default());
9815 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
9816 let responses = handler
9817 .handle_control(
9818 ConnectionId::new(1),
9819 hello_frame("aft", PROTOCOL_VERSION, 7),
9820 )
9821 .unwrap();
9822 let ack = parse_ack(&responses[0]);
9823 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9824
9825 let described = handler
9826 .handle_control_frame(
9827 &route_ctx(ConnectionId::new(2)).0,
9828 Frame::build(
9829 FrameType::Request,
9830 control_flags(),
9831 0,
9832 0,
9833 9,
9834 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9835 )
9836 .unwrap(),
9837 )
9838 .await
9839 .unwrap();
9840 let ClientControlResponse::ServerDescribe { machine_id, .. } =
9841 serde_json::from_slice(&described[0].body).unwrap()
9842 else {
9843 panic!("server.describe answered with another shape");
9844 };
9845 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9846 }
9847
9848 #[test]
9849 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9850 let registry = Arc::new(Registry::default());
9853 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
9854 crate::daemon_config::StorageConfig::Sqlite {
9855 data_home: std::path::PathBuf::from("/data"),
9856 },
9857 ));
9858
9859 let responses = handler
9860 .handle_control(
9861 ConnectionId::new(1),
9862 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9863 )
9864 .unwrap();
9865 let ack = parse_ack(&responses[0]);
9866 assert_eq!(
9867 ack.storage,
9868 Some(serde_json::json!({
9869 "module_id": "alfonso-routing",
9870 "storage_namespace": "default",
9871 "isolation": { "kind": "module" },
9872 "backend": {
9873 "backend": "sqlite",
9874 "path": "/data/cortexkit/alfonso-routing/store.db"
9875 }
9876 })),
9877 "the delivered descriptor is the module's own sqlite store path"
9878 );
9879 }
9880
9881 #[test]
9882 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9883 let registry = Arc::new(Registry::default());
9884 let handler = ControlHandler::new(Arc::clone(®istry));
9885 let conn = ConnectionId::new(1);
9886 let responses = handler
9887 .handle_control(
9888 conn,
9889 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9890 )
9891 .unwrap();
9892 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9893 let registration = registry.get_module("aft").unwrap().unwrap();
9894 assert_eq!(registration.control_ops, module_baseline_control_ops());
9895
9896 let frame =
9897 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9898 assert!(handler
9899 .guard_module_control_op(&frame, "aft", "route.bind")
9900 .unwrap()
9901 .is_none());
9902 let error = handler
9903 .guard_module_control_op(&frame, "aft", "test.synthetic")
9904 .unwrap()
9905 .expect("synthetic ungranted op should be rejected");
9906 assert_eq!(error.header.ty, FrameType::Error);
9907 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9908 }
9909
9910 #[test]
9911 fn hello_control_ops_some_adds_optional_grants() {
9912 let registry = Arc::new(Registry::default());
9913 let handler = ControlHandler::new(Arc::clone(®istry));
9914 handler
9915 .handle_control(
9916 ConnectionId::new(1),
9917 hello_frame_with_control_ops(
9918 "aft",
9919 PROTOCOL_VERSION,
9920 7,
9921 Some(vec![
9922 "future.synthetic".to_string(),
9923 "route.bind".to_string(),
9924 ]),
9925 ),
9926 )
9927 .unwrap();
9928 let registration = registry.get_module("aft").unwrap().unwrap();
9929 assert_eq!(
9930 registration.control_ops,
9931 vec![
9932 "route.bind".to_string(),
9933 "route.status".to_string(),
9934 "future.synthetic".to_string(),
9935 ]
9936 );
9937 let frame =
9938 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9939 assert!(handler
9940 .guard_module_control_op(&frame, "aft", "future.synthetic")
9941 .unwrap()
9942 .is_none());
9943 }
9944
9945 #[tokio::test]
9946 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9947 let registry = Arc::new(Registry::default());
9948 let forwarding = Arc::new(ForwardingTable::default());
9949 let handler =
9950 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9951 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9952 hello_via_sink(
9953 &handler,
9954 &module_ctx,
9955 &mut module_rx,
9956 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9957 )
9958 .await;
9959
9960 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9961 let responses = handler
9962 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9963 .await
9964 .unwrap();
9965 assert_eq!(responses.len(), 1);
9966 assert_eq!(responses[0].header.ty, FrameType::Error);
9967 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9968 assert!(module_rx.try_recv().is_err());
9969 }
9970
9971 #[tokio::test]
9972 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9973 let registry = Arc::new(Registry::default());
9974 let forwarding = Arc::new(ForwardingTable::default());
9975 let handler =
9976 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9977 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9978 hello_via_sink(
9979 &handler,
9980 &module_ctx,
9981 &mut module_rx,
9982 hello_frame_with_control_ops(
9983 "aft",
9984 PROTOCOL_VERSION,
9985 7,
9986 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9987 ),
9988 )
9989 .await;
9990
9991 let project_root = unique_project_root("demux");
9992 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9993 let route_handler = handler.clone();
9994 let route_task = tokio::spawn(async move {
9995 route_handler
9996 .handle_control_frame(
9997 &route_client_ctx,
9998 route_open_frame(100, "aft", project_root),
9999 )
10000 .await
10001 .unwrap()
10002 });
10003 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10004 .await
10005 .unwrap()
10006 .unwrap();
10007 assert!(matches!(
10008 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10009 ModuleControlRequest::RouteBind { .. }
10010 ));
10011
10012 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10013 let health_handler = handler.clone();
10014 let health_task = tokio::spawn(async move {
10015 health_handler
10016 .handle_control_frame(
10017 &health_client_ctx,
10018 supervisor_health_probe_frame(101, "aft"),
10019 )
10020 .await
10021 .unwrap()
10022 });
10023 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10024 .await
10025 .unwrap()
10026 .unwrap();
10027 assert_eq!(
10028 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10029 ModuleControlRequest::HealthCheck {}
10030 );
10031
10032 handler
10033 .handle_control_frame(
10034 &module_ctx,
10035 health_response(health_frame.header.corr, HealthStatus::Degraded),
10036 )
10037 .await
10038 .unwrap();
10039 let health_response = health_task.await.unwrap();
10040 assert_eq!(health_response.len(), 1);
10041 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10042 ClientControlResponse::SupervisorHealthProbe {
10043 module_id,
10044 status,
10045 detail,
10046 metrics,
10047 } => {
10048 assert_eq!(module_id, "aft");
10049 assert_eq!(status, HealthStatus::Degraded);
10050 assert_eq!(detail.as_deref(), Some("warming"));
10051 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10052 }
10053 other => panic!("unexpected health response: {other:?}"),
10054 }
10055
10056 handler
10057 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10058 .await
10059 .unwrap();
10060 let route_response = route_task.await.unwrap();
10061 assert!(route_response.is_empty());
10062 let published = route_client_rx.recv().await.unwrap();
10063 assert!(matches!(
10064 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10065 ClientControlResponse::RouteOpen { .. }
10066 ));
10067 }
10068
10069 async fn relay_route_open(
10074 handler: &ControlHandler,
10075 client_connection: ConnectionId,
10076 client_egress: &FrameSink,
10077 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10078 corr: u64,
10079 module_id: &str,
10080 project_root_label: &str,
10081 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10082 let ctx = RouteCtx {
10083 connection_id: client_connection,
10084 egress: client_egress.clone(),
10085 };
10086 let handler = handler.clone();
10087 let project_root = unique_project_root(project_root_label);
10088 let module_id = module_id.to_string();
10089 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10090 let task = tokio::spawn(async move {
10091 let _guard = tracing::dispatcher::set_default(&dispatch);
10092 handler
10093 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10094 .await
10095 .unwrap()
10096 });
10097 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10098 .await
10099 .expect("module receives the relayed route.bind")
10100 .expect("module egress is open");
10101 (task, bind.frame)
10102 }
10103
10104 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10105 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10106 ModuleControlRequest::RouteBind {
10107 route_channel,
10108 epoch,
10109 ..
10110 } => (route_channel, epoch),
10111 other => panic!("expected a route.bind request, got {other:?}"),
10112 }
10113 }
10114
10115 fn published_route(frame: &Frame) -> (u16, u32) {
10116 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10117 ClientControlResponse::RouteOpen {
10118 route_channel,
10119 route_epoch,
10120 } => (route_channel, route_epoch),
10121 other => panic!("expected a route.open response, got {other:?}"),
10122 }
10123 }
10124
10125 #[tokio::test]
10145 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10146 let registry = Arc::new(Registry::default());
10147 let forwarding = Arc::new(ForwardingTable::default());
10148 let handler =
10149 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10150
10151 let module_connection = ConnectionId::new(30);
10152 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10153 hello_via_sink(
10154 &handler,
10155 &module_ctx,
10156 &mut module_rx,
10157 hello_frame("aft", PROTOCOL_VERSION, 7),
10158 )
10159 .await;
10160
10161 let dying_client = ConnectionId::new(31);
10162 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10163
10164 let (first_task, first_bind) = relay_route_open(
10167 &handler,
10168 dying_client,
10169 &dying_ctx.egress,
10170 &mut module_rx,
10171 100,
10172 "aft",
10173 "closing-first",
10174 )
10175 .await;
10176 handler
10177 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10178 .await
10179 .unwrap();
10180 assert!(first_task.await.unwrap().is_empty());
10181 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10182
10183 let (second_task, second_bind) = relay_route_open(
10185 &handler,
10186 dying_client,
10187 &dying_ctx.egress,
10188 &mut module_rx,
10189 101,
10190 "aft",
10191 "closing-second",
10192 )
10193 .await;
10194 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10195
10196 assert!(forwarding
10199 .escalate_client_delivery_failure(
10200 dying_client,
10201 first_channel,
10202 first_epoch,
10203 CloseReason::new(
10204 "module_to_client_delivery_failed",
10205 "client egress refused a module frame",
10206 ),
10207 crate::forwarding::UndeliveredFrame {
10208 module_id: None,
10209 sink: &dying_ctx.egress,
10210 },
10211 )
10212 .unwrap());
10213 assert!(!dying_ctx.egress.is_closed());
10214
10215 let ack = handler
10217 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10218 .await;
10219 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10220 if module_loop_error.is_some() {
10221 handler.cleanup_connection(module_connection).unwrap();
10225 }
10226 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10230 .await
10231 .ok()
10232 .flatten();
10233
10234 assert!(
10236 registry
10237 .get_module_by_connection(module_connection)
10238 .unwrap()
10239 .is_some(),
10240 "one client's closing connection ended the shared module connection: \
10241 {module_loop_error:?}"
10242 );
10243 let cotenant = ConnectionId::new(32);
10245 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10246 let (cotenant_task, cotenant_bind) = relay_route_open(
10247 &handler,
10248 cotenant,
10249 &cotenant_ctx.egress,
10250 &mut module_rx,
10251 102,
10252 "aft",
10253 "closing-cotenant",
10254 )
10255 .await;
10256 handler
10257 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10258 .await
10259 .unwrap();
10260 assert!(cotenant_task.await.unwrap().is_empty());
10261 let (cotenant_channel, cotenant_epoch) =
10262 published_route(&cotenant_rx.recv().await.unwrap());
10263 assert!(matches!(
10264 forwarding
10265 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10266 .unwrap(),
10267 DataRoute::Client(DataRouteState::Bound(_))
10268 ));
10269
10270 let goodbye = post_ack_module_frame
10273 .expect("module receives a GOODBYE for the abandoned route channel");
10274 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10275 assert_eq!(goodbye.header.channel, abandoned_channel);
10276 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10277
10278 assert!(dying_rx.try_recv().is_err());
10282 let second_response = second_task.await.unwrap();
10283 assert_eq!(second_response.len(), 1);
10284 assert_eq!(
10285 parse_error(&second_response[0])["code"],
10286 "target_unavailable"
10287 );
10288 }
10289
10290 async fn catalog_snapshot(
10293 handler: &ControlHandler,
10294 client_ctx: &RouteCtx,
10295 corr: u64,
10296 ) -> (u64, Vec<(String, bool)>) {
10297 let body =
10298 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10299 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10300 let reply = handler
10301 .handle_control_frame(client_ctx, request)
10302 .await
10303 .unwrap()
10304 .pop()
10305 .unwrap();
10306 let ClientControlResponse::CatalogList {
10307 generation,
10308 modules,
10309 ..
10310 } = serde_json::from_slice(&reply.body).unwrap()
10311 else {
10312 panic!("not a catalog.list reply");
10313 };
10314 (
10315 generation,
10316 modules
10317 .into_iter()
10318 .map(|entry| (entry.module_id, entry.ready))
10319 .collect(),
10320 )
10321 }
10322
10323 fn hello_held_before_forwarding() -> (
10329 ControlHandler,
10330 Arc<std::sync::Barrier>,
10331 std::thread::JoinHandle<Vec<Frame>>,
10332 RouteCtx,
10333 mpsc::Receiver<crate::router::OutboundFrame>,
10334 ) {
10335 let hold = Arc::new(std::sync::Barrier::new(2));
10336 let handler = ControlHandler::with_forwarding(
10337 Arc::new(Registry::default()),
10338 Arc::new(ForwardingTable::default()),
10339 )
10340 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10341 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10342 let hello = {
10343 let handler = handler.clone();
10344 let module_ctx = module_ctx.clone();
10345 std::thread::spawn(move || {
10346 handler
10347 .handle_hello(
10348 module_ctx.connection_id,
10349 Some(module_ctx.egress.clone()),
10350 hello_frame("aft", PROTOCOL_VERSION, 7),
10351 )
10352 .unwrap()
10353 })
10354 };
10355 (handler, hold, hello, module_ctx, module_rx)
10356 }
10357
10358 #[tokio::test]
10367 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10368 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10369 let client = ConnectionId::new(41);
10370 let (client_ctx, mut client_rx) = route_ctx(client);
10371
10372 hold.wait();
10373 assert!(handler.registry.get_module("aft").unwrap().is_some());
10375 assert!(!handler
10376 .forwarding
10377 .has_live_module_connection("aft")
10378 .unwrap());
10379 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10380 assert!(
10381 listed.is_empty(),
10382 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10383 );
10384 let refused = handler
10385 .handle_control_frame(
10386 &client_ctx,
10387 route_open_frame(301, "aft", unique_project_root("listed-window")),
10388 )
10389 .await
10390 .unwrap();
10391 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10392 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10393 let duplicate = handler
10394 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10395 .await
10396 .unwrap();
10397 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10398
10399 hold.wait();
10400 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10401 let ack = module_rx.try_recv().unwrap().frame;
10402 assert_eq!(ack.header.ty, FrameType::HelloAck);
10403 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10404 assert_eq!(listed, vec![("aft".to_string(), true)]);
10405
10406 let (open_task, bind) = relay_route_open(
10407 &handler,
10408 client,
10409 &client_ctx.egress,
10410 &mut module_rx,
10411 303,
10412 "aft",
10413 "listed-after",
10414 )
10415 .await;
10416 assert!(handler
10417 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10418 .await
10419 .unwrap()
10420 .is_empty());
10421 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10422 published_route(&client_rx.try_recv().unwrap().frame);
10423 }
10424
10425 #[tokio::test]
10432 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10433 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10434 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10435 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10436
10437 hold.wait();
10438 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10439 assert!(listed.is_empty());
10440 assert_eq!(
10441 during, before,
10442 "the generation moved while the module was not listed"
10443 );
10444
10445 hold.wait();
10446 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10447 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10448 assert_eq!(listed, vec![("aft".to_string(), true)]);
10449 assert_eq!(
10450 after,
10451 before + 1,
10452 "the module appeared without exactly one generation bump"
10453 );
10454 }
10455
10456 #[tokio::test]
10461 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10462 let registry = Arc::new(Registry::default());
10463 let handler = ControlHandler::with_forwarding(
10464 Arc::clone(®istry),
10465 Arc::new(ForwardingTable::default()),
10466 );
10467 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10468 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10469 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10470 drop(module_rx);
10471
10472 let replies = handler
10473 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10474 .await
10475 .unwrap();
10476 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10477 assert!(registry.get_module("aft").unwrap().is_none());
10478 assert!(registry
10479 .get_module_by_connection(module_ctx.connection_id)
10480 .unwrap()
10481 .is_none());
10482 assert_eq!(
10483 catalog_snapshot(&handler, &client_ctx, 301).await,
10484 (before, Vec::new())
10485 );
10486 }
10487
10488 async fn route_open_held_after_bind_send() -> (
10492 ControlHandler,
10493 Arc<tokio::sync::Barrier>,
10494 RouteCtx,
10495 mpsc::Receiver<crate::router::OutboundFrame>,
10496 ) {
10497 let hold = Arc::new(tokio::sync::Barrier::new(2));
10498 let handler = ControlHandler::with_forwarding(
10499 Arc::new(Registry::default()),
10500 Arc::new(ForwardingTable::default()),
10501 )
10502 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10503 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10504 hello_via_sink(
10505 &handler,
10506 &module_ctx,
10507 &mut module_rx,
10508 hello_frame("aft", PROTOCOL_VERSION, 7),
10509 )
10510 .await;
10511 (handler, hold, module_ctx, module_rx)
10512 }
10513
10514 #[tokio::test]
10521 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10522 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10523 let client = ConnectionId::new(41);
10524 let (client_ctx, mut client_rx) = route_ctx(client);
10525
10526 let (open_task, bind) = relay_route_open(
10527 &handler,
10528 client,
10529 &client_ctx.egress,
10530 &mut module_rx,
10531 200,
10532 "aft",
10533 "acked-before-mark",
10534 )
10535 .await;
10536 let (module_channel, module_epoch) = route_bind_channel(&bind);
10537 hold.wait().await;
10539
10540 assert!(handler
10542 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10543 .await
10544 .unwrap()
10545 .is_empty());
10546 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10547
10548 hold.wait().await;
10549 assert!(
10550 open_task.await.unwrap().is_empty(),
10551 "route.open answers accepted"
10552 );
10553
10554 match module_rx.try_recv() {
10557 Err(mpsc::error::TryRecvError::Empty) => {}
10558 Ok(outbound) => panic!(
10559 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10560 outbound.frame.header.ty,
10561 outbound.frame.header.channel,
10562 outbound.frame.header.epoch
10563 ),
10564 Err(err) => panic!("module egress closed: {err}"),
10565 }
10566
10567 let router = Router::with_control_handler(Arc::new(handler.clone()));
10570 let request = Frame::build(
10571 FrameType::Request,
10572 Flags::new(true, Priority::Interactive, false),
10573 client_channel,
10574 client_epoch,
10575 201,
10576 b"first request".to_vec(),
10577 )
10578 .unwrap();
10579 router
10580 .route_for_connection(&client_ctx, request)
10581 .await
10582 .unwrap();
10583 let delivered = module_rx.try_recv().unwrap().frame;
10584 assert_eq!(delivered.header.ty, FrameType::Request);
10585 assert_eq!(
10586 (delivered.header.channel, delivered.header.epoch),
10587 (module_channel, module_epoch)
10588 );
10589 assert_eq!(delivered.body, b"first request");
10590 }
10591
10592 #[tokio::test]
10598 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10599 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10600 let client = ConnectionId::new(42);
10601 let (client_ctx, _client_rx) = route_ctx(client);
10602
10603 let (open_task, bind) = relay_route_open(
10604 &handler,
10605 client,
10606 &client_ctx.egress,
10607 &mut module_rx,
10608 300,
10609 "aft",
10610 "released-before-mark",
10611 )
10612 .await;
10613 let (module_channel, module_epoch) = route_bind_channel(&bind);
10614 hold.wait().await;
10615
10616 handler.cleanup_connection(client).unwrap();
10617 assert!(
10618 module_rx.try_recv().is_err(),
10619 "client teardown must not GOODBYE a bind route.open is still queueing"
10620 );
10621
10622 hold.wait().await;
10623 let response = open_task.await.unwrap();
10624 assert_eq!(response.len(), 1);
10625 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10626
10627 let goodbye = module_rx
10628 .try_recv()
10629 .expect("module is told to drop the binding the daemon abandoned")
10630 .frame;
10631 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10632 assert_eq!(
10633 (goodbye.header.channel, goodbye.header.epoch),
10634 (module_channel, module_epoch)
10635 );
10636 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10637 }
10638
10639 #[tokio::test]
10644 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10645 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10646 let client = ConnectionId::new(43);
10647 let (client_ctx, client_rx) = route_ctx(client);
10648
10649 let (open_task, bind) = relay_route_open(
10650 &handler,
10651 client,
10652 &client_ctx.egress,
10653 &mut module_rx,
10654 400,
10655 "aft",
10656 "gone-client-before-mark",
10657 )
10658 .await;
10659 let (module_channel, module_epoch) = route_bind_channel(&bind);
10660 hold.wait().await;
10661
10662 drop(client_rx);
10663 assert!(client_ctx.egress.is_closed());
10664 handler
10665 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10666 .await
10667 .unwrap();
10668
10669 hold.wait().await;
10670 let response = open_task.await.unwrap();
10671 assert_eq!(response.len(), 1);
10672 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10673
10674 let goodbye = module_rx
10675 .try_recv()
10676 .expect("module is told to drop the binding it acked")
10677 .frame;
10678 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10679 assert_eq!(
10680 (goodbye.header.channel, goodbye.header.epoch),
10681 (module_channel, module_epoch)
10682 );
10683 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10684 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10685 }
10686
10687 #[test]
10694 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10695 let handler = ControlHandler::default();
10696 let module_connection = ConnectionId::new(30);
10697 let client_connection = ConnectionId::new(31);
10698
10699 handler
10700 .refuse_to_end_module_connection_for_a_client(
10701 module_connection,
10702 77,
10703 ForwardingError::ConnectionClosing {
10704 connection_id: client_connection,
10705 },
10706 )
10707 .expect("a closing client must never end the module connection");
10708
10709 assert!(matches!(
10710 handler.refuse_to_end_module_connection_for_a_client(
10711 module_connection,
10712 78,
10713 ForwardingError::ConnectionClosing {
10714 connection_id: module_connection,
10715 },
10716 ),
10717 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10718 connection_id
10719 })) if connection_id == module_connection
10720 ));
10721 assert!(matches!(
10722 handler.refuse_to_end_module_connection_for_a_client(
10723 module_connection,
10724 79,
10725 ForwardingError::Poisoned,
10726 ),
10727 Err(RouterError::Forwarding(ForwardingError::Poisoned))
10728 ));
10729 assert!(matches!(
10730 handler.refuse_to_end_module_connection_for_a_client(
10731 module_connection,
10732 80,
10733 ForwardingError::StaleModuleEndpoint,
10734 ),
10735 Err(RouterError::Forwarding(
10736 ForwardingError::StaleModuleEndpoint
10737 ))
10738 ));
10739 }
10740
10741 #[tokio::test]
10754 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10755 let registry = Arc::new(Registry::default());
10756 let forwarding = Arc::new(ForwardingTable::default());
10757 let supervisor = SupervisorHandle::new();
10758 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10759 let handler =
10760 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10761 .with_supervisor(supervisor);
10762
10763 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10764 hello_via_sink(
10765 &handler,
10766 &target_ctx,
10767 &mut target_rx,
10768 hello_frame("target", PROTOCOL_VERSION, 1),
10769 )
10770 .await;
10771
10772 let wrong_nonce = handler
10776 .handle_control_frame(
10777 &route_ctx(ConnectionId::new(91)).0,
10778 route_open_frame_with_admission_facts(
10779 20,
10780 "target",
10781 unique_project_root("admission-facts"),
10782 Some(subc_control::ConsumerIdentity {
10783 module_id: "fed".to_string(),
10784 launch_nonce: "not-the-real-nonce".to_string(),
10785 }),
10786 None,
10787 ),
10788 )
10789 .await
10790 .unwrap();
10791 assert_eq!(
10792 parse_error(&wrong_nonce[0])["code"],
10793 "bad_consumer_identity",
10794 "a mismatched launch nonce must be refused, not stamped Reserved"
10795 );
10796
10797 let never_spawned = handler
10801 .handle_control_frame(
10802 &route_ctx(ConnectionId::new(92)).0,
10803 route_open_frame_with_admission_facts(
10804 21,
10805 "target",
10806 unique_project_root("admission-facts"),
10807 Some(subc_control::ConsumerIdentity {
10808 module_id: "never-spawned".to_string(),
10809 launch_nonce: "any-nonce".to_string(),
10810 }),
10811 None,
10812 ),
10813 )
10814 .await
10815 .unwrap();
10816 assert_eq!(
10817 parse_error(&never_spawned[0])["code"],
10818 "bad_consumer_identity",
10819 "an unspawned module_id must be refused rather than accepted for lack of a record"
10820 );
10821 }
10822
10823 #[tokio::test]
10846 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10847 let registry = Arc::new(Registry::default());
10848 let forwarding = Arc::new(ForwardingTable::default());
10849 let supervisor = SupervisorHandle::new();
10850 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10851 let handler =
10852 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10853 .with_supervisor(supervisor);
10854
10855 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10856 hello_via_sink(
10857 &handler,
10858 &target_ctx,
10859 &mut target_rx,
10860 hello_frame("target", PROTOCOL_VERSION, 1),
10861 )
10862 .await;
10863
10864 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10865 let route_handler = handler.clone();
10866 let route_task = tokio::spawn(async move {
10867 route_handler
10868 .handle_control_frame(
10869 &client_ctx,
10870 route_open_frame_with_admission_facts(
10871 30,
10872 "target",
10873 unique_project_root("admission-facts"),
10874 Some(subc_control::ConsumerIdentity {
10875 module_id: "fed".to_string(),
10876 launch_nonce: "fed-nonce".to_string(),
10877 }),
10878 None,
10879 ),
10880 )
10881 .await
10882 .unwrap()
10883 });
10884
10885 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10892 .await
10893 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10894 .expect("module control channel closed before route.bind");
10895 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10896 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10897 panic!("expected route.bind")
10898 };
10899 assert_eq!(
10900 principal,
10901 Some(Principal::Reserved {
10902 module_id: "fed".to_string()
10903 }),
10904 "a correctly attested consumer must be stamped Reserved for its own id"
10905 );
10906
10907 handler
10908 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10909 .await
10910 .unwrap();
10911 assert!(route_task.await.unwrap().is_empty());
10912 assert!(
10913 matches!(
10914 serde_json::from_slice::<ClientControlResponse>(
10915 &client_rx.recv().await.unwrap().body
10916 )
10917 .unwrap(),
10918 ClientControlResponse::RouteOpen { .. }
10919 ),
10920 "the route must actually open, not merely avoid an error"
10921 );
10922 }
10923
10924 #[tokio::test(start_paused = true)]
10925 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10926 let registry = Arc::new(Registry::default());
10927 let forwarding = Arc::new(ForwardingTable::default());
10928 let supervisor = SupervisorHandle::new();
10929 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10930 let handler =
10931 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10932 .with_supervisor(supervisor);
10933
10934 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10935 hello_via_sink(
10936 &handler,
10937 &target_ctx,
10938 &mut target_rx,
10939 hello_frame("target", PROTOCOL_VERSION, 1),
10940 )
10941 .await;
10942
10943 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10944 let direct_handler = handler.clone();
10945 let direct_open = tokio::spawn(async move {
10946 direct_handler
10947 .handle_control_frame(
10948 &direct_ctx,
10949 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10950 )
10951 .await
10952 .unwrap()
10953 });
10954 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10955 .await
10956 .expect("no direct route.bind within 5s")
10957 .expect("target control channel closed before direct route.bind");
10958 handler
10959 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10960 .await
10961 .unwrap();
10962 assert!(direct_open.await.unwrap().is_empty());
10963 let _ = direct_rx.recv().await.unwrap();
10964
10965 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10966 let reserved_handler = handler.clone();
10967 let reserved_open = tokio::spawn(async move {
10968 reserved_handler
10969 .handle_control_frame(
10970 &reserved_ctx,
10971 route_open_frame_with_admission_facts(
10972 3,
10973 "target",
10974 unique_project_root("admission-facts"),
10975 Some(ConsumerIdentity {
10976 module_id: "fed".to_string(),
10977 launch_nonce: "fed-nonce".to_string(),
10978 }),
10979 None,
10980 ),
10981 )
10982 .await
10983 .unwrap()
10984 });
10985 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10986 .await
10987 .expect("no reserved route.bind within 5s")
10988 .expect("target control channel closed before reserved route.bind");
10989 handler
10990 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10991 .await
10992 .unwrap();
10993 assert!(reserved_open.await.unwrap().is_empty());
10994 let _ = reserved_rx.recv().await.unwrap();
10995
10996 forwarding
10997 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10998 .unwrap();
10999 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11000 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11001 module_id: Some("target".to_string()),
11002 })
11003 .unwrap();
11004 let census_frame =
11005 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11006 let response = handler
11007 .handle_control_frame(&census_ctx, census_frame)
11008 .await
11009 .unwrap()
11010 .pop()
11011 .unwrap();
11012 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11013 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11014 assert!(matches!(
11015 decoded,
11016 ClientControlResponse::SupervisorRoutes { .. }
11017 ));
11018 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11019 assert_eq!(routes.len(), 2);
11020 assert!(routes.iter().all(|route| route["draining"] == true));
11021 assert!(
11024 routes.iter().all(|route| route["drain_reason"] == "reload"),
11025 "draining routes must name the drain's reason: {routes:?}"
11026 );
11027 assert!(routes.iter().any(|route| {
11028 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11029 }));
11030 assert!(routes.iter().any(|route| {
11031 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11032 }));
11033
11034 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11035 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11036 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11037 std::fs::write(
11038 &golden_path,
11039 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11040 )
11041 .unwrap();
11042 }
11043 let expected: Value =
11044 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11045 assert_eq!(actual, expected);
11046 }
11047
11048 async fn query_live_roots(
11049 handler: &ControlHandler,
11050 module_ctx: &RouteCtx,
11051 ) -> ModuleControlResponseToModule {
11052 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11053 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11054 let response = handler
11055 .handle_control_frame(module_ctx, frame)
11056 .await
11057 .unwrap()
11058 .pop()
11059 .unwrap();
11060 serde_json::from_slice(&response.body).unwrap()
11061 }
11062
11063 #[tokio::test(start_paused = true)]
11064 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11065 let registry = Arc::new(Registry::default());
11066 let forwarding = Arc::new(ForwardingTable::default());
11067 let handler = ControlHandler::with_forwarding(registry, forwarding);
11068 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11069 hello_via_sink(
11070 &handler,
11071 &target_ctx,
11072 &mut target_rx,
11073 hello_frame("target", PROTOCOL_VERSION, 1),
11074 )
11075 .await;
11076 let root = unique_project_root("live-roots-known");
11077 let path = ProjectRootId::from_path_allowing_missing(root.path())
11078 .unwrap()
11079 .as_path()
11080 .to_path_buf();
11081 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11082 let open_handler = handler.clone();
11083 let opened = tokio::spawn(async move {
11084 open_handler
11085 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11086 .await
11087 .unwrap()
11088 });
11089 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11090 .await
11091 .unwrap()
11092 .unwrap();
11093 handler
11094 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11095 .await
11096 .unwrap();
11097 assert!(opened.await.unwrap().is_empty());
11098 let _ = client_rx.recv().await.unwrap();
11099
11100 let root = unique_project_root("live-roots-pending");
11101 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11102 .unwrap()
11103 .as_path()
11104 .to_path_buf();
11105 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11106 let open_handler = handler.clone();
11107 let pending = tokio::spawn(async move {
11108 open_handler
11109 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11110 .await
11111 .unwrap()
11112 });
11113 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11114 .await
11115 .unwrap()
11116 .unwrap();
11117 let actual = query_live_roots(&handler, &target_ctx).await;
11118 let ModuleControlResponseToModule::LiveRoots {
11119 roots,
11120 unknown_root_bindings,
11121 total_bindings,
11122 } = actual
11123 else {
11124 panic!("expected live roots")
11125 };
11126 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11127 assert_eq!(unknown_root_bindings, 0);
11128 assert_eq!(
11129 roots.len(),
11130 2,
11131 "root-known arm must retain each canonical root"
11132 );
11133 assert_eq!(
11134 total_bindings,
11135 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11136 );
11137 let counts = roots
11138 .iter()
11139 .map(|root| (root.project_root.clone(), root.bound, root.pending))
11140 .collect::<Vec<_>>();
11141 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11142 expected.sort_by(|a, b| a.0.cmp(&b.0));
11143 assert_eq!(
11144 counts, expected,
11145 "roots must sort by path and count pending separately"
11146 );
11147 handler
11148 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11149 .await
11150 .unwrap();
11151 assert!(pending.await.unwrap().is_empty());
11152 }
11153
11154 #[tokio::test(start_paused = true)]
11155 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11156 let forwarding = Arc::new(ForwardingTable::default());
11157 let handler =
11158 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11159 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11160 hello_via_sink(
11161 &handler,
11162 &target_ctx,
11163 &mut target_rx,
11164 hello_frame("target", PROTOCOL_VERSION, 1),
11165 )
11166 .await;
11167 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11168 let pending = forwarding
11169 .begin_route_bind_relay_for_test(
11170 client_ctx.connection_id,
11171 client_ctx.egress.clone(),
11172 2,
11173 "target",
11174 )
11175 .unwrap();
11176 forwarding
11177 .complete_pending_relay(
11178 target_ctx.connection_id,
11179 pending.corr,
11180 RouteBindRelayOutcome::Accepted,
11181 )
11182 .unwrap();
11183 let actual = query_live_roots(&handler, &target_ctx).await;
11184 let ModuleControlResponseToModule::LiveRoots {
11185 roots,
11186 unknown_root_bindings,
11187 total_bindings,
11188 } = actual
11189 else {
11190 panic!("expected live roots")
11191 };
11192 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11193 assert_eq!(
11194 unknown_root_bindings, 1,
11195 "unknown-root arm must not read as no bindings"
11196 );
11197 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11198 assert_eq!(
11199 total_bindings,
11200 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11201 );
11202 }
11203
11204 #[tokio::test(start_paused = true)]
11212 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11213 let forwarding = Arc::new(ForwardingTable::default());
11214 let handler =
11215 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11216 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11217 let replies = handler
11218 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11219 .await
11220 .unwrap();
11221 let queued_by_hello = module_rx.len();
11222
11223 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11224 let open_handler = handler.clone();
11225 let open = tokio::spawn(async move {
11226 open_handler
11227 .handle_control_frame(
11228 &client_ctx,
11229 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11230 )
11231 .await
11232 .unwrap()
11233 });
11234 let mut spins = 0;
11236 while module_rx.len() == queued_by_hello {
11237 spins += 1;
11238 assert!(spins < 10_000, "route.open never queued a route.bind");
11239 tokio::task::yield_now().await;
11240 }
11241
11242 for reply in replies {
11244 module_ctx.egress.send(reply).await.unwrap();
11245 }
11246
11247 let first = module_rx.recv().await.unwrap().frame;
11248 assert_eq!(
11249 first.header.ty,
11250 FrameType::HelloAck,
11251 "the first frame a registering module reads must be its HELLO_ACK"
11252 );
11253 assert_eq!(first.header.corr, 7);
11254 let second = module_rx.recv().await.unwrap().frame;
11255 assert_eq!(second.header.ty, FrameType::Request);
11256 assert!(
11257 matches!(
11258 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11259 ModuleControlRequest::RouteBind { .. }
11260 ),
11261 "the route.bind follows the ack"
11262 );
11263 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11264
11265 handler
11266 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11267 .await
11268 .unwrap();
11269 assert!(open.await.unwrap().is_empty());
11270 let _ = client_rx.recv().await.unwrap();
11271 }
11272
11273 #[tokio::test(start_paused = true)]
11274 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11275 let handler = ControlHandler::with_forwarding(
11276 Arc::new(Registry::default()),
11277 Arc::new(ForwardingTable::default()),
11278 );
11279 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11280 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11281 hello_via_sink(
11282 &handler,
11283 &first_ctx,
11284 &mut first_rx,
11285 hello_frame("first", PROTOCOL_VERSION, 1),
11286 )
11287 .await;
11288 hello_via_sink(
11289 &handler,
11290 &second_ctx,
11291 &mut second_rx,
11292 hello_frame("second", PROTOCOL_VERSION, 2),
11293 )
11294 .await;
11295 let root = unique_project_root("second-only");
11296 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11297 let cloned = handler.clone();
11298 let open = tokio::spawn(async move {
11299 cloned
11300 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11301 .await
11302 .unwrap()
11303 });
11304 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11305 .await
11306 .unwrap()
11307 .unwrap();
11308 let first = query_live_roots(&handler, &first_ctx).await;
11309 let second = query_live_roots(&handler, &second_ctx).await;
11310 assert!(
11311 matches!(
11312 first,
11313 ModuleControlResponseToModule::LiveRoots {
11314 total_bindings: 0,
11315 ..
11316 }
11317 ),
11318 "cross-module scope must not expose another module's roots"
11319 );
11320 assert!(
11321 matches!(
11322 second,
11323 ModuleControlResponseToModule::LiveRoots {
11324 total_bindings: 1,
11325 ..
11326 }
11327 ),
11328 "second module must see its pending route"
11329 );
11330 handler
11331 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11332 .await
11333 .unwrap();
11334 assert!(open.await.unwrap().is_empty());
11335 }
11336
11337 #[tokio::test(start_paused = true)]
11338 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11339 let handler = ControlHandler::with_forwarding(
11340 Arc::new(Registry::default()),
11341 Arc::new(ForwardingTable::default()),
11342 );
11343 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11344 hello_via_sink(
11345 &handler,
11346 &target_ctx,
11347 &mut target_rx,
11348 hello_frame("target", PROTOCOL_VERSION, 1),
11349 )
11350 .await;
11351 let actual = query_live_roots(&handler, &target_ctx).await;
11352 let ModuleControlResponseToModule::LiveRoots {
11353 roots,
11354 unknown_root_bindings,
11355 total_bindings,
11356 } = actual
11357 else {
11358 panic!("expected live roots")
11359 };
11360 assert!(roots.is_empty());
11361 assert_eq!(unknown_root_bindings, 0);
11362 assert_eq!(total_bindings, 0);
11363 assert_eq!(
11364 total_bindings,
11365 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11366 );
11367 }
11368
11369 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11376 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11377 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11378 let text = std::fs::read_to_string(&path)
11379 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11380 let vectors: Vec<(String, Value)> = text
11381 .lines()
11382 .filter(|line| !line.trim().is_empty())
11383 .map(|line| {
11384 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11385 let id = entry["corpus_id"]
11386 .as_str()
11387 .expect("every vector carries a corpus_id")
11388 .to_string();
11389 (id, entry["package"].clone())
11390 })
11391 .collect();
11392 assert_eq!(
11396 vectors.len(),
11397 3,
11398 "vendored fed corpus changed size; re-sync from subc-federation"
11399 );
11400
11401 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11416 let richest = vectors
11417 .iter()
11418 .filter_map(|(_, package)| package.as_object())
11419 .map(|object| {
11420 object
11421 .keys()
11422 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11423 .count()
11424 })
11425 .max()
11426 .unwrap_or(0);
11427 assert!(
11428 richest >= 2,
11429 "vendored corpus no longer carries a package with unmodelled fields, \
11430 so the relay test can no longer distinguish a verbatim relay from a lossy one"
11431 );
11432
11433 vectors
11434 }
11435
11436 #[tokio::test]
11444 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11445 for (corpus_id, package) in fed_admission_facts_vectors() {
11446 let registry = Arc::new(Registry::default());
11447 let forwarding = Arc::new(ForwardingTable::default());
11448 let supervisor = SupervisorHandle::new();
11449 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11450 let handler =
11451 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11452 .with_supervisor(supervisor)
11453 .with_admission_facts_config(
11454 Some("fed".to_string()),
11455 Some(vec!["target".to_string()]),
11456 );
11457
11458 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11459 hello_via_sink(
11460 &handler,
11461 &target_ctx,
11462 &mut target_rx,
11463 hello_frame("target", PROTOCOL_VERSION, 1),
11464 )
11465 .await;
11466
11467 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11468 let route_handler = handler.clone();
11469 let expected = package.clone();
11470 let route_task = tokio::spawn(async move {
11471 route_handler
11472 .handle_control_frame(
11473 &client_ctx,
11474 route_open_frame_with_admission_facts(
11475 20,
11476 "target",
11477 unique_project_root("admission-facts"),
11478 Some(subc_control::ConsumerIdentity {
11479 module_id: "fed".to_string(),
11480 launch_nonce: "fed-nonce".to_string(),
11481 }),
11482 Some(package),
11483 ),
11484 )
11485 .await
11486 .unwrap()
11487 });
11488
11489 let bind_frame = target_rx.recv().await.unwrap();
11490 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11491 let ModuleControlRequest::RouteBind {
11492 admission_facts, ..
11493 } = bind
11494 else {
11495 panic!("{corpus_id}: expected route.bind")
11496 };
11497 assert_eq!(
11498 admission_facts,
11499 Some(expected),
11500 "{corpus_id}: relay must not add, drop or reshape any field"
11501 );
11502
11503 handler
11504 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11505 .await
11506 .unwrap();
11507 route_task.await.unwrap();
11508 }
11509 }
11510
11511 #[tokio::test]
11512 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11513 let registry = Arc::new(Registry::default());
11514 let forwarding = Arc::new(ForwardingTable::default());
11515 let supervisor = SupervisorHandle::new();
11516 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11517 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11518 let handler =
11519 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11520 .with_supervisor(supervisor)
11521 .with_admission_facts_config(
11522 Some("fed".to_string()),
11523 Some(vec!["target".to_string()]),
11524 );
11525
11526 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11527 hello_via_sink(
11528 &handler,
11529 &target_ctx,
11530 &mut target_rx,
11531 hello_frame("target", PROTOCOL_VERSION, 1),
11532 )
11533 .await;
11534 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11535 hello_via_sink(
11536 &handler,
11537 &other_ctx,
11538 &mut other_rx,
11539 hello_frame("other", PROTOCOL_VERSION, 2),
11540 )
11541 .await;
11542
11543 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11544 let expected_facts = facts.clone();
11545 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11546 let route_handler = handler.clone();
11547 let route_task = tokio::spawn(async move {
11548 route_handler
11549 .handle_control_frame(
11550 &client_ctx,
11551 route_open_frame_with_admission_facts(
11552 10,
11553 "target",
11554 unique_project_root("admission-facts"),
11555 Some(subc_control::ConsumerIdentity {
11556 module_id: "fed".to_string(),
11557 launch_nonce: "fed-nonce".to_string(),
11558 }),
11559 Some(facts.clone()),
11560 ),
11561 )
11562 .await
11563 .unwrap()
11564 });
11565 let bind_frame = target_rx.recv().await.unwrap();
11566 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11567 let ModuleControlRequest::RouteBind {
11568 admission_facts, ..
11569 } = bind
11570 else {
11571 panic!("expected route.bind")
11572 };
11573 assert_eq!(admission_facts, Some(expected_facts));
11574 handler
11575 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11576 .await
11577 .unwrap();
11578 assert!(route_task.await.unwrap().is_empty());
11579 assert!(matches!(
11580 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11581 .unwrap(),
11582 ClientControlResponse::RouteOpen { .. }
11583 ));
11584
11585 let direct = handler
11586 .handle_control_frame(
11587 &route_ctx(ConnectionId::new(73)).0,
11588 route_open_frame_with_admission_facts(
11589 11,
11590 "target",
11591 unique_project_root("admission-facts"),
11592 None,
11593 Some(json!({"x": 1})),
11594 ),
11595 )
11596 .await
11597 .unwrap();
11598 assert_eq!(
11599 parse_error(&direct[0])["code"],
11600 "admission_facts_not_permitted"
11601 );
11602
11603 let different_reserved = handler
11604 .handle_control_frame(
11605 &route_ctx(ConnectionId::new(77)).0,
11606 route_open_frame_with_admission_facts(
11607 15,
11608 "target",
11609 unique_project_root("admission-facts"),
11610 Some(subc_control::ConsumerIdentity {
11611 module_id: "other".to_string(),
11612 launch_nonce: "other-nonce".to_string(),
11613 }),
11614 Some(json!({"x": 1})),
11615 ),
11616 )
11617 .await
11618 .unwrap();
11619 assert_eq!(
11620 parse_error(&different_reserved[0])["code"],
11621 "admission_facts_not_permitted"
11622 );
11623
11624 let other_target = handler
11625 .handle_control_frame(
11626 &route_ctx(ConnectionId::new(74)).0,
11627 route_open_frame_with_admission_facts(
11628 12,
11629 "other",
11630 unique_project_root("admission-facts"),
11631 Some(subc_control::ConsumerIdentity {
11632 module_id: "fed".to_string(),
11633 launch_nonce: "fed-nonce".to_string(),
11634 }),
11635 Some(json!({"x": 1})),
11636 ),
11637 )
11638 .await
11639 .unwrap();
11640 assert_eq!(
11641 parse_error(&other_target[0])["code"],
11642 "admission_facts_target_not_allowed"
11643 );
11644
11645 let nonexistent = handler
11646 .handle_control_frame(
11647 &route_ctx(ConnectionId::new(75)).0,
11648 route_open_frame_with_admission_facts(
11649 13,
11650 "missing",
11651 unique_project_root("admission-facts"),
11652 None,
11653 Some(json!({"x": 1})),
11654 ),
11655 )
11656 .await
11657 .unwrap();
11658 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
11659
11660 let described = handler
11661 .handle_control_frame(
11662 &route_ctx(ConnectionId::new(76)).0,
11663 Frame::build(
11664 FrameType::Request,
11665 control_flags(),
11666 0,
11667 0,
11668 14,
11669 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11670 )
11671 .unwrap(),
11672 )
11673 .await
11674 .unwrap();
11675 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11676 serde_json::from_slice(&described[0].body).unwrap()
11677 else {
11678 panic!("expected server.describe response")
11679 };
11680 assert!(capabilities
11681 .iter()
11682 .any(|cap| cap == "admission_facts_relay_v1"));
11683 }
11684
11685 #[tokio::test]
11686 async fn admission_facts_without_configured_carrier_are_rejected() {
11687 let registry = Arc::new(Registry::default());
11688 let forwarding = Arc::new(ForwardingTable::default());
11689 let handler = ControlHandler::with_forwarding(registry, forwarding);
11690 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11691 hello_via_sink(
11692 &handler,
11693 &target_ctx,
11694 &mut target_rx,
11695 hello_frame("target", PROTOCOL_VERSION, 1),
11696 )
11697 .await;
11698
11699 let responses = handler
11700 .handle_control_frame(
11701 &route_ctx(ConnectionId::new(79)).0,
11702 route_open_frame_with_admission_facts(
11703 16,
11704 "target",
11705 unique_project_root("admission-facts"),
11706 None,
11707 Some(json!({"x": 1})),
11708 ),
11709 )
11710 .await
11711 .unwrap();
11712 assert_eq!(
11713 parse_error(&responses[0])["code"],
11714 "admission_facts_not_permitted"
11715 );
11716 }
11717
11718 #[tokio::test]
11719 async fn route_open_relays_consumer_capabilities_verbatim() {
11720 let registry = Arc::new(Registry::default());
11721 let forwarding = Arc::new(ForwardingTable::default());
11722 let handler =
11723 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11724 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11725 hello_via_sink(
11726 &handler,
11727 &module_ctx,
11728 &mut module_rx,
11729 hello_frame("aft", PROTOCOL_VERSION, 7),
11730 )
11731 .await;
11732
11733 let expected = vec!["elicitation".to_string(), "roots".to_string()];
11734 let expected_for_request = expected.clone();
11735 let project_root = unique_project_root("consumer-capabilities-present");
11736 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11737 let route_handler = handler.clone();
11738 let route_task = tokio::spawn(async move {
11739 route_handler
11740 .handle_control_frame(
11741 &client_ctx,
11742 route_open_frame_with_consumer_capabilities(
11743 401,
11744 "aft",
11745 project_root,
11746 Some(expected_for_request),
11747 ),
11748 )
11749 .await
11750 .unwrap()
11751 });
11752 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11753 .await
11754 .unwrap()
11755 .unwrap();
11756 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11757 let ModuleControlRequest::RouteBind {
11758 consumer_capabilities,
11759 ..
11760 } = bind
11761 else {
11762 panic!("expected route.bind request, got {bind:?}");
11763 };
11764 assert_eq!(consumer_capabilities, Some(expected.clone()));
11765
11766 handler
11767 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11768 .await
11769 .unwrap();
11770 let route_response = route_task.await.unwrap();
11771 assert!(route_response.is_empty());
11772 let published = client_rx.recv().await.unwrap();
11773 assert!(matches!(
11774 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11775 ClientControlResponse::RouteOpen { .. }
11776 ));
11777 }
11778
11779 #[tokio::test]
11780 async fn route_open_without_consumer_capabilities_relays_none() {
11781 let registry = Arc::new(Registry::default());
11782 let forwarding = Arc::new(ForwardingTable::default());
11783 let handler =
11784 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11785 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11786 hello_via_sink(
11787 &handler,
11788 &module_ctx,
11789 &mut module_rx,
11790 hello_frame("aft", PROTOCOL_VERSION, 7),
11791 )
11792 .await;
11793
11794 let project_root = unique_project_root("consumer-capabilities-absent");
11795 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11796 let route_handler = handler.clone();
11797 let route_task = tokio::spawn(async move {
11798 route_handler
11799 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11800 .await
11801 .unwrap()
11802 });
11803 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11804 .await
11805 .unwrap()
11806 .unwrap();
11807 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11808 let ModuleControlRequest::RouteBind {
11809 consumer_capabilities,
11810 ..
11811 } = bind
11812 else {
11813 panic!("expected route.bind request, got {bind:?}");
11814 };
11815 assert_eq!(consumer_capabilities, None);
11816
11817 handler
11818 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11819 .await
11820 .unwrap();
11821 let route_response = route_task.await.unwrap();
11822 assert!(route_response.is_empty());
11823 let published = client_rx.recv().await.unwrap();
11824 assert!(matches!(
11825 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11826 ClientControlResponse::RouteOpen { .. }
11827 ));
11828 }
11829
11830 async fn bind_role_versions_for(
11834 sent: Option<BTreeMap<String, String>>,
11835 connection: u64,
11836 ) -> Option<BTreeMap<String, String>> {
11837 let registry = Arc::new(Registry::default());
11838 let forwarding = Arc::new(ForwardingTable::default());
11839 let handler =
11840 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11841 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11842 hello_via_sink(
11843 &handler,
11844 &module_ctx,
11845 &mut module_rx,
11846 hello_frame("aft", PROTOCOL_VERSION, 7),
11847 )
11848 .await;
11849 let project_root = unique_project_root("role-versions");
11850 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11851 let route_handler = handler.clone();
11852 let route_task = tokio::spawn(async move {
11853 route_handler
11854 .handle_control_frame(
11855 &client_ctx,
11856 route_open_frame_with_role_versions(403, "aft", project_root, sent),
11857 )
11858 .await
11859 .unwrap()
11860 });
11861 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11862 .await
11863 .expect("a well-formed route.open reaches the module as a bind")
11864 .unwrap();
11865 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11866 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11867 panic!("expected route.bind request, got {bind:?}");
11868 };
11869 handler
11870 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11871 .await
11872 .unwrap();
11873 assert!(route_task.await.unwrap().is_empty());
11874 let published = client_rx.recv().await.unwrap();
11875 assert!(matches!(
11876 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11877 ClientControlResponse::RouteOpen { .. }
11878 ));
11879 role_versions
11880 }
11881
11882 #[tokio::test]
11883 async fn route_open_relays_role_versions_verbatim() {
11884 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11885 assert_eq!(
11886 bind_role_versions_for(Some(sent.clone()), 141).await,
11887 Some(sent)
11888 );
11889 }
11890
11891 #[tokio::test]
11894 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11895 assert_eq!(bind_role_versions_for(None, 143).await, None);
11896 assert_eq!(
11897 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11898 None
11899 );
11900 }
11901
11902 #[tokio::test]
11903 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11904 let registry = Arc::new(Registry::default());
11905 let forwarding = Arc::new(ForwardingTable::default());
11906 let handler =
11907 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11908 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11909 hello_via_sink(
11910 &handler,
11911 &module_ctx,
11912 &mut module_rx,
11913 hello_frame("aft", PROTOCOL_VERSION, 7),
11914 )
11915 .await;
11916
11917 let nine: BTreeMap<String, String> = (0..9)
11918 .map(|index| (format!("role-{index}"), "v1".to_string()))
11919 .collect();
11920 for (label, malformed) in [
11921 (
11922 "invalid role name",
11923 role_versions(&[("Tool_Provider", "v1")]),
11924 ),
11925 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11926 ("nine entries", nine),
11927 ] {
11928 let responses = tokio::time::timeout(
11931 Duration::from_secs(1),
11932 handler.handle_control_frame(
11933 &route_ctx(ConnectionId::new(148)).0,
11934 route_open_frame_with_role_versions(
11935 404,
11936 "aft",
11937 unique_project_root("role-versions-malformed"),
11938 Some(malformed),
11939 ),
11940 ),
11941 )
11942 .await
11943 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11944 .unwrap();
11945 assert_eq!(responses.len(), 1, "{label}");
11946 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11947 let error = parse_error(&responses[0]);
11948 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11949 assert_eq!(
11950 error["detail"]["field"], "role_versions",
11951 "{label}: {error}"
11952 );
11953 assert!(
11954 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11955 "{label}: a malformed declaration is terminal"
11956 );
11957 assert!(
11958 module_rx.try_recv().is_err(),
11959 "{label}: the module must never see a bind"
11960 );
11961 }
11962 }
11963
11964 #[tokio::test]
11968 async fn route_role_versions_capability_is_advertised() {
11969 let handler = ControlHandler::new(Arc::new(Registry::default()));
11970 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11971 let ack = hello_via_sink(
11972 &handler,
11973 &ctx,
11974 &mut rx,
11975 hello_frame("m", PROTOCOL_VERSION, 1),
11976 )
11977 .await;
11978 let ack = parse_ack(&ack);
11979 assert!(
11980 ack.subc_capabilities
11981 .iter()
11982 .any(|c| c == "route-role-versions/v1"),
11983 "{:?}",
11984 ack.subc_capabilities
11985 );
11986
11987 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11988 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11989 let reply = handler
11990 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11991 .await
11992 .unwrap()
11993 .pop()
11994 .unwrap();
11995 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11996 serde_json::from_slice(&reply.body).unwrap()
11997 else {
11998 panic!("not a server.describe reply");
11999 };
12000 assert!(
12001 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12002 "{capabilities:?}"
12003 );
12004 }
12005
12006 #[tokio::test]
12010 async fn catalog_events_capability_is_advertised() {
12011 let handler = ControlHandler::new(Arc::new(Registry::default()));
12012 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12013 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12014 let reply = handler
12015 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12016 .await
12017 .unwrap()
12018 .pop()
12019 .unwrap();
12020 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12021 serde_json::from_slice(&reply.body).unwrap()
12022 else {
12023 panic!("not a server.describe reply");
12024 };
12025 assert!(
12026 capabilities
12027 .iter()
12028 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12029 "{capabilities:?}"
12030 );
12031 }
12032
12033 #[tokio::test]
12034 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12035 let handler = ControlHandler::new(Arc::new(Registry::default()));
12036 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12037 let ack = hello_via_sink(
12038 &handler,
12039 &module,
12040 &mut module_rx,
12041 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12042 )
12043 .await;
12044 let ack = parse_ack(&ack);
12045 assert!(
12046 ack.subc_capabilities
12047 .iter()
12048 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12049 "{:?}",
12050 ack.subc_capabilities
12051 );
12052 assert!(ack
12053 .subc_capabilities
12054 .iter()
12055 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12056
12057 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12058 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12059 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12060 let reply = handler
12061 .handle_control_frame(&client, frame)
12062 .await
12063 .unwrap()
12064 .pop()
12065 .unwrap();
12066 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12067 serde_json::from_slice(&reply.body).unwrap()
12068 else {
12069 panic!("not a server.describe reply");
12070 };
12071 assert!(
12072 capabilities
12073 .iter()
12074 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12075 "{capabilities:?}"
12076 );
12077 assert!(capabilities
12078 .iter()
12079 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12080 }
12081
12082 #[tokio::test]
12083 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12084 let handler = ControlHandler::new(Arc::new(Registry::default()));
12085 let (module, mut rx) = route_ctx(ConnectionId::new(156));
12086 let ack = hello_via_sink(
12087 &handler,
12088 &module,
12089 &mut rx,
12090 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12091 )
12092 .await;
12093 assert!(parse_ack(&ack)
12094 .subc_capabilities
12095 .iter()
12096 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12097 assert!(parse_ack(&ack)
12098 .subc_capabilities
12099 .iter()
12100 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12101 let (client, _rx) = route_ctx(ConnectionId::new(157));
12102 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12103 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12104 let reply = handler
12105 .handle_control_frame(&client, frame)
12106 .await
12107 .unwrap()
12108 .pop()
12109 .unwrap();
12110 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12111 serde_json::from_slice(&reply.body).unwrap()
12112 else {
12113 panic!("server.describe reply")
12114 };
12115 assert!(capabilities
12116 .iter()
12117 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12118 assert!(capabilities
12119 .iter()
12120 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12121 }
12122
12123 #[tokio::test]
12124 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12125 let handler = ControlHandler::new(Arc::new(Registry::default()));
12126 let (module, mut rx) = route_ctx(ConnectionId::new(158));
12127 let ack = hello_via_sink(
12128 &handler,
12129 &module,
12130 &mut rx,
12131 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12132 )
12133 .await;
12134 assert!(parse_ack(&ack)
12135 .subc_capabilities
12136 .iter()
12137 .any(|cap| cap == "scope-workdir/v1"));
12138 let (client, _rx) = route_ctx(ConnectionId::new(159));
12139 let frame = Frame::build(
12140 FrameType::Request,
12141 control_flags(),
12142 0,
12143 0,
12144 5,
12145 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12146 )
12147 .unwrap();
12148 let reply = handler
12149 .handle_control_frame(&client, frame)
12150 .await
12151 .unwrap()
12152 .remove(0);
12153 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12154 serde_json::from_slice(&reply.body).unwrap()
12155 else {
12156 panic!("server.describe reply")
12157 };
12158 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12159 }
12160
12161 #[tokio::test]
12162 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12163 let handler = ControlHandler::new(Arc::new(Registry::default()));
12164 let (module, mut rx) = route_ctx(ConnectionId::new(154));
12165 let ack = hello_via_sink(
12166 &handler,
12167 &module,
12168 &mut rx,
12169 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12170 )
12171 .await;
12172 assert!(parse_ack(&ack)
12173 .subc_capabilities
12174 .iter()
12175 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12176 let (client, _rx) = route_ctx(ConnectionId::new(155));
12177 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12178 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12179 let reply = handler
12180 .handle_control_frame(&client, frame)
12181 .await
12182 .unwrap()
12183 .pop()
12184 .unwrap();
12185 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12186 serde_json::from_slice(&reply.body).unwrap()
12187 else {
12188 panic!("not a server.describe reply")
12189 };
12190 assert!(capabilities
12191 .iter()
12192 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12193 }
12194
12195 #[tokio::test]
12196 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12197 let registry = Arc::new(Registry::default());
12198 let forwarding = Arc::new(ForwardingTable::default());
12199 let handler =
12200 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12201 .with_health_probe_timeout(Duration::from_secs(5));
12202 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12203 hello_via_sink(
12204 &handler,
12205 &module_ctx,
12206 &mut module_rx,
12207 non_routable_hello_frame_with_control_ops(
12208 "mcp",
12209 300,
12210 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12211 ),
12212 )
12213 .await;
12214 assert!(registry
12215 .get_module("mcp")
12216 .unwrap()
12217 .unwrap()
12218 .manifest
12219 .provides
12220 .is_empty());
12221
12222 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12223 let route_response = handler
12224 .handle_control_frame(
12225 &route_client_ctx,
12226 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12227 )
12228 .await
12229 .unwrap();
12230 assert_eq!(route_response[0].header.ty, FrameType::Error);
12231 assert_eq!(
12232 parse_error(&route_response[0])["code"],
12233 "target_unavailable"
12234 );
12235 assert!(parse_error(&route_response[0])["message"]
12236 .as_str()
12237 .unwrap()
12238 .contains("does not provide the requested target"));
12239 assert!(module_rx.try_recv().is_err());
12240
12241 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12242 let health_handler = handler.clone();
12243 let health_task = tokio::spawn(async move {
12244 health_handler
12245 .handle_control_frame(
12246 &health_client_ctx,
12247 supervisor_health_probe_frame(302, "mcp"),
12248 )
12249 .await
12250 .unwrap()
12251 });
12252 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12253 .await
12254 .unwrap()
12255 .unwrap();
12256 assert_eq!(
12257 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12258 ModuleControlRequest::HealthCheck {}
12259 );
12260 handler
12261 .handle_control_frame(
12262 &module_ctx,
12263 health_response(health_frame.header.corr, HealthStatus::Ok),
12264 )
12265 .await
12266 .unwrap();
12267 let health_response = health_task.await.unwrap();
12268 assert_eq!(health_response[0].header.ty, FrameType::Response);
12269 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12270 ClientControlResponse::SupervisorHealthProbe {
12271 module_id, status, ..
12272 } => {
12273 assert_eq!(module_id, "mcp");
12274 assert_eq!(status, HealthStatus::Ok);
12275 }
12276 other => panic!("unexpected health response: {other:?}"),
12277 }
12278
12279 forwarding
12284 .cleanup_connection(module_ctx.connection_id)
12285 .unwrap();
12286 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12287 let cleanup_response = tokio::time::timeout(
12288 Duration::from_millis(200),
12289 handler.handle_control_frame(
12290 &cleanup_probe_ctx,
12291 supervisor_health_probe_frame(303, "mcp"),
12292 ),
12293 )
12294 .await
12295 .expect("probe should fail immediately when the control lane is gone")
12296 .unwrap();
12297 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12298 assert_eq!(
12299 parse_error(&cleanup_response[0])["code"],
12300 "target_unavailable"
12301 );
12302 assert!(parse_error(&cleanup_response[0])["message"]
12303 .as_str()
12304 .unwrap()
12305 .contains("no module connection"));
12306
12307 handler
12308 .cleanup_connection(module_ctx.connection_id)
12309 .unwrap();
12310 }
12311
12312 #[tokio::test]
12313 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12314 let registry = Arc::new(Registry::default());
12315 let supervisor_handle = SupervisorHandle::new();
12316 let supervisor =
12317 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12318 .with_handle(supervisor_handle.clone())
12319 .with_connection_file_path(
12320 std::env::temp_dir()
12321 .join(format!("subc-route-open-warming-{}", std::process::id())),
12322 );
12323 let module = supervisor
12324 .supervise_configured(
12325 ModuleSpec {
12326 module_id: "warming".to_string(),
12327 program: fake_aft_stub_path(),
12328 args: Vec::new(),
12329 env: Vec::new(),
12330 reserved: false,
12331 reserved_prefixes: Vec::new(),
12332 protocol: ModuleProtocol::Subc,
12333 overlap: Default::default(),
12334 },
12335 true,
12336 )
12337 .unwrap();
12338 assert_eq!(module.state().unwrap(), ModuleState::Running);
12339
12340 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12341 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12342 let response = handler
12343 .handle_control_frame(
12344 &ctx,
12345 route_open_frame(304, "warming", unique_project_root("warming")),
12346 )
12347 .await
12348 .unwrap();
12349 module.stop().await.unwrap();
12350
12351 assert_eq!(response[0].header.ty, FrameType::Error);
12352 let error = parse_error(&response[0]);
12353 assert_eq!(error["code"], "module_warming");
12354 assert!(error["message"]
12355 .as_str()
12356 .unwrap()
12357 .contains("state=running, enabled=true, live=false"));
12358 }
12359
12360 #[test]
12361 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12362 let handler = ControlHandler::new(Arc::new(Registry::default()));
12363 let capture = EventCapture::default();
12364 let _subscriber =
12365 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12366 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12367 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12368 let pending = (0..limit).collect::<Vec<_>>();
12369 let response = handler
12370 .route_open_capacity_refusal(
12371 &ctx,
12372 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12373 "busy",
12374 pending.len(),
12375 limit,
12376 )
12377 .unwrap();
12378 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12379 let event = capture
12380 .events()
12381 .into_iter()
12382 .find(|event| {
12383 event.target == "control"
12384 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12385 })
12386 .expect("connection admission refusal event");
12387 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12388 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12389 }
12390
12391 #[test]
12392 fn route_open_target_cap_logs_admission_reason_and_capacity() {
12393 let handler = ControlHandler::new(Arc::new(Registry::default()));
12394 let capture = EventCapture::default();
12395 let _subscriber =
12396 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12397 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12398 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12399 let guards = (0..limit)
12400 .map(|_| {
12401 handler
12402 .route_bind_concurrency
12403 .try_admit("busy", limit)
12404 .unwrap()
12405 })
12406 .collect::<Vec<_>>();
12407 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12408 Err(in_flight) => in_flight,
12409 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12410 };
12411 let response = handler
12412 .route_open_target_capacity_refusal(
12413 &ctx,
12414 &route_open_frame(397, "busy", unique_project_root("target-cap")),
12415 "busy",
12416 in_flight,
12417 )
12418 .unwrap();
12419 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12420 let event = capture
12421 .events()
12422 .into_iter()
12423 .find(|event| {
12424 event.target == "control"
12425 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12426 })
12427 .expect("target admission refusal event");
12428 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12429 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12430 drop(guards);
12431 }
12432
12433 #[tokio::test]
12434 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
12435 let registry = Arc::new(Registry::default());
12436 let handle = SupervisorHandle::new();
12437 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
12438 .with_handle(handle.clone());
12439 supervisor
12440 .supervise_configured(
12441 crate::ModuleSpec {
12442 module_id: "drift".into(),
12443 program: PathBuf::from("disabled-module"),
12444 args: Vec::new(),
12445 env: Vec::new(),
12446 reserved: false,
12447 reserved_prefixes: Vec::new(),
12448 protocol: ModuleProtocol::Subc,
12449 overlap: Default::default(),
12450 },
12451 false,
12452 )
12453 .unwrap();
12454 let handler = ControlHandler::new(registry).with_supervisor(handle);
12455 for configured in [None, Some(false), Some(true)] {
12456 handler.capability_evaluator.configure(
12457 configured.map(|enabled| ("drift".to_string(), enabled)),
12458 BTreeMap::new(),
12459 );
12460 let frame =
12461 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12462 let replies = handler
12463 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
12464 .await
12465 .unwrap();
12466 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12467 assert!(body.get("configured_enabled").is_none());
12468 assert_eq!(body["op"], "supervisor.ack");
12469 let frame =
12470 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12471 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12472 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12473 assert_eq!(
12474 body["modules"][0].get("configured_enabled"),
12475 configured.map(Value::Bool).as_ref()
12476 );
12477 }
12478 }
12479
12480 #[tokio::test(flavor = "current_thread")]
12481 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12482 let handler = ControlHandler::new(Arc::new(Registry::default()));
12483 let capture = EventCapture::default();
12484 let _subscriber =
12485 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12486 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12487 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12488
12489 let _ = handler
12490 .handle_client_control_request(
12491 &ctx,
12492 frame,
12493 ClientControlRequest::SupervisorSetEnabled {
12494 module_id: "broca".to_string(),
12495 enabled: false,
12496 },
12497 &mut None,
12498 )
12499 .await
12500 .unwrap();
12501
12502 let events = capture
12503 .events()
12504 .into_iter()
12505 .filter(|event| {
12506 event.target == "control"
12507 && event.fields.get("message").map(String::as_str)
12508 == Some("supervisor request received")
12509 })
12510 .collect::<Vec<_>>();
12511 assert_eq!(events.len(), 1, "one supervisor request log line");
12512 let event = &events[0];
12513 assert_eq!(event.level, tracing::Level::INFO);
12514 assert_eq!(
12515 event.fields.get("op"),
12516 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12517 );
12518 assert_eq!(
12519 event.fields.get("module_id"),
12520 Some(&"\"broca\"".to_string())
12521 );
12522 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12523 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12524 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12525 }
12526
12527 #[tokio::test(flavor = "current_thread")]
12528 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12529 let registry = Arc::new(Registry::default());
12530 let module_connection = ConnectionId::new(303);
12531 registry
12532 .register_with_control_ops(
12533 manifest("aft", PROTOCOL_VERSION),
12534 PROTOCOL_VERSION,
12535 module_connection,
12536 module_baseline_control_ops(),
12537 )
12538 .unwrap();
12539 let handler = ControlHandler::new(registry);
12540 let capture = EventCapture::default();
12541 let _subscriber =
12542 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12543 let (ctx, _rx) = route_ctx(module_connection);
12544 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12545
12546 let _ = handler
12547 .handle_client_control_request(
12548 &ctx,
12549 frame,
12550 ClientControlRequest::SupervisorSetEnabled {
12551 module_id: "broca".to_string(),
12552 enabled: false,
12553 },
12554 &mut None,
12555 )
12556 .await
12557 .unwrap();
12558
12559 let callers = capture
12560 .events()
12561 .into_iter()
12562 .filter(|event| {
12563 event.fields.get("message").map(String::as_str)
12564 == Some("supervisor request received")
12565 })
12566 .map(|event| event.fields.get("caller").cloned())
12567 .collect::<Vec<_>>();
12568 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12569 }
12570
12571 #[tokio::test(flavor = "current_thread")]
12572 async fn supervisor_list_does_not_log_request_received() {
12573 let handler = ControlHandler::new(Arc::new(Registry::default()));
12574 let capture = EventCapture::default();
12575 let _subscriber =
12576 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12577 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12578 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12579
12580 handler
12581 .handle_client_control_request(
12582 &ctx,
12583 frame,
12584 ClientControlRequest::SupervisorList {},
12585 &mut None,
12586 )
12587 .await
12588 .unwrap();
12589
12590 assert!(capture.events().into_iter().all(|event| {
12591 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12592 }));
12593 }
12594
12595 #[tokio::test(flavor = "current_thread")]
12596 async fn supervisor_rescan_preview_does_not_log_request_received() {
12597 let handler = ControlHandler::new(Arc::new(Registry::default()));
12598 let capture = EventCapture::default();
12599 let _subscriber =
12600 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12601 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
12602 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12603
12604 handler
12605 .handle_client_control_request(
12606 &ctx,
12607 frame,
12608 ClientControlRequest::SupervisorRescan { preview: true },
12609 &mut None,
12610 )
12611 .await
12612 .unwrap();
12613
12614 assert!(capture.events().into_iter().all(|event| {
12615 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12616 }));
12617 }
12618
12619 struct ApproveOperator;
12620 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
12621 fn prompt(
12622 &self,
12623 _: &str,
12624 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
12625 ) -> crate::operator_confirm::ProviderResult {
12626 crate::operator_confirm::ProviderResult::Approved
12627 }
12628 }
12629
12630 #[tokio::test]
12631 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
12632 assert_eq!(
12633 ControlHandler::default().privileged_operator_prompt,
12634 PrivilegedOperatorPrompt::Log
12635 );
12636 for (initial, changed, running) in [
12637 ("log", "enforce", PrivilegedOperatorPrompt::Log),
12638 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
12639 ] {
12640 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
12641 let path = dir.join("subc.jsonc");
12642 std::fs::write(
12643 &path,
12644 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
12645 )
12646 .unwrap();
12647 let config = crate::daemon_config::load(&path).unwrap().unwrap();
12648 let registry = Arc::new(Registry::default());
12649 let handle = SupervisorHandle::new();
12650 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
12651 .with_handle(handle.clone());
12652 let handler = ControlHandler::new(registry)
12653 .with_supervisor(handle)
12654 .with_privileged_operator_prompt(config.privileged_operator_prompt)
12655 .with_supervisor_rescan(supervisor, &path, config.port);
12656 handler.forwarding.operator_confirms().configure(
12657 Arc::new(ApproveOperator),
12658 crate::operator_confirm::OperatorLimits::default(),
12659 );
12660 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
12661
12662 for (disk_mode, needs_restart) in [
12663 (Some(initial), false),
12664 (Some(changed), true),
12665 (Some(changed), true),
12666 (None, running == PrivilegedOperatorPrompt::Enforce),
12667 (Some(initial), false),
12668 ] {
12669 let mut doc = json!({"version":1});
12670 if let Some(mode) = disk_mode {
12671 doc["privileged_operator_prompt"] = json!(mode);
12672 }
12673 std::fs::write(&path, doc.to_string()).unwrap();
12674 for preview in [true, false] {
12675 let request = ClientControlRequest::SupervisorRescan { preview };
12676 let frame = Frame::build(
12677 FrameType::Request,
12678 control_flags(),
12679 0,
12680 0,
12681 1,
12682 serde_json::to_vec(&request).unwrap(),
12683 )
12684 .unwrap();
12685 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12686 assert_eq!(frames.len(), 1);
12687 let ClientControlResponse::SupervisorRescan { result } =
12688 serde_json::from_slice(&frames[0].body).unwrap()
12689 else {
12690 panic!("expected a rescan response");
12691 };
12692 let expected = if needs_restart {
12693 vec!["privileged_operator_prompt"]
12694 } else {
12695 Vec::new()
12696 };
12697 assert_eq!(result.restart_required, expected);
12698 assert_eq!(handler.privileged_operator_prompt, running);
12699 assert_eq!(
12700 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
12701 running
12702 );
12703 }
12704 }
12705 }
12706 }
12707
12708 #[tokio::test]
12709 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
12710 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
12711 let path = dir.join("subc.jsonc");
12712 let registry = Arc::new(Registry::default());
12713 let handle = SupervisorHandle::new();
12714 let supervisor =
12715 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
12716 let handler = ControlHandler::new(registry)
12717 .with_supervisor(handle)
12718 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
12719 .with_supervisor_rescan(supervisor, &path, None);
12720 handler.forwarding.operator_confirms().configure(
12721 Arc::new(ApproveOperator),
12722 crate::operator_confirm::OperatorLimits::default(),
12723 );
12724 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
12725 for preview in [true, false] {
12726 std::fs::write(
12727 &path,
12728 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
12729 )
12730 .unwrap();
12731 let request = ClientControlRequest::SupervisorRescan { preview };
12732 let frame = Frame::build(
12733 FrameType::Request,
12734 control_flags(),
12735 0,
12736 0,
12737 1,
12738 serde_json::to_vec(&request).unwrap(),
12739 )
12740 .unwrap();
12741 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12742 assert_eq!(frames.len(), 1);
12743 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
12744 assert_eq!(
12745 handler.privileged_operator_prompt,
12746 PrivilegedOperatorPrompt::Enforce
12747 );
12748 }
12749 }
12750
12751 #[tokio::test]
12756 async fn route_open_refusal_names_the_check_that_refused() {
12757 let handler = ControlHandler::new(Arc::new(Registry::default()));
12758 let capture = EventCapture::default();
12759 let _subscriber =
12760 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12761 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12762 let response = handler
12763 .handle_control_frame(
12764 &ctx,
12765 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
12766 )
12767 .await
12768 .unwrap();
12769
12770 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12771 let event = capture
12772 .events()
12773 .into_iter()
12774 .find(|event| {
12775 event.target == "control"
12776 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12777 })
12778 .expect("route.open refusal event");
12779 assert_eq!(
12780 event.fields.get("reason"),
12781 Some(&"\"not_registered\"".to_string())
12782 );
12783 }
12784
12785 #[tokio::test]
12786 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12787 let registry = Arc::new(Registry::default());
12788 let supervisor_handle = SupervisorHandle::new();
12789 let supervisor =
12790 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12791 .with_handle(supervisor_handle.clone())
12792 .with_connection_file_path(std::env::temp_dir().join(format!(
12793 "subc-route-open-refusal-info-{}",
12794 std::process::id()
12795 )));
12796 let module = supervisor
12797 .supervise_configured(
12798 ModuleSpec {
12799 module_id: "warming".to_string(),
12800 program: fake_aft_stub_path(),
12801 args: Vec::new(),
12802 env: Vec::new(),
12803 reserved: false,
12804 reserved_prefixes: Vec::new(),
12805 protocol: ModuleProtocol::Subc,
12806 overlap: Default::default(),
12807 },
12808 true,
12809 )
12810 .unwrap();
12811 assert_eq!(module.state().unwrap(), ModuleState::Running);
12812
12813 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12814 assert!(handler
12815 .counters()
12816 .snapshot()
12817 .get("route_open_refused_by_code")
12818 .is_none());
12819 let capture = EventCapture::default();
12820 let _subscriber =
12821 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12822 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
12823 let response = handler
12824 .handle_control_frame(
12825 &ctx,
12826 route_open_frame(394, "warming", unique_project_root("refusal-info")),
12827 )
12828 .await
12829 .unwrap();
12830 module.stop().await.unwrap();
12831
12832 assert_eq!(parse_error(&response[0])["code"], "module_warming");
12833 let event = capture
12834 .events()
12835 .into_iter()
12836 .find(|event| {
12837 event.target == "control"
12838 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
12839 })
12840 .expect("route.open refusal event");
12841 assert_eq!(
12842 event.fields.get("module_id"),
12843 Some(&"\"warming\"".to_string())
12844 );
12845 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
12846 assert_eq!(
12847 event.fields.get("reason"),
12848 Some(&"\"supervised_not_registered\"".to_string())
12849 );
12850 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
12851 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
12852 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
12853 assert_eq!(
12854 handler.counters().snapshot()["route_open_refused_by_code"],
12855 json!({ "module_warming": 1 })
12856 );
12857 }
12858
12859 const OUTAGE_START: &str = "route.open refusing module: not serving";
12860 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
12861
12862 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
12863 capture
12864 .events()
12865 .into_iter()
12866 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
12867 .collect()
12868 }
12869
12870 fn supervise_stub(
12871 registry: &Arc<Registry>,
12872 module_id: &str,
12873 enabled: bool,
12874 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
12875 let supervisor_handle = SupervisorHandle::new();
12876 let supervisor =
12877 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
12878 .with_handle(supervisor_handle.clone())
12879 .with_connection_file_path(std::env::temp_dir().join(format!(
12880 "subc-route-outage-{module_id}-{}",
12881 std::process::id()
12882 )));
12883 let module = supervisor
12884 .supervise_configured(
12885 ModuleSpec {
12886 module_id: module_id.to_string(),
12887 program: fake_aft_stub_path(),
12888 args: Vec::new(),
12889 env: Vec::new(),
12890 reserved: false,
12891 reserved_prefixes: Vec::new(),
12892 protocol: ModuleProtocol::Subc,
12893 overlap: Default::default(),
12894 },
12895 enabled,
12896 )
12897 .unwrap();
12898 (supervisor_handle, module)
12899 }
12900
12901 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12902 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12903 module_id: module_id.to_string(),
12904 drain_timeout_ms: Some(50),
12905 })
12906 .unwrap();
12907 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12908 }
12909
12910 #[test]
12914 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12915 let registry = Arc::new(Registry::default());
12916 let forwarding = Arc::new(ForwardingTable::default());
12917 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12918 let second = ControlHandler::with_forwarding(registry, forwarding);
12919 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12920 }
12921
12922 #[tokio::test(flavor = "current_thread")]
12926 async fn route_open_unknown_module_refusals_add_no_outage_state() {
12927 let handler = ControlHandler::new(Arc::new(Registry::default()));
12928 let capture = EventCapture::default();
12929 let _subscriber =
12930 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12931 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12932 for corr in 0..8 {
12933 let response = handler
12934 .handle_control_frame(
12935 &ctx,
12936 route_open_frame(
12937 380 + corr,
12938 &format!("nobody-{corr}"),
12939 unique_project_root("outage-unknown"),
12940 ),
12941 )
12942 .await
12943 .unwrap();
12944 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12945 }
12946
12947 assert_eq!(handler.route_outages.tracked_module_count(), 0);
12948 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12949 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12950 }
12951
12952 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12957 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12958 let registry = Arc::new(Registry::default());
12959 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
12960 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12961 let capture = EventCapture::default();
12962 let _subscriber =
12963 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12964 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12965 handler.route_outages.record_accepted("outage-restart");
12968
12969 let response = handler
12970 .handle_control_frame(
12971 &ctx,
12972 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12973 )
12974 .await
12975 .unwrap();
12976 assert_eq!(response[0].header.ty, FrameType::Error);
12977 let starts = outage_lines(&capture, OUTAGE_START);
12978 assert_eq!(starts.len(), 1, "{starts:?}");
12979 assert_eq!(starts[0].level, tracing::Level::WARN);
12980 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12981 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12982 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12983 handler.route_outages.record_accepted("outage-restart");
12984 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12985
12986 let restart = handler
12987 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12988 .await
12989 .unwrap();
12990 assert_eq!(
12991 restart[0].header.ty,
12992 FrameType::Response,
12993 "{:?}",
12994 parse_error(&restart[0])
12995 );
12996 handler
12997 .handle_control_frame(
12998 &ctx,
12999 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13000 )
13001 .await
13002 .unwrap();
13003 module.stop().await.unwrap();
13004
13005 let starts = outage_lines(&capture, OUTAGE_START);
13006 assert_eq!(starts.len(), 2, "{starts:?}");
13007 assert_eq!(starts[1].level, tracing::Level::INFO);
13008 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13009 }
13010
13011 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13015 async fn failed_operator_restart_leaves_no_operator_mark() {
13016 let registry = Arc::new(Registry::default());
13017 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
13018 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13019 let capture = EventCapture::default();
13020 let _subscriber =
13021 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13022 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13023 handler.route_outages.record_accepted("outage-disabled");
13024
13025 let restart = handler
13026 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13027 .await
13028 .unwrap();
13029 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13030 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13031
13032 handler
13033 .handle_control_frame(
13034 &ctx,
13035 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13036 )
13037 .await
13038 .unwrap();
13039 let starts = outage_lines(&capture, OUTAGE_START);
13040 assert_eq!(starts.len(), 1, "{starts:?}");
13041 assert_eq!(starts[0].level, tracing::Level::WARN);
13042 }
13043
13044 #[tokio::test(flavor = "current_thread")]
13045 async fn route_open_unknown_module_escapes_target_module_id() {
13046 let handler = ControlHandler::new(Arc::new(Registry::default()));
13047 let capture = EventCapture::default();
13048 let _subscriber =
13049 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13050 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13051 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13052 let response = handler
13053 .handle_control_frame(
13054 &ctx,
13055 route_open_frame(
13056 395,
13057 hostile_module_id,
13058 unique_project_root("hostile-target-module-id"),
13059 ),
13060 )
13061 .await
13062 .unwrap();
13063
13064 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13065 let event = capture
13066 .events()
13067 .into_iter()
13068 .find(|event| {
13069 event.target == "control"
13070 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13071 })
13072 .expect("route.open unknown-module refusal event");
13073 let logged = event.fields.get("module_id").expect("module_id field");
13074 assert!(!logged.bytes().any(|byte| byte < 0x20));
13075 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13076 }
13077
13078 #[tokio::test(flavor = "current_thread")]
13079 async fn route_open_module_rejection_uses_daemon_counter_key() {
13080 let registry = Arc::new(Registry::default());
13081 let forwarding = Arc::new(ForwardingTable::default());
13082 let handler =
13083 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13084 let module_connection = ConnectionId::new(95);
13085 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13086 hello_via_sink(
13087 &handler,
13088 &module_ctx,
13089 &mut module_rx,
13090 hello_frame("aft", PROTOCOL_VERSION, 395),
13091 )
13092 .await;
13093
13094 let client_connection = ConnectionId::new(96);
13095 let (client_ctx, _client_rx) = route_ctx(client_connection);
13096 let capture = EventCapture::default();
13097 let _subscriber =
13098 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13099 let (route_task, bind) = relay_route_open(
13100 &handler,
13101 client_connection,
13102 &client_ctx.egress,
13103 &mut module_rx,
13104 396,
13105 "aft",
13106 "hostile-module-code",
13107 )
13108 .await;
13109 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13110 let rejection = Frame::build(
13111 FrameType::Error,
13112 control_flags(),
13113 0,
13114 0,
13115 bind.header.corr,
13116 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13117 )
13118 .unwrap();
13119 handler
13120 .handle_control_frame(&module_ctx, rejection)
13121 .await
13122 .unwrap();
13123
13124 let response = route_task.await.unwrap();
13125 assert_eq!(parse_error(&response[0])["code"], hostile_code);
13126 let counters = handler.counters().snapshot();
13127 assert_eq!(
13128 counters["route_open_refused_by_code"],
13129 json!({ "module_rejected": 1 })
13130 );
13131 assert!(counters["route_open_refused_by_code"]
13132 .get(hostile_code)
13133 .is_none());
13134
13135 let event = capture
13136 .events()
13137 .into_iter()
13138 .find(|event| {
13139 event.target == "control"
13140 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13141 })
13142 .expect("route.open module-rejection refusal event");
13143 let logged = event.fields.get("module_code").expect("module_code field");
13144 assert!(!logged.bytes().any(|byte| byte < 0x20));
13145 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13146 }
13147
13148 #[tokio::test]
13149 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13150 let registry = Arc::new(Registry::default());
13151 let supervisor_handle = SupervisorHandle::new();
13152 let missing_program = std::env::temp_dir().join(format!(
13153 "subc-route-open-missing-program-{}",
13154 std::process::id()
13155 ));
13156 let supervisor =
13157 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13158 .with_handle(supervisor_handle.clone());
13159 let module = supervisor
13160 .supervise_configured(
13161 ModuleSpec {
13162 module_id: "failed".to_string(),
13163 program: missing_program,
13164 args: Vec::new(),
13165 env: Vec::new(),
13166 reserved: false,
13167 reserved_prefixes: Vec::new(),
13168 protocol: ModuleProtocol::Subc,
13169 overlap: Default::default(),
13170 },
13171 true,
13172 )
13173 .unwrap();
13174 assert_eq!(module.state().unwrap(), ModuleState::Failed);
13175
13176 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13177 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13178 let response = handler
13179 .handle_control_frame(
13180 &ctx,
13181 route_open_frame(305, "failed", unique_project_root("failed")),
13182 )
13183 .await
13184 .unwrap();
13185
13186 assert_eq!(response[0].header.ty, FrameType::Error);
13187 let error = parse_error(&response[0]);
13188 assert_eq!(error["code"], "target_unavailable");
13189 assert!(error["message"]
13190 .as_str()
13191 .unwrap()
13192 .contains("state=failed, enabled=true, live=false"));
13193 }
13194
13195 #[tokio::test]
13196 async fn route_open_role_mismatch_remains_target_unavailable() {
13197 let registry = Arc::new(Registry::default());
13198 let handler = ControlHandler::new(Arc::clone(®istry));
13199 handler
13200 .handle_control(
13201 ConnectionId::new(41),
13202 non_routable_hello_frame_with_control_ops("health-only", 306, None),
13203 )
13204 .unwrap();
13205
13206 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13207 let response = handler
13208 .handle_control_frame(
13209 &ctx,
13210 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13211 )
13212 .await
13213 .unwrap();
13214
13215 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13216 assert!(parse_error(&response[0])["message"]
13217 .as_str()
13218 .unwrap()
13219 .contains("does not provide the requested target"));
13220 }
13221
13222 #[tokio::test]
13223 async fn route_open_inactive_registration_remains_target_unavailable() {
13224 let registry = Arc::new(Registry::default());
13225 let handler = ControlHandler::new(Arc::clone(®istry));
13226 handler
13227 .handle_control(
13228 ConnectionId::new(43),
13229 hello_frame("inactive", PROTOCOL_VERSION, 308),
13230 )
13231 .unwrap();
13232 assert!(registry
13233 .set_module_state_for_test("inactive", ChannelState::Closed)
13234 .unwrap());
13235
13236 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13237 let response = handler
13238 .handle_control_frame(
13239 &ctx,
13240 route_open_frame(309, "inactive", unique_project_root("inactive")),
13241 )
13242 .await
13243 .unwrap();
13244
13245 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13246 assert!(parse_error(&response[0])["message"]
13247 .as_str()
13248 .unwrap()
13249 .contains("is not active"));
13250 }
13251
13252 #[tokio::test]
13253 async fn late_health_reply_is_recorded_through_the_module_response_path() {
13254 let registry = Arc::new(Registry::default());
13255 let forwarding = Arc::new(ForwardingTable::default());
13256 let supervisor_handle = SupervisorHandle::new();
13257 let supervisor =
13258 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
13259 .with_forwarding(Arc::clone(&forwarding))
13260 .with_handle(supervisor_handle.clone());
13261 let module = supervisor
13262 .supervise_configured(
13263 crate::ModuleSpec {
13264 module_id: "late-health-response".to_string(),
13265 program: PathBuf::from("disabled-module"),
13266 args: Vec::new(),
13267 env: Vec::new(),
13268 reserved: false,
13269 reserved_prefixes: Vec::new(),
13270 protocol: ModuleProtocol::Subc,
13271 overlap: Default::default(),
13272 },
13273 false,
13274 )
13275 .unwrap();
13276 let handler =
13277 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13278 .with_supervisor(supervisor_handle);
13279 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13280 handler
13281 .handle_control_frame(
13282 &module_ctx,
13283 hello_frame_with_control_ops(
13284 "late-health-response",
13285 PROTOCOL_VERSION,
13286 7,
13287 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13288 ),
13289 )
13290 .await
13291 .unwrap();
13292 let probe_started_at = Instant::now() - Duration::from_millis(80);
13293 let pending = forwarding
13294 .begin_health_probe_rpc_for(
13295 "late-health-response",
13296 MODULE_CONTROL_OP_HEALTH_CHECK,
13297 probe_started_at,
13298 Instant::now() - Duration::from_millis(1),
13299 )
13300 .unwrap();
13301 assert!(forwarding
13302 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13303 .unwrap());
13304
13305 let responses = handler
13306 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13307 .await
13308 .unwrap();
13309
13310 assert!(responses.is_empty());
13311 let health = module.status().unwrap().health;
13312 assert_eq!(health.late_answer_count, 1);
13313 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13314 }
13315
13316 #[tokio::test]
13317 async fn health_probe_timeout_and_module_death_are_typed() {
13318 let registry = Arc::new(Registry::default());
13319 let forwarding = Arc::new(ForwardingTable::default());
13320 let handler =
13321 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13322 .with_health_probe_timeout(Duration::from_millis(50));
13323 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13324 hello_via_sink(
13325 &handler,
13326 &module_ctx,
13327 &mut module_rx,
13328 hello_frame_with_control_ops(
13329 "aft",
13330 PROTOCOL_VERSION,
13331 7,
13332 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13333 ),
13334 )
13335 .await;
13336
13337 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13338 let responses = handler
13339 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13340 .await
13341 .unwrap();
13342 assert_eq!(responses[0].header.ty, FrameType::Error);
13343 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13344 let _ = module_rx.try_recv();
13345
13346 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13347 let health_handler = handler.clone();
13348 let death_task = tokio::spawn(async move {
13349 health_handler
13350 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13351 .await
13352 .unwrap()
13353 });
13354 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13355 .await
13356 .unwrap()
13357 .unwrap();
13358 handler
13359 .cleanup_connection(module_ctx.connection_id)
13360 .unwrap();
13361 let responses = death_task.await.unwrap();
13362 assert_eq!(responses[0].header.ty, FrameType::Error);
13363 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13364 }
13365
13366 #[test]
13367 fn hello_requires_exact_protocol_version() {
13368 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13369 let registry = Arc::new(Registry::default());
13370 let handler = ControlHandler::new(Arc::clone(®istry));
13371 let responses = handler
13372 .handle_control(
13373 ConnectionId::new(connection),
13374 hello_frame("aft", offered, 9),
13375 )
13376 .unwrap();
13377
13378 assert_eq!(responses.len(), 1);
13379 assert_eq!(responses[0].header.ty, FrameType::Error);
13380 let error = parse_error(&responses[0]);
13381 assert_eq!(error["code"], "version_unsupported");
13382 assert!(registry.get_module("aft").unwrap().is_none());
13383 assert_eq!(registry.active_registration_count().unwrap(), 0);
13384 }
13385 }
13386
13387 #[test]
13388 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13389 let registry = Arc::new(Registry::default());
13390 let forwarding = Arc::new(ForwardingTable::default());
13391 let handler =
13392 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13393 let module_connection = ConnectionId::new(301);
13394 let registration = registry
13395 .register_with_control_ops(
13396 manifest("aft-push", PROTOCOL_VERSION),
13397 PROTOCOL_VERSION,
13398 module_connection,
13399 module_baseline_control_ops(),
13400 )
13401 .unwrap();
13402 let (module_tx, _module_rx) = mpsc::channel(8);
13403 let endpoint = forwarding
13404 .register_module_connection(
13405 module_connection,
13406 "aft-push".to_string(),
13407 PROTOCOL_VERSION,
13408 manifest_concurrency(®istration.manifest),
13409 FrameSink::new(module_tx),
13410 )
13411 .unwrap();
13412
13413 let unknown = Frame::build(
13415 FrameType::Push,
13416 control_flags(),
13417 0,
13418 0,
13419 5,
13420 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13421 )
13422 .unwrap();
13423 let out = handler.handle_status_update(endpoint, unknown).unwrap();
13424 assert!(
13425 out.is_empty(),
13426 "unknown push op must be ignored, got {out:?}"
13427 );
13428
13429 let malformed = Frame::build(
13431 FrameType::Push,
13432 control_flags(),
13433 0,
13434 0,
13435 6,
13436 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13437 )
13438 .unwrap();
13439 let out = handler.handle_status_update(endpoint, malformed).unwrap();
13440 assert_eq!(out.len(), 1);
13441 assert_eq!(out[0].header.ty, FrameType::Error);
13442 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13443 }
13444
13445 #[test]
13446 fn hello_rejected_when_connection_already_owns_client_routes() {
13447 let registry = Arc::new(Registry::default());
13448 let forwarding = Arc::new(ForwardingTable::default());
13449 let handler =
13450 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13451 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
13453 let client_connection = ConnectionId::new(202);
13454
13455 let responses = handler
13458 .handle_control(
13459 client_connection,
13460 hello_frame("aft-second", PROTOCOL_VERSION, 9),
13461 )
13462 .unwrap();
13463 assert_eq!(responses[0].header.ty, FrameType::Error);
13464 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13465 assert!(registry.get_module("aft-second").unwrap().is_none());
13466 }
13467
13468 #[tokio::test]
13469 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13470 let registry = Arc::new(Registry::default());
13471 let forwarding = Arc::new(ForwardingTable::default());
13472 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13473 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13474 hello_via_sink(
13475 &handler,
13476 &module_ctx,
13477 &mut module_rx,
13478 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13479 )
13480 .await;
13481 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13482 let pending = forwarding
13483 .begin_route_bind_relay_for_test(
13484 client_ctx.connection_id,
13485 client_ctx.egress.clone(),
13486 2,
13487 "alpha",
13488 )
13489 .unwrap();
13490 forwarding
13491 .complete_pending_relay(
13492 module_ctx.connection_id,
13493 pending.corr,
13494 RouteBindRelayOutcome::Accepted,
13495 )
13496 .unwrap();
13497 client_rx.try_recv().unwrap();
13498 for module_id in ["beta", "alpha"] {
13499 let replies = handler
13500 .handle_control_frame(
13501 &module_ctx,
13502 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13503 )
13504 .await
13505 .unwrap();
13506 assert_eq!(replies.len(), 1, "second HELLO must be refused");
13507 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13508 }
13509 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13510 assert!(registry.get_module("beta").unwrap().is_none());
13511 assert!(matches!(
13512 forwarding
13513 .lookup_data_route(
13514 client_ctx.connection_id,
13515 pending.client_channel,
13516 pending.client_epoch,
13517 )
13518 .unwrap(),
13519 DataRoute::Client(DataRouteState::Bound(_))
13520 ));
13521 assert!(handler
13522 .hello_launch_nonces
13523 .lock()
13524 .unwrap()
13525 .presented(module_ctx.connection_id, Some("alpha-nonce")));
13526 assert!(module_rx.try_recv().is_err());
13527 }
13528
13529 #[test]
13530 fn reserved_module_hello_requires_matching_launch_nonce() {
13531 let registry = Arc::new(Registry::default());
13532 let supervisor = SupervisorHandle::new();
13533 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13536 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13537
13538 let no_nonce = handler
13540 .handle_control(
13541 ConnectionId::new(1),
13542 hello_frame("vault", PROTOCOL_VERSION, 1),
13543 )
13544 .unwrap();
13545 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13546 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13547 assert!(registry.get_module("vault").unwrap().is_none());
13548
13549 let wrong = handler
13551 .handle_control(
13552 ConnectionId::new(2),
13553 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13554 )
13555 .unwrap();
13556 assert_eq!(wrong[0].header.ty, FrameType::Error);
13557 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13558 assert!(registry.get_module("vault").unwrap().is_none());
13559
13560 let ok = handler
13562 .handle_control(
13563 ConnectionId::new(3),
13564 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13565 )
13566 .unwrap();
13567 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13568 assert!(registry.get_module("vault").unwrap().is_some());
13569 }
13570
13571 #[test]
13572 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13573 let registry = Arc::new(Registry::default());
13574 let supervisor = SupervisorHandle::new();
13575 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13576 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13577 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13578
13579 let squat = handler
13580 .handle_control(
13581 ConnectionId::new(1),
13582 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13583 )
13584 .unwrap();
13585 assert_eq!(squat[0].header.ty, FrameType::Error);
13586 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
13587 assert!(parse_error(&squat[0])["message"]
13588 .as_str()
13589 .unwrap()
13590 .contains("fed:"));
13591
13592 let accepted_peer = handler
13593 .handle_control(
13594 ConnectionId::new(2),
13595 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
13596 )
13597 .unwrap();
13598 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
13599
13600 let accepted_short = handler
13601 .handle_control(
13602 ConnectionId::new(3),
13603 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
13604 )
13605 .unwrap();
13606 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
13607
13608 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
13609 let response = handler
13610 .handle_control(
13611 ConnectionId::new(conn),
13612 hello_frame(module_id, PROTOCOL_VERSION, conn),
13613 )
13614 .unwrap();
13615 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
13616 }
13617 }
13618
13619 #[test]
13620 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
13621 let registry = Arc::new(Registry::default());
13622 let supervisor = SupervisorHandle::new();
13623 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13624 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13625 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
13626 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13627
13628 let owner_nonce = handler
13629 .handle_control(
13630 ConnectionId::new(1),
13631 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
13632 )
13633 .unwrap();
13634 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
13635 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
13636 assert!(registry.get_module("fed:special").unwrap().is_none());
13637
13638 let exact_nonce = handler
13639 .handle_control(
13640 ConnectionId::new(2),
13641 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
13642 )
13643 .unwrap();
13644 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
13645 assert!(registry.get_module("fed:special").unwrap().is_some());
13646 }
13647
13648 #[test]
13649 fn non_reserved_module_ignores_launch_nonce() {
13650 let registry = Arc::new(Registry::default());
13651 let handler = ControlHandler::new(Arc::clone(®istry));
13654 let no_nonce = handler
13655 .handle_control(
13656 ConnectionId::new(1),
13657 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
13658 )
13659 .unwrap();
13660 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
13661 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
13662
13663 let echoed_nonce = handler
13664 .handle_control(
13665 ConnectionId::new(2),
13666 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
13667 )
13668 .unwrap();
13669 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
13670 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
13671 }
13672
13673 #[test]
13674 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
13675 let handler = ControlHandler::default();
13676 let conn = ConnectionId::new(1);
13677 let malformed = Frame::build(
13678 FrameType::Hello,
13679 control_flags(),
13680 0,
13681 0,
13682 3,
13683 b"{not json".to_vec(),
13684 )
13685 .unwrap();
13686
13687 let error = handler.handle_control(conn, malformed).unwrap();
13688 assert_eq!(error[0].header.ty, FrameType::Error);
13689 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
13690
13691 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13692 let pong = handler.handle_control(conn, ping).unwrap();
13693 assert_eq!(pong[0].header.ty, FrameType::Pong);
13694 assert_eq!(pong[0].header.corr, 4);
13695 }
13696
13697 #[test]
13698 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
13699 let registry = Arc::new(Registry::default());
13700 let handler = ControlHandler::new(Arc::clone(®istry));
13701
13702 handler
13703 .handle_control(
13704 ConnectionId::new(1),
13705 hello_frame("aft", PROTOCOL_VERSION, 1),
13706 )
13707 .unwrap();
13708 let duplicate = handler
13709 .handle_control(
13710 ConnectionId::new(2),
13711 hello_frame("aft", PROTOCOL_VERSION, 2),
13712 )
13713 .unwrap();
13714
13715 assert_eq!(duplicate[0].header.ty, FrameType::Error);
13716 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13717 let registration = registry.get_module("aft").unwrap().unwrap();
13718 assert_eq!(registration.connection_id, ConnectionId::new(1));
13719 }
13720
13721 #[test]
13722 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
13723 let registry = Arc::new(Registry::default());
13724 let forwarding = Arc::new(ForwardingTable::default());
13725 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
13726 let handler =
13727 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13728 .with_process_liveness(process_liveness);
13729 let (ctx, route_channel, route_epoch) =
13730 bind_liveness_route(®istry, &forwarding, "aft-dead");
13731 let responses = handler
13732 .handle_route_poll(
13733 &ctx,
13734 route_poll_frame(41, PollKind::Liveness, route_channel),
13735 route_channel,
13736 route_epoch,
13737 PollKind::Liveness,
13738 )
13739 .unwrap();
13740
13741 assert_eq!(responses.len(), 1);
13742 assert_eq!(responses[0].header.ty, FrameType::Response);
13743 assert_route_poll_liveness(&responses[0], false);
13744 }
13745
13746 #[test]
13747 fn liveness_poll_without_process_source_uses_bound_route() {
13748 let registry = Arc::new(Registry::default());
13749 let forwarding = Arc::new(ForwardingTable::default());
13750 let handler =
13751 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13752 let (ctx, route_channel, route_epoch) =
13753 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
13754 let responses = handler
13755 .handle_route_poll(
13756 &ctx,
13757 route_poll_frame(42, PollKind::Liveness, route_channel),
13758 route_channel,
13759 route_epoch,
13760 PollKind::Liveness,
13761 )
13762 .unwrap();
13763
13764 assert_route_poll_liveness(&responses[0], true);
13765 }
13766
13767 #[test]
13768 fn liveness_poll_untracked_process_source_uses_bound_route() {
13769 let registry = Arc::new(Registry::default());
13770 let forwarding = Arc::new(ForwardingTable::default());
13771 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13772 let handler =
13773 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13774 .with_process_liveness(process_liveness);
13775 let (ctx, route_channel, route_epoch) =
13776 bind_liveness_route(®istry, &forwarding, "aft-untracked");
13777 let responses = handler
13778 .handle_route_poll(
13779 &ctx,
13780 route_poll_frame(43, PollKind::Liveness, route_channel),
13781 route_channel,
13782 route_epoch,
13783 PollKind::Liveness,
13784 )
13785 .unwrap();
13786
13787 assert_route_poll_liveness(&responses[0], true);
13788 }
13789
13790 #[tokio::test]
13791 async fn unknown_op_returns_unknown_control_op() {
13792 let handler = ControlHandler::default();
13793 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13794 let request = Frame::build(
13795 FrameType::Request,
13796 control_flags(),
13797 0,
13798 0,
13799 55,
13800 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13801 )
13802 .unwrap();
13803
13804 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13805
13806 assert_eq!(response.len(), 1);
13807 assert_eq!(response[0].header.ty, FrameType::Error);
13808 assert_eq!(response[0].header.corr, 55);
13809 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
13810 }
13811
13812 #[tokio::test]
13813 async fn supervisor_provenance_rejects_unknown_exact_module() {
13814 let handler = ControlHandler::default();
13815 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
13816 let request = Frame::build(
13817 FrameType::Request,
13818 control_flags(),
13819 0,
13820 0,
13821 57,
13822 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
13823 )
13824 .unwrap();
13825
13826 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13827
13828 assert_eq!(response.len(), 1);
13829 assert_eq!(response[0].header.ty, FrameType::Error);
13830 assert_eq!(response[0].header.corr, 57);
13831 let error = parse_error(&response[0]);
13832 assert_eq!(error["code"], "unknown_module");
13833 assert_eq!(error["message"], "module_id 'missing' is not supervised");
13834 }
13835
13836 #[test]
13837 fn provenance_probe_override_keeps_handler_tests_deterministic() {
13838 let expected = subc_control::RunningImageAgreement::Unavailable {
13839 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13840 };
13841 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
13842 assert_eq!(handler.provenance_probe_override, Some(expected));
13843 }
13844
13845 #[test]
13846 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
13847 let verdict = reload_verdict(
13848 std::path::Path::new("/bin/new"),
13849 Some(std::path::Path::new("/bin/old")),
13850 subc_control::RunningImageAgreement::Unavailable {
13851 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13852 },
13853 );
13854 assert!(matches!(
13855 verdict.path,
13856 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
13857 if configured == std::path::Path::new("/bin/new")
13858 && spawned_from == std::path::Path::new("/bin/old")
13859 ));
13860 }
13861
13862 #[test]
13863 fn launch_spec_without_running_digest_is_unavailable_not_match() {
13864 use subc_control::{
13865 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
13866 };
13867 let configured = crate::supervise::ModuleSpec {
13868 module_id: "legacy".into(),
13869 program: "/module".into(),
13870 args: Vec::new(),
13871 env: Vec::new(),
13872 reserved: false,
13873 reserved_prefixes: Vec::new(),
13874 protocol: subc_control::ModuleProtocol::Subc,
13875 overlap: Default::default(),
13876 };
13877 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
13878 assert_eq!(
13879 launch_spec_agreement(&configured, false, Some(&digest)),
13880 Agreement::Unavailable {
13881 reason: Reason::NotRunning
13882 }
13883 );
13884 assert_eq!(
13885 launch_spec_agreement(&configured, true, None),
13886 Agreement::Unavailable {
13887 reason: Reason::SpawnedSpecUnavailable
13888 }
13889 );
13890 assert_eq!(
13891 launch_spec_agreement(&configured, true, Some(&digest)),
13892 Agreement::Match
13893 );
13894 }
13895
13896 #[cfg(windows)]
13897 #[test]
13898 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13899 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13900 let configured = dir.join("module.exe");
13901 std::fs::write(&configured, b"executable path fixture").unwrap();
13902 let spawned = std::fs::canonicalize(&configured).unwrap();
13903 assert_ne!(
13904 configured, spawned,
13905 "fixture must exercise different path spellings"
13906 );
13907 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13908 let lower_case = configured.to_string_lossy().to_lowercase();
13909 for spelling in [
13910 &configured,
13911 Path::new(&forward_slashes),
13912 Path::new(&lower_case),
13913 ] {
13914 let verdict = reload_verdict(
13915 spelling,
13916 Some(&spawned),
13917 subc_control::RunningImageAgreement::Unavailable {
13918 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13919 },
13920 );
13921 assert_eq!(
13922 verdict.path,
13923 ReloadPathAgreement::Match,
13924 "configured {spelling:?}, spawned {spawned:?}"
13925 );
13926 }
13927 }
13928
13929 #[cfg(windows)]
13930 #[test]
13931 fn windows_reload_path_keeps_different_existing_program_pending() {
13932 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13933 let old = dir.join("old.exe");
13934 let new = dir.join("new.exe");
13935 std::fs::write(&old, b"same executable content").unwrap();
13936 std::fs::copy(&old, &new).unwrap();
13937 let spawned = std::fs::canonicalize(&old).unwrap();
13938 let image = subc_control::RunningImageAgreement::Unavailable {
13939 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13940 };
13941 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13942 assert!(
13943 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13944 if configured == new && spawned_from == spawned)
13945 );
13946 let missing = dir.join("missing.exe");
13947 assert!(matches!(
13948 reload_verdict(&missing, Some(&spawned), image).path,
13949 ReloadPathAgreement::Mismatch { .. }
13950 ));
13951 }
13952
13953 #[test]
13954 fn reload_verdict_detects_replaced_image_at_same_path() {
13955 let image = subc_control::RunningImageAgreement::Mismatch {
13956 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13957 digest: "old".into(),
13958 },
13959 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13960 digest: "new".into(),
13961 },
13962 };
13963 let verdict = reload_verdict(
13964 std::path::Path::new("/bin/same"),
13965 Some(std::path::Path::new("/bin/same")),
13966 image.clone(),
13967 );
13968 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13969 assert_eq!(verdict.image, image);
13970 }
13971
13972 #[test]
13973 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13974 let image = subc_control::RunningImageAgreement::Unavailable {
13975 reason: subc_control::RunningImageUnavailableReason::NotRunning,
13976 };
13977 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13978 assert_eq!(
13979 verdict.path,
13980 subc_control::ReloadPathAgreement::Unavailable {
13981 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13982 }
13983 );
13984 assert_eq!(verdict.image, image);
13985
13986 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13987 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13988 };
13989 let verdict = reload_verdict(
13990 std::path::Path::new("/bin/same"),
13991 Some(std::path::Path::new("/bin/same")),
13992 unconfirmed.clone(),
13993 );
13994 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13995 assert_eq!(verdict.image, unconfirmed);
13996 }
13997
13998 #[test]
13999 fn reload_verdict_preserves_each_image_unavailability_reason() {
14000 use subc_control::RunningImageUnavailableReason as Reason;
14001
14002 for reason in [
14003 Reason::NotRunning,
14004 Reason::UnsupportedPlatform,
14005 Reason::RunningExecutableUnreadable,
14006 Reason::SpawnedPathUnreadable,
14007 Reason::HashFailed,
14008 Reason::ProcessIdentityUnconfirmed,
14009 Reason::Unknown("future_probe_reason".to_string()),
14010 ] {
14011 let image = subc_control::RunningImageAgreement::Unavailable {
14012 reason: reason.clone(),
14013 };
14014 let verdict = reload_verdict(
14015 std::path::Path::new("/bin/same"),
14016 Some(std::path::Path::new("/bin/same")),
14017 image.clone(),
14018 );
14019 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14020 assert_eq!(verdict.image, image, "{reason:?}");
14021 }
14022 }
14023
14024 #[tokio::test]
14025 async fn malformed_control_bodies_return_invalid_control_body() {
14026 let handler = ControlHandler::default();
14027 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14028
14029 for (corr, body) in [
14030 (56, br#"{"route_channel":1}"#.as_slice()),
14031 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14032 (
14033 58,
14034 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14035 ),
14036 ] {
14037 let request = Frame::build(
14038 FrameType::Request,
14039 control_flags(),
14040 0,
14041 0,
14042 corr,
14043 body.to_vec(),
14044 )
14045 .unwrap();
14046 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14047
14048 assert_eq!(response.len(), 1);
14049 assert_eq!(response[0].header.ty, FrameType::Error);
14050 assert_eq!(response[0].header.corr, corr);
14051 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14052 }
14053 }
14054
14055 #[tokio::test]
14056 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14057 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14058 let registry = Arc::new(Registry::default());
14059 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14060 let router = Router::with_control_handler(Arc::clone(&control));
14061 let connection = router.begin_connection();
14062 let (ctx, mut rx) = route_ctx(connection.id());
14063
14064 router
14065 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14066 .await
14067 .unwrap();
14068 let response = rx.recv().await.unwrap();
14069 let ack = parse_ack(&response);
14070 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14071 let channel = 1;
14072
14073 let goodbye =
14074 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14075 router.route_for_connection(&ctx, goodbye).await.unwrap();
14076 assert!(rx.try_recv().is_err());
14077 assert!(registry.get_module("aft").unwrap().is_none());
14078
14079 router
14080 .route_for_connection(&ctx, channel_request(channel, 13))
14081 .await
14082 .unwrap();
14083 let error_frame = rx.recv().await.unwrap();
14084 assert_eq!(error_frame.header.ty, FrameType::Error);
14085 assert_eq!(error_frame.header.channel, channel);
14086 let captured = crate::router::test_log::captured_logs(&logs);
14087 assert_eq!(
14088 captured
14089 .lines()
14090 .filter(|line| {
14091 line.contains("module_id=aft")
14092 && line.contains("reason=explicit_goodbye")
14093 && line.contains("module registration ended")
14094 })
14095 .count(),
14096 1,
14097 "unexpected GOODBYE registry log: {captured}"
14098 );
14099 }
14100
14101 #[tokio::test]
14102 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14103 let registry = Arc::new(Registry::default());
14104 let handler = ControlHandler::new(registry).with_capability_config(
14105 [("prov".to_string(), true), ("cons".to_string(), true)],
14106 BTreeMap::new(),
14107 );
14108 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14109 register_capability_manifest(
14110 &handler,
14111 &provider_ctx,
14112 &mut provider_rx,
14113 capability_manifest("prov", &["thing/v1"], &[]),
14114 1,
14115 )
14116 .await;
14117 let mut consumer = capability_manifest("cons", &[], &[]);
14118 consumer.capabilities.as_mut().unwrap().requires.push(
14119 subc_protocol::manifest::CapabilityRequirement {
14120 capability: "thing/v1".to_string(),
14121 need: subc_protocol::manifest::CapabilityNeed::Required,
14122 },
14123 );
14124 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14125 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14126 assert_eq!(
14127 handler.capability_evaluator.verdict("cons", "thing/v1"),
14128 Some(CapabilityVerdict::Provided)
14129 );
14130 let (mut client_rx, _) = open_route_for_capability_test(
14131 &handler,
14132 &provider_ctx,
14133 &mut provider_rx,
14134 703,
14135 3,
14136 "prov",
14137 None,
14138 )
14139 .await;
14140 let goodbye =
14141 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14142 handler
14143 .handle_control_frame(&provider_ctx, goodbye)
14144 .await
14145 .unwrap();
14146 assert_eq!(
14147 handler.capability_evaluator.verdict("cons", "thing/v1"),
14148 Some(CapabilityVerdict::NeverProvided)
14149 );
14150 let closed = client_rx
14151 .try_recv()
14152 .expect("GOODBYE pushes route.closed before route GOODBYE");
14153 assert!(
14154 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14155 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14156 );
14157 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14158 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14159 }
14160
14161 struct RouteClosedFixture {
14162 handler: ControlHandler,
14163 #[cfg(unix)]
14164 supervisor: Supervisor,
14165 module_ctx: RouteCtx,
14166 #[cfg(unix)]
14167 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14168 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14169 route: Arc<crate::forwarding::RouteBinding>,
14170 }
14171
14172 async fn route_closed_fixture() -> RouteClosedFixture {
14173 let registry = Arc::new(Registry::default());
14174 let handler = ControlHandler::new(Arc::clone(®istry));
14175 #[cfg(unix)]
14176 let supervisor = Supervisor::new(registry, RestartPolicy::default())
14177 .with_forwarding(Arc::clone(&handler.forwarding));
14178 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14179 register_capability_manifest(
14180 &handler,
14181 &module_ctx,
14182 &mut module_rx,
14183 capability_manifest("prov", &[], &[]),
14184 1,
14185 )
14186 .await;
14187 let (client_rx, bind) = open_route_for_capability_test(
14188 &handler,
14189 &module_ctx,
14190 &mut module_rx,
14191 702,
14192 2,
14193 "prov",
14194 None,
14195 )
14196 .await;
14197 let ModuleControlRequest::RouteBind {
14198 route_channel: channel,
14199 epoch: route_epoch,
14200 ..
14201 } = bind
14202 else {
14203 panic!("expected route.bind");
14204 };
14205 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14206 handler
14207 .forwarding
14208 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14209 .unwrap()
14210 else {
14211 panic!("the acknowledged route must be bound");
14212 };
14213 RouteClosedFixture {
14214 handler,
14215 #[cfg(unix)]
14216 supervisor,
14217 module_ctx,
14218 #[cfg(unix)]
14219 module_rx,
14220 client_rx,
14221 route,
14222 }
14223 }
14224
14225 impl RouteClosedFixture {
14226 #[cfg(unix)]
14227 async fn cut(&mut self) {
14228 let notices = async {
14229 let mut draining = self.module_rx.recv().await.unwrap();
14230 assert!(matches!(
14231 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14232 &draining.body
14233 )
14234 .unwrap(),
14235 subc_protocol::session::ModuleControlCommand::Draining {
14236 reason: RouteCloseReason::Restart,
14237 ..
14238 }
14239 ));
14240 draining.flushed.take().unwrap().send(()).unwrap();
14241 let mut closing = self.client_rx.recv().await.unwrap();
14242 assert!(matches!(
14243 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14244 ClientControlPush::RouteClosing {
14245 reason: RouteCloseReason::Restart,
14246 ..
14247 }
14248 ));
14249 closing.flushed.take().unwrap().send(()).unwrap();
14250 };
14251 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14252 result.unwrap();
14253 }
14254
14255 fn disconnect(&mut self) -> ClientControlPush {
14256 self.handler
14257 .cleanup_connection(self.module_ctx.connection_id)
14258 .unwrap();
14259 let closed = self.client_rx.try_recv().unwrap();
14260 assert_eq!(closed.header.ty, FrameType::Push);
14261 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14262 assert!(matches!(
14263 &push,
14264 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14265 if channels == &[self.route.client_channel]
14266 ));
14267 assert_eq!(
14268 self.client_rx.try_recv().unwrap().header.ty,
14269 FrameType::Goodbye
14270 );
14271 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14272 push
14273 }
14274 }
14275
14276 #[cfg(unix)]
14277 #[tokio::test(start_paused = true)]
14278 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14279 let mut fixture = route_closed_fixture().await;
14280 fixture.cut().await;
14281 assert!(matches!(
14282 fixture.disconnect(),
14283 ClientControlPush::RouteClosed {
14284 reason: RouteCloseReason::Restart,
14285 drained: true,
14286 ..
14287 }
14288 ));
14289 }
14290
14291 #[cfg(unix)]
14292 #[tokio::test(start_paused = true)]
14293 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14294 let mut fixture = route_closed_fixture().await;
14295 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14296 let started = tokio::time::Instant::now();
14297 fixture.cut().await;
14298 assert!(started.elapsed() >= Duration::from_secs(2));
14299 assert_eq!(fixture.route.flow.in_flight(), 1);
14300 assert!(matches!(
14301 fixture.disconnect(),
14302 ClientControlPush::RouteClosed {
14303 reason: RouteCloseReason::Restart,
14304 drained: false,
14305 ..
14306 }
14307 ));
14308 }
14309
14310 #[tokio::test]
14311 async fn module_connection_crash_reports_crash_and_not_drained() {
14312 let mut fixture = route_closed_fixture().await;
14313 assert!(matches!(
14317 fixture.disconnect(),
14318 ClientControlPush::RouteClosed {
14319 reason: RouteCloseReason::Crash,
14320 drained: false,
14321 ..
14322 }
14323 ));
14324 }
14325
14326 #[tokio::test]
14327 async fn dropping_router_connection_releases_registration() {
14328 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14329 let registry = Arc::new(Registry::default());
14330 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14331 let router = Router::with_control_handler(Arc::clone(&control));
14332 let connection = router.begin_connection();
14333 let connection_id = connection.id();
14334 let (ctx, mut rx) = route_ctx(connection_id);
14335
14336 router
14337 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14338 .await
14339 .unwrap();
14340 let response = rx.recv().await.unwrap();
14341 let ack = parse_ack(&response);
14342 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14343 assert!(registry.get_module("aft").unwrap().is_some());
14344
14345 drop(connection);
14346
14347 assert!(registry.get_module("aft").unwrap().is_none());
14348 assert_eq!(registry.active_registration_count().unwrap(), 0);
14349
14350 control
14351 .cleanup_connection(ConnectionId::new(u64::MAX))
14352 .unwrap();
14353 let captured = crate::router::test_log::captured_logs(&logs);
14354 println!("captured registry lifecycle logs:\n{captured}");
14355 let events: Vec<_> = captured
14356 .lines()
14357 .filter(|line| {
14358 line.contains("module registered module_id=aft ")
14359 || line.contains("module registration ended")
14360 })
14361 .collect();
14362 assert_eq!(
14363 events.len(),
14364 2,
14365 "unexpected registry lifecycle logs: {captured}"
14366 );
14367 assert!(events[0].contains("module registered module_id=aft "));
14368 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14369 assert!(events[1].contains(&format!(
14370 "module_id=aft connection_id={}",
14371 connection_id.get()
14372 )));
14373 assert!(events[1].contains("reason=connection_closed"));
14374 assert!(events[1].contains("module registration ended"));
14375 assert_eq!(
14376 captured
14377 .lines()
14378 .filter(|line| line.contains("module registered module_id=aft "))
14379 .count(),
14380 1,
14381 "legacy registration admission line must appear once: {captured}"
14382 );
14383 }
14384
14385 #[tokio::test]
14386 async fn hello_registration_keeps_legacy_module_registered_line_once() {
14387 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14388 let registry = Arc::new(Registry::default());
14389 let control = ControlHandler::new(Arc::clone(®istry));
14390 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14391 hello_via_sink(
14392 &control,
14393 &ctx,
14394 &mut rx,
14395 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14396 )
14397 .await;
14398
14399 let captured = crate::router::test_log::captured_logs(&logs);
14400 let admissions: Vec<_> = captured
14401 .lines()
14402 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14403 .collect();
14404 assert_eq!(
14405 admissions.len(),
14406 1,
14407 "legacy admission line must remain exactly once: {captured}"
14408 );
14409 assert!(admissions[0].contains("routable_provider=true"));
14410 assert!(admissions[0].contains("connection_id=777"));
14411 }
14412
14413 fn capability_manifest(
14414 module_id: &str,
14415 provides: &[&str],
14416 must_never_reach: &[&str],
14417 ) -> ModuleManifest {
14418 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14419 manifest.capabilities = Some(CapabilityDeclarations {
14420 provides: provides
14421 .iter()
14422 .map(|capability| (*capability).to_string())
14423 .collect(),
14424 requires: Vec::new(),
14425 must_never_reach: must_never_reach
14426 .iter()
14427 .map(|capability| (*capability).to_string())
14428 .collect(),
14429 });
14430 manifest
14431 }
14432
14433 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14434 Frame::build(
14435 FrameType::Hello,
14436 control_flags(),
14437 0,
14438 0,
14439 corr,
14440 serde_json::to_vec(&ModuleHelloBody {
14441 protocol_ver: manifest.protocol_ver,
14442 manifest,
14443 control_ops: None,
14444 launch_nonce: None,
14445 })
14446 .expect("capability test HELLO serializes"),
14447 )
14448 .expect("capability test HELLO frame builds")
14449 }
14450
14451 fn catalog_update_with_capabilities_frame(
14452 corr: u64,
14453 capabilities: CapabilityDeclarations,
14454 ) -> Frame {
14455 Frame::build(
14456 FrameType::Request,
14457 control_flags(),
14458 0,
14459 0,
14460 corr,
14461 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14462 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14463 capabilities: Some(capabilities),
14464 ready: None,
14465 })
14466 .expect("capability catalog.update serializes"),
14467 )
14468 .expect("capability catalog.update frame builds")
14469 }
14470
14471 async fn register_capability_manifest(
14472 handler: &ControlHandler,
14473 ctx: &RouteCtx,
14474 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14475 manifest: ModuleManifest,
14476 corr: u64,
14477 ) {
14478 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14479 }
14480
14481 async fn open_route_for_capability_test(
14482 handler: &ControlHandler,
14483 target_ctx: &RouteCtx,
14484 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14485 client_connection_id: u64,
14486 corr: u64,
14487 target_module_id: &str,
14488 consumer_identity: Option<ConsumerIdentity>,
14489 ) -> (
14490 mpsc::Receiver<crate::router::OutboundFrame>,
14491 ModuleControlRequest,
14492 ) {
14493 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14494 let route_handler = handler.clone();
14495 let target_module_id = target_module_id.to_string();
14496 let route_task = tokio::spawn(async move {
14497 route_handler
14498 .handle_control_frame(
14499 &client_ctx,
14500 route_open_frame_with_admission_facts(
14501 corr,
14502 &target_module_id,
14503 unique_project_root("admission-facts"),
14504 consumer_identity,
14505 None,
14506 ),
14507 )
14508 .await
14509 .expect("capability test route.open succeeds")
14510 });
14511 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14512 .await
14513 .expect("capability test route.open must reach route.bind")
14514 .expect("target control receiver stays open");
14515 let bind_request: ModuleControlRequest =
14516 serde_json::from_slice(&bind.body).expect("route.bind decodes");
14517 handler
14518 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14519 .await
14520 .expect("capability test route.bind ACK succeeds");
14521 assert!(route_task.await.expect("route.open task joins").is_empty());
14522 let opened = client_rx
14523 .recv()
14524 .await
14525 .expect("successful route.open publishes a response");
14526 assert!(matches!(
14527 serde_json::from_slice::<ClientControlResponse>(&opened.body),
14528 Ok(ClientControlResponse::RouteOpen { .. })
14529 ));
14530 (client_rx, bind_request)
14531 }
14532
14533 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
14534 assert_eq!(frame.header.ty, FrameType::Push);
14535 assert_eq!(frame.header.channel, 0);
14536 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14537 .expect("route.closed control push decodes");
14538 let ClientControlPush::RouteClosed { channels, .. } = &push else {
14539 panic!("expected route.closed");
14540 };
14541 assert_eq!(channels.len(), 1, "exactly one violating route closed");
14542 let channels = channels.clone();
14543 assert_eq!(
14544 push,
14545 ClientControlPush::RouteClosed {
14546 module_id: target_module_id.to_string(),
14547 channels,
14548 reason: RouteCloseReason::CapabilityDenied,
14549 drained: false,
14550 abandoned: 0,
14551 excluded_subscriptions: 0,
14552 terminal: Some(false),
14553 }
14554 );
14555 }
14556
14557 #[tokio::test]
14558 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14559 let registry = Arc::new(Registry::default());
14560 let forwarding = Arc::new(ForwardingTable::default());
14561 let supervisor = SupervisorHandle::new();
14562 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14563 let handler =
14564 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14565 .with_supervisor(supervisor);
14566 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14567 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14568 register_capability_manifest(
14569 &handler,
14570 &target_ctx,
14571 &mut target_rx,
14572 capability_manifest("target", &["credentials-provider/v1"], &[]),
14573 1,
14574 )
14575 .await;
14576 register_capability_manifest(
14577 &handler,
14578 &opener_ctx,
14579 &mut opener_rx,
14580 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14581 2,
14582 )
14583 .await;
14584
14585 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14586 let replies = handler
14587 .handle_control_frame(
14588 &client_ctx,
14589 route_open_frame_with_admission_facts(
14590 3,
14591 "target",
14592 unique_project_root("admission-facts"),
14593 Some(ConsumerIdentity {
14594 module_id: "opener".to_string(),
14595 launch_nonce: "opener-nonce".to_string(),
14596 }),
14597 None,
14598 ),
14599 )
14600 .await
14601 .expect("denied route.open returns a typed frame");
14602 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14603 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14604 assert!(
14605 target_rx.try_recv().is_err(),
14606 "forbidden route.open must not relay route.bind"
14607 );
14608 }
14609
14610 #[tokio::test]
14611 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
14612 let registry = Arc::new(Registry::default());
14613 let forwarding = Arc::new(ForwardingTable::default());
14614 let supervisor = SupervisorHandle::new();
14615 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14616 let handler =
14617 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14618 .with_supervisor(supervisor);
14619 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
14620 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
14621 register_capability_manifest(
14622 &handler,
14623 &target_ctx,
14624 &mut target_rx,
14625 capability_manifest("target", &["credentials-provider/v1"], &[]),
14626 1,
14627 )
14628 .await;
14629 register_capability_manifest(
14630 &handler,
14631 &old_opener_ctx,
14632 &mut old_opener_rx,
14633 capability_manifest("opener", &[], &[]),
14634 2,
14635 )
14636 .await;
14637 let (mut client_rx, _) = open_route_for_capability_test(
14638 &handler,
14639 &target_ctx,
14640 &mut target_rx,
14641 712,
14642 3,
14643 "target",
14644 Some(ConsumerIdentity {
14645 module_id: "opener".to_string(),
14646 launch_nonce: "opener-nonce".to_string(),
14647 }),
14648 )
14649 .await;
14650 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14651
14652 handler
14653 .cleanup_connection(old_opener_ctx.connection_id)
14654 .expect("old opener registration cleans up");
14655 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
14656 register_capability_manifest(
14657 &handler,
14658 &new_opener_ctx,
14659 &mut new_opener_rx,
14660 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14661 4,
14662 )
14663 .await;
14664
14665 assert_capability_denied_push(
14666 client_rx
14667 .try_recv()
14668 .expect("HELLO deny addition must emit route.closed")
14669 .frame,
14670 "target",
14671 );
14672 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14673 assert!(matches!(
14674 target_rx.try_recv(),
14675 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14676 ));
14677 }
14678
14679 #[tokio::test]
14680 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
14681 let registry = Arc::new(Registry::default());
14682 let forwarding = Arc::new(ForwardingTable::default());
14683 let supervisor = SupervisorHandle::new();
14684 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14685 let handler =
14686 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14687 .with_supervisor(supervisor);
14688 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
14689 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
14690 register_capability_manifest(
14691 &handler,
14692 &target_ctx,
14693 &mut target_rx,
14694 capability_manifest("target", &[], &[]),
14695 1,
14696 )
14697 .await;
14698 register_capability_manifest(
14699 &handler,
14700 &opener_ctx,
14701 &mut opener_rx,
14702 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14703 2,
14704 )
14705 .await;
14706 let (mut client_rx, _) = open_route_for_capability_test(
14707 &handler,
14708 &target_ctx,
14709 &mut target_rx,
14710 722,
14711 3,
14712 "target",
14713 Some(ConsumerIdentity {
14714 module_id: "opener".to_string(),
14715 launch_nonce: "opener-nonce".to_string(),
14716 }),
14717 )
14718 .await;
14719 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14720
14721 let replies = handler
14722 .handle_control_frame(
14723 &target_ctx,
14724 catalog_update_with_capabilities_frame(
14725 4,
14726 CapabilityDeclarations {
14727 provides: vec!["credentials-provider/v1".to_string()],
14728 requires: Vec::new(),
14729 must_never_reach: Vec::new(),
14730 },
14731 ),
14732 )
14733 .await
14734 .expect("claim catalog.update succeeds");
14735 assert!(matches!(
14736 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
14737 Ok(ModuleControlResponseToModule::CatalogUpdate {})
14738 ));
14739 assert_capability_denied_push(
14740 client_rx
14741 .try_recv()
14742 .expect("claim addition must emit route.closed")
14743 .frame,
14744 "target",
14745 );
14746 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14747 assert!(matches!(
14748 target_rx.try_recv(),
14749 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14750 ));
14751 }
14752
14753 #[tokio::test]
14754 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
14755 let registry = Arc::new(Registry::default());
14756 let forwarding = Arc::new(ForwardingTable::default());
14757 let supervisor = SupervisorHandle::new();
14758 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14759 let handler =
14760 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14761 .with_supervisor(supervisor);
14762 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
14763 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
14764 register_capability_manifest(
14765 &handler,
14766 &target_ctx,
14767 &mut target_rx,
14768 capability_manifest("target", &["credentials-provider/v1"], &[]),
14769 1,
14770 )
14771 .await;
14772 register_capability_manifest(
14773 &handler,
14774 &opener_ctx,
14775 &mut opener_rx,
14776 capability_manifest("opener", &[], &[]),
14777 2,
14778 )
14779 .await;
14780 let (mut client_rx, _) = open_route_for_capability_test(
14781 &handler,
14782 &target_ctx,
14783 &mut target_rx,
14784 732,
14785 3,
14786 "target",
14787 Some(ConsumerIdentity {
14788 module_id: "opener".to_string(),
14789 launch_nonce: "opener-nonce".to_string(),
14790 }),
14791 )
14792 .await;
14793 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14794
14795 handler
14796 .handle_control_frame(
14797 &target_ctx,
14798 catalog_update_with_capabilities_frame(
14799 4,
14800 CapabilityDeclarations {
14801 provides: Vec::new(),
14802 requires: Vec::new(),
14803 must_never_reach: Vec::new(),
14804 },
14805 ),
14806 )
14807 .await
14808 .expect("claim removal catalog.update succeeds");
14809 assert_eq!(
14810 forwarding.active_binding_count().unwrap(),
14811 1,
14812 "removing an attested target claim must leave the route census unchanged"
14813 );
14814 assert!(
14815 client_rx.try_recv().is_err(),
14816 "claim removal must not emit route.closed capability_denied"
14817 );
14818 assert!(
14819 target_rx.try_recv().is_err(),
14820 "claim removal must not send the target a route GOODBYE"
14821 );
14822 }
14823
14824 #[tokio::test]
14827 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
14828 let registry = Arc::new(Registry::default());
14829 let forwarding = Arc::new(ForwardingTable::default());
14830 let supervisor = SupervisorHandle::new();
14831 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14832 let handler =
14833 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14834 .with_supervisor(supervisor);
14835 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
14836 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
14837 register_capability_manifest(
14838 &handler,
14839 &target_ctx,
14840 &mut target_rx,
14841 capability_manifest("target", &["credentials-provider/v1"], &[]),
14842 1,
14843 )
14844 .await;
14845 register_capability_manifest(
14846 &handler,
14847 &opener_ctx,
14848 &mut opener_rx,
14849 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14850 2,
14851 )
14852 .await;
14853
14854 let (_client_rx, bind) = open_route_for_capability_test(
14855 &handler,
14856 &target_ctx,
14857 &mut target_rx,
14858 742,
14859 3,
14860 "target",
14861 None,
14862 )
14863 .await;
14864 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14865 panic!("direct scope-honesty route must bind");
14866 };
14867 assert_eq!(principal, Some(Principal::Direct));
14868 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14869 }
14870
14871 #[tokio::test]
14874 async fn must_never_reach_self_route_is_capability_forbidden() {
14875 let registry = Arc::new(Registry::default());
14876 let forwarding = Arc::new(ForwardingTable::default());
14877 let supervisor = SupervisorHandle::new();
14878 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
14879 let handler =
14880 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14881 .with_supervisor(supervisor);
14882 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
14883 register_capability_manifest(
14884 &handler,
14885 &self_ctx,
14886 &mut self_rx,
14887 capability_manifest(
14888 "self-provider",
14889 &["credentials-provider/v1"],
14890 &["credentials-provider/v1"],
14891 ),
14892 1,
14893 )
14894 .await;
14895
14896 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14897 let replies = handler
14898 .handle_control_frame(
14899 &client_ctx,
14900 route_open_frame_with_admission_facts(
14901 2,
14902 "self-provider",
14903 unique_project_root("admission-facts"),
14904 Some(ConsumerIdentity {
14905 module_id: "self-provider".to_string(),
14906 launch_nonce: "self-nonce".to_string(),
14907 }),
14908 None,
14909 ),
14910 )
14911 .await
14912 .expect("self-route refusal returns a typed frame");
14913 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14914 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14915 assert!(
14916 self_rx.try_recv().is_err(),
14917 "self denial must not relay route.bind"
14918 );
14919 }
14920
14921 #[test]
14922 fn unsupported_channel_zero_frame_returns_error() {
14923 let handler = ControlHandler::default();
14924 let request = Frame::build(
14925 FrameType::Request,
14926 control_flags(),
14927 0,
14928 0,
14929 21,
14930 b"opaque".to_vec(),
14931 )
14932 .unwrap();
14933
14934 let response = handler
14935 .handle_control(ConnectionId::new(1), request)
14936 .unwrap();
14937
14938 assert_eq!(response[0].header.ty, FrameType::Error);
14939 assert_eq!(
14940 parse_error(&response[0])["code"],
14941 "unsupported_control_frame"
14942 );
14943 }
14944
14945 mod swap {
14950 use super::*;
14951
14952 const INCUMBENT: ConnectionId = ConnectionId::new(30);
14953 const CANDIDATE: ConnectionId = ConnectionId::new(40);
14954
14955 struct Swap {
14956 registry: Arc<Registry>,
14957 forwarding: Arc<ForwardingTable>,
14958 handler: ControlHandler,
14959 incumbent_ctx: RouteCtx,
14960 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14961 candidate_ctx: RouteCtx,
14962 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14963 }
14964
14965 async fn swap_with_incumbent() -> Swap {
14966 let registry = Arc::new(Registry::default());
14967 let forwarding = Arc::new(ForwardingTable::default());
14968 let handler =
14969 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14970 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14971 hello_via_sink(
14972 &handler,
14973 &incumbent_ctx,
14974 &mut incumbent_rx,
14975 hello_frame("aft", PROTOCOL_VERSION, 7),
14976 )
14977 .await;
14978 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14979 Swap {
14980 registry,
14981 forwarding,
14982 handler,
14983 incumbent_ctx,
14984 incumbent_rx,
14985 candidate_ctx,
14986 candidate_rx,
14987 }
14988 }
14989
14990 fn register_candidate(swap: &Swap, ready: Option<bool>) {
14991 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14992 candidate_manifest.ready = ready;
14993 let registration = swap
14994 .registry
14995 .register_candidate_with_control_ops(
14996 candidate_manifest,
14997 PROTOCOL_VERSION,
14998 CANDIDATE,
14999 module_baseline_control_ops(),
15000 )
15001 .unwrap();
15002 swap.forwarding
15003 .register_candidate_module_connection(
15004 CANDIDATE,
15005 "aft".to_string(),
15006 PROTOCOL_VERSION,
15007 manifest_concurrency(®istration.manifest),
15008 swap.candidate_ctx.egress.clone(),
15009 )
15010 .unwrap();
15011 }
15012
15013 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15014 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15015 swap.registry.promote_candidate("aft").unwrap().unwrap();
15016 cutover.incumbent.unwrap()
15017 }
15018
15019 #[tokio::test]
15028 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15029 let mut swap = swap_with_incumbent().await;
15030 let hold = Arc::new(std::sync::Barrier::new(2));
15031 let handler = swap
15032 .handler
15033 .clone()
15034 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15035 handler
15036 .supervisor
15037 .open_swap("aft", "candidate-nonce".to_string());
15038 let hello = {
15039 let handler = handler.clone();
15040 let candidate_ctx = swap.candidate_ctx.clone();
15041 std::thread::spawn(move || {
15042 handler
15043 .handle_hello(
15044 candidate_ctx.connection_id,
15045 Some(candidate_ctx.egress.clone()),
15046 hello_frame_with_nonce(
15047 "aft",
15048 PROTOCOL_VERSION,
15049 8,
15050 Some("candidate-nonce"),
15051 ),
15052 )
15053 .unwrap()
15054 })
15055 };
15056
15057 hold.wait();
15058 assert!(swap
15061 .registry
15062 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15063 .unwrap()
15064 .is_some());
15065 assert!(swap
15066 .registry
15067 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15068 .unwrap()
15069 .is_none());
15070 assert!(swap
15071 .forwarding
15072 .module_endpoint_for_connection(CANDIDATE)
15073 .unwrap()
15074 .is_none());
15075
15076 hold.wait();
15077 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15078 assert_eq!(
15079 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15080 FrameType::HelloAck
15081 );
15082 let published = swap
15083 .registry
15084 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15085 .unwrap()
15086 .expect("candidate is published once its HELLO completes");
15087 assert_eq!(published.connection_id, CANDIDATE);
15088 assert!(swap
15089 .forwarding
15090 .module_endpoint_for_connection(CANDIDATE)
15091 .unwrap()
15092 .is_some());
15093 }
15094
15095 fn keyed_total(counters: &Value, key: &str) -> u64 {
15096 counters[key]
15097 .as_object()
15098 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15099 .unwrap_or(0)
15100 }
15101
15102 #[tokio::test]
15103 async fn replacement_logs_old_end_and_new_admission_once() {
15104 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15105 let mut swap = swap_with_incumbent().await;
15106 swap.handler
15107 .supervisor
15108 .open_swap("aft", "candidate-nonce".to_string());
15109 hello_via_sink(
15110 &swap.handler,
15111 &swap.candidate_ctx,
15112 &mut swap.candidate_rx,
15113 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15114 )
15115 .await;
15116 cutover(&swap);
15117 swap.handler.cleanup_connection(INCUMBENT).unwrap();
15118
15119 let captured = crate::router::test_log::captured_logs(&logs);
15120 println!("captured replacement registry logs:\n{captured}");
15121 let new_admissions: Vec<_> = captured
15122 .lines()
15123 .filter(|line| {
15124 line.contains("connection_id=40")
15125 && line.contains("swap candidate registered; not routable until cutover")
15126 })
15127 .collect();
15128 assert_eq!(
15129 new_admissions.len(),
15130 1,
15131 "unexpected admission logs: {captured}"
15132 );
15133 assert!(new_admissions[0].contains("module_id=aft"));
15134 assert!(new_admissions[0].contains("connection_id=40"));
15135 assert!(
15136 new_admissions[0].contains("swap candidate registered; not routable until cutover")
15137 );
15138 assert!(new_admissions[0].contains("ready=true"));
15139
15140 let old_admissions: Vec<_> = captured
15141 .lines()
15142 .filter(|line| {
15143 line.contains("module registered module_id=aft ")
15144 && line.contains("connection_id=30")
15145 })
15146 .collect();
15147 assert_eq!(
15148 old_admissions.len(),
15149 1,
15150 "unexpected incumbent admission: {captured}"
15151 );
15152
15153 let old_ends: Vec<_> = captured
15154 .lines()
15155 .filter(|line| {
15156 line.contains("connection_id=30") && line.contains("module registration ended")
15157 })
15158 .collect();
15159 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15160 assert!(old_ends[0].contains("module_id=aft"));
15161 assert!(old_ends[0].contains("connection_id=30"));
15162 assert!(old_ends[0].contains("reason=replaced"));
15163 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15164 assert_eq!(
15165 captured
15166 .lines()
15167 .filter(|line| line.contains("module registration promoted"))
15168 .count(),
15169 1,
15170 "unexpected promotion logs: {captured}"
15171 );
15172 }
15173
15174 #[tokio::test]
15180 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15181 let mut swap = swap_with_incumbent().await;
15182 let handler = swap.handler.clone();
15183
15184 let cotenant = ConnectionId::new(31);
15186 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15187 let (cotenant_task, cotenant_bind) = relay_route_open(
15188 &handler,
15189 cotenant,
15190 &cotenant_ctx.egress,
15191 &mut swap.incumbent_rx,
15192 100,
15193 "aft",
15194 "swap-cotenant",
15195 )
15196 .await;
15197 handler
15198 .handle_control_frame(
15199 &swap.incumbent_ctx,
15200 route_bind_ack(cotenant_bind.header.corr),
15201 )
15202 .await
15203 .unwrap();
15204 assert!(cotenant_task.await.unwrap().is_empty());
15205 let (cotenant_channel, cotenant_epoch) =
15206 published_route(&cotenant_rx.recv().await.unwrap());
15207
15208 let caller = ConnectionId::new(32);
15210 let (caller_ctx, mut caller_rx) = route_ctx(caller);
15211 let (caller_task, caller_bind) = relay_route_open(
15212 &handler,
15213 caller,
15214 &caller_ctx.egress,
15215 &mut swap.incumbent_rx,
15216 101,
15217 "aft",
15218 "swap-caller",
15219 )
15220 .await;
15221 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15222
15223 register_candidate(&swap, None);
15224 cutover(&swap);
15225
15226 let ack = handler
15228 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15229 .await;
15230 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15231 if module_loop_error.is_some() {
15232 handler.cleanup_connection(INCUMBENT).unwrap();
15235 }
15236
15237 assert!(
15239 cotenant_rx.try_recv().is_err(),
15240 "the co-tenant route on the incumbent was torn down by one late ack: \
15241 {module_loop_error:?}"
15242 );
15243 assert!(matches!(
15244 swap.forwarding
15245 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15246 .unwrap(),
15247 DataRoute::Client(DataRouteState::Bound(_))
15248 ));
15249 assert_eq!(module_loop_error, None);
15250 assert!(swap
15251 .registry
15252 .get_module_by_connection(INCUMBENT)
15253 .unwrap()
15254 .is_some());
15255
15256 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15258 .await
15259 .expect("the incumbent is told to drop the abandoned binding")
15260 .unwrap()
15261 .frame;
15262 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15263 assert_eq!(goodbye.header.channel, abandoned_channel);
15264 assert_eq!(goodbye.header.epoch, abandoned_epoch);
15265 assert!(swap.incumbent_rx.try_recv().is_err());
15266
15267 let response = caller_task.await.unwrap();
15269 assert_eq!(response.len(), 1);
15270 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15271 assert!(caller_rx.try_recv().is_err());
15272
15273 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15277 let counters = handler.counters().snapshot();
15278 assert_eq!(
15279 keyed_total(&counters, "route_open_accepted_by_principal"),
15280 1
15281 );
15282 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15283 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15284 }
15285
15286 #[tokio::test]
15290 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15291 let mut swap = swap_with_incumbent().await;
15292 register_candidate(&swap, None);
15293 let incumbent = cutover(&swap);
15294 swap.forwarding
15295 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15296 .unwrap()
15297 .expect("the incumbent is still registered");
15298
15299 let client = ConnectionId::new(33);
15300 let (client_ctx, mut client_rx) = route_ctx(client);
15301 let route_handler = swap.handler.clone();
15302 let open_ctx = RouteCtx {
15303 connection_id: client,
15304 egress: client_ctx.egress.clone(),
15305 };
15306 let mut route_task = tokio::spawn(async move {
15307 route_handler
15308 .handle_control_frame(
15309 &open_ctx,
15310 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15311 )
15312 .await
15313 .unwrap()
15314 });
15315 let bind = tokio::select! {
15316 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15317 response = &mut route_task => {
15318 let response = response.unwrap();
15319 panic!(
15320 "post-cutover route.open was refused instead of relayed to the candidate: {}",
15321 parse_error(&response[0])["code"]
15322 );
15323 }
15324 };
15325 swap.handler
15326 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15327 .await
15328 .unwrap();
15329 assert!(route_task.await.unwrap().is_empty());
15330 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15331 match swap
15332 .forwarding
15333 .lookup_data_route(client, channel, epoch)
15334 .unwrap()
15335 {
15336 DataRoute::Client(DataRouteState::Bound(route)) => {
15337 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15338 }
15339 other => panic!("expected a bound route on the candidate, got {other:?}"),
15340 }
15341 assert!(swap.incumbent_rx.try_recv().is_err());
15342 }
15343
15344 #[tokio::test]
15349 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15350 let swap = swap_with_incumbent().await;
15351 register_candidate(&swap, Some(false));
15352 let update = Frame::build(
15353 FrameType::Request,
15354 control_flags(),
15355 0,
15356 0,
15357 55,
15358 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15359 provides: manifest("aft", PROTOCOL_VERSION).provides,
15360 capabilities: None,
15361 ready: Some(true),
15362 })
15363 .unwrap(),
15364 )
15365 .unwrap();
15366
15367 let replies = swap
15368 .handler
15369 .handle_control_frame(&swap.candidate_ctx, update)
15370 .await
15371 .unwrap();
15372
15373 assert_eq!(replies.len(), 1);
15374 assert_eq!(
15375 replies[0].header.ty,
15376 FrameType::Response,
15377 "candidate catalog.update was refused: {:?}",
15378 serde_json::from_slice::<Value>(&replies[0].body).ok()
15379 );
15380 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15381 assert_eq!(
15382 swap.registry
15383 .get_module("aft")
15384 .unwrap()
15385 .unwrap()
15386 .connection_id,
15387 INCUMBENT
15388 );
15389 }
15390 }
15391
15392 mod swap_admission {
15396 use super::*;
15397
15398 const INCUMBENT_NONCE: &str = "incumbent-nonce";
15399 const CANDIDATE_NONCE: &str = "candidate-nonce";
15400
15401 fn handler_with_incumbent(
15402 module_id: &str,
15403 reserved: bool,
15404 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15405 let registry = Arc::new(Registry::default());
15406 let supervisor = SupervisorHandle::new();
15407 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15408 if reserved {
15409 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15410 }
15411 let handler =
15412 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
15413 let incumbent = handler
15414 .handle_control(
15415 ConnectionId::new(1),
15416 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15417 )
15418 .unwrap();
15419 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15420 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15421 (registry, supervisor, handler)
15422 }
15423
15424 #[test]
15430 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15431 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15432
15433 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15434 let replies = handler
15435 .handle_control(
15436 ConnectionId::new(connection),
15437 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15438 )
15439 .unwrap();
15440 assert_eq!(replies[0].header.ty, FrameType::Error);
15441 assert_eq!(
15442 parse_error(&replies[0])["code"],
15443 "swap_token_invalid",
15444 "nonce {nonce:?}"
15445 );
15446 }
15447 assert!(registry.get_candidate("aft").unwrap().is_none());
15448 assert_eq!(
15449 registry.get_module("aft").unwrap().unwrap().connection_id,
15450 ConnectionId::new(1)
15451 );
15452
15453 let admitted = handler
15456 .handle_control(
15457 ConnectionId::new(4),
15458 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15459 )
15460 .unwrap();
15461 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15462 assert_eq!(
15463 registry
15464 .get_candidate("aft")
15465 .unwrap()
15466 .unwrap()
15467 .connection_id,
15468 ConnectionId::new(4)
15469 );
15470 assert_eq!(
15471 registry.get_module("aft").unwrap().unwrap().connection_id,
15472 ConnectionId::new(1),
15473 "the candidate must not take the active slot"
15474 );
15475 let replayed = handler
15476 .handle_control(
15477 ConnectionId::new(5),
15478 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15479 )
15480 .unwrap();
15481 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15482
15483 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15487 let squatter = handler
15488 .handle_control(
15489 ConnectionId::new(6),
15490 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15491 )
15492 .unwrap();
15493 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15494 assert!(
15495 registry.get_module("aft").unwrap().is_none(),
15496 "a squatter took the active slot of an id being swapped"
15497 );
15498 }
15499
15500 #[test]
15505 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15506 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15507
15508 let replies = handler
15509 .handle_control(
15510 ConnectionId::new(2),
15511 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15512 )
15513 .unwrap();
15514
15515 assert_eq!(
15516 replies[0].header.ty,
15517 FrameType::HelloAck,
15518 "reserved candidate refused: {:?}",
15519 serde_json::from_slice::<Value>(&replies[0].body).ok()
15520 );
15521 assert_eq!(
15522 registry
15523 .get_candidate("vault")
15524 .unwrap()
15525 .unwrap()
15526 .connection_id,
15527 ConnectionId::new(2)
15528 );
15529 }
15530
15531 #[test]
15534 fn without_an_open_swap_the_ordinary_gates_decide() {
15535 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15536 supervisor.close_swap("vault");
15537
15538 let candidate = handler
15539 .handle_control(
15540 ConnectionId::new(2),
15541 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15542 )
15543 .unwrap();
15544 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15545 let duplicate = handler
15546 .handle_control(
15547 ConnectionId::new(3),
15548 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15549 )
15550 .unwrap();
15551 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15552 assert!(registry.get_candidate("vault").unwrap().is_none());
15553 }
15554 }
15555
15556 mod scopes {
15560 mod idle_tests {
15561 use super::*;
15562 include!("scope_idle_tests.rs");
15563 }
15564 mod route_open_timings {
15565 use super::*;
15566 include!("route_open_timing_tests.rs");
15567 }
15568 mod activity {
15569 include!("activity_scope_tests.rs");
15570 }
15571 mod delta {
15572 include!("scope_handler_delta_tests.rs");
15573 }
15574 mod call_modes {
15575 include!("scope_call_mode_tests.rs");
15576 }
15577 #[tokio::test]
15578 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15579 let rig = rig().await;
15580 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15581 "s".into(),
15582 1,
15583 "tool".into(),
15584 ))
15585 .unwrap();
15586 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
15587 let (direct, _rx) = wide_ctx(999);
15588 for (ctx, expected) in [
15589 (&rig.owner, FrameType::Response),
15590 (&direct, FrameType::Error),
15591 ] {
15592 let frame =
15593 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
15594 .unwrap();
15595 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
15596 assert_eq!(replies[0].header.ty, expected);
15597 if expected == FrameType::Response {
15598 let reply: subc_control::ScopeServedByReply =
15599 serde_json::from_slice(&replies[0].body).unwrap();
15600 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
15601 } else {
15602 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
15603 }
15604 }
15605 let reply = call(
15606 &rig.handler,
15607 &rig.owner,
15608 &ModuleControlRequestFromModule::ScopeDescribe {
15609 owner: Principal::Reserved {
15610 module_id: OWNER.into(),
15611 },
15612 scope_ref: "s".into(),
15613 },
15614 )
15615 .await;
15616 assert!(matches!(
15617 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
15618 ModuleControlResponseToModule::ScopeDescribe { .. }
15619 ));
15620 }
15621
15622 use subc_protocol::scope::{
15623 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
15624 ScopeStatus,
15625 };
15626
15627 use super::*;
15628
15629 const OWNER: &str = "prefrontal-core";
15630
15631 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
15632 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
15633 }
15634
15635 async fn call(
15636 handler: &ControlHandler,
15637 ctx: &RouteCtx,
15638 request: &ModuleControlRequestFromModule,
15639 ) -> Frame {
15640 let body = serde_json::to_vec(request).unwrap();
15641 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
15642 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
15643 assert_eq!(replies.len(), 1, "{replies:?}");
15644 replies.pop().unwrap()
15645 }
15646
15647 async fn sync(
15648 handler: &ControlHandler,
15649 ctx: &RouteCtx,
15650 generation: u64,
15651 scopes: Vec<ScopeRecord>,
15652 ) -> Result<ModuleControlResponseToModule, String> {
15653 let reply = call(
15654 handler,
15655 ctx,
15656 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
15657 )
15658 .await;
15659 match reply.header.ty {
15660 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
15661 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
15662 }
15663 }
15664
15665 async fn describe(
15666 handler: &ControlHandler,
15667 ctx: &RouteCtx,
15668 owner: &str,
15669 scope_ref: &str,
15670 ) -> ModuleControlResponseToModule {
15671 let reply = call(
15672 handler,
15673 ctx,
15674 &ModuleControlRequestFromModule::ScopeDescribe {
15675 owner: Principal::Reserved {
15676 module_id: owner.to_string(),
15677 },
15678 scope_ref: scope_ref.to_string(),
15679 },
15680 )
15681 .await;
15682 assert_eq!(
15683 reply.header.ty,
15684 FrameType::Response,
15685 "{:?}",
15686 parse_error(&reply)
15687 );
15688 serde_json::from_slice(&reply.body).unwrap()
15689 }
15690
15691 #[tokio::test]
15692 async fn scope_describe_never_echoes_bind_proof_or_hash() {
15693 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15694 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15695 let mut rig = rig().await;
15696 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15697 .await;
15698 let _bound = rig
15699 .bound(
15700 Some(AFT),
15701 PLEXUS,
15702 Some(
15703 rig.selector("s", Some(1))
15704 .with_bind_proof(Some(PROOF.into())),
15705 ),
15706 )
15707 .await;
15708 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
15709 let output = serde_json::to_string(&reply).unwrap();
15710 assert!(output.contains("\"status\":\"live\""));
15711 assert!(!output.contains(PROOF));
15712 assert!(!output.contains(HASH));
15713 }
15714
15715 async fn module(
15717 handler: &ControlHandler,
15718 connection: u64,
15719 module_id: &str,
15720 nonce: Option<&str>,
15721 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15722 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
15723 hello_via_sink(
15724 handler,
15725 &ctx,
15726 &mut rx,
15727 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
15728 )
15729 .await;
15730 (ctx, rx)
15731 }
15732
15733 #[tokio::test]
15736 async fn a_client_connection_cannot_sync_or_describe() {
15737 let handler = ControlHandler::new(Arc::new(Registry::default()));
15738 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
15739 for request in [
15740 ModuleControlRequestFromModule::ScopeSync {
15741 generation: 1,
15742 scopes: vec![head("s", 1)],
15743 },
15744 ModuleControlRequestFromModule::ScopeDescribe {
15745 owner: Principal::Direct,
15746 scope_ref: "s".to_string(),
15747 },
15748 ] {
15749 let reply = call(&handler, &ctx, &request).await;
15750 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
15751 }
15752 assert!(
15753 !handler
15754 .scopes
15755 .read()
15756 .unwrap()
15757 .describe(
15758 &Principal::Reserved {
15759 module_id: OWNER.to_string()
15760 },
15761 "s"
15762 )
15763 .owner_synced
15764 );
15765 }
15766
15767 #[tokio::test]
15770 async fn a_module_without_a_supervised_launch_cannot_sync() {
15771 let handler = ControlHandler::new(Arc::new(Registry::default()));
15772 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
15773 assert_eq!(
15774 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
15775 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15776 );
15777 }
15778
15779 #[tokio::test]
15780 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
15781 let supervisor = SupervisorHandle::new();
15782 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15783 let handler = ControlHandler::new(Arc::new(Registry::default()))
15784 .with_supervisor(supervisor.clone());
15785 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15786 sync(&handler, &incumbent, 1, vec![head("s", 1)])
15787 .await
15788 .expect("the current launch syncs");
15789
15790 supervisor.open_swap(OWNER, "n2".to_string());
15793 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
15794 assert_eq!(
15795 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15796 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15797 );
15798 sync(&handler, &incumbent, 2, vec![head("s", 1)])
15799 .await
15800 .expect("the serving owner syncs during the swap");
15801
15802 supervisor.close_swap(OWNER);
15805 assert_eq!(
15806 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15807 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15808 );
15809 sync(&handler, &incumbent, 3, vec![head("s", 1)])
15810 .await
15811 .expect("the serving owner syncs after the rollback");
15812 handler.cleanup_connection(candidate.connection_id).unwrap();
15813
15814 supervisor.open_swap(OWNER, "n3".to_string());
15819 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
15820 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
15821 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
15822 .await
15823 .expect("the promoted launch takes authority");
15824 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15825 panic!("unexpected reply {reply:?}");
15826 };
15827 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
15828 assert_eq!(
15829 sync(&handler, &incumbent, 4, Vec::new()).await,
15830 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15831 );
15832 }
15833
15834 #[tokio::test]
15837 async fn closing_the_authority_connection_frees_sync_authority() {
15838 let supervisor = SupervisorHandle::new();
15839 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15840 let handler = ControlHandler::new(Arc::new(Registry::default()))
15841 .with_supervisor(supervisor.clone());
15842 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15843 sync(&handler, &first, 10, vec![head("s", 1)])
15844 .await
15845 .unwrap();
15846 handler.cleanup_connection(first.connection_id).unwrap();
15847
15848 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
15849 sync(&handler, &second, 1, vec![head("s", 1)])
15850 .await
15851 .expect("the next connection takes the released authority");
15852 }
15853
15854 #[tokio::test]
15855 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
15856 let supervisor = SupervisorHandle::new();
15857 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15858 let handler =
15859 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
15860 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
15861 sync(&handler, &first, 10, vec![head("s", 1)])
15862 .await
15863 .unwrap();
15864 handler
15865 .handle_control_frame(
15866 &first,
15867 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
15868 )
15869 .await
15870 .unwrap();
15871 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
15872 sync(&handler, &second, 1, vec![head("s", 1)])
15873 .await
15874 .expect("GOODBYE releases authority even if the old socket remains open");
15875 }
15876
15877 #[tokio::test]
15878 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
15879 let registry = Arc::new(Registry::default());
15880 let supervisor_handle = SupervisorHandle::new();
15881 let supervisor =
15882 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15883 .with_handle(supervisor_handle.clone())
15884 .with_daemon_incarnation("incarnation-7".to_string());
15885 supervisor
15888 .supervise_configured(
15889 ModuleSpec {
15890 module_id: OWNER.to_string(),
15891 program: PathBuf::from("/nonexistent/prefrontal-core"),
15892 args: Vec::new(),
15893 env: Vec::new(),
15894 reserved: false,
15895 reserved_prefixes: Vec::new(),
15896 protocol: ModuleProtocol::Subc,
15897 overlap: Default::default(),
15898 },
15899 false,
15900 )
15901 .unwrap();
15902 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
15903 let handler =
15904 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15905 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
15906
15907 let ModuleControlResponseToModule::ScopeDescribe {
15909 status,
15910 daemon_incarnation,
15911 owner_synced,
15912 owner_configured,
15913 scope,
15914 ..
15915 } = describe(&handler, &reader, OWNER, "s").await
15916 else {
15917 panic!("not a describe reply");
15918 };
15919 assert_eq!(status, ScopeStatus::NotLive);
15920 assert_eq!(daemon_incarnation, "incarnation-7");
15921 assert!(!owner_synced);
15922 assert!(owner_configured);
15923 assert!(scope.is_none());
15924
15925 let ModuleControlResponseToModule::ScopeDescribe {
15928 status,
15929 owner_configured,
15930 ..
15931 } = describe(&handler, &reader, "ghost", "s").await
15932 else {
15933 panic!("not a describe reply");
15934 };
15935 assert_eq!(status, ScopeStatus::NotLive);
15936 assert!(!owner_configured);
15937
15938 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15940 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15941 let ModuleControlResponseToModule::ScopeDescribe {
15942 status,
15943 scope_epoch,
15944 owner_synced,
15945 scope,
15946 ..
15947 } = describe(&handler, &reader, OWNER, "s").await
15948 else {
15949 panic!("not a describe reply");
15950 };
15951 assert_eq!(status, ScopeStatus::Live);
15952 assert_eq!(scope_epoch, Some(4));
15953 assert!(owner_synced);
15954 let stamp = scope.expect("a live scope carries its stamp");
15955 assert!(
15956 stamp.owner_authorized,
15957 "prefrontal-core is the default authority"
15958 );
15959 assert_eq!(stamp.kind, ScopeKind::Head);
15960 }
15961
15962 #[tokio::test]
15963 async fn scope_authority_owners_decides_owner_authorized() {
15964 let supervisor = SupervisorHandle::new();
15965 supervisor.set_spawn_nonce("broca", "b1".to_string());
15966 let handler = ControlHandler::new(Arc::new(Registry::default()))
15967 .with_supervisor(supervisor)
15968 .with_scope_authority_owners(vec!["broca".to_string()]);
15969 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15970 let mut gated = head("s", 1);
15971 gated.attributes.agent_id = Some("agent".to_string());
15972 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15973 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15974 describe(&handler, &broca, "broca", "s").await
15975 else {
15976 panic!("not a describe reply");
15977 };
15978 assert!(scope.unwrap().owner_authorized);
15979 }
15980
15981 #[tokio::test]
15985 async fn scope_ops_and_the_scopes_capability_are_advertised() {
15986 let handler = ControlHandler::new(Arc::new(Registry::default()));
15987 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15988 let ack = hello_via_sink(
15989 &handler,
15990 &ctx,
15991 &mut rx,
15992 hello_frame("m", PROTOCOL_VERSION, 1),
15993 )
15994 .await;
15995 let ack = parse_ack(&ack);
15996 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15997 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15998 }
15999 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16000
16001 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16002 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16003 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16004 let reply = handler
16005 .handle_control_frame(&client, frame)
16006 .await
16007 .unwrap()
16008 .pop()
16009 .unwrap();
16010 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16011 serde_json::from_slice(&reply.body).unwrap()
16012 else {
16013 panic!("not a server.describe reply");
16014 };
16015 assert!(
16016 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16017 "{capabilities:?}"
16018 );
16019 }
16020
16021 const PLEXUS: &str = "plexus";
16024 const OTHER: &str = "other";
16025 const AFT: &str = "aft";
16026 const BROCA: &str = "broca";
16027 const MAGIC: &str = "magic-context";
16028
16029 fn nonce(module_id: &str) -> String {
16030 format!("nonce-{module_id}")
16031 }
16032
16033 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16034 let (tx, rx) = mpsc::channel(64);
16035 (
16036 RouteCtx {
16037 connection_id: ConnectionId::new(connection),
16038 egress: FrameSink::new(tx),
16039 },
16040 rx,
16041 )
16042 }
16043
16044 struct Rig {
16048 handler: ControlHandler,
16049 forwarding: Arc<ForwardingTable>,
16050 owner: RouteCtx,
16051 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16052 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16053 generation: u64,
16054 next_connection: u64,
16055 _supervisor: Supervisor,
16056 }
16057
16058 async fn rig() -> Rig {
16059 rig_with_flow_support(true).await
16060 }
16061
16062 async fn rig_with_flow_support(flow_support: bool) -> Rig {
16063 rig_with_scope_capabilities(flow_support, false).await
16064 }
16065
16066 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16067 let mut provides = Vec::new();
16068 if flow_support {
16069 provides.push("flow-scopes/v1");
16070 }
16071 if run_support {
16072 provides.push("agent-run-scopes/v1");
16073 }
16074 rig_with_target_capabilities(&provides).await
16075 }
16076
16077 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16078 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16079 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16080 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16083 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16084 Frame::build(
16085 FrameType::Hello,
16086 control_flags(),
16087 0,
16088 0,
16089 connection,
16090 serde_json::to_vec(&body).unwrap(),
16091 )
16092 .unwrap()
16093 }
16094
16095 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16096 let registry = Arc::new(Registry::default());
16097 let forwarding = Arc::new(ForwardingTable::default());
16098 let supervisor_handle = SupervisorHandle::new();
16099 let supervisor =
16100 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16101 .with_handle(supervisor_handle.clone());
16102 supervisor
16103 .supervise_configured(
16104 ModuleSpec {
16105 module_id: OWNER.to_string(),
16106 program: PathBuf::from("/nonexistent/prefrontal-core"),
16107 args: Vec::new(),
16108 env: Vec::new(),
16109 reserved: false,
16110 reserved_prefixes: Vec::new(),
16111 protocol: ModuleProtocol::Subc,
16112 overlap: Default::default(),
16113 },
16114 false,
16115 )
16116 .unwrap();
16117 for module_id in [OWNER, AFT, BROCA, MAGIC] {
16118 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16119 }
16120 let handler =
16121 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16122 .with_supervisor(supervisor_handle);
16123 let (owner, mut owner_rx) = wide_ctx(1);
16124 hello_via_sink(
16125 &handler,
16126 &owner,
16127 &mut owner_rx,
16128 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16129 )
16130 .await;
16131 let mut modules = BTreeMap::new();
16132 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16133 let (ctx, mut rx) = wide_ctx(connection);
16134 let hello = hello_with_capabilities(module_id, connection, provides);
16135 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16136 modules.insert(module_id.to_string(), (ctx, rx));
16137 }
16138 Rig {
16139 handler,
16140 forwarding,
16141 owner,
16142 _owner_rx: owner_rx,
16143 modules,
16144 generation: 0,
16145 next_connection: 100,
16146 _supervisor: supervisor,
16147 }
16148 }
16149
16150 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16151 ScopeCarrier::new(Principal::Reserved {
16152 module_id: module_id.to_string(),
16153 })
16154 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16155 }
16156
16157 fn session(scope_epoch: u64) -> ScopeRecord {
16160 let mut record = head("s", scope_epoch);
16161 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16162 record.attributes.agent_id = Some("agent-1".to_string());
16163 record.attributes.delegates = true;
16164 record
16165 }
16166
16167 impl Rig {
16168 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16169 self.generation += 1;
16170 sync(&self.handler, &self.owner, self.generation, scopes)
16171 .await
16172 .expect("the owner's sync is accepted");
16173 }
16174
16175 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16176 ScopeSelector::new(
16177 Principal::Reserved {
16178 module_id: OWNER.to_string(),
16179 },
16180 scope_ref.to_string(),
16181 scope_epoch,
16182 )
16183 }
16184
16185 fn open_frame(
16186 &mut self,
16187 opener: Option<&str>,
16188 target: &str,
16189 scope: Option<ScopeSelector>,
16190 ) -> (
16191 RouteCtx,
16192 mpsc::Receiver<crate::router::OutboundFrame>,
16193 Frame,
16194 ) {
16195 self.next_connection += 1;
16196 let (ctx, rx) = wide_ctx(self.next_connection);
16197 let root = unique_project_root("scoped-open");
16198 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16199 target: RouteTarget::ToolProvider {
16200 module_id: target.to_string(),
16201 },
16202 identity: BindIdentity::new(
16203 root.path().to_path_buf(),
16204 "unit".to_string(),
16205 "session".to_string(),
16206 ),
16207 consumer_identity: opener.map(|module_id| ConsumerIdentity {
16208 module_id: module_id.to_string(),
16209 launch_nonce: nonce(module_id),
16210 }),
16211 consumer_capabilities: None,
16212 role_versions: None,
16213 admission_facts: None,
16214 scope,
16215 })
16216 .unwrap();
16217 let frame = Frame::build(
16218 FrameType::Request,
16219 control_flags(),
16220 0,
16221 0,
16222 self.next_connection,
16223 body,
16224 )
16225 .unwrap();
16226 (ctx, rx, frame)
16227 }
16228
16229 async fn refused(
16231 &mut self,
16232 opener: Option<&str>,
16233 target: &str,
16234 scope: Option<ScopeSelector>,
16235 ) -> String {
16236 self.refusal_body(opener, target, scope).await["code"]
16237 .as_str()
16238 .unwrap()
16239 .to_string()
16240 }
16241
16242 async fn refusal_body(
16243 &mut self,
16244 opener: Option<&str>,
16245 target: &str,
16246 scope: Option<ScopeSelector>,
16247 ) -> Value {
16248 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16249 let replies = tokio::time::timeout(
16250 Duration::from_secs(2),
16251 self.handler.handle_control_frame(&ctx, frame),
16252 )
16253 .await
16254 .expect("the open must be refused before waiting for a bind ack")
16255 .unwrap();
16256 assert_eq!(replies.len(), 1, "{replies:?}");
16257 assert_eq!(replies[0].header.ty, FrameType::Error);
16258 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16259 assert!(
16260 module_rx.try_recv().is_err(),
16261 "a refused open relays nothing"
16262 );
16263 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16264 parse_error(&replies[0])
16265 }
16266
16267 async fn relayed(
16269 &mut self,
16270 opener: Option<&str>,
16271 target: &str,
16272 scope: Option<ScopeSelector>,
16273 ) -> Relayed {
16274 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16275 let handler = self.handler.clone();
16276 let task_ctx = ctx.clone();
16277 let task = tokio::spawn(async move {
16278 handler
16279 .handle_control_frame(&task_ctx, frame)
16280 .await
16281 .unwrap()
16282 });
16283 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16284 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16285 .await
16286 .expect("the target receives the relayed route.bind")
16287 .unwrap()
16288 .frame;
16289 Relayed {
16290 target: target.to_string(),
16291 client: ctx,
16292 client_rx: rx,
16293 task,
16294 bind,
16295 }
16296 }
16297
16298 async fn ack(&self, relayed: &Relayed) {
16299 let (module, _) = &self.modules[&relayed.target];
16300 self.handler
16301 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16302 .await
16303 .unwrap();
16304 }
16305
16306 async fn bound(
16308 &mut self,
16309 opener: Option<&str>,
16310 target: &str,
16311 scope: Option<ScopeSelector>,
16312 ) -> Bound {
16313 let relayed = self.relayed(opener, target, scope).await;
16314 self.ack(&relayed).await;
16315 let Relayed {
16316 target,
16317 client,
16318 mut client_rx,
16319 task,
16320 bind,
16321 } = relayed;
16322 assert!(
16323 task.await.unwrap().is_empty(),
16324 "the open is answered by commit"
16325 );
16326 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16327 Bound {
16328 target,
16329 client,
16330 client_rx,
16331 channel,
16332 epoch,
16333 bind,
16334 }
16335 }
16336
16337 fn live(&self, route: &Bound) -> bool {
16338 matches!(
16339 self.forwarding
16340 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16341 .unwrap(),
16342 DataRoute::Client(DataRouteState::Bound(_))
16343 )
16344 }
16345 }
16346
16347 struct Relayed {
16348 target: String,
16349 client: RouteCtx,
16350 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16351 task: tokio::task::JoinHandle<Vec<Frame>>,
16352 bind: Frame,
16353 }
16354
16355 struct Bound {
16356 target: String,
16357 client: RouteCtx,
16358 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16359 channel: u16,
16360 epoch: u32,
16361 bind: Frame,
16362 }
16363
16364 impl Bound {
16365 fn closed_reason(&mut self) -> RouteCloseReason {
16368 let mut reason = None;
16369 let mut goodbye = false;
16370 while let Ok(outbound) = self.client_rx.try_recv() {
16371 let frame = outbound.frame;
16372 match frame.header.ty {
16373 FrameType::Goodbye => {
16374 assert_eq!(
16375 (frame.header.channel, frame.header.epoch),
16376 (self.channel, self.epoch)
16377 );
16378 goodbye = true;
16379 }
16380 FrameType::Push => {
16381 let ClientControlPush::RouteClosed {
16382 reason: r,
16383 module_id,
16384 ..
16385 } = serde_json::from_slice(&frame.body).unwrap()
16386 else {
16387 panic!("unexpected push");
16388 };
16389 assert_eq!(module_id, self.target);
16390 reason = Some(r);
16391 }
16392 other => panic!("unexpected frame {other:?}"),
16393 }
16394 }
16395 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16396 reason.expect("the client is told why the route closed")
16397 }
16398
16399 fn untouched(&mut self) -> bool {
16400 self.client_rx.try_recv().is_err()
16401 }
16402
16403 fn stamp(&self) -> Option<ScopeStamp> {
16404 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16405 ModuleControlRequest::RouteBind { scope, .. } => scope,
16406 other => panic!("expected a route.bind, got {other:?}"),
16407 }
16408 }
16409 }
16410
16411 #[tokio::test]
16412 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16413 ) {
16414 let mut rig = rig().await;
16415 let mut record = session(1);
16416 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16417 record.child_owners = vec![Principal::Reserved {
16418 module_id: MAGIC.to_string(),
16419 }];
16420 rig.sync(vec![record]).await;
16421 let scope = || Some(rig_selector("s", Some(1)));
16422
16423 rig.bound(Some(OWNER), PLEXUS, scope()).await;
16426 rig.bound(Some(AFT), OTHER, scope()).await;
16427 rig.bound(Some(BROCA), PLEXUS, scope()).await;
16428
16429 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16433 assert_eq!(
16434 rig.refused(opener, target, scope()).await,
16435 error_codes::SCOPE_NOT_CARRIER,
16436 "{opener:?} -> {target}"
16437 );
16438 }
16439 }
16440
16441 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16442 ScopeSelector::new(
16443 Principal::Reserved {
16444 module_id: OWNER.to_string(),
16445 },
16446 scope_ref.to_string(),
16447 scope_epoch,
16448 )
16449 }
16450
16451 #[tokio::test]
16452 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16453 let mut rig = rig().await;
16454 rig.sync(vec![session(1)]).await;
16455 for opener in [OWNER, AFT] {
16456 assert_eq!(
16457 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16458 .await,
16459 error_codes::SCOPE_EPOCH_REQUIRED,
16460 "{opener}"
16461 );
16462 }
16463 }
16464
16465 #[tokio::test]
16466 async fn admission_separates_not_synced_not_live_and_ended() {
16467 let mut rig = rig().await;
16468 let code = rig
16470 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16471 .await;
16472 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16473 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16474
16475 let ghost = ScopeSelector::new(
16477 Principal::Reserved {
16478 module_id: "ghost".to_string(),
16479 },
16480 "s".to_string(),
16481 Some(1),
16482 );
16483 assert_eq!(
16484 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16485 error_codes::SCOPE_NOT_LIVE
16486 );
16487
16488 rig.sync(vec![session(2)]).await;
16489 assert_eq!(
16490 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16491 .await,
16492 error_codes::SCOPE_NOT_LIVE
16493 );
16494 for epoch in [1, 3] {
16495 assert_eq!(
16496 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16497 .await,
16498 error_codes::SCOPE_ENDED,
16499 "epoch {epoch}"
16500 );
16501 }
16502 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16504 .await;
16505 }
16506
16507 #[tokio::test]
16508 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16509 let mut rig = rig().await;
16510 rig.sync(vec![session(1)]).await;
16511 let route = rig
16512 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16513 .await;
16514 let stamp = route.stamp().expect("a scoped bind carries the stamp");
16515 assert_eq!(stamp.scope_ref, "s");
16516 assert_eq!(stamp.scope_epoch, 1);
16517 assert_eq!(stamp.kind, ScopeKind::Head);
16518 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16519 assert!(stamp.attributes.delegates);
16520 assert!(stamp.owner_authorized);
16521 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16522 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16523
16524 let (broca, mut broca_rx) = wide_ctx(50);
16527 hello_via_sink(
16528 &rig.handler,
16529 &broca,
16530 &mut broca_rx,
16531 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16532 )
16533 .await;
16534 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16535 .await
16536 .unwrap();
16537 let own = ScopeSelector::new(
16538 Principal::Reserved {
16539 module_id: BROCA.to_string(),
16540 },
16541 "b".to_string(),
16542 Some(1),
16543 );
16544 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16545 assert!(!route.stamp().unwrap().owner_authorized);
16546 }
16547
16548 #[tokio::test]
16549 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16550 let mut rig = rig().await;
16551 let mut record = head("s", 1);
16552 record.carriers = vec![carrier(AFT, None)];
16553 let flow_id = "Flow:run-7/step_2!~";
16554 record.attributes.flow_id = Some(flow_id.to_string());
16555 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16556 .await
16557 .unwrap();
16558 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16559 panic!("not a sync reply");
16560 };
16561 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16562 let route = rig
16563 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16564 .await;
16565 assert!(rig.live(&route), "the stamped bind committed");
16566 let stamp = route.stamp().expect("a flow scope carries a stamp");
16567 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16568 assert_eq!(stamp.attributes.agent_id, None);
16569 assert!(!stamp.attributes.delegates);
16570 assert!(stamp.owner_authorized);
16571 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16572 assert_eq!(unscoped.stamp(), None);
16573 }
16574
16575 #[tokio::test]
16576 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16577 let mut rig = rig_with_flow_support(false).await;
16578 let mut record = session(1);
16579 record.attributes.flow_id = Some("flow:7".to_string());
16580 rig.sync(vec![record]).await;
16581 for opener in [OWNER, AFT] {
16582 let body = rig
16583 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16584 .await;
16585 assert_eq!(body["code"], "target_flow_unsupported");
16586 let message = body["message"].as_str().unwrap();
16587 for required in [PLEXUS, "flow-scopes/v1"] {
16588 assert!(message.contains(required), "{message}");
16589 }
16590 }
16591 }
16592
16593 #[tokio::test]
16594 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
16595 let mut rig = rig_with_flow_support(true).await;
16596 let mut record = session(1);
16597 record.attributes.flow_id = Some("flow:7".to_string());
16598 rig.sync(vec![record]).await;
16599 for opener in [OWNER, AFT] {
16600 let route = rig
16601 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16602 .await;
16603 assert!(rig.live(&route));
16604 assert_eq!(
16605 route.stamp().unwrap().attributes.flow_id.as_deref(),
16606 Some("flow:7")
16607 );
16608 }
16609 }
16610
16611 #[tokio::test]
16612 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
16613 use std::future::Future;
16614
16615 let mut rig = rig_with_flow_support(true).await;
16616 let mut record = session(1);
16617 record.attributes.flow_id = Some("flow:7".to_string());
16618 rig.sync(vec![record]).await;
16619 let (client, mut client_rx, frame) =
16620 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
16621 for _ in 0..64 {
16624 client.egress.try_send(route_bind_ack(1)).unwrap();
16625 }
16626 let handler = rig.handler.clone();
16627 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16628 std::future::poll_fn(|cx| {
16629 assert!(open.as_mut().poll(cx).is_pending());
16630 std::task::Poll::Ready(())
16631 })
16632 .await;
16633 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16634
16635 let old_connection = rig.modules[PLEXUS].0.connection_id;
16636 rig.handler.cleanup_connection(old_connection).unwrap();
16637 let (replacement, mut replacement_rx) = wide_ctx(200);
16638 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
16639 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16640 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16641 let hello = Frame::build(
16642 FrameType::Hello,
16643 control_flags(),
16644 0,
16645 0,
16646 200,
16647 serde_json::to_vec(&body).unwrap(),
16648 )
16649 .unwrap();
16650 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
16651
16652 client_rx.try_recv().unwrap();
16653 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16654 .await
16655 .expect("the replacement is refused without waiting for a bind ack")
16656 .unwrap();
16657 assert_eq!(replies.len(), 1);
16658 let body = parse_error(&replies[0]);
16659 assert_eq!(body["code"], "target_flow_unsupported");
16660 for required in [PLEXUS, "flow-scopes/v1"] {
16661 assert!(body["message"].as_str().unwrap().contains(required));
16662 }
16663 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
16664 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16665 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16666 assert!(rig
16667 .handler
16668 .registry
16669 .get_module_by_connection(replacement.connection_id)
16670 .unwrap()
16671 .is_some());
16672 }
16673
16674 #[tokio::test]
16675 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
16676 use std::future::Future;
16677 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16678 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16679 const NEXT_HASH: &str =
16680 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16681 let mut rig = rig().await;
16682 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
16683 record.carriers.push(ScopeCarrier::new(Principal::Direct));
16684 rig.sync(vec![record.clone()]).await;
16685 let (client, mut rx, frame) = rig.open_frame(
16686 None,
16687 PLEXUS,
16688 Some(
16689 rig.selector("s", Some(1))
16690 .with_bind_proof(Some(PROOF.into())),
16691 ),
16692 );
16693 for _ in 0..64 {
16696 client.egress.try_send(route_bind_ack(1)).unwrap();
16697 }
16698 let handler = rig.handler.clone();
16699 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16700 std::future::poll_fn(|cx| {
16701 assert!(open.as_mut().poll(cx).is_pending());
16702 std::task::Poll::Ready(())
16703 })
16704 .await;
16705 record.bind_proof_sha256 = Some(NEXT_HASH.into());
16706 rig.sync(vec![record]).await;
16707 rx.try_recv().unwrap();
16708 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16709 .await
16710 .unwrap()
16711 .unwrap();
16712 assert_eq!(
16713 parse_error(&replies[0])["code"],
16714 error_codes::SCOPE_BIND_PROOF_MISMATCH
16715 );
16716 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
16717 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16718 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16719 }
16720
16721 #[tokio::test]
16722 async fn scope_bind_proof_reserved_openers_ignore_proof() {
16723 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16724 let mut rig = rig().await;
16725 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16726 .await;
16727 for opener in [OWNER, AFT] {
16728 for proof in [None, Some("malformed+proof=")] {
16729 let route = rig
16730 .bound(
16731 Some(opener),
16732 PLEXUS,
16733 Some(
16734 rig.selector("s", Some(1))
16735 .with_bind_proof(proof.map(str::to_string)),
16736 ),
16737 )
16738 .await;
16739 assert!(rig.live(&route));
16740 }
16741 }
16742 }
16743
16744 #[tokio::test]
16745 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
16746 use std::future::Future;
16747 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16748 const NEXT_HASH: &str =
16749 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16750 let mut rig = rig().await;
16751 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16752 .await;
16753 let (client, mut rx, frame) =
16754 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
16755 for _ in 0..64 {
16756 client.egress.try_send(route_bind_ack(1)).unwrap();
16757 }
16758 let handler = rig.handler.clone();
16759 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16760 std::future::poll_fn(|cx| {
16761 assert!(open.as_mut().poll(cx).is_pending());
16762 std::task::Poll::Ready(())
16763 })
16764 .await;
16765 rig.sync(vec![
16766 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
16767 ])
16768 .await;
16769 rx.try_recv().unwrap();
16770 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16771 let bind = std::future::poll_fn(|cx| {
16772 assert!(open.as_mut().poll(cx).is_pending());
16773 module_rx.poll_recv(cx)
16774 })
16775 .await
16776 .unwrap()
16777 .frame;
16778 handler
16779 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
16780 .await
16781 .unwrap();
16782 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16783 .await
16784 .unwrap()
16785 .unwrap();
16786 assert_eq!(replies.len(), 1);
16787 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
16790 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16791 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
16792 let route = rig
16793 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
16794 .await;
16795 assert!(rig.live(&route));
16796 }
16797
16798 #[tokio::test]
16799 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
16800 {
16801 let mut rig = rig_with_flow_support(false).await;
16802 rig.sync(vec![session(1)]).await;
16803 let route = rig
16804 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16805 .await;
16806 assert!(rig.live(&route));
16807 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
16808 let mut record = session(1);
16809 record.attributes.flow_id = Some("flow:7".to_string());
16810 rig.sync(vec![record]).await;
16811 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
16812 assert!(rig.live(&unscoped));
16813 assert_eq!(unscoped.stamp(), None);
16814 }
16815
16816 #[tokio::test]
16817 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
16818 let mut rig = rig().await;
16819 let mut record = session(1);
16820 record.attributes.flow_id = Some("flow:7".to_string());
16821 rig.sync(vec![record.clone()]).await;
16822 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16823 let mut owner_route = rig
16824 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16825 .await;
16826 let mut carrier_route = rig
16827 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
16828 .await;
16829 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16830 rig.sync(vec![record.clone()]).await;
16831 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16832 assert!(rig.live(&owner_route) && owner_route.untouched());
16833 assert!(rig.live(&carrier_route) && carrier_route.untouched());
16834
16835 record.attributes.flow_id = Some("flow:8".to_string());
16836 rig.sync(vec![record]).await;
16837 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
16838 let before = before.unwrap();
16839 assert_eq!(after.scope_epoch, before.scope_epoch);
16840 assert!(after.version > before.version);
16841 for route in [&mut owner_route, &mut carrier_route] {
16842 assert!(!rig.live(route));
16843 assert_eq!(
16844 route.closed_reason(),
16845 RouteCloseReason::ScopeDelegationChanged
16846 );
16847 }
16848 assert!(rig.live(&unscoped) && unscoped.untouched());
16849 for target in [PLEXUS, OTHER] {
16852 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
16853 let goodbye = module_rx
16854 .try_recv()
16855 .expect("the provider sees the drain")
16856 .frame;
16857 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16858 assert!(module_rx.try_recv().is_err());
16859 }
16860 let rebound = rig
16861 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16862 .await;
16863 assert_eq!(
16864 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
16865 Some("flow:8")
16866 );
16867 }
16868
16869 #[tokio::test]
16874 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
16875 let mut rig = rig().await;
16876 rig.sync(vec![session(1)]).await;
16877 let mut cotenant = rig
16878 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16879 .await;
16880
16881 let mut changed = session(1);
16882 changed.child_owners.push(Principal::Reserved {
16883 module_id: MAGIC.to_string(),
16884 });
16885 let mut ended = None;
16886 for (code, next) in [
16887 (error_codes::SCOPE_CHANGED, vec![changed]),
16888 (error_codes::SCOPE_ENDED, Vec::new()),
16889 ] {
16890 let relayed = rig
16891 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16892 .await;
16893 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
16894 ended = Some(next.is_empty());
16895 rig.sync(next).await;
16896 rig.ack(&relayed).await;
16897 let replies = relayed.task.await.unwrap();
16898 assert_eq!(replies.len(), 1, "{replies:?}");
16899 assert_eq!(parse_error(&replies[0])["code"], code);
16900 assert_eq!(
16901 subc_protocol::error_codes::is_retryable_route_open(code),
16902 code == error_codes::SCOPE_CHANGED
16903 );
16904 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16905 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16907 let mut goodbyes = Vec::new();
16910 while let Ok(outbound) = plexus_rx.try_recv() {
16911 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
16912 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
16913 }
16914 assert!(
16915 goodbyes.contains(&(bind_channel, bind_epoch)),
16916 "{goodbyes:?}"
16917 );
16918 assert!(rig
16919 .handler
16920 .registry
16921 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
16922 .unwrap()
16923 .is_some());
16924 }
16925 assert_eq!(ended, Some(true));
16926 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
16929 }
16930
16931 #[tokio::test]
16934 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
16935 struct Case {
16936 name: &'static str,
16937 change: fn(&mut ScopeRecord),
16938 closed: [Option<RouteCloseReason>; 4],
16941 }
16942 use RouteCloseReason::*;
16943 let cases = [
16944 Case {
16945 name: "a carrier entry removed",
16946 change: |r| {
16947 r.carriers.retain(|c| {
16948 c.principal
16949 != Principal::Reserved {
16950 module_id: AFT.to_string(),
16951 }
16952 })
16953 },
16954 closed: [None, Some(ScopeCarrierRemoved), None, None],
16955 },
16956 Case {
16957 name: "a target removed from a carrier",
16958 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16959 closed: [None, None, None, Some(ScopeCarrierRemoved)],
16960 },
16961 Case {
16962 name: "a bare carrier narrowed to targets",
16963 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16964 closed: [None, Some(ScopeCarrierRemoved), None, None],
16965 },
16966 Case {
16967 name: "delegates turned off",
16968 change: |r| r.attributes.delegates = false,
16969 closed: [Some(ScopeDelegationChanged); 4],
16970 },
16971 Case {
16972 name: "agent_id changed",
16973 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16974 closed: [Some(ScopeDelegationChanged); 4],
16975 },
16976 Case {
16977 name: "a carrier added, child owners changed, the record re-sent",
16978 change: |r| {
16979 r.carriers.push(carrier(MAGIC, None));
16980 r.child_owners.push(Principal::Reserved {
16981 module_id: MAGIC.to_string(),
16982 });
16983 },
16984 closed: [None; 4],
16985 },
16986 Case {
16987 name: "a target added",
16988 change: |r| {
16989 r.carriers[1]
16990 .targets
16991 .as_mut()
16992 .unwrap()
16993 .push("third".to_string())
16994 },
16995 closed: [None; 4],
16996 },
16997 Case {
16998 name: "delegates turned on",
16999 change: |r| r.attributes.delegates = true,
17000 closed: [None; 4],
17001 },
17002 ];
17003 for case in cases {
17004 let mut rig = rig().await;
17005 rig.sync(vec![session(1)]).await;
17006 let scope = || Some(rig_selector("s", Some(1)));
17007 let mut routes = [
17008 rig.bound(Some(OWNER), PLEXUS, scope()).await,
17009 rig.bound(Some(AFT), PLEXUS, scope()).await,
17010 rig.bound(Some(BROCA), PLEXUS, scope()).await,
17011 rig.bound(Some(BROCA), OTHER, scope()).await,
17012 ];
17013 let mut record = session(1);
17014 (case.change)(&mut record);
17015 rig.sync(vec![record]).await;
17016 for (index, expected) in case.closed.iter().enumerate() {
17017 let route = &mut routes[index];
17018 match expected {
17019 Some(reason) => {
17020 assert!(!rig.live(route), "{}: route {index} still live", case.name);
17021 assert_eq!(
17022 route.closed_reason(),
17023 *reason,
17024 "{}: route {index}",
17025 case.name
17026 );
17027 }
17028 None => {
17029 assert!(rig.live(route), "{}: route {index} closed", case.name);
17030 assert!(
17031 route.untouched(),
17032 "{}: route {index} was told something",
17033 case.name
17034 );
17035 }
17036 }
17037 }
17038 }
17039 }
17040
17041 #[tokio::test]
17042 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17043 for next in [Vec::new(), vec![session(2)]] {
17045 let mut rig = rig().await;
17046 rig.sync(vec![session(1)]).await;
17047 let mut route = rig
17048 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17049 .await;
17050 rig.sync(next).await;
17051 assert!(!rig.live(&route));
17052 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17053 }
17054
17055 let mut rig = rig().await;
17058 let mut child = session(1);
17059 child.scope_ref = "child".to_string();
17060 child.kind = ScopeKind::Worker;
17061 child.parent = Some(ScopeParent::new(
17062 Principal::Reserved {
17063 module_id: OWNER.to_string(),
17064 },
17065 "s".to_string(),
17066 1,
17067 ));
17068 rig.sync(vec![session(1), child.clone()]).await;
17069 let mut child_route = rig
17070 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17071 .await;
17072 assert_eq!(
17073 child_route.stamp().unwrap().parent_state,
17074 Some(ParentState::Linked)
17075 );
17076 rig.sync(vec![child]).await;
17077 assert!(!rig.live(&child_route));
17078 assert_eq!(
17079 child_route.closed_reason(),
17080 RouteCloseReason::ScopeParentEnded
17081 );
17082 }
17083
17084 #[tokio::test]
17085 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17086 ) {
17087 let mut rig = rig().await;
17088 rig.sync(vec![session(1)]).await;
17089 let mut route = rig
17090 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17091 .await;
17092 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17093 rig.sync(vec![session(1)]).await;
17094 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17095 assert!(rig.live(&route) && route.untouched());
17096
17097 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17101 .forwarding
17102 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17103 .unwrap()
17104 else {
17105 panic!("the route is bound");
17106 };
17107 binding.flow.acquire_tagged(9, false).await.unwrap();
17108 let mut widened = session(1);
17109 widened.carriers.push(carrier(MAGIC, None));
17110 rig.sync(vec![widened]).await;
17111 assert!(rig.live(&route) && route.untouched());
17112 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17113 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17114 assert_eq!(binding.flow.in_flight(), 1);
17117 binding
17118 .flow
17119 .acquire_tagged(10, false)
17120 .await
17121 .expect("the flow is still open");
17122 }
17123
17124 #[tokio::test]
17127 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17128 let mut rig = rig().await;
17129 rig.sync(vec![session(1)]).await;
17130 let mut on_incumbent = rig
17131 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17132 .await;
17133
17134 let (candidate, _candidate_rx) = wide_ctx(9);
17137 let registration = rig
17138 .handler
17139 .registry
17140 .register_candidate_with_control_ops(
17141 manifest(PLEXUS, PROTOCOL_VERSION),
17142 PROTOCOL_VERSION,
17143 candidate.connection_id,
17144 module_baseline_control_ops(),
17145 )
17146 .unwrap();
17147 rig.forwarding
17148 .register_candidate_module_connection(
17149 candidate.connection_id,
17150 PLEXUS.to_string(),
17151 PROTOCOL_VERSION,
17152 manifest_concurrency(®istration.manifest),
17153 candidate.egress.clone(),
17154 )
17155 .unwrap();
17156 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17157 rig.handler
17158 .registry
17159 .promote_candidate(PLEXUS)
17160 .unwrap()
17161 .unwrap();
17162 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17163
17164 rig.sync(Vec::new()).await;
17165 assert!(!rig.live(&on_incumbent));
17166 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17167 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17168 let goodbye = incumbent_rx
17169 .try_recv()
17170 .expect("the superseded endpoint is told")
17171 .frame;
17172 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17173 }
17174 }
17175}
17176
17177#[cfg(test)]
17178mod concurrency_default_exposure_tests {
17179 use super::*;
17180
17181 fn hello_body(role_json: &str) -> Vec<u8> {
17182 format!(
17183 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17184 )
17185 .into_bytes()
17186 }
17187
17188 fn manifest_from(body: &[u8]) -> ModuleManifest {
17189 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17190 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17191 .expect("manifest parses")
17192 }
17193
17194 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17195
17196 #[test]
17197 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17198 let body = hello_body(&format!(
17199 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17200 ));
17201 let manifest = manifest_from(&body);
17202 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17205 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17206 }
17207
17208 #[test]
17209 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17210 let body = hello_body(&format!(
17211 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17212 ));
17213 let manifest = manifest_from(&body);
17214 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17215 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17216 }
17217
17218 #[test]
17219 fn non_management_roles_are_never_reported() {
17220 let body = hello_body(
17221 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17222 );
17223 let manifest = manifest_from(&body);
17224 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17225 }
17226}