1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_rich_ui_descriptor, validate_hello_self_signal_declarations,
26 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
27 ProviderRole,
28 },
29 scope::{
30 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32 SCOPE_SYNC_OP,
33 },
34 session::{
35 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38 ROLE_VERSIONS_FIELD,
39 },
40 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41 Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47 caller_identity::CallerIdentity,
48 capability_requirements::{
49 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51 RuntimeModule,
52 },
53 child_roster::DaemonShutdownFlag,
54 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55 forwarding::{
56 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59 },
60 observability::{
61 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62 },
63 privileged_prompt::{prompt_text, PrivilegedOperation},
64 provenance::{
65 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66 },
67 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68 router::{RouteCtx, RouterError},
69 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71 stderr_tail::{CaptureState, TailEntry},
72 supervise::{
73 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74 SupervisorHandle, SwapHelloAdmission,
75 },
76 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93 ops::ACTIVITY_RECORD,
94 ops::ACTIVITY_QUERY,
95 ops::ACTIVITY_SUBSCRIBE,
96 ops::SERVER_DESCRIBE,
97 ops::CATALOG_LIST,
98 ops::ROUTE_OPEN,
99 ops::ROUTE_POLL,
100 ops::ROUTE_CLOSING,
101 ops::ROUTE_CLOSED,
102 ops::SUPERVISOR_LIST,
103 ops::SUPERVISOR_RESTART,
104 ops::SUPERVISOR_SWAP,
105 ops::SUPERVISOR_RELOAD,
106 ops::SUPERVISOR_RESCAN,
107 ops::SUPERVISOR_RELEASE_RESERVED,
108 ops::SUPERVISOR_SET_ENABLED,
109 ops::SUPERVISOR_HEALTH_PROBE,
110 ops::SUPERVISOR_HEALTH,
111 ops::SUPERVISOR_STDERR_TAIL,
112 ops::SUPERVISOR_TERMINALS,
113 ops::SUPERVISOR_ROUTES,
114 ops::SUPERVISOR_PROVENANCE,
115 ops::SUPERVISOR_SPAWN_SNAPSHOT,
116 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132 "supervisor.live_roots",
133 SCOPE_SYNC_OP,
134 SCOPE_APPLY_OP,
135 SCOPE_DESCRIBE_OP,
136 "operator.confirm",
137];
138
139const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194 configured: &Path,
195 spawned_from: Option<&Path>,
196 image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198 let path = match spawned_from {
199 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200 ReloadPathAgreement::Match
201 }
202 Some(spawned_from) => ReloadPathAgreement::Mismatch {
203 configured: configured.to_path_buf(),
204 spawned_from: spawned_from.to_path_buf(),
205 },
206 None => ReloadPathAgreement::Unavailable {
207 reason: if matches!(
208 image,
209 subc_control::RunningImageAgreement::Unavailable {
210 reason: subc_control::RunningImageUnavailableReason::NotRunning
211 }
212 ) {
213 ReloadPathUnavailableReason::NotRunning
214 } else {
215 ReloadPathUnavailableReason::SpawnedPathUnavailable
216 },
217 },
218 };
219 PendingReloadVerdict {
220 path,
221 image,
222 spec: None,
223 }
224}
225
226fn launch_spec_agreement(
227 configured: &crate::supervise::ModuleSpec,
228 running: bool,
229 spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232 if !running {
233 return Agreement::Unavailable {
234 reason: Reason::NotRunning,
235 };
236 }
237 match spawned {
238 None => Agreement::Unavailable {
239 reason: Reason::SpawnedSpecUnavailable,
240 },
241 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242 Agreement::Match
243 }
244 Some(_) => Agreement::Mismatch,
245 }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249 if configured == spawned_from {
250 return true;
251 }
252 #[cfg(windows)]
253 {
254 match (
260 std::fs::canonicalize(configured),
261 std::fs::canonicalize(spawned_from),
262 ) {
263 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264 _ => false,
265 }
266 }
267 #[cfg(not(windows))]
268 {
269 false
270 }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275 build: DaemonBuildProvenance,
276 pid: Option<u32>,
277 started_at_ms: Option<u64>,
278 start_clock: Option<crate::clock::StartClock>,
279 executable_path: Option<PathBuf>,
280 executable_identity: Option<SpawnedFileIdentity>,
281 process_start_time: Option<u64>,
282 probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286 fn default() -> Self {
287 Self {
288 build: DaemonBuildProvenance {
289 build_git_sha: None,
290 build_lock_digest: None,
291 },
292 pid: None,
293 started_at_ms: None,
294 start_clock: None,
295 executable_path: None,
296 executable_identity: None,
297 process_start_time: None,
298 probe: ExecutableIdentityProbe::default(),
299 }
300 }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305 supervisor: Supervisor,
306 config_path: PathBuf,
307 configured_port: Option<u16>,
308 storage_config: Option<crate::daemon_config::StorageConfig>,
309 admission_facts_carrier_module_id: Option<String>,
310 admission_facts_targets: Option<Vec<String>>,
311 scope_authority_owners: Vec<String>,
312 privileged_operator_prompt: PrivilegedOperatorPrompt,
313 generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342 "reloading",
343 "supervisor_not_live",
344 "registration_not_active",
345 "no_forwarding_connection",
346 "relay_send_failed",
347];
348
349#[derive(Clone)]
351pub struct ControlHandler {
352 registry: Arc<Registry>,
353 activity: Option<crate::activity::ActivityLedger>,
354 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355 forwarding: Arc<ForwardingTable>,
356 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357 supervisor: SupervisorHandle,
358 subc_capabilities: Arc<[String]>,
359 route_bind_relay_timeout: Duration,
363 route_bind_relay_timeouts: BTreeMap<String, Duration>,
367 route_bind_breakers: RouteBindBreakers,
370 route_bind_concurrency: RouteBindConcurrency,
373 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378 route_bind_breaker_threshold: u32,
380 route_bind_breaker_cooldown: Duration,
382 health_probe_timeout: Duration,
383 storage_config: Option<crate::daemon_config::StorageConfig>,
386 machine_id: Option<crate::machine_id::MachineId>,
390 admission_facts_carrier_module_id: Option<String>,
391 admission_facts_targets: Option<Vec<String>>,
392 scopes: Arc<RwLock<ScopeTable>>,
396 scope_deadline_added: Arc<tokio::sync::Notify>,
400 scope_authority_owners: Vec<String>,
403 privileged_operator_prompt: PrivilegedOperatorPrompt,
405 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408 #[cfg(test)]
409 caller_lookup: Option<TestCallerLookup>,
410 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413 rescan: Option<SupervisorRescanContext>,
414 connected_clients: ConnectedClients,
415 counters: DaemonCounters,
416 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417 daemon_shutdown: DaemonShutdownFlag,
422 daemon_provenance: DaemonProvenanceFacts,
423 #[cfg(test)]
424 control_dispatch_delay: Option<Duration>,
425 #[cfg(test)]
426 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427 #[cfg(test)]
431 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432 #[cfg(test)]
438 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443 f.debug_struct("ControlHandler")
444 .field("registry", &self.registry)
445 .field("forwarding", &self.forwarding)
446 .field("process_liveness", &self.process_liveness.is_some())
447 .field("supervisor", &self.supervisor)
448 .field("subc_capabilities", &self.subc_capabilities)
449 .finish()
450 }
451}
452
453struct RouteOpenRequest {
454 target: RouteTarget,
455 identity: BindIdentity,
456 consumer_identity: Option<ConsumerIdentity>,
457 consumer_capabilities: Option<Vec<String>>,
458 role_versions: Option<BTreeMap<String, String>>,
459 admission_facts: Option<serde_json::Value>,
460 scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465 Admission,
466 ProjectRoot,
467 EgressReservation,
468 BindSend,
469 BindAckWait,
470}
471
472struct RouteOpenTimings {
476 module_id: String,
477 principal: String,
478 admission: Duration,
479 project_root: Duration,
480 egress_reservation: Duration,
481 bind_send: Duration,
482 bind_ack_wait: Duration,
483 phase: Option<RouteOpenPhase>,
484 phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488 fn new() -> Self {
489 Self {
490 module_id: String::new(),
491 principal: "unresolved".to_string(),
493 admission: Duration::ZERO,
494 project_root: Duration::ZERO,
495 egress_reservation: Duration::ZERO,
496 bind_send: Duration::ZERO,
497 bind_ack_wait: Duration::ZERO,
498 phase: Some(RouteOpenPhase::Admission),
499 phase_started_at: Instant::now(),
500 }
501 }
502
503 fn finish(&mut self) {
504 let elapsed = self.phase_started_at.elapsed();
505 match self.phase.take() {
506 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511 None => {}
512 }
513 }
514
515 fn enter(&mut self, phase: RouteOpenPhase) {
516 self.finish();
517 self.phase = Some(phase);
518 self.phase_started_at = Instant::now();
519 }
520}
521
522struct RouteBindReservationGuard {
523 forwarding: Arc<ForwardingTable>,
524 endpoint: ModuleEndpointId,
525 relay_corr: u64,
526 armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530 forwarding: Arc<ForwardingTable>,
531 endpoint: ModuleEndpointId,
532 corr: u64,
533 armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538 Self {
539 forwarding,
540 endpoint,
541 corr,
542 armed: true,
543 }
544 }
545
546 fn disarm(&mut self) {
547 self.armed = false;
548 }
549}
550
551impl Drop for ModuleControlRpcGuard {
552 fn drop(&mut self) {
553 if self.armed {
554 let _ = self
555 .forwarding
556 .cancel_module_control_rpc(self.endpoint, self.corr);
557 }
558 }
559}
560
561impl RouteBindReservationGuard {
562 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563 Self {
564 forwarding,
565 endpoint,
566 relay_corr,
567 armed: true,
568 }
569 }
570
571 fn release_and_disarm(&mut self) {
572 if !self.armed {
573 return;
574 }
575 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576 self.endpoint,
577 self.relay_corr,
578 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579 ) {
580 send_goodbye_target_best_effort(
581 &self.forwarding.counters(),
582 &target,
583 "canceled route.bind",
584 );
585 }
586 self.armed = false;
587 }
588
589 fn disarm(&mut self) {
590 self.armed = false;
591 }
592}
593
594impl Drop for RouteBindReservationGuard {
595 fn drop(&mut self) {
596 self.release_and_disarm();
597 }
598}
599
600#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636 modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640 concurrency: RouteBindConcurrency,
641 module_id: String,
642}
643
644impl RouteBindConcurrency {
645 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648 let mut modules = self
649 .modules
650 .lock()
651 .expect("route.bind concurrency mutex poisoned");
652 let in_flight = modules.entry(module_id.to_string()).or_default();
653 if *in_flight >= limit {
654 return Err(*in_flight);
655 }
656 *in_flight += 1;
657 Ok(RouteBindConcurrencyGuard {
658 concurrency: self.clone(),
659 module_id: module_id.to_string(),
660 })
661 }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665 fn drop(&mut self) {
666 let mut modules = self
667 .concurrency
668 .modules
669 .lock()
670 .expect("route.bind concurrency mutex poisoned");
671 let remove = {
672 let in_flight = modules
673 .get_mut(&self.module_id)
674 .expect("admitted route.bind has a concurrency entry");
675 *in_flight -= 1;
676 *in_flight == 0
677 };
678 if remove {
679 modules.remove(&self.module_id);
680 }
681 }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686 consecutive_timeouts: u32,
688 cooldown_until: Option<Instant>,
691 probe_in_flight: Option<Arc<()>>,
696}
697
698enum RouteBindAdmission<'a> {
700 Admitted {
701 guard: RouteBindBreakerGuard<'a>,
702 probe: bool,
705 },
706 Refused {
707 consecutive_timeouts: u32,
708 retry_in: Duration,
712 probe_in_flight: bool,
713 },
714}
715
716struct RouteBindBreakerGuard<'a> {
723 breakers: RouteBindBreakers,
724 module_id: &'a str,
725 probe_token: Option<Arc<()>>,
726 settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730 fn record_accepted(&mut self) -> bool {
734 self.settled = true;
735 self.breakers.record_accepted(self.module_id)
736 }
737
738 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741 self.settled = true;
742 self.breakers.record_timeout(
743 self.module_id,
744 self.probe_token.as_ref(),
745 threshold,
746 cooldown,
747 )
748 }
749
750 fn record_inconclusive(&mut self) {
759 self.settled = true;
760 self.breakers
761 .record_inconclusive(self.module_id, self.probe_token.as_ref());
762 }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766 fn drop(&mut self) {
767 if !self.settled {
768 self.breakers
769 .record_inconclusive(self.module_id, self.probe_token.as_ref());
770 }
771 }
772}
773
774struct BreakerOpened {
778 consecutive_timeouts: u32,
779 reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786 self.modules
787 .lock()
788 .expect("route.bind breaker mutex poisoned")
789 }
790
791 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795 probe: probe_token.is_some(),
796 guard: RouteBindBreakerGuard {
797 breakers: self.clone(),
798 module_id,
799 probe_token,
800 settled: false,
801 },
802 };
803
804 let mut modules = self.lock();
805 let Some(state) = modules.get_mut(module_id) else {
806 return admitted(None);
807 };
808 let Some(cooldown_until) = state.cooldown_until else {
809 return admitted(None);
810 };
811 if state.probe_in_flight.is_some() {
812 return RouteBindAdmission::Refused {
813 consecutive_timeouts: state.consecutive_timeouts,
814 retry_in: Duration::ZERO,
815 probe_in_flight: true,
816 };
817 }
818 let now = Instant::now();
819 if now < cooldown_until {
820 return RouteBindAdmission::Refused {
821 consecutive_timeouts: state.consecutive_timeouts,
822 retry_in: cooldown_until - now,
823 probe_in_flight: false,
824 };
825 }
826 let token = Arc::new(());
827 state.probe_in_flight = Some(Arc::clone(&token));
828 admitted(Some(token))
829 }
830
831 fn record_accepted(&self, module_id: &str) -> bool {
832 self.lock()
833 .remove(module_id)
834 .is_some_and(|state| state.cooldown_until.is_some())
835 }
836
837 fn record_timeout(
838 &self,
839 module_id: &str,
840 probe_token: Option<&Arc<()>>,
841 threshold: u32,
842 cooldown: Duration,
843 ) -> Option<BreakerOpened> {
844 let mut modules = self.lock();
845 let state = modules.entry(module_id.to_string()).or_default();
846 let was_open = state.cooldown_until.is_some();
847 let was_probe = Self::owns_probe(state, probe_token);
848 if was_probe {
849 state.probe_in_flight = None;
850 }
851 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852 if state.consecutive_timeouts < threshold {
853 return None;
854 }
855 state.cooldown_until = Some(Instant::now() + cooldown);
856 Some(BreakerOpened {
857 consecutive_timeouts: state.consecutive_timeouts,
858 reopened_after_probe: was_open && was_probe,
859 })
860 }
861
862 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863 state
867 .probe_in_flight
868 .as_ref()
869 .zip(token)
870 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871 }
872
873 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874 if let Some(state) = self.lock().get_mut(module_id) {
875 if Self::owns_probe(state, probe_token) {
876 state.probe_in_flight = None;
877 }
878 }
879 }
880
881 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896 self.lock()
897 .remove(module_id)
898 .map(|state| state.consecutive_timeouts)
899 .filter(|discarded| *discarded > 0)
900 }
901
902 fn open_snapshot(&self) -> Option<serde_json::Value> {
909 let now = Instant::now();
910 let modules = self.lock();
911 let open = modules
912 .iter()
913 .filter_map(|(module_id, state)| {
914 let cooldown_until = state.cooldown_until?;
915 Some((
916 module_id.clone(),
917 serde_json::json!({
918 "consecutive_timeouts": state.consecutive_timeouts,
919 "cooldown_remaining_ms":
920 cooldown_until.saturating_duration_since(now).as_millis() as u64,
921 "probe_in_flight": state.probe_in_flight.is_some(),
922 }),
923 ))
924 })
925 .collect::<serde_json::Map<String, serde_json::Value>>();
926 (!open.is_empty()).then_some(serde_json::Value::Object(open))
927 }
928}
929
930impl ControlHandler {
931 pub fn new(registry: Arc<Registry>) -> Self {
932 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933 }
934
935 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936 let counters = forwarding.counters();
937 let route_bind_breakers = forwarding.route_bind_breakers();
941 let route_bind_concurrency = forwarding.route_bind_concurrency();
942 let route_outages = forwarding.route_outages();
943 Self {
944 registry,
945 activity: None,
946 activity_streams: Arc::new(Mutex::new(HashMap::new())),
947 forwarding,
948 process_liveness: None,
949 supervisor: SupervisorHandle::new(),
950 subc_capabilities: Arc::from([
951 CAP_MANIFEST_REGISTRATION.to_string(),
952 CAP_CHANNEL_LIFECYCLE.to_string(),
953 CAP_PING_PONG.to_string(),
954 CAP_SESSION_ATTACH.to_string(),
955 CAP_ADMISSION_FACTS_RELAY.to_string(),
956 CAP_SCOPES_V1.to_string(),
957 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
964 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
965 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
966 ]),
967 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
968 route_bind_relay_timeouts: BTreeMap::new(),
969 route_bind_breakers,
970 route_bind_concurrency,
971 route_outages,
972 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
973 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
974 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
975 storage_config: None,
976 machine_id: None,
977 admission_facts_carrier_module_id: None,
978 admission_facts_targets: None,
979 scopes: Arc::new(RwLock::new(ScopeTable::new(
980 crate::daemon_config::default_scope_authority_owners(),
981 ))),
982 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
983 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
984 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
985 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
986 #[cfg(test)]
987 caller_lookup: None,
988 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
989 rescan: None,
990 connected_clients: ConnectedClients::new(),
991 counters,
992 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
993 daemon_shutdown: DaemonShutdownFlag::default(),
994 daemon_provenance: DaemonProvenanceFacts::default(),
995 #[cfg(test)]
996 control_dispatch_delay: None,
997 #[cfg(test)]
998 provenance_probe_override: None,
999 #[cfg(test)]
1000 route_bind_relay_sent_hold: None,
1001 #[cfg(test)]
1002 hello_registry_to_forwarding_hold: None,
1003 }
1004 }
1005
1006 pub fn with_storage_config(
1009 mut self,
1010 storage_config: Option<crate::daemon_config::StorageConfig>,
1011 ) -> Self {
1012 self.storage_config = storage_config;
1013 self
1014 }
1015
1016 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1019 self.activity = Some(crate::activity::ActivityLedger::open(
1020 run_dir.join("activity.jsonl"),
1021 incarnation,
1022 ));
1023 self.supervisor.set_activity(self.activity.clone());
1024 self
1025 }
1026
1027 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1030 self.machine_id = machine_id;
1031 self
1032 }
1033
1034 pub fn with_admission_facts_config(
1038 mut self,
1039 carrier_module_id: Option<String>,
1040 targets: Option<Vec<String>>,
1041 ) -> Self {
1042 self.admission_facts_carrier_module_id = carrier_module_id;
1043 self.admission_facts_targets = targets;
1044 self
1045 }
1046
1047 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1051 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1052 self.scope_authority_owners = owners;
1053 self
1054 }
1055
1056 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1058 self.privileged_operator_prompt = mode;
1059 self
1060 }
1061
1062 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1065 self.route_bind_relay_timeout = timeout;
1066 self
1067 }
1068
1069 pub fn with_route_bind_relay_timeouts(
1075 mut self,
1076 timeouts: impl IntoIterator<Item = (String, Duration)>,
1077 ) -> Self {
1078 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1079 self
1080 }
1081
1082 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1088 self.route_bind_relay_timeouts
1089 .get(module_id)
1090 .copied()
1091 .unwrap_or(self.route_bind_relay_timeout)
1092 }
1093
1094 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1102 self.route_bind_breaker_threshold = threshold.max(1);
1103 self.route_bind_breaker_cooldown = cooldown;
1104 self
1105 }
1106
1107 #[cfg(test)]
1108 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1109 self.health_probe_timeout = timeout;
1110 self
1111 }
1112
1113 #[cfg(test)]
1114 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1115 self.control_dispatch_delay = Some(delay);
1116 self
1117 }
1118
1119 #[cfg(test)]
1123 pub(crate) fn with_route_bind_relay_sent_hold(
1124 mut self,
1125 hold: Arc<tokio::sync::Barrier>,
1126 ) -> Self {
1127 self.route_bind_relay_sent_hold = Some(hold);
1128 self
1129 }
1130
1131 #[cfg(test)]
1136 pub(crate) fn with_hello_registry_to_forwarding_hold(
1137 mut self,
1138 hold: Arc<std::sync::Barrier>,
1139 ) -> Self {
1140 self.hello_registry_to_forwarding_hold = Some(hold);
1141 self
1142 }
1143
1144 #[cfg(test)]
1145 fn hold_between_registry_and_forwarding(&self) {
1146 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1147 hold.wait();
1148 hold.wait();
1149 }
1150 }
1151
1152 pub fn with_process_liveness(
1153 mut self,
1154 process_liveness: Arc<dyn ModuleProcessLiveness>,
1155 ) -> Self {
1156 self.process_liveness = Some(process_liveness);
1157 self
1158 }
1159
1160 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1161 self.supervisor = supervisor;
1162 self.supervisor.set_activity(self.activity.clone());
1163 self
1164 }
1165
1166 pub fn with_daemon_provenance(
1167 mut self,
1168 pid: u32,
1169 started_at_ms: u64,
1170 executable_path: Option<PathBuf>,
1171 build_git_sha: Option<String>,
1172 build_lock_digest: Option<String>,
1173 ) -> Self {
1174 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1175 let process_start_time = process_start_time(pid);
1176 self.daemon_provenance = DaemonProvenanceFacts {
1177 build: DaemonBuildProvenance {
1178 build_git_sha,
1179 build_lock_digest,
1180 },
1181 pid: Some(pid),
1182 started_at_ms: Some(started_at_ms),
1183 start_clock: None,
1184 executable_path,
1185 executable_identity,
1186 process_start_time,
1187 probe: ExecutableIdentityProbe::default(),
1188 };
1189 self
1190 }
1191
1192 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1193 self.daemon_provenance.start_clock = Some(clock);
1194 self
1195 }
1196
1197 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1199 self.scopes
1200 .write()
1201 .unwrap_or_else(|poisoned| poisoned.into_inner())
1202 .set_wall_clock(Arc::new(clock));
1203 self
1204 }
1205
1206 #[cfg(test)]
1207 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1208 self.provenance_probe_override = Some(result);
1209 self
1210 }
1211
1212 pub fn with_capability_config(
1216 self,
1217 modules: impl IntoIterator<Item = (String, bool)>,
1218 reserved_capabilities: BTreeMap<String, String>,
1219 ) -> Self {
1220 self.capability_evaluator
1221 .configure(modules, reserved_capabilities);
1222 self
1223 }
1224
1225 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1229 self.daemon_shutdown = flag;
1230 self
1231 }
1232
1233 pub fn with_supervisor_rescan(
1234 mut self,
1235 supervisor: Supervisor,
1236 config_path: impl Into<PathBuf>,
1237 configured_port: Option<u16>,
1238 ) -> Self {
1239 self.rescan = Some(SupervisorRescanContext {
1240 supervisor,
1241 config_path: config_path.into(),
1242 configured_port,
1243 storage_config: self.storage_config.clone(),
1244 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1245 admission_facts_targets: self.admission_facts_targets.clone(),
1246 scope_authority_owners: self.scope_authority_owners.clone(),
1247 privileged_operator_prompt: self.privileged_operator_prompt,
1248 generation: Arc::default(),
1249 });
1250 self
1251 }
1252
1253 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1254 self.connected_clients = connected_clients;
1255 self
1256 }
1257
1258 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1259 Arc::clone(&self.forwarding)
1260 }
1261
1262 pub(crate) fn counters(&self) -> DaemonCounters {
1263 self.counters.clone()
1264 }
1265
1266 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1269 tokio::spawn(async move {
1270 loop {
1271 self.capability_evaluator
1272 .wait_for_change_or_deadline()
1273 .await;
1274 self.refresh_capability_requirements();
1275 }
1276 });
1277 }
1278
1279 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1285 tokio::spawn(async move {
1286 loop {
1287 let has_deadlines = self
1288 .scopes
1289 .read()
1290 .unwrap_or_else(|poisoned| poisoned.into_inner())
1291 .has_deadlines();
1292 if !has_deadlines {
1293 self.scope_deadline_added.notified().await;
1294 continue;
1295 }
1296 tokio::time::sleep(Duration::from_secs(1)).await;
1297 if let Err(error) = self.sweep_expired_scopes() {
1298 warn!(%error, "scope expiry sweep failed");
1299 }
1300 }
1301 });
1302 }
1303
1304 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1305 let mut table = self
1306 .scopes
1307 .write()
1308 .unwrap_or_else(|poisoned| poisoned.into_inner());
1309 let swept = table.sweep_expired();
1310 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1311 drop(table);
1312 self.close_scope_drained_routes(drained);
1313 Ok(())
1314 }
1315
1316 fn publish_scope_changes(
1317 &self,
1318 changes: &[crate::scopes::ScopeTagChange],
1319 expired: &[crate::scopes::ScopeExpired],
1320 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1321 if changes.is_empty() {
1322 return Ok(Vec::new());
1323 }
1324 let drained = self
1325 .forwarding
1326 .publish_scope_changes(changes)
1327 .map_err(RouterError::Forwarding)?;
1328 let mut counts = HashMap::new();
1329 for route in &drained {
1330 *counts
1331 .entry((
1332 route.scope.owner.as_str(),
1333 route.scope.scope_ref.as_str(),
1334 route.scope.tag.scope_epoch,
1335 ))
1336 .or_insert(0usize) += 1;
1337 }
1338 for scope in expired {
1339 let routes_closed = counts
1340 .get(&(
1341 scope.owner.as_str(),
1342 scope.scope_ref.as_str(),
1343 scope.scope_epoch,
1344 ))
1345 .copied()
1346 .unwrap_or(0);
1347 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1348 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1349 routes_closed, "scope expired");
1350 }
1351 Ok(drained)
1352 }
1353
1354 fn runtime_capability_snapshot(
1355 &self,
1356 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1357 let runtime = self
1358 .supervisor
1359 .list()
1360 .into_iter()
1361 .map(|module| {
1362 let status = module.status().map_err(|err| {
1363 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1364 })?;
1365 Ok(RuntimeModule {
1366 module_id: status.module_id,
1367 state: status.state,
1368 enabled: status.enabled,
1369 })
1370 })
1371 .collect::<Result<Vec<_>, RouterError>>()?;
1372 let registrations = self.registry.list_registered_modules().map_err(|err| {
1378 RouterError::backend(
1379 0,
1380 0,
1381 format!("failed to list capability registrations: {err}"),
1382 )
1383 })?;
1384 let registrations = registrations
1385 .into_iter()
1386 .map(|registration| RegisteredModule {
1387 module_id: registration.manifest.module_id,
1388 module_version: registration.manifest.module_version,
1389 capabilities: registration.manifest.capabilities,
1390 })
1391 .collect();
1392 Ok((runtime, registrations))
1393 }
1394
1395 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1402 let cached_registration = RegisteredModule {
1403 module_id: registration.manifest.module_id.clone(),
1404 module_version: registration.manifest.module_version.clone(),
1405 capabilities: registration.manifest.capabilities.clone(),
1406 };
1407 if self.capability_evaluator.record_hello(&cached_registration) {
1408 warn!(
1409 module_id = %cached_registration.module_id,
1410 "capability claims drifted from the cached manifest"
1411 );
1412 }
1413 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1414 self.enforce_capability_denies();
1415 }
1416 self.refresh_capability_requirements();
1417 }
1418
1419 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1423 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1424 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1425 self.supervisor.set_swap_promotion_observer(observer);
1426 }
1427
1428 pub fn refresh_capability_requirements(&self) {
1429 match self.runtime_capability_snapshot() {
1430 Ok((runtime, registrations)) => {
1431 log_requirement_events(
1432 self.capability_evaluator
1433 .evaluate_now(&runtime, ®istrations),
1434 &self.daemon_shutdown,
1435 );
1436 }
1437 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1438 }
1439 }
1440
1441 fn enforce_capability_denies(&self) {
1445 let registrations = match self.registry.list_registered_modules() {
1446 Ok(snapshot) => snapshot,
1447 Err(err) => {
1448 warn!(error = %err, "failed to read registrations for capability deny census");
1449 return;
1450 }
1451 };
1452 let manifests = registrations
1453 .into_iter()
1454 .map(|registration| {
1455 (
1456 registration.manifest.module_id.clone(),
1457 registration.manifest,
1458 )
1459 })
1460 .collect::<BTreeMap<_, _>>();
1461 let census = match self.forwarding.route_census(None) {
1462 Ok(census) => census,
1463 Err(err) => {
1464 warn!(error = %err, "failed to read route census for capability deny enforcement");
1465 return;
1466 }
1467 };
1468
1469 for (target_module_id, routes) in census {
1470 let Some(target_manifest) = manifests.get(&target_module_id) else {
1471 continue;
1472 };
1473 let mut closed_routes = Vec::new();
1474 let mut module_goodbyes = Vec::new();
1475 for route in routes {
1476 let Principal::Reserved {
1477 module_id: opening_module_id,
1478 } = &route.principal
1479 else {
1480 continue;
1481 };
1482 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1483 continue;
1484 };
1485 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1486 continue;
1487 };
1488
1489 match self.forwarding.release_client_route(
1490 route.goodbye_target.connection_id,
1491 route.goodbye_target.channel,
1492 route.goodbye_target.epoch,
1493 ) {
1494 Ok(RouteRelease::Removed(module_goodbye)) => {
1495 warn!(
1496 opening_module_id,
1497 target_module_id,
1498 capability,
1499 "force-closing route because an attested capability deny edge now matches"
1500 );
1501 closed_routes.push(route);
1502 module_goodbyes.push(module_goodbye);
1503 }
1504 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1505 Err(err) => warn!(
1506 opening_module_id,
1507 target_module_id,
1508 capability,
1509 error = %err,
1510 "failed to force-close capability-denied route"
1511 ),
1512 }
1513 }
1514
1515 if closed_routes.is_empty() {
1516 continue;
1517 }
1518 send_route_control_pushes(
1519 &self.forwarding,
1520 closed_routes,
1521 ClientControlPush::RouteClosed {
1522 module_id: target_module_id,
1523 channels: Vec::new(),
1524 reason: RouteCloseReason::CapabilityDenied,
1525 drained: false,
1526 abandoned: 0,
1527 excluded_subscriptions: 0,
1528 terminal: Some(false),
1529 },
1530 );
1531 self.emit_route_goodbyes(module_goodbyes);
1532 }
1533 }
1534
1535 fn not_ready_reason(
1541 &self,
1542 registration: &crate::registry::ModuleRegistration,
1543 ) -> Option<NotReadyReason> {
1544 if !registration.ready {
1545 return Some(NotReadyReason {
1546 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1547 capability: None,
1548 });
1549 }
1550 self.first_unprovided_required_capability(registration)
1551 .map(|capability| NotReadyReason {
1552 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1553 capability: Some(capability),
1554 })
1555 }
1556
1557 fn first_unprovided_required_capability(
1575 &self,
1576 registration: &crate::registry::ModuleRegistration,
1577 ) -> Option<String> {
1578 let required = registration
1579 .manifest
1580 .capabilities
1581 .iter()
1582 .flat_map(|declarations| declarations.requires.iter())
1583 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1584 .map(|requirement| requirement.capability.as_str())
1585 .collect::<BTreeSet<_>>();
1586 if required.is_empty() {
1587 return None;
1588 }
1589 let module_id = registration.manifest.module_id.as_str();
1590 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1591 if required
1592 .iter()
1593 .any(|capability| verdict(capability).is_none())
1594 {
1595 self.refresh_capability_requirements();
1596 }
1597 required
1598 .into_iter()
1599 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1600 .map(str::to_string)
1601 }
1602
1603 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1604 self.capability_evaluator
1605 .statuses()
1606 .into_iter()
1607 .map(capability_requirement_status)
1608 .collect()
1609 }
1610
1611 fn deregister_connection(
1618 &self,
1619 connection_id: ConnectionId,
1620 reason: RegistrationEndReason,
1621 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1622 self.registry
1623 .deregister_connection_with_reason(connection_id, reason)
1624 }
1625
1626 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1627 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1628 return None;
1629 }
1630 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1631 parse_client_control_request(&frame.body)
1632 else {
1633 return None;
1634 };
1635 Some(target_module_id(&target).to_string())
1636 }
1637
1638 pub(crate) fn route_open_capacity_refusal(
1639 &self,
1640 ctx: &RouteCtx,
1641 frame: &Frame,
1642 target_module_id: &str,
1643 in_flight: usize,
1644 limit: usize,
1645 ) -> Result<Frame, RouterError> {
1646 self.route_open_admission_refusal_frame(
1647 ctx,
1648 frame,
1649 target_module_id,
1650 "open_admission_full",
1651 (in_flight, limit),
1652 format!(
1653 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1654 ),
1655 )
1656 }
1657
1658 fn route_open_target_capacity_refusal(
1659 &self,
1660 ctx: &RouteCtx,
1661 frame: &Frame,
1662 target_module_id: &str,
1663 in_flight: usize,
1664 ) -> Result<Frame, RouterError> {
1665 self.route_open_admission_refusal_frame(
1666 ctx,
1667 frame,
1668 target_module_id,
1669 "target_binds_full",
1670 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1671 format!(
1672 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1673 ),
1674 )
1675 }
1676
1677 fn route_open_admission_refusal_frame(
1685 &self,
1686 ctx: &RouteCtx,
1687 frame: &Frame,
1688 target_module_id: &str,
1689 reason: &'static str,
1690 (in_flight, limit): (usize, usize),
1691 message: impl Into<String>,
1692 ) -> Result<Frame, RouterError> {
1693 let code = error_codes::TARGET_UNAVAILABLE;
1694 self.counters.increment_route_open_refused(code);
1695 info!(
1696 target: "control",
1697 code,
1698 reason,
1699 module_id = ?target_module_id,
1700 connection_id = ctx.connection_id.get(),
1701 in_flight,
1702 limit,
1703 "route.open refused"
1704 );
1705 control_error_frame(frame, code, message.into())
1706 }
1707
1708 #[cfg(test)]
1714 pub fn handle_control(
1715 &self,
1716 connection_id: ConnectionId,
1717 frame: Frame,
1718 ) -> Result<Vec<Frame>, RouterError> {
1719 match frame.header.ty {
1720 FrameType::Ping => Ok(vec![pong(&frame)?]),
1721 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1722 FrameType::Goodbye => self.handle_goodbye(connection_id),
1723 ty => Ok(vec![control_error_frame(
1724 &frame,
1725 "unsupported_control_frame",
1726 format!("unsupported channel-0 frame {ty:?}"),
1727 )?]),
1728 }
1729 }
1730
1731 pub async fn handle_control_frame(
1732 &self,
1733 ctx: &RouteCtx,
1734 frame: Frame,
1735 ) -> Result<Vec<Frame>, RouterError> {
1736 self.handle_control_frame_timed(ctx, frame, None).await
1737 }
1738
1739 pub(crate) fn record_caller_endpoints(
1740 &self,
1741 connection: ConnectionId,
1742 endpoints: Option<(SocketAddr, SocketAddr)>,
1743 ) {
1744 let mut callers = self
1745 .caller_endpoints
1746 .lock()
1747 .unwrap_or_else(|p| p.into_inner());
1748 callers.remove(&connection);
1749 if let Some(endpoints) = endpoints {
1750 callers.insert(connection, endpoints);
1751 }
1752 }
1753
1754 #[cfg(test)]
1755 pub(crate) fn with_caller_lookup_for_test(
1756 mut self,
1757 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1758 ) -> Self {
1759 self.caller_lookup = Some(Arc::new(lookup));
1760 self
1761 }
1762
1763 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1764 let endpoints = self
1765 .caller_endpoints
1766 .lock()
1767 .unwrap_or_else(|p| p.into_inner())
1768 .get(&connection)
1769 .copied();
1770 #[cfg(test)]
1771 let lookup = self.caller_lookup.clone();
1772 tokio::task::spawn_blocking(move || {
1773 #[cfg(test)]
1774 if let Some(lookup) = lookup {
1775 return lookup(endpoints);
1776 }
1777 endpoints
1778 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1779 .unwrap_or_default()
1780 })
1781 .await
1782 .unwrap_or_default()
1783 }
1784
1785 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1786 frame.header.channel == 0
1787 && frame.header.ty == FrameType::Request
1788 && self
1789 .forwarding
1790 .module_endpoint_for_connection(ctx.connection_id)
1791 .is_ok_and(|endpoint| endpoint.is_none())
1792 && parse_client_control_request(&frame.body)
1793 .ok()
1794 .is_some_and(|request| privileged_operation(&request).is_some())
1795 }
1796
1797 pub(crate) async fn handle_control_frame_timed(
1798 &self,
1799 ctx: &RouteCtx,
1800 frame: Frame,
1801 dispatch_started_at: Option<StdInstant>,
1802 ) -> Result<Vec<Frame>, RouterError> {
1803 match frame.header.ty {
1804 FrameType::Ping => Ok(vec![pong(&frame)?]),
1805 FrameType::Hello => {
1806 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1807 }
1808 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1809 FrameType::Cancel => {
1810 if self
1813 .forwarding
1814 .operator_confirms()
1815 .cancel(ctx.connection_id, frame.header.corr)
1816 || self
1817 .supervisor
1818 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1819 || self.cancel_activity_subscription(ctx, &frame)?
1820 {
1821 Ok(Vec::new())
1822 } else {
1823 Ok(vec![control_error_frame(
1824 &frame,
1825 "unknown_subscription",
1826 "no subscription has this correlation id",
1827 )?])
1828 }
1829 }
1830 FrameType::Request => {
1831 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1834 if op
1835 .as_ref()
1836 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1837 {
1838 return self.handle_activity_record(ctx, frame).await;
1839 }
1840 if op.as_ref().is_some_and(|probe| {
1841 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1842 }) {
1843 return self.handle_activity_read(ctx, frame).await;
1844 }
1845 if op
1846 .as_ref()
1847 .is_some_and(|probe| probe.op == "operator.confirm")
1848 {
1849 return self.handle_operator_confirm(ctx, frame);
1850 }
1851 if op
1852 .as_ref()
1853 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1854 {
1855 return self.handle_scope_served_by(ctx.connection_id, frame);
1856 }
1857 if self
1858 .forwarding
1859 .module_endpoint_for_connection(ctx.connection_id)
1860 .map_err(RouterError::Forwarding)?
1861 .is_some()
1862 {
1863 if !is_known_module_request_op(&frame.body) {
1864 return Ok(vec![control_error_frame(
1865 &frame,
1866 "unsupported_control_frame",
1867 "module-originated channel-0 REQUEST is not supported",
1868 )?]);
1869 }
1870 let request = match parse_module_control_request_from_module(&frame.body) {
1871 Ok(request) => request,
1872 Err((err, ControlRequestBodyError::UnknownOp)) => {
1873 return Ok(vec![control_error_frame(
1874 &frame,
1875 "unsupported_control_frame",
1876 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1877 )?])
1878 }
1879 Err((err, ControlRequestBodyError::InvalidBody)) => {
1880 return Ok(vec![control_error_frame(
1881 &frame,
1882 "invalid_control_body",
1883 format!("malformed module control body: {err}"),
1884 )?])
1885 }
1886 };
1887 let op = module_control_request_op(&request);
1888 let corr = frame.header.corr;
1889 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1890 let result =
1891 self.handle_module_control_request(ctx.connection_id, frame, request);
1892 log_slow_control_dispatch(
1893 dispatch_started_at,
1894 op,
1895 ctx.connection_id,
1896 corr,
1897 None,
1898 );
1899 return result;
1900 }
1901
1902 if is_known_module_request_op(&frame.body) {
1903 return Ok(vec![control_error_frame(
1904 &frame,
1905 "not_registered",
1906 "catalog.update requires an active module registration owned by this connection",
1907 )?]);
1908 }
1909
1910 let request = match parse_client_control_request(&frame.body) {
1911 Ok(request) => request,
1912 Err((err, ControlRequestBodyError::UnknownOp)) => {
1913 return Ok(vec![control_error_frame(
1914 &frame,
1915 "unknown_control_op",
1916 format!("unknown client control op: {err}"),
1917 )?])
1918 }
1919 Err((err, ControlRequestBodyError::InvalidBody)) => {
1920 return Ok(vec![control_error_frame(
1921 &frame,
1922 "invalid_control_body",
1923 format!("malformed client control body: {err}"),
1924 )?])
1925 }
1926 };
1927 let op = client_control_request_op(&request);
1928 let corr = frame.header.corr;
1929 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1930 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1931 #[cfg(test)]
1932 if let Some(delay) = self.control_dispatch_delay {
1933 tokio::time::sleep(delay).await;
1934 }
1935 let result = self
1936 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1937 .await;
1938 if let Some(timings) = route_open_timings.as_mut() {
1939 timings.finish();
1940 }
1941 log_slow_control_dispatch(
1942 dispatch_started_at,
1943 op,
1944 ctx.connection_id,
1945 corr,
1946 route_open_timings.as_ref(),
1947 );
1948 result
1949 }
1950 FrameType::Push => {
1951 let Some(endpoint) = self
1952 .forwarding
1953 .module_endpoint_for_connection(ctx.connection_id)
1954 .map_err(RouterError::Forwarding)?
1955 else {
1956 return Ok(vec![control_error_frame(
1957 &frame,
1958 "unsupported_control_frame",
1959 "client-originated channel-0 PUSH is not supported",
1960 )?]);
1961 };
1962 self.handle_status_update(endpoint, frame)
1963 }
1964 FrameType::Response | FrameType::Error
1965 if self
1966 .forwarding
1967 .module_endpoint_for_connection(ctx.connection_id)
1968 .map_err(RouterError::Forwarding)?
1969 .is_some() =>
1970 {
1971 self.handle_module_relay_response(ctx.connection_id, frame)
1972 }
1973 ty => Ok(vec![control_error_frame(
1974 &frame,
1975 "unsupported_control_frame",
1976 format!("unsupported channel-0 frame {ty:?}"),
1977 )?]),
1978 }
1979 }
1980
1981 pub fn cleanup_connection(
1982 &self,
1983 connection_id: ConnectionId,
1984 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1985 self.caller_endpoints
1986 .lock()
1987 .unwrap_or_else(|p| p.into_inner())
1988 .remove(&connection_id);
1989 self.forwarding
1990 .operator_confirms()
1991 .connection_closed(connection_id);
1992 self.cleanup_connection_with_end_reason(
1993 connection_id,
1994 RegistrationEndReason::ConnectionClosed,
1995 )
1996 }
1997
1998 fn cleanup_connection_with_end_reason(
1999 &self,
2000 connection_id: ConnectionId,
2001 requested_reason: RegistrationEndReason,
2002 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2003 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2004 self.registry
2005 .get_module_by_connection(connection_id)?
2006 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2007 .and_then(|module| module.registration_end_reason().ok().flatten())
2008 .unwrap_or(requested_reason)
2009 } else {
2010 requested_reason
2011 };
2012 let crash_closed = self
2013 .registry
2014 .get_module_by_connection(connection_id)?
2015 .and_then(|registration| {
2016 self.forwarding
2017 .module_endpoint_for_connection(connection_id)
2018 .ok()
2019 .flatten()
2020 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2021 .map(|routes| (registration.manifest.module_id, routes))
2022 });
2023 let crash_closed = crash_closed.map(|(module_id, routes)| {
2024 let terminal = match self.supervisor.get(&module_id) {
2025 None => false,
2026 Some(module) => match module.will_recover_after_connection_loss() {
2027 Ok(will_recover) => !will_recover,
2028 Err(err) => {
2029 warn!(
2030 %module_id,
2031 error = %err,
2032 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2033 );
2034 false
2035 }
2036 },
2037 };
2038 let (reason, drained) = match self
2042 .forwarding
2043 .daemon_drain_result_for_connection(connection_id)
2044 {
2045 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2046 Ok(None) => (RouteCloseReason::Crash, false),
2047 Err(err) => {
2048 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2049 (RouteCloseReason::Crash, false)
2050 }
2051 };
2052 (module_id, routes, reason, drained, terminal)
2053 });
2054 let registrations = self.deregister_connection(connection_id, end_reason);
2055 let cleanup = if crash_closed.is_some() {
2056 self.forwarding.cleanup_connection_counted(connection_id)
2057 } else {
2058 self.forwarding
2064 .cleanup_connection(connection_id)
2065 .map(|released| crate::forwarding::ConnectionCleanup {
2066 released,
2067 abandoned_relays: 0,
2068 })
2069 };
2070 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2075 let abandoned = cleanup
2076 .as_ref()
2077 .map_or(0, |cleanup| cleanup.abandoned_relays);
2078 send_route_control_pushes(
2079 &self.forwarding,
2080 routes,
2081 ClientControlPush::RouteClosed {
2082 module_id,
2083 channels: Vec::new(),
2084 reason,
2085 drained,
2086 abandoned,
2087 excluded_subscriptions: 0,
2088 terminal: Some(terminal),
2089 },
2090 );
2091 }
2092 if let Ok(cleanup) = cleanup {
2093 self.emit_route_goodbyes(cleanup.released);
2094 }
2095 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2099 crate::supervise::notify_registration_release();
2100 self.capability_evaluator.wake_deadline_loop();
2101 self.refresh_capability_requirements();
2102 }
2103 self.supervisor.remove_spawn_subscribers(connection_id);
2104 self.activity_streams
2105 .lock()
2106 .unwrap_or_else(|p| p.into_inner())
2107 .retain(|(id, _), task| {
2108 if *id == connection_id {
2109 task.abort();
2110 false
2111 } else {
2112 true
2113 }
2114 });
2115 self.hello_launch_nonces
2118 .lock()
2119 .unwrap_or_else(|poisoned| poisoned.into_inner())
2120 .forget(connection_id);
2121 self.scopes
2122 .write()
2123 .unwrap_or_else(|poisoned| poisoned.into_inner())
2124 .release_connection(connection_id);
2125 registrations
2126 }
2127
2128 pub(crate) fn handle_route_goodbye(
2129 &self,
2130 connection_id: ConnectionId,
2131 route_channel: u16,
2132 route_epoch: u32,
2133 ) -> Result<bool, RouterError> {
2134 debug!(
2135 connection_id = connection_id.get(),
2136 route_channel, route_epoch, "handling route GOODBYE"
2137 );
2138 let RouteRelease::Removed(released_route) = self
2139 .forwarding
2140 .release_client_route(connection_id, route_channel, route_epoch)
2141 .map_err(RouterError::Forwarding)?
2142 else {
2143 return Ok(false);
2144 };
2145 self.emit_route_goodbyes(vec![released_route]);
2146 Ok(true)
2147 }
2148
2149 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2150 for released in released_routes {
2151 let frame = match Frame::build_with_version(
2152 released.negotiated_ver,
2153 FrameType::Goodbye,
2154 control_flags(),
2155 released.channel,
2156 released.epoch,
2157 0,
2158 Vec::new(),
2159 ) {
2160 Ok(frame) => frame,
2161 Err(err) => {
2162 warn!(
2163 route_channel = released.channel,
2164 error = %err,
2165 "failed to build route GOODBYE frame"
2166 );
2167 continue;
2168 }
2169 };
2170 if !released.close_on_delivery_failure() {
2171 crate::forwarding::send_module_route_goodbye(
2172 &self.counters,
2173 &released.sink,
2174 frame,
2175 released.module_id.as_deref(),
2176 "client route released",
2177 );
2178 continue;
2179 }
2180 if let Err(err) = released.sink.try_send(frame) {
2181 warn!(
2182 target_connection_id = released.connection_id.get(),
2183 route_channel = released.channel,
2184 error = %err,
2185 "route GOODBYE was not delivered to client; closing target connection"
2186 );
2187 if self
2188 .forwarding
2189 .escalate_client_delivery_failure(
2190 released.connection_id,
2191 released.channel,
2192 released.epoch,
2193 CloseReason::new(
2194 "route_goodbye_delivery_failed",
2195 format!(
2196 "failed to enqueue route GOODBYE for channel {}: {err}",
2197 released.channel
2198 ),
2199 ),
2200 crate::forwarding::UndeliveredFrame {
2201 module_id: released.module_id.as_deref(),
2202 sink: &released.sink,
2203 },
2204 )
2205 .unwrap_or(false)
2206 {
2207 self.counters.increment_goodbye_relay_client_failed();
2208 }
2209 }
2210 }
2211 }
2212
2213 fn send_abandoned_route_bind_goodbye(
2225 &self,
2226 module_sink: &crate::FrameSink,
2227 negotiated_ver: u8,
2228 module_channel: u16,
2229 module_epoch: u32,
2230 ) {
2231 let frame = match Frame::build_with_version(
2232 negotiated_ver,
2233 FrameType::Goodbye,
2234 control_flags(),
2235 module_channel,
2236 module_epoch,
2237 0,
2238 Vec::new(),
2239 ) {
2240 Ok(frame) => frame,
2241 Err(err) => {
2242 warn!(
2243 route_channel = module_channel,
2244 error = %err,
2245 "failed to build GOODBYE for abandoned route.bind"
2246 );
2247 return;
2248 }
2249 };
2250 crate::forwarding::send_module_route_goodbye(
2251 &self.counters,
2252 module_sink,
2253 frame,
2254 None,
2255 "abandoned route.bind",
2256 );
2257 }
2258
2259 fn handle_hello(
2260 &self,
2261 connection_id: ConnectionId,
2262 sink: Option<crate::FrameSink>,
2263 frame: Frame,
2264 ) -> Result<Vec<Frame>, RouterError> {
2265 debug!(
2266 connection_id = connection_id.get(),
2267 corr = frame.header.corr,
2268 "handling HELLO"
2269 );
2270 if self
2274 .registry
2275 .get_module_by_connection(connection_id)
2276 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2277 .is_some()
2278 {
2279 return Ok(vec![control_error_frame(
2280 &frame,
2281 "invalid_hello",
2282 "connection is already registered as a module",
2283 )?]);
2284 }
2285 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2286 Ok(value) => value,
2287 Err(err) => {
2288 return Ok(vec![control_error_frame(
2289 &frame,
2290 "invalid_hello",
2291 format!("malformed HELLO body: {err}"),
2292 )?])
2293 }
2294 };
2295 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2296 return Ok(vec![control_error_frame(
2297 &frame,
2298 "invalid_capability_grammar",
2299 err.to_string(),
2300 )?]);
2301 }
2302 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2303 return Ok(vec![control_error_frame(
2304 &frame,
2305 "invalid_manifest",
2306 err.to_string(),
2307 )?]);
2308 }
2309 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2310 return Ok(vec![control_error_frame(
2311 &frame,
2312 "invalid_event_declaration",
2313 err.to_string(),
2314 )?]);
2315 }
2316 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2317 return Ok(vec![control_error_frame(
2318 &frame,
2319 "invalid_manifest",
2320 err.to_string(),
2321 )?]);
2322 }
2323 if let Some(provenance) = hello_value
2324 .get("manifest")
2325 .and_then(|manifest| manifest.get("provenance"))
2326 {
2327 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2328 return Ok(vec![control_error_frame(
2329 &frame,
2330 "invalid_manifest",
2331 format!("malformed manifest provenance: {err}"),
2332 )?]);
2333 }
2334 }
2335 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2336 Ok(hello) => hello,
2337 Err(err) => {
2338 return Ok(vec![control_error_frame(
2339 &frame,
2340 "invalid_hello",
2341 format!("malformed HELLO body: {err}"),
2342 )?])
2343 }
2344 };
2345
2346 if hello.protocol_ver != hello.manifest.protocol_ver {
2347 return Ok(vec![control_error_frame(
2348 &frame,
2349 "invalid_manifest",
2350 format!(
2351 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2352 hello.protocol_ver, hello.manifest.protocol_ver
2353 ),
2354 )?]);
2355 }
2356
2357 if hello.manifest.module_id.trim().is_empty() {
2358 return Ok(vec![control_error_frame(
2359 &frame,
2360 "invalid_manifest",
2361 "manifest module_id must not be empty",
2362 )?]);
2363 }
2364
2365 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2366 Ok(negotiated_ver) => negotiated_ver,
2367 Err(message) => {
2368 return Ok(vec![control_error_frame(
2369 &frame,
2370 "version_unsupported",
2371 message,
2372 )?])
2373 }
2374 };
2375
2376 let swap_admission = self
2384 .supervisor
2385 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2386 if swap_admission == SwapHelloAdmission::Refused {
2387 warn!(
2388 module_id = %hello.manifest.module_id,
2389 connection_id = connection_id.get(),
2390 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2391 );
2392 return Ok(vec![control_error_frame(
2393 &frame,
2394 "swap_token_invalid",
2395 format!(
2396 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2397 hello.manifest.module_id
2398 ),
2399 )?]);
2400 }
2401 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2402
2403 let nonce_verified = if swap_candidate {
2408 true
2409 } else {
2410 match self.supervisor.reserved_hello_verification(
2411 &hello.manifest.module_id,
2412 hello.launch_nonce.as_deref(),
2413 ) {
2414 Ok(verified) => verified,
2415 Err(rejection) => {
2416 let message = match rejection {
2417 ReservedHelloRejection::Exact { module_id } => format!(
2418 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2419 ),
2420 ReservedHelloRejection::Prefix {
2421 prefix,
2422 owner_module_id,
2423 } => format!(
2424 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2425 hello.manifest.module_id
2426 ),
2427 };
2428 return Ok(vec![control_error_frame(
2429 &frame,
2430 "reserved_module",
2431 message,
2432 )?]);
2433 }
2434 }
2435 };
2436
2437 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2438 &hello.manifest.module_id,
2439 hello.manifest.capabilities.as_ref(),
2440 );
2441 if let Some(refusal) = reserved_capability_refusals.first() {
2442 let capability = refusal.capability.clone();
2443 let bound_module = refusal.claimants[0].clone();
2444 log_duplicate_claim_events(reserved_capability_refusals);
2445 return Ok(vec![control_error_frame(
2446 &frame,
2447 "reserved_capability",
2448 format!(
2449 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2450 capability, bound_module, hello.manifest.module_id
2451 ),
2452 )?]);
2453 }
2454
2455 if self
2458 .forwarding
2459 .connection_has_client_routes(connection_id)
2460 .map_err(RouterError::Forwarding)?
2461 {
2462 return Ok(vec![control_error_frame(
2463 &frame,
2464 "invalid_hello",
2465 "connection has open client routes and cannot also register as a module",
2466 )?]);
2467 }
2468
2469 self.hello_launch_nonces
2474 .lock()
2475 .unwrap_or_else(|poisoned| poisoned.into_inner())
2476 .record(connection_id, hello.launch_nonce.as_deref());
2477 let control_ops = effective_module_control_ops(hello.control_ops);
2478 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2481 if swap_candidate {
2482 return self.register_swap_candidate(
2483 connection_id,
2484 sink,
2485 &frame,
2486 hello.manifest,
2487 negotiated_ver,
2488 control_ops,
2489 hello_ack,
2490 );
2491 }
2492 let registration = match self
2499 .registry
2500 .register_unpublished_verified_with_control_ops(
2501 hello.manifest,
2502 negotiated_ver,
2503 connection_id,
2504 control_ops,
2505 nonce_verified,
2506 ) {
2507 Ok(registration) => registration,
2508 Err(RegistryError::DuplicateModuleId { module_id }) => {
2509 return Ok(vec![control_error_frame(
2510 &frame,
2511 "duplicate_module_id",
2512 format!(
2513 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2514 ),
2515 )?])
2516 }
2517 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2518 return Ok(vec![control_error_frame(
2519 &frame,
2520 "invalid_module_id",
2521 err.to_string(),
2522 )?])
2523 }
2524 Err(err) => {
2525 return Ok(vec![control_error_frame(
2526 &frame,
2527 "registry_error",
2528 err.to_string(),
2529 )?])
2530 }
2531 };
2532 #[cfg(test)]
2533 self.hold_between_registry_and_forwarding();
2534
2535 let reply = if let Some(sink) = sink {
2536 let concurrency = manifest_concurrency(®istration.manifest);
2552 if let Err(err) = self.forwarding.register_module_connection_acked(
2553 connection_id,
2554 registration.manifest.module_id.clone(),
2555 negotiated_ver,
2556 concurrency,
2557 sink,
2558 hello_ack,
2559 ) {
2560 if matches!(
2565 self.deregister_connection(
2566 connection_id,
2567 RegistrationEndReason::RegistrationFailed,
2568 ),
2569 Ok(r) if !r.is_empty()
2570 ) {
2571 crate::supervise::notify_registration_release();
2572 }
2573 return Ok(vec![control_error_frame(
2574 &frame,
2575 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2576 "invalid_hello"
2577 } else {
2578 forwarding_error_code(&err)
2579 },
2580 err.to_string(),
2581 )?]);
2582 }
2583 Vec::new()
2584 } else {
2585 vec![hello_ack]
2588 };
2589
2590 if !self
2595 .registry
2596 .publish_registration(connection_id)
2597 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2598 {
2599 debug!(
2600 module_id = %registration.manifest.module_id,
2601 connection_id = connection_id.get(),
2602 "registration was removed before it could be published"
2603 );
2604 }
2605
2606 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2615 info!(
2616 module_id = %registration.manifest.module_id,
2617 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2618 );
2619 }
2620
2621 self.apply_registration_capabilities(®istration);
2622
2623 self.supervisor
2624 .record_activity_registration(®istration.manifest.module_id);
2625
2626 info!(
2627 module_id = %registration.manifest.module_id,
2628 module_version = %registration.manifest.module_version,
2629 negotiated_ver,
2630 routable_provider = manifest_provides_routable_role(®istration.manifest),
2631 connection_id = connection_id.get(),
2632 "module registered"
2633 );
2634
2635 Ok(reply)
2636 }
2637
2638 #[allow(clippy::too_many_arguments)]
2655 fn register_swap_candidate(
2656 &self,
2657 connection_id: ConnectionId,
2658 sink: Option<crate::FrameSink>,
2659 frame: &Frame,
2660 manifest: ModuleManifest,
2661 negotiated_ver: u8,
2662 control_ops: Vec<String>,
2663 hello_ack: Frame,
2664 ) -> Result<Vec<Frame>, RouterError> {
2665 let module_id = manifest.module_id.clone();
2666 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2667 manifest,
2668 negotiated_ver,
2669 connection_id,
2670 control_ops,
2671 ) {
2672 Ok(registration) => registration,
2673 Err(RegistryError::DuplicateModuleId { module_id }) => {
2674 return Ok(vec![control_error_frame(
2675 frame,
2676 "duplicate_module_id",
2677 format!(
2678 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2679 ),
2680 )?])
2681 }
2682 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2683 return Ok(vec![control_error_frame(
2684 frame,
2685 "invalid_module_id",
2686 err.to_string(),
2687 )?])
2688 }
2689 Err(err) => {
2690 return Ok(vec![control_error_frame(
2691 frame,
2692 "registry_error",
2693 err.to_string(),
2694 )?])
2695 }
2696 };
2697 #[cfg(test)]
2698 self.hold_between_registry_and_forwarding();
2699 let reply = if let Some(sink) = sink {
2700 let concurrency = manifest_concurrency(®istration.manifest);
2704 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2705 connection_id,
2706 module_id.clone(),
2707 negotiated_ver,
2708 concurrency,
2709 sink,
2710 hello_ack,
2711 ) {
2712 if matches!(
2713 self.deregister_connection(
2714 connection_id,
2715 RegistrationEndReason::RegistrationFailed,
2716 ),
2717 Ok(r) if !r.is_empty()
2718 ) {
2719 crate::supervise::notify_registration_release();
2720 }
2721 return Ok(vec![control_error_frame(
2722 frame,
2723 forwarding_error_code(&err),
2724 err.to_string(),
2725 )?]);
2726 }
2727 Vec::new()
2728 } else {
2729 vec![hello_ack]
2730 };
2731 if !self
2732 .registry
2733 .publish_registration(connection_id)
2734 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2735 {
2736 debug!(
2737 module_id = %module_id,
2738 connection_id = connection_id.get(),
2739 "swap candidate registration was removed before it could be published"
2740 );
2741 }
2742 self.supervisor.mark_swap_candidate_admitted(&module_id);
2743 info!(
2744 module_id = %module_id,
2745 module_version = %registration.manifest.module_version,
2746 negotiated_ver,
2747 ready = registration.ready,
2748 connection_id = connection_id.get(),
2749 "swap candidate registered; not routable until cutover"
2750 );
2751 Ok(reply)
2752 }
2753
2754 fn build_hello_ack(
2755 &self,
2756 frame: &Frame,
2757 negotiated_ver: u8,
2758 module_id: &str,
2759 ) -> Result<Frame, RouterError> {
2760 let ack = ModuleHelloAckBody {
2761 negotiated_ver,
2762 subc_ops: module_subc_ops(),
2763 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2764 storage: self
2765 .storage_config
2766 .as_ref()
2767 .map(|cfg| cfg.descriptor_for(module_id)),
2768 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2769 };
2770 let body = serde_json::to_vec(&ack).map_err(|err| {
2771 RouterError::backend(
2772 0,
2773 frame.header.corr,
2774 format!("failed to encode HELLO_ACK: {err}"),
2775 )
2776 })?;
2777
2778 Frame::build_with_version(
2779 negotiated_ver,
2780 FrameType::HelloAck,
2781 control_flags(),
2782 0,
2783 0,
2784 frame.header.corr,
2785 body,
2786 )
2787 .map_err(RouterError::FrameBuild)
2788 }
2789
2790 async fn handle_client_control_request(
2791 &self,
2792 ctx: &RouteCtx,
2793 frame: Frame,
2794 request: ClientControlRequest,
2795 route_open_timings: &mut Option<RouteOpenTimings>,
2796 ) -> Result<Vec<Frame>, RouterError> {
2797 if let Some(operation) = privileged_operation(&request) {
2798 let caller = self.resolve_caller(ctx.connection_id).await;
2799 let op = client_control_request_op(&request);
2800 let text = prompt_text(operation, &caller);
2801 match self.privileged_operator_prompt {
2802 PrivilegedOperatorPrompt::Log => {
2803 info!(target: "subc_daemon::privileged_prompt", operation = op,
2804 target = ?operation, caller = %caller.description(), summary = %text,
2805 "privileged request would prompt the operator");
2806 }
2807 PrivilegedOperatorPrompt::Enforce => {
2808 let key =
2809 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2810 let outcome = self
2811 .forwarding
2812 .operator_confirms()
2813 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2814 .await;
2815 if outcome.code != "confirmed" {
2816 return Ok(vec![outcome.frame(&frame)]);
2817 }
2818 }
2819 }
2820 }
2821 match request {
2822 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2823 ClientControlRequest::CatalogList { module_id } => {
2824 self.handle_catalog_list(frame, module_id)
2825 }
2826 ClientControlRequest::RouteOpen {
2827 target,
2828 identity,
2829 consumer_identity,
2830 consumer_capabilities,
2831 role_versions,
2832 admission_facts,
2833 scope,
2834 } => {
2835 self.handle_route_open(
2836 ctx,
2837 frame,
2838 RouteOpenRequest {
2839 target,
2840 identity,
2841 consumer_identity,
2842 consumer_capabilities,
2843 role_versions,
2844 admission_facts,
2845 scope,
2846 },
2847 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2848 )
2849 .await
2850 }
2851 ClientControlRequest::RoutePoll {
2852 route_channel,
2853 route_epoch,
2854 kind,
2855 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2856 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2857 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2858 self.handle_supervisor_spawn_snapshot(frame)
2859 }
2860 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2861 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2862 }
2863 ClientControlRequest::SupervisorRestart {
2864 module_id,
2865 drain_timeout_ms,
2866 } => {
2867 self.log_supervisor_request_received(
2868 ctx,
2869 frame.header.corr,
2870 ops::SUPERVISOR_RESTART,
2871 Some(&module_id),
2872 None,
2873 )?;
2874 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2875 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2876 .await
2877 }
2878 ClientControlRequest::SupervisorSwap {
2879 module_id,
2880 ready_timeout_ms,
2881 } => {
2882 self.log_supervisor_request_received(
2883 ctx,
2884 frame.header.corr,
2885 ops::SUPERVISOR_SWAP,
2886 Some(&module_id),
2887 None,
2888 )?;
2889 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2890 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2891 .await
2892 }
2893 ClientControlRequest::SupervisorReload { module_id } => {
2894 self.log_supervisor_request_received(
2895 ctx,
2896 frame.header.corr,
2897 ops::SUPERVISOR_RELOAD,
2898 Some(&module_id),
2899 None,
2900 )?;
2901 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2902 self.handle_supervisor_reload(frame, module_id, requester)
2903 .await
2904 }
2905 ClientControlRequest::SupervisorRescan { preview } => {
2906 if !preview {
2907 self.log_supervisor_request_received(
2908 ctx,
2909 frame.header.corr,
2910 ops::SUPERVISOR_RESCAN,
2911 None,
2912 None,
2913 )?;
2914 }
2915 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2916 self.handle_supervisor_rescan(
2917 frame,
2918 if preview {
2919 RescanMode::Preview
2920 } else {
2921 RescanMode::Execute
2922 },
2923 requester,
2924 )
2925 .await
2926 }
2927 ClientControlRequest::SupervisorRescanChecked {
2928 preview,
2929 candidate,
2930 expect,
2931 } => {
2932 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2933 .await
2934 }
2935 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2936 self.log_supervisor_request_received(
2937 ctx,
2938 frame.header.corr,
2939 ops::SUPERVISOR_RELEASE_RESERVED,
2940 Some(&module_id),
2941 None,
2942 )?;
2943 self.handle_supervisor_release_reserved(frame, module_id)
2944 .await
2945 }
2946 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2947 self.log_supervisor_request_received(
2948 ctx,
2949 frame.header.corr,
2950 ops::SUPERVISOR_SET_ENABLED,
2951 Some(&module_id),
2952 Some(enabled),
2953 )?;
2954 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2955 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2956 .await
2957 }
2958 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2959 self.handle_supervisor_health_probe(frame, module_id).await
2960 }
2961 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2962 ClientControlRequest::SupervisorRoutes { module_id } => {
2963 self.handle_supervisor_routes(frame, module_id)
2964 }
2965 ClientControlRequest::SupervisorProvenance { module_id } => {
2966 self.handle_supervisor_provenance(frame, module_id).await
2967 }
2968 ClientControlRequest::SupervisorStderrTail {
2969 module_id,
2970 max_lines,
2971 max_bytes,
2972 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2973 ClientControlRequest::SupervisorTerminals { module_id } => {
2974 self.handle_supervisor_terminals(frame, module_id).await
2975 }
2976 }
2977 }
2978
2979 fn log_supervisor_request_received(
2980 &self,
2981 ctx: &RouteCtx,
2982 corr: u64,
2983 op: &'static str,
2984 module_id: Option<&str>,
2985 enabled: Option<bool>,
2986 ) -> Result<(), RouterError> {
2987 let caller = self.supervisor_requester(ctx, corr)?;
2988 let caller = principal_label(&caller);
2989 info!(
2992 target: "control",
2993 op,
2994 module_id,
2995 enabled,
2996 connection_id = ctx.connection_id.get(),
2997 caller = %caller,
2998 "supervisor request received"
2999 );
3000 Ok(())
3001 }
3002
3003 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3004 Ok(self
3005 .registry
3006 .get_module_by_connection(ctx.connection_id)
3007 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3008 .map(|registration| Principal::Reserved {
3009 module_id: registration.manifest.module_id,
3010 })
3011 .unwrap_or(Principal::Direct))
3012 }
3013
3014 fn handle_module_control_request(
3015 &self,
3016 connection_id: ConnectionId,
3017 frame: Frame,
3018 request: ModuleControlRequestFromModule,
3019 ) -> Result<Vec<Frame>, RouterError> {
3020 match request {
3021 ModuleControlRequestFromModule::CatalogUpdate {
3022 provides,
3023 capabilities,
3024 ready,
3025 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3026 ModuleControlRequestFromModule::LiveRoots {} => {
3027 let registered = self
3028 .registry
3029 .get_module_by_connection(connection_id)
3030 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3031 let Some(registration) = registered else {
3032 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3033 };
3034 let response = self
3035 .forwarding
3036 .live_roots(®istration.manifest.module_id)
3037 .map_err(RouterError::Forwarding)?;
3038 Ok(vec![control_response_body_frame(
3039 &frame,
3040 &response,
3041 "ModuleControlResponseToModule::LiveRoots",
3042 )?])
3043 }
3044 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3045 self.handle_scope_sync(connection_id, frame, generation, scopes)
3046 }
3047 ModuleControlRequestFromModule::ScopeApply {
3048 generation,
3049 upsert,
3050 end,
3051 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3052 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3053 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3054 }
3055 }
3056 }
3057
3058 fn handle_scope_served_by(
3059 &self,
3060 connection_id: ConnectionId,
3061 frame: Frame,
3062 ) -> Result<Vec<Frame>, RouterError> {
3063 let registered = self
3064 .registry
3065 .get_module_by_connection(connection_id)
3066 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3067 let Some(registered) = registered else {
3068 return Ok(vec![control_error_frame(
3069 &frame,
3070 "not_registered",
3071 "scope.served_by requires an active module registration owned by this connection",
3072 )?]);
3073 };
3074 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3075 {
3076 Ok(request) => request,
3077 Err(err) => {
3078 return Ok(vec![control_error_frame(
3079 &frame,
3080 "invalid_control_body",
3081 format!("malformed scope.served_by body: {err}"),
3082 )?])
3083 }
3084 };
3085 let answer = self
3086 .scopes
3087 .read()
3088 .unwrap_or_else(|poisoned| poisoned.into_inner())
3089 .served_by(
3090 ®istered.manifest.module_id,
3091 &request.scope_ref,
3092 request.scope_epoch,
3093 &request.module_id,
3094 );
3095 let reply = subc_control::ScopeServedByReply::new(
3096 answer,
3097 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3098 );
3099 Ok(vec![control_response_body_frame(
3100 &frame,
3101 &reply,
3102 "ScopeServedByReply",
3103 )?])
3104 }
3105
3106 fn handle_operator_confirm(
3107 &self,
3108 ctx: &RouteCtx,
3109 frame: Frame,
3110 ) -> Result<Vec<Frame>, RouterError> {
3111 use crate::operator_confirm::{audit, Outcome};
3112 let registration = self
3113 .registry
3114 .get_module_by_connection(ctx.connection_id)
3115 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3116 let Some(registration) = registration else {
3117 let outcome = Outcome::refusal("not_registered");
3118 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3119 return Ok(vec![outcome.frame(&frame)]);
3120 };
3121 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3122 Ok(request) => request,
3123 Err(_) => {
3124 let outcome = Outcome::refusal("invalid_control_body");
3125 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3126 return Ok(vec![outcome.frame(&frame)]);
3127 }
3128 };
3129 let module_id = registration.manifest.module_id;
3130 let nonce = self
3134 .hello_launch_nonces
3135 .lock()
3136 .unwrap_or_else(|p| p.into_inner())
3137 .nonce(ctx.connection_id)
3138 .map(str::to_owned);
3139 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3140 self.supervisor
3141 .spawned_consumer_authorized(&module_id, nonce)
3142 });
3143 let confirms = self.forwarding.operator_confirms();
3144 self.forwarding
3145 .with_operator_route(
3146 ctx.connection_id,
3147 request.route_channel,
3148 request.route_epoch,
3149 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3150 )
3151 .map_err(RouterError::Forwarding)
3152 }
3153
3154 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3157 let mut scopes = ScopeRecords::from(scopes);
3158 for (index, record) in scopes.records.iter_mut().enumerate() {
3159 if let Some(refusal) = Self::canonicalize_workdir(record) {
3160 scopes.refusals.insert(index, refusal);
3161 }
3162 }
3163 scopes
3164 }
3165
3166 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3167 let workdir = record.workdir.as_mut()?;
3168 let path = std::path::Path::new(workdir);
3169 if !path.is_absolute() {
3170 return Some((
3171 error_codes::INVALID_WORKDIR,
3172 "workdir must be absolute".to_string(),
3173 ));
3174 }
3175 match ProjectRootId::from_path_allowing_missing(path) {
3176 Ok(root) => match root.as_path().to_str() {
3177 Some(canonical) => {
3178 *workdir = canonical.to_string();
3179 None
3180 }
3181 None => Some((
3182 error_codes::INVALID_WORKDIR,
3183 "canonical workdir is not UTF-8".to_string(),
3184 )),
3185 },
3186 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3187 }
3188 }
3189
3190 fn handle_scope_sync(
3194 &self,
3195 connection_id: ConnectionId,
3196 frame: Frame,
3197 generation: u64,
3198 scopes: Vec<ScopeRecord>,
3199 ) -> Result<Vec<Frame>, RouterError> {
3200 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3201 }
3202
3203 fn handle_scope_change(
3204 &self,
3205 connection_id: ConnectionId,
3206 frame: Frame,
3207 generation: u64,
3208 scopes: Vec<ScopeRecord>,
3209 end: Option<Vec<ScopeEnd>>,
3210 ) -> Result<Vec<Frame>, RouterError> {
3211 let is_apply = end.is_some();
3212 let Some(registration) = self
3213 .registry
3214 .get_module_by_connection(connection_id)
3215 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3216 else {
3217 return Ok(vec![control_error_frame(
3218 &frame,
3219 "not_registered",
3220 "scope.sync and scope.apply require an active module registration owned by this connection",
3221 )?]);
3222 };
3223 let owner = registration.manifest.module_id;
3224 let scopes = Self::prepare_scope_records(scopes);
3225 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3226 let is_current_launch = |connection: ConnectionId| {
3227 self.hello_launch_nonces
3228 .lock()
3229 .unwrap_or_else(|poisoned| poisoned.into_inner())
3230 .presented(connection, current_nonce.as_deref())
3231 };
3232 let mut table = self
3237 .scopes
3238 .write()
3239 .unwrap_or_else(|poisoned| poisoned.into_inner());
3240 let outcome = match end {
3241 Some(end) => table.apply(
3242 &owner,
3243 connection_id,
3244 is_current_launch,
3245 generation,
3246 scopes,
3247 end,
3248 ),
3249 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3250 };
3251 let drained = match &outcome {
3252 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3253 Err(_) => Vec::new(),
3254 };
3255 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3256 drop(table);
3257 if has_deadlines {
3258 self.scope_deadline_added.notify_one();
3259 }
3260 match outcome {
3261 Ok(applied) => {
3262 let counts = ScopeOutcomeCounts::of(&applied.results);
3263 info!(
3264 owner = %owner,
3265 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3266 generation,
3267 records = applied.results.len(),
3268 created = counts.created,
3269 replaced = counts.replaced,
3270 updated = counts.updated,
3271 unchanged = counts.unchanged,
3272 refused = counts.refused,
3273 ended = applied.ended.len(),
3274 tag_changes = applied.tag_changes.len(),
3275 routes_closed = drained.len(),
3276 "scope change accepted"
3277 );
3278 for refused in applied
3284 .results
3285 .iter()
3286 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3287 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3288 {
3289 warn!(
3290 owner = %owner,
3291 generation,
3292 scope_ref = %refused.scope_ref,
3293 scope_epoch = refused.scope_epoch,
3294 code = refused.code.as_deref().unwrap_or(""),
3295 "scope record refused"
3296 );
3297 }
3298 self.close_scope_drained_routes(drained);
3299 let response = if is_apply {
3300 ModuleControlResponseToModule::ScopeApply {
3301 generation,
3302 results: applied.results,
3303 end_results: applied.end_results,
3304 ended: applied.ended,
3305 }
3306 } else {
3307 ModuleControlResponseToModule::ScopeSync {
3308 generation,
3309 results: applied.results,
3310 ended: applied.ended,
3311 }
3312 };
3313 Ok(vec![control_response_body_frame(
3314 &frame,
3315 &response,
3316 "scope change response",
3317 )?])
3318 }
3319 Err(refusal) => {
3320 info!(
3321 owner = %owner,
3322 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3323 generation,
3324 code = refusal.code,
3325 "scope change refused"
3326 );
3327 Ok(vec![control_error_frame(
3328 &frame,
3329 refusal.code,
3330 refusal.message,
3331 )?])
3332 }
3333 }
3334 }
3335
3336 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3342 if drained.is_empty() {
3343 return;
3344 }
3345 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3346 BTreeMap::new();
3347 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3348 for route in drained {
3349 warn!(
3350 module_id = %route.module_id,
3351 reason = ?route.reason,
3352 client_connection_id = route.client.connection_id.get(),
3353 route_channel = route.client.channel,
3354 "closing route because its scope changed"
3355 );
3356 pushes
3357 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3358 .or_insert_with(|| (route.reason, Vec::new()))
3359 .1
3360 .push(EndpointRoute {
3361 goodbye_target: route.client.clone(),
3362 principal: Principal::Unverified,
3363 bound_at: Instant::now(),
3364 draining: false,
3365 drain_reason: None,
3366 });
3367 goodbyes.push(route.module);
3368 goodbyes.push(route.client);
3369 }
3370 for ((module_id, _), (reason, routes)) in pushes {
3371 send_route_control_pushes(
3372 &self.forwarding,
3373 routes,
3374 ClientControlPush::RouteClosed {
3375 module_id,
3376 channels: Vec::new(),
3377 reason,
3378 drained: false,
3379 abandoned: 0,
3380 excluded_subscriptions: 0,
3381 terminal: Some(false),
3382 },
3383 );
3384 }
3385 self.emit_route_goodbyes(goodbyes);
3386 }
3387
3388 fn handle_scope_describe(
3391 &self,
3392 connection_id: ConnectionId,
3393 frame: Frame,
3394 owner: Principal,
3395 scope_ref: String,
3396 ) -> Result<Vec<Frame>, RouterError> {
3397 let registered = self
3398 .registry
3399 .get_module_by_connection(connection_id)
3400 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3401 let Some(registered) = registered else {
3402 return Ok(vec![control_error_frame(
3403 &frame,
3404 "not_registered",
3405 "scope.describe requires an active module registration owned by this connection",
3406 )?]);
3407 };
3408 let mut description = self
3409 .scopes
3410 .read()
3411 .unwrap_or_else(|poisoned| poisoned.into_inner())
3412 .describe(&owner, &scope_ref);
3413 if !registered
3416 .manifest
3417 .capabilities
3418 .as_ref()
3419 .is_some_and(|caps| {
3420 caps.provides
3421 .iter()
3422 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3423 })
3424 {
3425 if let Some(stamp) = description.stamp.as_mut() {
3426 stamp.attributes.on_behalf_of_agent = None;
3427 }
3428 }
3429 let owner_configured = match &owner {
3430 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3435 _ => false,
3436 };
3437 let response = ModuleControlResponseToModule::ScopeDescribe {
3438 status: description.status,
3439 scope_epoch: description.scope_epoch,
3440 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3441 owner_synced: description.owner_synced,
3442 owner_configured,
3443 scope: description.stamp,
3444 };
3445 Ok(vec![control_response_body_frame(
3446 &frame,
3447 &response,
3448 "ModuleControlResponseToModule::ScopeDescribe",
3449 )?])
3450 }
3451
3452 fn handle_catalog_update(
3453 &self,
3454 connection_id: ConnectionId,
3455 frame: Frame,
3456 provides: Vec<ProviderRole>,
3457 capabilities: Option<CapabilityDeclarations>,
3458 ready: Option<bool>,
3459 ) -> Result<Vec<Frame>, RouterError> {
3460 self.refresh_capability_requirements();
3461 let Some(registration) = self
3462 .registry
3463 .get_module_by_connection(connection_id)
3464 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3465 else {
3466 return Ok(vec![control_error_frame(
3467 &frame,
3468 "not_registered",
3469 "catalog.update requires an active module registration owned by this connection",
3470 )?]);
3471 };
3472
3473 if let Some(message) =
3474 catalog_update_frozen_field_message(®istration.manifest, &provides)
3475 {
3476 return Ok(vec![control_error_frame(
3477 &frame,
3478 "catalog_update_frozen_field",
3479 message,
3480 )?]);
3481 }
3482
3483 let mut candidate = registration.manifest.clone();
3484 candidate.provides = provides.clone();
3485 candidate.capabilities = capabilities
3486 .clone()
3487 .or_else(|| registration.manifest.capabilities.clone());
3488 if let Some(rich_ui) = &candidate.rich_ui {
3489 if let Err(err) = rich_ui.validate() {
3490 return Ok(vec![control_error_frame(
3491 &frame,
3492 "invalid_manifest",
3493 err.to_string(),
3494 )?]);
3495 }
3496 }
3497 if let Err(err) = candidate.validate_capability_grammar() {
3498 return Ok(vec![control_error_frame(
3499 &frame,
3500 "invalid_capability_grammar",
3501 err.to_string(),
3502 )?]);
3503 }
3504
3505 let mut conflicts = self
3508 .capability_evaluator
3509 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3510 if let Some(conflict) = conflicts.first() {
3511 let message = format!(
3512 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3513 conflict.capability, conflict.claimants[0], candidate.module_id
3514 );
3515 for conflict in &mut conflicts {
3516 conflict.source = DuplicateClaimSource::CatalogUpdate;
3517 }
3518 log_duplicate_claim_events(conflicts);
3519 return Ok(vec![control_error_frame(
3520 &frame,
3521 "reserved_capability",
3522 message,
3523 )?]);
3524 }
3525
3526 let updated = self
3527 .registry
3528 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3529 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3530 if updated.is_none() {
3531 return Ok(vec![control_error_frame(
3532 &frame,
3533 "not_registered",
3534 "catalog.update requires an active module registration owned by this connection",
3535 )?]);
3536 }
3537 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3538 log_duplicate_claim_events(
3539 self.capability_evaluator
3540 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3541 );
3542 }
3543 if capability_census_trigger(
3544 registration.manifest.capabilities.as_ref(),
3545 updated
3546 .as_ref()
3547 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3548 ) {
3549 self.enforce_capability_denies();
3550 }
3551 self.refresh_capability_requirements();
3552
3553 let response = ModuleControlResponseToModule::CatalogUpdate {};
3554 control_response_body_frame(
3555 &frame,
3556 &response,
3557 "ModuleControlResponseToModule::CatalogUpdate",
3558 )
3559 .map(|frame| vec![frame])
3560 }
3561
3562 pub(crate) fn with_activity_ledger_config(
3563 mut self,
3564 run_dir: &Path,
3565 incarnation: String,
3566 config: crate::activity::ActivityConfig,
3567 ) -> Self {
3568 self.activity = Some(crate::activity::ActivityLedger::open_config(
3569 run_dir.join("activity.jsonl"),
3570 incarnation,
3571 config,
3572 ));
3573 self.supervisor.set_activity(self.activity.clone());
3574 self
3575 }
3576
3577 pub(crate) async fn configure_activity(
3578 &self,
3579 config: crate::activity::ActivityConfig,
3580 ) -> Result<(), &'static str> {
3581 match &self.activity {
3582 Some(ledger) => ledger.configure(config).await,
3583 None => Ok(()),
3584 }
3585 }
3586
3587 fn cancel_activity_subscription(
3588 &self,
3589 ctx: &RouteCtx,
3590 frame: &Frame,
3591 ) -> Result<bool, RouterError> {
3592 let task = self
3593 .activity_streams
3594 .lock()
3595 .unwrap_or_else(|p| p.into_inner())
3596 .remove(&(ctx.connection_id, frame.header.corr));
3597 let Some(task) = task else {
3598 return Ok(false);
3599 };
3600 task.abort();
3601 let end = Frame::build_with_version(
3602 response_version(frame),
3603 FrameType::StreamEnd,
3604 control_flags(),
3605 0,
3606 0,
3607 frame.header.corr,
3608 Vec::new(),
3609 )
3610 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3611 let sink = ctx.egress.clone();
3612 tokio::spawn(async move {
3613 let _ = sink.send(end).await;
3614 });
3615 Ok(true)
3616 }
3617
3618 async fn handle_activity_read(
3619 &self,
3620 ctx: &RouteCtx,
3621 frame: Frame,
3622 ) -> Result<Vec<Frame>, RouterError> {
3623 use subc_control::{
3624 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3625 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3626 };
3627 #[derive(Deserialize)]
3628 struct Identity {
3629 #[serde(default)]
3630 consumer_identity: Option<ConsumerIdentity>,
3631 }
3632 let principal = match self
3636 .forwarding
3637 .module_id_for_connection(ctx.connection_id)
3638 .map_err(RouterError::Forwarding)?
3639 {
3640 Some(module_id) => Ok(Principal::Reserved { module_id }),
3641 None => {
3642 let identity: Identity = match serde_json::from_slice(&frame.body) {
3643 Ok(identity) => identity,
3644 Err(_) => {
3645 return Ok(vec![control_error_frame(
3646 &frame,
3647 ACTIVITY_MALFORMED,
3648 "malformed activity read",
3649 )?])
3650 }
3651 };
3652 self.route_open_principal(&frame, identity.consumer_identity)?
3653 }
3654 };
3655 let admitted = matches!(principal, Ok(Principal::Direct))
3656 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3657 if !admitted {
3658 return Ok(vec![control_error_frame(
3659 &frame,
3660 ACTIVITY_READ_FORBIDDEN,
3661 "activity reads require an operator or trusted mirror principal",
3662 )?]);
3663 }
3664 let Some(ledger) = &self.activity else {
3665 return Ok(vec![control_error_frame(
3666 &frame,
3667 ACTIVITY_UNAVAILABLE,
3668 "activity ledger unavailable",
3669 )?]);
3670 };
3671 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3672 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3673 {
3674 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3675 Ok(request) => ledger.query(request).await,
3676 Err(_) => Err(ACTIVITY_MALFORMED),
3677 };
3678 return match result {
3679 Ok(reply) => Ok(vec![control_response_body_frame(
3680 &frame,
3681 &reply,
3682 ops::ACTIVITY_QUERY,
3683 )?]),
3684 Err(code) => Ok(vec![control_error_frame(
3685 &frame,
3686 code,
3687 "activity query refused",
3688 )?]),
3689 };
3690 }
3691 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3692 Ok(request) => request,
3693 Err(_) => {
3694 return Ok(vec![control_error_frame(
3695 &frame,
3696 ACTIVITY_MALFORMED,
3697 "malformed activity subscription",
3698 )?])
3699 }
3700 };
3701 let mut subscription = match ledger.subscribe(request.since).await {
3702 Ok(subscription) => subscription,
3703 Err(code) => {
3704 return Ok(vec![control_error_frame(
3705 &frame,
3706 code,
3707 "activity subscription refused",
3708 )?])
3709 }
3710 };
3711 let key = (ctx.connection_id, frame.header.corr);
3712 let sink = ctx.egress.clone();
3713 let mut streams = self
3714 .activity_streams
3715 .lock()
3716 .unwrap_or_else(|p| p.into_inner());
3717 streams.retain(|_, task| !task.is_finished());
3718 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3719 return Ok(vec![control_error_frame(
3720 &frame,
3721 "duplicate_subscription",
3722 "activity subscription correlation id already in use",
3723 )?]);
3724 }
3725 let task = tokio::spawn(async move {
3726 loop {
3727 let entry = if let Some(entry) = subscription.replay.pop_front() {
3728 Some(entry)
3729 } else {
3730 subscription.receiver.recv().await
3731 };
3732 let Some(entry) = entry else {
3733 break;
3734 };
3735 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3736 return;
3737 };
3738 let Ok(next) = Frame::build_with_version(
3739 response_version(&frame),
3740 FrameType::StreamData,
3741 control_flags(),
3742 0,
3743 0,
3744 frame.header.corr,
3745 body,
3746 ) else {
3747 return;
3748 };
3749 if sink.send(next).await.is_err() {
3750 return;
3751 }
3752 }
3753 if let Some(code) = subscription.terminal_code() {
3756 if let Ok(end) =
3757 control_error_frame(&frame, code, "activity subscriber fell behind")
3758 {
3759 let _ = sink.send(end).await;
3760 }
3761 }
3762 });
3763 streams.insert(key, task.abort_handle());
3764 Ok(Vec::new())
3765 }
3766
3767 async fn handle_activity_record(
3768 &self,
3769 ctx: &RouteCtx,
3770 frame: Frame,
3771 ) -> Result<Vec<Frame>, RouterError> {
3772 use subc_control::{
3773 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3774 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3775 };
3776 let result = async {
3777 let module = self
3778 .forwarding
3779 .module_id_for_connection(ctx.connection_id)
3780 .ok()
3781 .flatten()
3782 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3783 let request: ActivityRecordRequest =
3784 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3785 crate::activity::validate(&module, &request)?;
3786 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3787 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3788 let at_ms = std::time::SystemTime::now()
3789 .duration_since(std::time::UNIX_EPOCH)
3790 .unwrap_or_default()
3791 .as_millis() as u64;
3792 ledger.record(module, request, actor, at_ms).await
3793 }
3794 .await;
3795 match result {
3796 Ok(entry) => Ok(vec![control_response_body_frame(
3797 &frame,
3798 &ActivityRecordResponse { seq: entry.seq },
3799 "activity.record",
3800 )?]),
3801 Err(code) => Ok(vec![control_error_frame(
3802 &frame,
3803 code,
3804 "activity recording refused",
3805 )?]),
3806 }
3807 }
3808
3809 fn activity_actor(
3810 &self,
3811 connection: ConnectionId,
3812 module: &str,
3813 on: Option<&subc_control::ActivityOn>,
3814 ) -> Result<subc_control::ActivityActor, &'static str> {
3815 use crate::forwarding::{DataRoute, DataRouteState};
3816 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3817 let principal = |id: &str| Principal::Reserved {
3818 module_id: id.to_owned(),
3819 };
3820 let mut actor = ActivityActor {
3821 principal: principal(module),
3822 scope_owner: None,
3823 agent_id: None,
3824 run_id: None,
3825 flow_id: None,
3826 project_root: None,
3827 workdir: None,
3828 project_id: None,
3829 };
3830 let scope = match on {
3831 None => return Ok(actor),
3832 Some(ActivityOn::Scope(scope)) => {
3833 let selector = subc_protocol::scope::ScopeSelector::new(
3834 principal(module),
3835 scope.scope_ref.clone(),
3836 Some(scope.epoch),
3837 );
3838 self.scopes
3839 .read()
3840 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3841 .attribute(&principal(module), module, &selector, true)
3842 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3843 }
3844 Some(ActivityOn::Route(route)) => {
3845 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3846 .forwarding
3847 .lookup_data_route(connection, route.channel, route.epoch)
3848 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3849 else {
3850 return Err(ACTIVITY_ACTOR_UNKNOWN);
3851 };
3852 if binding.module_id != module {
3853 return Err(ACTIVITY_ACTOR_UNKNOWN);
3854 }
3855 actor.principal = binding.principal.clone();
3856 actor.project_root = binding
3857 .project_root
3858 .as_ref()
3859 .map(|root| root.as_path().to_path_buf());
3860 let Some(bound) = &binding.scope else {
3861 return Ok(actor);
3862 };
3863 let owner = principal(&bound.owner);
3864 let selector = subc_protocol::scope::ScopeSelector::new(
3865 owner.clone(),
3866 bound.scope_ref.clone(),
3867 Some(bound.tag.scope_epoch),
3868 );
3869 let admitted = self
3870 .scopes
3871 .read()
3872 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3873 .attribute(&owner, module, &selector, true)
3874 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3875 if admitted.tag != bound.tag {
3876 return Err(ACTIVITY_ACTOR_UNKNOWN);
3877 }
3878 admitted
3879 }
3880 };
3881 actor.scope_owner = Some(scope.stamp.owner);
3882 actor.agent_id = scope.stamp.attributes.agent_id;
3883 actor.run_id = scope.stamp.attributes.run_id;
3884 actor.flow_id = scope.stamp.attributes.flow_id;
3885 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3886 actor.project_id = scope.stamp.project_id;
3887 Ok(actor)
3888 }
3889
3890 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3891 self.refresh_capability_requirements();
3892 let mut counters = self.counters.snapshot();
3898 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3899 obj.insert(
3900 "activity_unparseable_lines".into(),
3901 ledger.read_errors().into(),
3902 );
3903 obj.insert(
3904 "activity_lifecycle_append_failures".into(),
3905 ledger.lifecycle_failures().into(),
3906 );
3907 obj.insert(
3908 "activity_started_digest_skips".into(),
3909 ledger.lifecycle_digest_skips().into(),
3910 );
3911 }
3912 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3913 self.forwarding.client_route_concentration(),
3914 counters.as_object_mut(),
3915 ) {
3916 obj.insert(
3917 "client_connections_with_routes".into(),
3918 connections_with_routes.into(),
3919 );
3920 obj.insert("max_routes_on_one_connection".into(), max.into());
3921 }
3922 if let (Some(open_breakers), Some(obj)) = (
3928 self.route_bind_breakers.open_snapshot(),
3929 counters.as_object_mut(),
3930 ) {
3931 obj.insert("route_bind_breakers_open".into(), open_breakers);
3932 }
3933 let response = ClientControlResponse::ServerDescribe {
3934 protocol_ver: PROTOCOL_VERSION,
3935 subc_ops: subc_ops(),
3936 capabilities: self.describe_capabilities(),
3937 connected_clients: self.connected_clients.count(),
3938 counters: Some(counters),
3939 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3940 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3941 capability_requirements: self.capability_requirement_statuses(),
3942 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3943 config_path: self
3944 .rescan
3945 .as_ref()
3946 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3947 };
3948 Ok(vec![control_response_body_frame(
3949 &frame,
3950 &response,
3951 "ClientControlResponse::ServerDescribe",
3952 )?])
3953 }
3954
3955 fn handle_catalog_list(
3956 &self,
3957 frame: Frame,
3958 module_id: Option<String>,
3959 ) -> Result<Vec<Frame>, RouterError> {
3960 let (generation, modules) = self
3961 .registry
3962 .list_modules_with_nonce_verification()
3963 .map_err(|err| {
3964 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3965 })?;
3966 let entries = modules
3967 .into_iter()
3968 .filter(|(registration, _)| {
3969 module_id
3970 .as_deref()
3971 .map(|wanted| registration.manifest.module_id == wanted)
3972 .unwrap_or(true)
3973 })
3974 .map(|(registration, nonce_verified)| {
3975 let not_ready = self.not_ready_reason(®istration);
3976 let roles = registration.manifest.provides;
3977 CatalogEntry::new(
3978 registration.manifest.module_id,
3979 roles,
3980 registration.control_ops,
3981 )
3982 .with_ready(not_ready.is_none())
3983 .with_not_ready(not_ready)
3984 .with_module_version(Some(registration.manifest.module_version))
3985 .with_capabilities(registration.manifest.capabilities)
3986 .with_self_signals(registration.manifest.self_signals)
3987 .with_events(registration.manifest.events)
3988 .with_operation_delivery(registration.manifest.operation_delivery)
3989 .with_operation_payload(registration.manifest.operation_payload)
3990 .with_rich_ui(registration.manifest.rich_ui)
3991 .with_nonce_verified(nonce_verified)
3992 })
3993 .collect();
3994 let response = ClientControlResponse::CatalogList {
3995 generation,
3996 modules: entries,
3997 subc_ops: subc_ops(),
3998 };
3999 Ok(vec![control_response_body_frame(
4000 &frame,
4001 &response,
4002 "ClientControlResponse::CatalogList",
4003 )?])
4004 }
4005
4006 fn route_open_principal(
4007 &self,
4008 frame: &Frame,
4009 consumer_identity: Option<ConsumerIdentity>,
4010 ) -> Result<Result<Principal, Frame>, RouterError> {
4011 let Some(consumer_identity) = consumer_identity else {
4012 return Ok(Ok(Principal::Direct));
4013 };
4014
4015 if self.supervisor.spawned_consumer_authorized(
4016 &consumer_identity.module_id,
4017 &consumer_identity.launch_nonce,
4018 ) {
4019 return Ok(Ok(Principal::Reserved {
4020 module_id: consumer_identity.module_id,
4021 }));
4022 }
4023
4024 Ok(Err(control_error_frame(
4025 frame,
4026 "bad_consumer_identity",
4027 format!(
4028 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4029 consumer_identity.module_id
4030 ),
4031 )?))
4032 }
4033
4034 fn route_open_refusal_frame(
4041 &self,
4042 ctx: &RouteCtx,
4043 frame: &Frame,
4044 module_id: &str,
4045 reason: &'static str,
4046 code: &'static str,
4047 message: impl Into<String>,
4048 ) -> Result<Frame, RouterError> {
4049 self.observe_route_open_refusal(ctx, module_id, reason, code);
4050 control_error_frame(frame, code, message.into())
4051 }
4052
4053 fn route_open_breaker_refusal_frame(
4071 &self,
4072 ctx: &RouteCtx,
4073 frame: &Frame,
4074 module_id: &str,
4075 consecutive_timeouts: u32,
4076 retry_in: Duration,
4077 probe_in_flight: bool,
4078 ) -> Result<Frame, RouterError> {
4079 self.counters
4080 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4081 debug!(
4082 target: "control",
4083 code = "module_timeout",
4084 module_id = ?module_id,
4085 connection_id = ctx.connection_id.get(),
4086 consecutive_timeouts,
4087 retry_in_ms = retry_in.as_millis() as u64,
4088 probe_in_flight,
4089 "route.open refused by open bind-relay breaker"
4090 );
4091 let detail = if probe_in_flight {
4098 "one test route is already being tried; retry once it settles".to_string()
4099 } else {
4100 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4101 };
4102 control_error_frame(
4103 frame,
4104 "module_timeout",
4105 format!(
4106 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4107 timed out in a row); {detail}; its established routes are unaffected"
4108 ),
4109 )
4110 }
4111
4112 fn observe_route_open_refusal(
4125 &self,
4126 ctx: &RouteCtx,
4127 module_id: &str,
4128 reason: &'static str,
4129 code: &'static str,
4130 ) {
4131 self.counters.increment_route_open_refused(code);
4132 info!(
4133 target: "control",
4134 code,
4135 reason,
4136 module_id = ?module_id,
4137 connection_id = ctx.connection_id.get(),
4138 "route.open refused"
4139 );
4140 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4141 self.route_outages.record_not_serving(module_id, reason);
4142 }
4143 }
4144
4145 fn observe_route_open_accept(
4190 &self,
4191 ctx: &RouteCtx,
4192 module_id: &str,
4193 principal: &str,
4194 scope: &str,
4195 ) {
4196 self.route_outages.record_accepted(module_id);
4197 self.counters.increment_route_open_accepted(principal);
4198 info!(
4199 target: "control",
4200 principal,
4201 module_id,
4202 connection_id = ctx.connection_id.get(),
4203 scope = %scope,
4204 "route.open accepted"
4205 );
4206 }
4207
4208 fn supervised_absent_route_open_refusal_frame(
4209 &self,
4210 ctx: &RouteCtx,
4211 frame: &Frame,
4212 module_id: &str,
4213 code: &'static str,
4214 status: &crate::supervise::ModuleStatus,
4215 ) -> Result<Frame, RouterError> {
4216 self.counters.increment_route_open_refused(code);
4217 info!(
4218 target: "control",
4219 code,
4220 reason = "supervised_not_registered",
4221 module_id = ?module_id,
4222 connection_id = ctx.connection_id.get(),
4223 state = %status.state,
4224 enabled = status.enabled,
4225 live = status.live,
4226 "route.open refused"
4227 );
4228 self.route_outages
4232 .record_not_serving(module_id, "supervised_not_registered");
4233 let message = if let Some(pid) = status.waiting_for_previous_pid {
4234 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4235 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4236 })
4237 } else {
4238 format!(
4239 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4240 status.state, status.enabled, status.live
4241 )
4242 };
4243 control_error_frame(frame, code, message)
4244 }
4245
4246 async fn handle_route_open(
4247 &self,
4248 ctx: &RouteCtx,
4249 frame: Frame,
4250 request: RouteOpenRequest,
4251 timings: &mut RouteOpenTimings,
4252 ) -> Result<Vec<Frame>, RouterError> {
4253 let RouteOpenRequest {
4254 target,
4255 mut identity,
4256 consumer_identity,
4257 consumer_capabilities,
4258 role_versions,
4259 admission_facts,
4260 scope,
4261 } = request;
4262 let target_module_id = target_module_id(&target).to_string();
4263 timings.module_id.clone_from(&target_module_id);
4264 if self
4265 .registry
4266 .get_module_by_connection(ctx.connection_id)
4267 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4268 .is_some()
4269 {
4270 return Ok(vec![control_error_frame(
4271 &frame,
4272 "invalid_request",
4273 "module connections cannot open client routes",
4274 )?]);
4275 }
4276 debug!(
4277 connection_id = ctx.connection_id.get(),
4278 corr = frame.header.corr,
4279 module_id = %target_module_id,
4280 "handling route.open"
4281 );
4282
4283 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4289 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4290 self.observe_route_open_refusal(
4291 ctx,
4292 &target_module_id,
4293 "invalid_role_versions",
4294 error_codes::INVALID_REQUEST,
4295 );
4296 return Ok(vec![control_error_body_frame(
4297 &frame,
4298 ErrorBody {
4299 code: error_codes::INVALID_REQUEST.to_string(),
4300 message: error.to_string(),
4301 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4302 },
4303 )?]);
4304 }
4305
4306 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4307 if status.waiting_for_previous_pid.is_some() {
4308 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4309 ctx,
4310 &frame,
4311 &target_module_id,
4312 "module_warming",
4313 &status,
4314 )?]);
4315 }
4316 }
4317
4318 let Some(registration) = self
4337 .registry
4338 .get_module(&target_module_id)
4339 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4340 else {
4341 if let Some((status, warming)) =
4342 self.supervisor_status(&target_module_id, frame.header.corr)?
4343 {
4344 if status.protocol == ModuleProtocol::None {
4351 return Ok(vec![self.route_open_refusal_frame(
4352 ctx,
4353 &frame,
4354 &target_module_id,
4355 "protocol_none",
4356 error_codes::MODULE_NO_PROTOCOL,
4357 format!(
4358 "module_id '{target_module_id}' is declared protocol: none; \
4359 it speaks no subc wire and serves no routes"
4360 ),
4361 )?]);
4362 }
4363 let code = if warming {
4364 "module_warming"
4365 } else {
4366 "target_unavailable"
4367 };
4368 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4369 ctx,
4370 &frame,
4371 &target_module_id,
4372 code,
4373 &status,
4374 )?]);
4375 }
4376 if let Some(removed_ago_ms) =
4377 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4378 {
4379 return Ok(vec![self.route_open_refusal_frame(
4380 ctx,
4381 &frame,
4382 &target_module_id,
4383 "removed",
4384 error_codes::MODULE_REMOVED,
4385 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4386 )?]);
4387 }
4388 return Ok(vec![self.route_open_refusal_frame(
4389 ctx,
4390 &frame,
4391 &target_module_id,
4392 "not_registered",
4393 error_codes::UNKNOWN_MODULE,
4394 format!("module_id '{target_module_id}' is not registered"),
4395 )?]);
4396 };
4397
4398 if !registration.ready {
4402 self.counters
4403 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4404 info!(
4405 target: "control",
4406 code = error_codes::MODULE_WARMING,
4407 module_id = ?target_module_id,
4408 connection_id = ctx.connection_id.get(),
4409 reason = "declared_not_ready",
4410 "route.open refused"
4411 );
4412 self.route_outages
4415 .record_not_serving(&target_module_id, "declared_not_ready");
4416 return Ok(vec![control_error_body_frame(
4417 &frame,
4418 ErrorBody {
4419 code: error_codes::MODULE_WARMING.to_string(),
4420 message: format!(
4421 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4422 ),
4423 detail: Some(serde_json::json!({
4424 "reason": "declared_not_ready"
4425 })),
4426 },
4427 )?]);
4428 }
4429
4430 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4449 self.counters
4450 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4451 info!(
4452 target: "control",
4453 code = error_codes::MODULE_WARMING,
4454 module_id = ?target_module_id,
4455 connection_id = ctx.connection_id.get(),
4456 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4457 capability = %capability,
4458 "route.open refused"
4459 );
4460 return Ok(vec![control_error_body_frame(
4461 &frame,
4462 ErrorBody {
4463 code: error_codes::MODULE_WARMING.to_string(),
4464 message: format!(
4465 "module_id '{target_module_id}' requires capability '{capability}', \
4466 which no registered module provides; retry"
4467 ),
4468 detail: Some(serde_json::json!({
4469 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4470 "capability": capability,
4471 })),
4472 },
4473 )?]);
4474 }
4475
4476 if !target_has_required_role(&target, ®istration.manifest.provides) {
4477 return Ok(vec![self.route_open_refusal_frame(
4478 ctx,
4479 &frame,
4480 &target_module_id,
4481 "role_not_provided",
4482 "target_unavailable",
4483 format!("module_id '{target_module_id}' does not provide the requested target"),
4484 )?]);
4485 }
4486
4487 if registration.state != ChannelState::Active {
4488 return Ok(vec![self.route_open_refusal_frame(
4489 ctx,
4490 &frame,
4491 &target_module_id,
4492 "registration_not_active",
4493 "target_unavailable",
4494 format!("module_id '{target_module_id}' is not active"),
4495 )?]);
4496 }
4497
4498 if self
4499 .forwarding
4500 .module_is_draining(&target_module_id)
4501 .map_err(RouterError::Forwarding)?
4502 {
4503 return Ok(vec![self.route_open_refusal_frame(
4504 ctx,
4505 &frame,
4506 &target_module_id,
4507 "reloading",
4508 "module_reloading",
4509 format!("module_id '{target_module_id}' is reloading"),
4510 )?]);
4511 }
4512
4513 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4514 process_liveness.process_live(&target_module_id) == Some(false)
4515 }) {
4516 if process_liveness.process_replacing(&target_module_id) {
4523 return Ok(vec![self.route_open_refusal_frame(
4524 ctx,
4525 &frame,
4526 &target_module_id,
4527 "reloading",
4528 "module_reloading",
4529 format!("module_id '{target_module_id}' is reloading"),
4530 )?]);
4531 }
4532 return Ok(vec![self.route_open_refusal_frame(
4533 ctx,
4534 &frame,
4535 &target_module_id,
4536 "supervisor_not_live",
4537 "target_unavailable",
4538 format!("module_id '{target_module_id}' is not live"),
4539 )?]);
4540 }
4541
4542 if !self
4543 .forwarding
4544 .has_live_module_connection(&target_module_id)
4545 .map_err(RouterError::Forwarding)?
4546 {
4547 return Ok(vec![self.route_open_refusal_frame(
4548 ctx,
4549 &frame,
4550 &target_module_id,
4551 "no_forwarding_connection",
4552 "target_unavailable",
4553 format!("module_id '{target_module_id}' has no live forwarding connection"),
4554 )?]);
4555 }
4556
4557 if let Some(error) =
4558 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4559 {
4560 self.observe_route_open_refusal(
4561 ctx,
4562 &target_module_id,
4563 "op_not_allowed",
4564 "op_not_allowed",
4565 );
4566 return Ok(vec![error]);
4567 }
4568
4569 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4570 Ok(principal) => principal,
4571 Err(error) => {
4572 self.observe_route_open_refusal(
4573 ctx,
4574 &target_module_id,
4575 "bad_consumer_identity",
4576 "bad_consumer_identity",
4577 );
4578 return Ok(vec![error]);
4579 }
4580 };
4581 timings.principal = principal_label(&principal);
4582
4583 if let Principal::Reserved {
4587 module_id: opening_module_id,
4588 } = &principal
4589 {
4590 if let Some(opening_registration) = self
4591 .registry
4592 .get_module(opening_module_id)
4593 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4594 {
4595 if let Some(capability) =
4596 denied_capability(&opening_registration.manifest, ®istration.manifest)
4597 {
4598 warn!(
4599 opening_module_id,
4600 target_module_id,
4601 capability,
4602 "refusing route.open because an attested capability deny edge matches"
4603 );
4604 return Ok(vec![self.route_open_refusal_frame(
4605 ctx,
4606 &frame,
4607 &target_module_id,
4608 "capability_deny_edge",
4609 "capability_forbidden",
4610 format!(
4611 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4612 ),
4613 )?]);
4614 }
4615 }
4616 }
4617
4618 if admission_facts.is_some() {
4619 let carrier_matches = matches!(
4620 &principal,
4621 Principal::Reserved { module_id }
4622 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4623 );
4624 if !carrier_matches {
4625 return Ok(vec![self.route_open_refusal_frame(
4626 ctx,
4627 &frame,
4628 &target_module_id,
4629 "admission_facts_carrier_not_permitted",
4630 "admission_facts_not_permitted",
4631 "admission facts may only be carried by the configured reserved module",
4632 )?]);
4633 }
4634
4635 let target_allowed = self
4636 .admission_facts_targets
4637 .as_ref()
4638 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4639 if !target_allowed {
4640 return Ok(vec![self.route_open_refusal_frame(
4641 ctx,
4642 &frame,
4643 &target_module_id,
4644 "admission_facts_target_not_listed",
4645 "admission_facts_target_not_allowed",
4646 format!(
4647 "admission facts are not permitted for target module_id '{target_module_id}'"
4648 ),
4649 )?]);
4650 }
4651
4652 }
4656
4657 let (bound_scope, mut scope_stamp) = match &scope {
4663 None => (None, None),
4664 Some(selector) => {
4665 let owner_configured = match &selector.owner {
4666 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4672 _ => false,
4673 };
4674 let mut scopes = self
4675 .scopes
4676 .write()
4677 .unwrap_or_else(|poisoned| poisoned.into_inner());
4678 let admitted = scopes
4679 .admit(&principal, &target_module_id, selector, owner_configured)
4680 .and_then(|admission| {
4681 crate::scopes::check_target_flow_support(
4682 &admission.stamp,
4683 &target_module_id,
4684 registration.manifest.capabilities.as_ref(),
4685 )?;
4686 crate::scopes::check_target_agent_run_support(
4687 &admission.stamp,
4688 &target_module_id,
4689 registration.manifest.capabilities.as_ref(),
4690 )?;
4691 let bound = scopes.reserve_bind(
4692 &admission,
4693 &principal,
4694 ctx.connection_id,
4695 selector.hold_presence,
4696 );
4697 Ok((admission, bound))
4698 });
4699 match admitted {
4700 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4701 Err(refusal) => {
4702 return Ok(vec![self.route_open_refusal_frame(
4703 ctx,
4704 &frame,
4705 &target_module_id,
4706 refusal.code,
4707 refusal.code,
4708 refusal.message,
4709 )?]);
4710 }
4711 }
4712 }
4713 };
4714
4715 timings.enter(RouteOpenPhase::ProjectRoot);
4737 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4738 Ok(project_root) => project_root,
4739 Err(err) => {
4740 return Ok(vec![control_error_frame(
4741 &frame,
4742 "invalid_project_root",
4743 err.to_string(),
4744 )?])
4745 }
4746 };
4747 timings.enter(RouteOpenPhase::Admission);
4748 identity.project_root = project_root.as_path().to_path_buf();
4749
4750 let _concurrency_guard = match self
4761 .route_bind_concurrency
4762 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4763 {
4764 Ok(guard) => guard,
4765 Err(in_flight) => {
4766 return Ok(vec![self.route_open_target_capacity_refusal(
4767 ctx,
4768 &frame,
4769 &target_module_id,
4770 in_flight,
4771 )?]);
4772 }
4773 };
4774
4775 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4778 RouteBindAdmission::Admitted { guard, probe } => {
4779 if probe {
4780 info!(
4781 module_id = %target_module_id,
4782 connection_id = ctx.connection_id.get(),
4783 "route.bind breaker half-open: admitting one probe"
4784 );
4785 }
4786 guard
4787 }
4788 RouteBindAdmission::Refused {
4789 consecutive_timeouts,
4790 retry_in,
4791 probe_in_flight,
4792 } => {
4793 return Ok(vec![self.route_open_breaker_refusal_frame(
4794 ctx,
4795 &frame,
4796 &target_module_id,
4797 consecutive_timeouts,
4798 retry_in,
4799 probe_in_flight,
4800 )?]);
4801 }
4802 };
4803
4804 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4809 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4810 timings.enter(RouteOpenPhase::EgressReservation);
4811 let pending = match self
4812 .forwarding
4813 .begin_route_bind_relay_for(
4814 ctx.connection_id,
4815 ctx.egress.clone(),
4816 response_version(&frame),
4817 frame.header.corr,
4818 &target_module_id,
4819 principal.clone(),
4820 bound_scope,
4821 Some(project_root),
4822 relay_deadline,
4823 )
4824 .await
4825 {
4826 Ok(pending) => pending,
4827 Err(err) => {
4828 return Ok(vec![self.route_open_refusal_frame(
4829 ctx,
4830 &frame,
4831 &target_module_id,
4832 "relay_reservation_failed",
4833 forwarding_error_code(&err),
4834 err.to_string(),
4835 )?])
4836 }
4837 };
4838 timings.enter(RouteOpenPhase::Admission);
4839 let crate::forwarding::PendingRouteBindRelay {
4840 endpoint,
4841 module_sink,
4842 negotiated_ver,
4843 client_channel,
4844 client_epoch,
4845 module_channel,
4846 module_epoch,
4847 corr: relay_corr,
4848 receiver,
4849 } = pending;
4850 let mut reservation =
4851 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4852
4853 let relay_registration = self
4858 .registry
4859 .get_module_by_connection(endpoint.connection_id)
4860 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4861 let relay_capabilities = relay_registration
4862 .as_ref()
4863 .and_then(|registration| registration.manifest.capabilities.as_ref());
4864 if let Some(selector) = &scope {
4865 if let Err(refusal) = self
4868 .scopes
4869 .read()
4870 .unwrap_or_else(|poisoned| poisoned.into_inner())
4871 .check_bind_proof(&principal, selector)
4872 {
4873 reservation.release_and_disarm();
4874 return Ok(vec![self.route_open_refusal_frame(
4875 ctx,
4876 &frame,
4877 &target_module_id,
4878 refusal.code,
4879 refusal.code,
4880 refusal.message,
4881 )?]);
4882 }
4883 }
4884 if let Some(stamp) = scope_stamp
4885 .as_ref()
4886 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4887 {
4888 if let Err(refusal) = crate::scopes::check_target_flow_support(
4889 stamp,
4890 &target_module_id,
4891 relay_capabilities,
4892 )
4893 .and_then(|()| {
4894 crate::scopes::check_target_agent_run_support(
4895 stamp,
4896 &target_module_id,
4897 relay_capabilities,
4898 )
4899 }) {
4900 reservation.release_and_disarm();
4901 return Ok(vec![self.route_open_refusal_frame(
4902 ctx,
4903 &frame,
4904 &target_module_id,
4905 refusal.code,
4906 refusal.code,
4907 refusal.message,
4908 )?]);
4909 }
4910 }
4911
4912 if !relay_capabilities.is_some_and(|caps| {
4915 caps.provides
4916 .iter()
4917 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
4918 }) {
4919 if let Some(stamp) = scope_stamp.as_mut() {
4920 stamp.workdir = None;
4921 stamp.project_id = None;
4922 }
4923 }
4924 if !relay_capabilities.is_some_and(|caps| {
4925 caps.provides
4926 .iter()
4927 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4928 }) {
4929 if let Some(stamp) = scope_stamp.as_mut() {
4930 stamp.attributes.on_behalf_of_agent = None;
4931 }
4932 }
4933 let call_mode = if relay_capabilities
4934 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4935 {
4936 Some(derive_call_mode(scope_stamp.as_ref()))
4937 } else {
4938 if let Some(stamp) = scope_stamp.as_mut() {
4939 stamp.expires_at_ms = None;
4940 }
4941 None
4942 };
4943
4944 debug!(
4945 connection_id = ctx.connection_id.get(),
4946 client_channel,
4947 client_epoch,
4948 module_channel,
4949 module_epoch,
4950 "reserved route handle pair"
4951 );
4952 let route_principal_label = principal_label(&principal);
4955 let scope_label = scope_stamp
4956 .as_ref()
4957 .map_or_else(|| "none".to_string(), route_open_scope_label);
4958 let relay = ModuleControlRequest::RouteBind {
4959 route_channel: module_channel,
4960 epoch: module_epoch,
4961 target,
4962 identity,
4963 principal: Some(principal),
4964 consumer_capabilities,
4965 role_versions,
4966 admission_facts,
4967 scope: scope_stamp,
4968 call_mode,
4969 };
4970 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4971 RouterError::backend(
4972 0,
4973 frame.header.corr,
4974 format!("failed to encode route.bind request: {err}"),
4975 )
4976 })?;
4977 let relay_frame = Frame::build_with_version(
4978 negotiated_ver,
4979 FrameType::Request,
4980 control_flags(),
4981 0,
4982 0,
4983 relay_corr,
4984 relay_body,
4985 )
4986 .map_err(RouterError::FrameBuild)?;
4987
4988 timings.enter(RouteOpenPhase::BindSend);
4989 let sent = module_sink.send(relay_frame).await;
4990 timings.enter(RouteOpenPhase::Admission);
4991 if let Err(err) = sent {
4992 reservation.release_and_disarm();
4993 return Ok(vec![self.route_open_refusal_frame(
4994 ctx,
4995 &frame,
4996 &target_module_id,
4997 "relay_send_failed",
4998 "target_unavailable",
4999 err.to_string(),
5000 )?]);
5001 }
5002
5003 #[cfg(test)]
5004 if let Some(hold) = &self.route_bind_relay_sent_hold {
5005 hold.wait().await;
5009 hold.wait().await;
5010 }
5011
5012 if self
5017 .forwarding
5018 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5019 .map_err(RouterError::Forwarding)?
5020 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5021 {
5022 self.send_abandoned_route_bind_goodbye(
5023 &module_sink,
5024 negotiated_ver,
5025 module_channel,
5026 module_epoch,
5027 );
5028 }
5029
5030 timings.enter(RouteOpenPhase::BindAckWait);
5031 let outcome = timeout_at(relay_deadline, receiver).await;
5032 timings.enter(RouteOpenPhase::Admission);
5033 match outcome {
5034 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5035 reservation.disarm();
5036 if breaker.record_accepted() {
5037 info!(
5038 module_id = %target_module_id,
5039 "route.bind breaker closed: the probe was accepted"
5040 );
5041 }
5042 self.observe_route_open_accept(
5043 ctx,
5044 &target_module_id,
5045 &route_principal_label,
5046 &scope_label,
5047 );
5048 Ok(Vec::new())
5049 }
5050 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5051 reservation.release_and_disarm();
5052 breaker.record_inconclusive();
5056 let scope_code = match body.code.as_str() {
5060 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5061 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5062 _ => None,
5063 };
5064 if let Some(code) = scope_code {
5065 self.observe_route_open_refusal(
5066 ctx,
5067 &target_module_id,
5068 "scope_changed_before_commit",
5069 code,
5070 );
5071 return Ok(vec![control_error_body_frame(&frame, body)?]);
5072 }
5073 self.counters
5074 .increment_route_open_refused("module_rejected");
5075 info!(
5076 target: "control",
5077 code = "module_rejected",
5078 module_code = ?body.code,
5079 module_id = ?target_module_id,
5080 connection_id = ctx.connection_id.get(),
5081 "route.open refused"
5082 );
5083 Ok(vec![control_error_body_frame(&frame, body)?])
5084 }
5085 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5086 reservation.release_and_disarm();
5087 breaker.record_inconclusive();
5088 tracing::warn!(
5095 module_id = %target_module_id,
5096 "route.bind relay abandoned: {message}"
5097 );
5098 Ok(vec![self.route_open_refusal_frame(
5099 ctx,
5100 &frame,
5101 &target_module_id,
5102 "relay_abandoned",
5103 "target_unavailable",
5104 message,
5105 )?])
5106 }
5107 Ok(Err(_)) => {
5108 reservation.release_and_disarm();
5109 breaker.record_inconclusive();
5110 Ok(vec![self.route_open_refusal_frame(
5111 ctx,
5112 &frame,
5113 &target_module_id,
5114 "relay_waiter_canceled",
5115 "target_unavailable",
5116 "route.bind relay waiter was canceled before the module responded",
5117 )?])
5118 }
5119 Err(_) => {
5120 reservation.release_and_disarm();
5121 if let Some(opened) = breaker.record_timeout(
5125 self.route_bind_breaker_threshold,
5126 self.route_bind_breaker_cooldown,
5127 ) {
5128 warn!(
5129 module_id = %target_module_id,
5130 consecutive_timeouts = opened.consecutive_timeouts,
5131 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5132 reopened_after_probe = opened.reopened_after_probe,
5133 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5134 );
5135 }
5136 tracing::warn!(
5144 module_id = %target_module_id,
5145 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5146 "route.bind relay timed out: module did not ack within budget"
5147 );
5148 Ok(vec![self.route_open_refusal_frame(
5149 ctx,
5150 &frame,
5151 &target_module_id,
5152 "relay_timed_out",
5153 "module_timeout",
5154 format!(
5155 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5156 route_bind_relay_timeout
5157 ),
5158 )?])
5159 }
5160 }
5161 }
5162
5163 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5164 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5165 snapshot: self.supervisor.spawn_snapshot(),
5166 };
5167 Ok(vec![control_response_body_frame(
5168 &frame,
5169 &response,
5170 "ClientControlResponse::SupervisorSpawnSnapshot",
5171 )?])
5172 }
5173
5174 fn handle_supervisor_spawn_subscribe(
5175 &self,
5176 ctx: &RouteCtx,
5177 frame: Frame,
5178 since: Option<SpawnCursor>,
5179 ) -> Result<Vec<Frame>, RouterError> {
5180 match self.supervisor.subscribe_spawns(
5181 ctx.connection_id,
5182 frame.header.corr,
5183 response_version(&frame),
5184 since,
5185 ctx.egress.clone(),
5186 ) {
5187 Ok(()) => Ok(Vec::new()),
5188 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5189 Ok(vec![control_error_body_frame(
5190 &frame,
5191 ErrorBody {
5192 code: "spawn_cursor_incarnation_mismatch".to_string(),
5193 message: "spawn cursor belongs to a different daemon incarnation"
5194 .to_string(),
5195 detail: Some(serde_json::json!({
5196 "current_daemon_incarnation": current
5197 })),
5198 },
5199 )?])
5200 }
5201 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5202 &frame,
5203 ErrorBody {
5204 code: "spawn_cursor_too_old".to_string(),
5205 message: "spawn cursor predates the retained event ring".to_string(),
5206 detail: Some(serde_json::json!({
5207 "oldest_retained_cursor": oldest
5208 })),
5209 },
5210 )?]),
5211 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5212 0,
5213 frame.header.corr,
5214 format!("failed to open supervisor spawn subscription: {error}"),
5215 )),
5216 }
5217 }
5218
5219 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5220 let generation = self
5221 .registry
5222 .generation()
5223 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5224 let mut modules = Vec::new();
5225 for module in self.supervisor.list() {
5226 let status = module.status_for_control("list").map_err(|err| {
5227 RouterError::backend(
5228 0,
5229 frame.header.corr,
5230 format!("failed to read supervisor status: {err}"),
5231 )
5232 })?;
5233 let (configured, _) = module.configuration().map_err(|err| {
5234 RouterError::backend(
5235 0,
5236 frame.header.corr,
5237 format!("failed to read module configuration: {err}"),
5238 )
5239 })?;
5240 let image = module.running_image_agreement().await;
5242 let resources = Some(module.child_resource_usage());
5245 let mut verdict =
5246 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5247 verdict.spec = Some(launch_spec_agreement(
5248 &configured,
5249 status.process_alive,
5250 status.launch_spec_digest.as_ref(),
5251 ));
5252 let pending_reload = Some(verdict);
5253 let configured_enabled = self
5254 .capability_evaluator
5255 .configured_enabled(&status.module_id);
5256 modules.push(
5259 SupervisorEntry::new(
5260 status.module_id,
5261 status.state.to_string(),
5262 status.enabled,
5263 status.live,
5264 status.health.status,
5265 )
5266 .with_configured_enabled(configured_enabled)
5267 .with_launch_nonce_env(Some(!cfg!(unix)))
5268 .with_launch_nonce_source(status.launch_nonce_source)
5269 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5270 .with_protocol(status.protocol)
5271 .with_pending_reload(pending_reload)
5272 .with_last_probe_ms(status.health.last_probe_ms)
5273 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5274 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5275 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5276 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5277 .with_restart_count(Some(status.restart_count))
5278 .with_max_restarts(Some(status.max_restarts))
5279 .with_lifetime_restarts(Some(status.lifetime_restarts))
5280 .with_spawn_generation(Some(status.spawn_generation))
5281 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5282 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5283 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5284 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5285 .with_resources(resources),
5286 );
5287 }
5288 let response = ClientControlResponse::SupervisorList {
5289 generation,
5290 modules,
5291 };
5292 Ok(vec![control_response_body_frame(
5293 &frame,
5294 &response,
5295 "ClientControlResponse::SupervisorList",
5296 )?])
5297 }
5298
5299 fn handle_supervisor_stderr_tail(
5300 &self,
5301 frame: Frame,
5302 module_id: String,
5303 max_lines: Option<u32>,
5304 max_bytes: Option<u32>,
5305 ) -> Result<Vec<Frame>, RouterError> {
5306 let Some(module) = self.supervisor.get(&module_id) else {
5307 return Ok(vec![control_error_frame(
5308 &frame,
5309 "unknown_module",
5310 format!("module_id '{module_id}' is not supervised"),
5311 )?]);
5312 };
5313
5314 let snapshot = module.stderr_tail(
5315 max_lines.map(|value| value as usize),
5316 max_bytes.map(|value| value as usize),
5317 );
5318
5319 let response = ClientControlResponse::SupervisorStderrTail {
5320 module_id,
5321 tail: StderrTail {
5322 capture: match snapshot.capture {
5323 CaptureState::Captured => StderrCaptureState::Captured,
5324 CaptureState::Incomplete { reason } => {
5325 StderrCaptureState::Incomplete { reason }
5326 }
5327 CaptureState::NotCaptured { reason } => {
5328 StderrCaptureState::NotCaptured { reason }
5329 }
5330 },
5331 entries: snapshot
5332 .entries
5333 .into_iter()
5334 .map(|entry| match entry {
5335 TailEntry::Line {
5336 text,
5337 truncated,
5338 at_ms,
5339 } => StderrTailEntry::Line {
5340 text,
5341 truncated,
5342 at_ms,
5343 },
5344 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5345 })
5346 .collect(),
5347 dropped_lines: snapshot.dropped_lines,
5348 },
5349 };
5350 Ok(vec![control_response_body_frame(
5351 &frame,
5352 &response,
5353 "ClientControlResponse::SupervisorStderrTail",
5354 )?])
5355 }
5356
5357 async fn handle_supervisor_terminals(
5358 &self,
5359 frame: Frame,
5360 module_id: String,
5361 ) -> Result<Vec<Frame>, RouterError> {
5362 let Some(module) = self.supervisor.get(&module_id) else {
5363 return Ok(vec![control_error_frame(
5364 &frame,
5365 "unknown_module",
5366 format!("module_id '{module_id}' is not supervised"),
5367 )?]);
5368 };
5369
5370 let terminals = module
5373 .read_durable_terminal_history()
5374 .await
5375 .map_err(|error| {
5376 RouterError::backend(
5377 0,
5378 frame.header.corr,
5379 format!("failed to read terminal history: {error}"),
5380 )
5381 })?;
5382 let response = ClientControlResponse::SupervisorTerminals {
5383 module_id,
5384 terminals,
5385 };
5386 Ok(vec![control_response_body_frame(
5387 &frame,
5388 &response,
5389 "ClientControlResponse::SupervisorTerminals",
5390 )?])
5391 }
5392
5393 fn handle_supervisor_routes(
5394 &self,
5395 frame: Frame,
5396 module_id: Option<String>,
5397 ) -> Result<Vec<Frame>, RouterError> {
5398 let modules = self
5399 .forwarding
5400 .route_census(module_id.as_deref())
5401 .map_err(RouterError::Forwarding)?
5402 .into_iter()
5403 .map(|(module_id, routes)| SupervisorRouteModule {
5404 module_id,
5405 routes: routes
5406 .into_iter()
5407 .map(|route| SupervisorRoute {
5408 consumer: match route.principal {
5409 Principal::Reserved { module_id } => {
5410 SupervisorRouteConsumer::Reserved { module_id }
5411 }
5412 Principal::Direct | Principal::Unverified => {
5413 SupervisorRouteConsumer::Direct {
5414 connection_id: route.goodbye_target.connection_id.get(),
5415 }
5416 }
5417 },
5418 age_ms: Instant::now()
5419 .saturating_duration_since(route.bound_at)
5420 .as_millis()
5421 .try_into()
5422 .unwrap_or(u64::MAX),
5423 draining: route.draining,
5424 drain_reason: route.drain_reason,
5425 })
5426 .collect(),
5427 })
5428 .collect();
5429 let response = ClientControlResponse::SupervisorRoutes { modules };
5430 Ok(vec![control_response_body_frame(
5431 &frame,
5432 &response,
5433 "ClientControlResponse::SupervisorRoutes",
5434 )?])
5435 }
5436
5437 async fn handle_supervisor_provenance(
5438 &self,
5439 frame: Frame,
5440 module_id: Option<String>,
5441 ) -> Result<Vec<Frame>, RouterError> {
5442 let mut selected = if let Some(module_id) = module_id {
5443 let Some(module) = self.supervisor.get(&module_id) else {
5444 return Ok(vec![control_error_frame(
5445 &frame,
5446 "unknown_module",
5447 format!("module_id '{module_id}' is not supervised"),
5448 )?]);
5449 };
5450 vec![module]
5451 } else {
5452 self.supervisor.list()
5453 };
5454
5455 let mut modules = Vec::with_capacity(selected.len());
5456 for module in selected.drain(..) {
5457 let (status, observed_image) = module
5458 .status_and_running_image_agreement()
5459 .await
5460 .map_err(|err| {
5461 RouterError::backend(
5462 0,
5463 frame.header.corr,
5464 format!("failed to read supervisor status: {err}"),
5465 )
5466 })?;
5467 let module_declared = self
5468 .registry
5469 .get_module(&status.module_id)
5470 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5471 .and_then(|registration| registration.manifest.provenance)
5472 .map(|build| ModuleDeclaredProvenance::Reported { build })
5473 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5474 #[cfg(test)]
5475 let running_image = match &self.provenance_probe_override {
5476 Some(result) => result.clone(),
5477 None => observed_image,
5478 };
5479 #[cfg(not(test))]
5480 let running_image = observed_image;
5481 modules.push(SupervisorModuleProvenance {
5482 module_id: status.module_id,
5483 module_declared,
5484 daemon_observed: SupervisorObservedProcess {
5485 launch_nonce_source: status.launch_nonce_source,
5486 pid: status.pid,
5487 spawned_at_ms: status.spawned_at_ms,
5488 spawned_from: status.spawned_from,
5489 running_image,
5490 },
5491 });
5492 }
5493 let daemon = SupervisorDaemonProvenance {
5494 daemon_build: self.daemon_provenance.build.clone(),
5495 daemon_observed: DaemonObservedProcess {
5496 pid: self.daemon_provenance.pid,
5497 started_at_ms: self
5498 .daemon_provenance
5499 .start_clock
5500 .map(|clock| clock.started_at_ms())
5501 .or(self.daemon_provenance.started_at_ms),
5502 running_image: self
5503 .daemon_provenance
5504 .probe
5505 .observe(
5506 self.daemon_provenance.pid,
5507 self.daemon_provenance.executable_path.as_deref(),
5508 self.daemon_provenance.executable_identity.clone(),
5509 self.daemon_provenance.process_start_time,
5510 )
5511 .await,
5512 },
5513 };
5514 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5515 Ok(vec![control_response_body_frame(
5516 &frame,
5517 &response,
5518 "ClientControlResponse::SupervisorProvenance",
5519 )?])
5520 }
5521
5522 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5523 self.refresh_capability_requirements();
5524 let generation = self
5525 .registry
5526 .generation()
5527 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5528 let modules = self
5529 .supervisor
5530 .list()
5531 .into_iter()
5532 .map(|module| {
5533 let status = module.status_for_control("health").map_err(|err| {
5534 RouterError::backend(
5535 0,
5536 frame.header.corr,
5537 format!("failed to read supervisor health: {err}"),
5538 )
5539 })?;
5540 let module_id = status.module_id;
5541 let capability_detail = self
5542 .capability_evaluator
5543 .required_problem_detail(&module_id);
5544 Ok(SupervisorHealthEntry {
5545 module_id,
5546 status: status.health.status,
5547 detail: append_capability_problem_detail(
5548 status.health.detail,
5549 capability_detail,
5550 ),
5551 metrics: status.health.metrics,
5552 consecutive_failures: status.health.consecutive_failures,
5553 late_answer_count: status.health.late_answer_count,
5554 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5555 last_action: status.health.last_action,
5556 last_action_ms: status.health.last_action_ms,
5557 last_probe_ms: status.health.last_probe_ms,
5558 })
5559 })
5560 .collect::<Result<Vec<_>, RouterError>>()?;
5561 let response = ClientControlResponse::SupervisorHealth {
5562 generation,
5563 modules,
5564 };
5565 Ok(vec![control_response_body_frame(
5566 &frame,
5567 &response,
5568 "ClientControlResponse::SupervisorHealth",
5569 )?])
5570 }
5571
5572 async fn handle_supervisor_restart(
5573 &self,
5574 frame: Frame,
5575 module_id: String,
5576 drain_timeout_ms: Option<u64>,
5577 requester: Principal,
5578 ) -> Result<Vec<Frame>, RouterError> {
5579 let operation_lock = self.supervisor.operation_lock();
5580 let _operation_guard = operation_lock.lock().await;
5581 let Some(module) = self.supervisor.get(&module_id) else {
5582 return Ok(vec![control_error_frame(
5583 &frame,
5584 "unknown_module",
5585 format!("module_id '{module_id}' is not supervised"),
5586 )?]);
5587 };
5588
5589 self.route_outages.mark_operator_action(&module_id);
5590 if let Err(err) = module
5591 .restart_requested(drain_timeout_ms, Some(requester))
5592 .await
5593 {
5594 self.route_outages
5595 .operator_action_ended_unrefused(&module_id);
5596 let (code, message) = match err {
5597 crate::supervise::SuperviseError::Disabled { .. } => {
5598 ("module_disabled", err.to_string())
5599 }
5600 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5601 ("swap_in_progress", err.to_string())
5602 }
5603 _ => (
5604 "target_unavailable",
5605 format!("failed to restart module_id '{module_id}': {err}"),
5606 ),
5607 };
5608 return Ok(vec![control_error_frame(&frame, code, message)?]);
5609 }
5610
5611 let response = ClientControlResponse::SupervisorAck {
5612 module_id,
5613 applied: true,
5614 };
5615 Ok(vec![control_response_body_frame(
5616 &frame,
5617 &response,
5618 "ClientControlResponse::SupervisorAck",
5619 )?])
5620 }
5621
5622 async fn handle_supervisor_swap(
5626 &self,
5627 frame: Frame,
5628 module_id: String,
5629 ready_timeout_ms: Option<u64>,
5630 requester: Principal,
5631 ) -> Result<Vec<Frame>, RouterError> {
5632 let module = {
5639 let operation_lock = self.supervisor.operation_lock();
5640 let _operation_guard = operation_lock.lock().await;
5641 self.supervisor.get(&module_id)
5642 };
5643 let Some(module) = module else {
5644 return Ok(vec![control_error_frame(
5645 &frame,
5646 "unknown_module",
5647 format!("module_id '{module_id}' is not supervised"),
5648 )?]);
5649 };
5650
5651 self.route_outages.mark_operator_action(&module_id);
5652 if let Err(err) = module
5653 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5654 .await
5655 {
5656 self.route_outages
5657 .operator_action_ended_unrefused(&module_id);
5658 use crate::supervise::SuperviseError;
5659 let message = err.to_string();
5660 let error = match err {
5661 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5662 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5663 code: "swap_refused".to_string(),
5664 message,
5665 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5666 },
5667 SuperviseError::SwapFailed {
5668 arm,
5669 candidate_exit,
5670 ..
5671 } => ErrorBody {
5672 code: "swap_failed".to_string(),
5673 message,
5674 detail: Some(serde_json::json!({
5675 "arm": arm.as_str(),
5676 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5677 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5678 })),
5679 },
5680 _ => ErrorBody::new(
5681 "target_unavailable",
5682 format!("failed to swap module_id '{module_id}': {message}"),
5683 ),
5684 };
5685 return Ok(vec![control_error_body_frame(&frame, error)?]);
5686 }
5687 self.route_outages
5691 .operator_action_ended_unrefused(&module_id);
5692
5693 let response = ClientControlResponse::SupervisorAck {
5694 module_id,
5695 applied: true,
5696 };
5697 Ok(vec![control_response_body_frame(
5698 &frame,
5699 &response,
5700 "ClientControlResponse::SupervisorAck",
5701 )?])
5702 }
5703
5704 async fn handle_supervisor_reload(
5705 &self,
5706 frame: Frame,
5707 module_id: String,
5708 requester: Principal,
5709 ) -> Result<Vec<Frame>, RouterError> {
5710 let operation_lock = self.supervisor.operation_lock();
5711 let _operation_guard = operation_lock.lock().await;
5712 let Some(module) = self.supervisor.get(&module_id) else {
5713 return Ok(vec![control_error_frame(
5714 &frame,
5715 "unknown_module",
5716 format!("module_id '{module_id}' is not supervised"),
5717 )?]);
5718 };
5719
5720 self.route_outages.mark_operator_action(&module_id);
5721 if let Err(err) = module.reload_requested(Some(requester)).await {
5722 self.route_outages
5723 .operator_action_ended_unrefused(&module_id);
5724 let (code, message) = match err {
5725 crate::supervise::SuperviseError::Disabled { .. } => {
5726 ("module_disabled", err.to_string())
5727 }
5728 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5729 ("swap_in_progress", err.to_string())
5730 }
5731 _ => (
5732 "reload_failed",
5733 format!("failed to reload module_id '{module_id}': {err}"),
5734 ),
5735 };
5736 return Ok(vec![control_error_frame(&frame, code, message)?]);
5737 }
5738
5739 let response = ClientControlResponse::SupervisorAck {
5740 module_id,
5741 applied: true,
5742 };
5743 Ok(vec![control_response_body_frame(
5744 &frame,
5745 &response,
5746 "ClientControlResponse::SupervisorAck",
5747 )?])
5748 }
5749
5750 async fn handle_supervisor_rescan(
5754 &self,
5755 frame: Frame,
5756 mode: RescanMode,
5757 requester: Principal,
5758 ) -> Result<Vec<Frame>, RouterError> {
5759 let preview = mode.previews();
5760 let Some(context) = self.rescan.clone() else {
5761 return Ok(vec![control_error_frame(
5762 &frame,
5763 "rescan_unavailable",
5764 "the daemon was not started with a reloadable config path".to_string(),
5765 )?]);
5766 };
5767
5768 let operation_lock = self.supervisor.operation_lock();
5769 let _operation_guard = operation_lock.lock().await;
5770 let loaded = match self.load_rescan_config(&mode, &context) {
5774 Ok(config) => config,
5775 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5776 };
5777 let Some((config, activity_config)) = loaded else {
5785 return Ok(vec![control_error_frame(
5786 &frame,
5787 "invalid_daemon_config",
5788 format!(
5789 "daemon config not found at {}; refusing to rescan (an absent config would \
5790 retire every supervised module)",
5791 context.config_path.display()
5792 ),
5793 )?]);
5794 };
5795 let (
5796 configured_port,
5797 storage_config,
5798 admission_facts_carrier_module_id,
5799 admission_facts_targets,
5800 scope_authority_owners,
5801 privileged_operator_prompt,
5802 modules,
5803 reserved_capabilities,
5804 ) = (
5805 config.port,
5806 config.storage,
5807 config.admission_facts_carrier_module_id,
5808 config.admission_facts_targets,
5809 config.scope_authority_owners,
5810 config.privileged_operator_prompt,
5811 config.modules,
5812 config.reserved_capabilities,
5813 );
5814
5815 let mut restart_required = Vec::new();
5824 for section in RestartRequiredSection::ALL {
5825 let changed = match section {
5826 RestartRequiredSection::Port => configured_port != context.configured_port,
5827 RestartRequiredSection::Storage => storage_config != context.storage_config,
5828 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5829 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5830 }
5831 RestartRequiredSection::AdmissionFactsTargets => {
5832 admission_facts_targets != context.admission_facts_targets
5833 }
5834 RestartRequiredSection::ScopeAuthorityOwners => {
5835 scope_authority_owners != context.scope_authority_owners
5836 }
5837 RestartRequiredSection::PrivilegedOperatorPrompt => {
5838 privileged_operator_prompt != context.privileged_operator_prompt
5839 }
5840 };
5841 if changed {
5842 restart_required.push(section.label().to_string());
5843 }
5844 }
5845 if !restart_required.is_empty() {
5846 warn!(
5847 config_path = %context.config_path.display(),
5848 sections = %restart_required.join(", "),
5849 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5850 );
5851 }
5852
5853 for configured in &modules {
5854 if let Err(err) = validate_spec(&configured.module_spec()) {
5855 return Ok(vec![control_error_frame(
5856 &frame,
5857 "invalid_daemon_config",
5858 format!("supervisor rescan rejected daemon config: {err}"),
5859 )?]);
5860 }
5861 }
5862
5863 let configured_capabilities = modules
5864 .iter()
5865 .map(|module| (module.module_id.clone(), module.enabled))
5866 .collect::<Vec<_>>();
5867 let preview_capability_warnings = if preview {
5868 let (_, registrations) = self.runtime_capability_snapshot()?;
5869 let current_modules = self
5870 .supervisor
5871 .list()
5872 .into_iter()
5873 .map(|module| module.module_id().to_string())
5874 .collect::<BTreeSet<_>>();
5875 let resulting_modules = configured_capabilities.clone();
5876 let removed = current_modules
5877 .into_iter()
5878 .filter(|module_id| {
5879 !resulting_modules
5880 .iter()
5881 .any(|(configured_id, _)| configured_id == module_id)
5882 })
5883 .collect::<Vec<_>>();
5884 let disabled = match &mode {
5887 RescanMode::Candidate(_) => resulting_modules
5888 .iter()
5889 .filter(|(_, enabled)| !enabled)
5890 .map(|(module_id, _)| module_id.clone())
5891 .collect::<Vec<_>>(),
5892 _ => Vec::new(),
5893 };
5894 self.capability_evaluator.preview_warnings(
5895 resulting_modules,
5896 &removed,
5897 &disabled,
5898 ®istrations,
5899 )
5900 } else {
5901 Vec::new()
5902 };
5903 let reconciled = match &mode {
5904 RescanMode::Candidate(candidate) => self
5905 .preview_candidate(&context, modules, candidate)
5906 .map(|result| (result, Vec::new())),
5907 _ => {
5908 self.reconcile_supervised_modules(
5909 &context.supervisor,
5910 modules,
5911 preview,
5912 requester.clone(),
5913 )
5914 .await
5915 }
5916 };
5917 let (result, changed) = match reconciled {
5918 Ok(result) => result,
5919 Err(message) => {
5920 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5921 }
5922 };
5923 if !preview {
5924 if let Some(ledger) = &self.activity {
5925 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
5926 ledger.config_changed(&result.added, &result.removed, &changed, requester);
5927 }
5928 if let Err(code) = self.configure_activity(activity_config).await {
5929 warn!(
5930 code,
5931 "activity retention could not be applied during rescan"
5932 );
5933 }
5934 self.capability_evaluator
5935 .configure(configured_capabilities, reserved_capabilities);
5936 self.capability_evaluator.wake_deadline_loop();
5937 self.refresh_capability_requirements();
5938 }
5939 let mut result = result;
5940 result.restart_required = restart_required;
5941 result.capability_warnings = preview_capability_warnings;
5942 let response = ClientControlResponse::SupervisorRescan { result };
5943 Ok(vec![control_response_body_frame(
5944 &frame,
5945 &response,
5946 "ClientControlResponse::SupervisorRescan",
5947 )?])
5948 }
5949
5950 async fn handle_supervisor_release_reserved(
5951 &self,
5952 frame: Frame,
5953 module_id: String,
5954 ) -> Result<Vec<Frame>, RouterError> {
5955 let Some(context) = self.rescan.clone() else {
5956 return Ok(vec![control_error_frame(
5957 &frame,
5958 "release_unavailable",
5959 "reserved-id release requires a daemon started with a reloadable config path",
5960 )?]);
5961 };
5962 let operation_lock = self.supervisor.operation_lock();
5963 let _operation_guard = operation_lock.lock().await;
5964 let loaded = match crate::daemon_config::load(&context.config_path) {
5965 Ok(Some(config)) => config,
5966 Ok(None) => {
5967 return Ok(vec![control_error_frame(
5968 &frame,
5969 "invalid_daemon_config",
5970 format!(
5971 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5972 context.config_path.display()
5973 ),
5974 )?])
5975 }
5976 Err(err) => {
5977 return Ok(vec![control_error_frame(
5978 &frame,
5979 "invalid_daemon_config",
5980 format!("unable to verify reserved-id release against daemon config: {err}"),
5981 )?])
5982 }
5983 };
5984 if loaded
5985 .modules
5986 .iter()
5987 .any(|configured| configured.module_id == module_id)
5988 {
5989 return Ok(vec![control_error_frame(
5990 &frame,
5991 "reserved_module_configured",
5992 format!(
5993 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5994 ),
5995 )?]);
5996 }
5997 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5998 return Ok(vec![control_error_frame(
5999 &frame,
6000 "reserved_gate_not_retained",
6001 format!(
6002 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6003 ),
6004 )?]);
6005 }
6006
6007 let response = ClientControlResponse::SupervisorAck {
6008 module_id,
6009 applied: true,
6010 };
6011 Ok(vec![control_response_body_frame(
6012 &frame,
6013 &response,
6014 "ClientControlResponse::SupervisorAck",
6015 )?])
6016 }
6017
6018 fn plan_rescan(
6022 &self,
6023 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6024 ) -> Result<RescanPlan, String> {
6025 let mut current = BTreeMap::new();
6026 for module in self.supervisor.list() {
6027 let (spec, health) = module.configuration().map_err(|err| {
6028 format!(
6029 "failed to read configuration for module_id '{}': {err}",
6030 module.module_id()
6031 )
6032 })?;
6033 let enabled = module
6034 .status()
6035 .map_err(|err| {
6036 format!(
6037 "failed to read status for module_id '{}': {err}",
6038 module.module_id()
6039 )
6040 })?
6041 .enabled;
6042 current.insert(
6043 module.module_id().to_string(),
6044 (module, spec, health, enabled),
6045 );
6046 }
6047 let configured = configured_modules
6048 .into_iter()
6049 .map(|module| (module.module_id.clone(), module))
6050 .collect::<BTreeMap<_, _>>();
6051
6052 let added = configured
6053 .keys()
6054 .filter(|module_id| !current.contains_key(*module_id))
6055 .cloned()
6056 .collect::<Vec<_>>();
6057 let removed = current
6058 .keys()
6059 .filter(|module_id| !configured.contains_key(*module_id))
6060 .cloned()
6061 .collect::<Vec<_>>();
6062 let mut changed_pending_reload = Vec::new();
6063 let mut configuration_changes = BTreeSet::new();
6064 let mut enabled_changes = BTreeSet::new();
6065 let mut unchanged = 0_u32;
6066
6067 for (module_id, configured_module) in &configured {
6068 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6069 else {
6070 continue;
6071 };
6072 let launch_changed = *current_spec != configured_module.module_spec();
6076 let configuration_changed =
6077 launch_changed || *current_health != configured_module.health;
6078 let enabled_changed = *current_enabled != configured_module.enabled;
6079 if configuration_changed {
6080 configuration_changes.insert(module_id.clone());
6081 }
6082 if launch_changed {
6083 changed_pending_reload.push(module_id.clone());
6084 }
6085 if enabled_changed {
6086 enabled_changes.insert(module_id.clone());
6087 }
6088 if !configuration_changed && !enabled_changed {
6089 unchanged = unchanged.saturating_add(1);
6090 }
6091 }
6092 Ok(RescanPlan {
6093 current,
6094 configured,
6095 added,
6096 removed,
6097 changed_pending_reload,
6098 configuration_changes,
6099 enabled_changes,
6100 unchanged,
6101 })
6102 }
6103
6104 async fn reconcile_supervised_modules(
6113 &self,
6114 supervisor: &Supervisor,
6115 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6116 preview: bool,
6117 requester: Principal,
6118 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6119 let plan = self.plan_rescan(configured_modules)?;
6120
6121 if preview {
6127 return Ok((plan.preview_result(), Vec::new()));
6128 }
6129 let changed = plan
6130 .configuration_changes
6131 .union(&plan.enabled_changes)
6132 .cloned()
6133 .collect::<Vec<_>>();
6134 let RescanPlan {
6135 current,
6136 configured,
6137 added,
6138 removed,
6139 changed_pending_reload,
6140 configuration_changes,
6141 enabled_changes,
6142 unchanged,
6143 } = plan;
6144 if !(added.is_empty()
6149 && removed.is_empty()
6150 && configuration_changes.is_empty()
6151 && enabled_changes.is_empty())
6152 {
6153 if let Some(context) = &self.rescan {
6154 context
6155 .generation
6156 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6157 }
6158 }
6159
6160 for module_id in &removed {
6161 let module = ¤t
6162 .get(module_id)
6163 .expect("removed module came from current supervisor state")
6164 .0;
6165 module
6166 .retire_requested(Some(requester.clone()))
6167 .await
6168 .map_err(|err| {
6169 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6170 })?;
6171 self.supervisor.record_rescan_removal(module_id);
6205 self.supervisor.retire(module_id);
6206 self.route_outages.forget(module_id);
6207 }
6208
6209 for module_id in configured.keys() {
6210 let Some((module, _, _, _)) = current.get(module_id) else {
6211 continue;
6212 };
6213 let configured_module = configured
6214 .get(module_id)
6215 .expect("configured module id came from configured map");
6216 if configuration_changes.contains(module_id) {
6217 module
6218 .update_configuration(
6219 configured_module.module_spec(),
6220 configured_module.health.clone(),
6221 configured_module.drain_timeout_ms,
6222 )
6223 .await
6224 .map_err(|err| {
6225 format!(
6226 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6227 )
6228 })?;
6229 }
6230 if enabled_changes.contains(module_id) {
6231 self.route_outages.mark_operator_action(module_id);
6234 module
6235 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6236 .await
6237 .map_err(|err| {
6238 self.route_outages.operator_action_ended_unrefused(module_id);
6239 format!(
6240 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6241 configured_module.enabled
6242 )
6243 })?;
6244 }
6245 }
6246
6247 for module_id in &added {
6248 let configured_module = configured
6249 .get(module_id)
6250 .expect("added module id came from configured map");
6251 supervisor
6252 .supervise_configured_requested(
6253 configured_module.module_spec(),
6254 configured_module.enabled,
6255 configured_module.health.clone(),
6256 configured_module.drain_timeout_ms,
6257 configured_module.restart,
6258 Some(requester.clone()),
6259 )
6260 .map_err(|err| {
6261 format!("failed to add module_id '{module_id}' during rescan: {err}")
6262 })?;
6263 }
6264
6265 Ok((
6266 SupervisorRescanResult {
6267 added,
6268 removed,
6269 changed_pending_reload,
6270 enabled_changes: enabled_changes.iter().cloned().collect(),
6271 unchanged,
6272 preview: false,
6273 restart_required: Vec::new(),
6276 capability_warnings: Vec::new(),
6277 module_actions: Vec::new(),
6278 earlier_pending: Vec::new(),
6279 expect: None,
6280 },
6281 changed,
6282 ))
6283 }
6284
6285 async fn handle_supervisor_set_enabled(
6286 &self,
6287 frame: Frame,
6288 module_id: String,
6289 enabled: bool,
6290 requester: Principal,
6291 ) -> Result<Vec<Frame>, RouterError> {
6292 let operation_lock = self.supervisor.operation_lock();
6293 let _operation_guard = operation_lock.lock().await;
6294 let Some(module) = self.supervisor.get(&module_id) else {
6295 return Ok(vec![control_error_frame(
6296 &frame,
6297 "unknown_module",
6298 format!("module_id '{module_id}' is not supervised"),
6299 )?]);
6300 };
6301
6302 self.route_outages.mark_operator_action(&module_id);
6305 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6306 Ok(applied) => applied,
6307 Err(err) => {
6308 self.route_outages
6309 .operator_action_ended_unrefused(&module_id);
6310 return Ok(vec![control_error_frame(
6311 &frame,
6312 "target_unavailable",
6313 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6314 )?]);
6315 }
6316 };
6317 if !applied {
6318 self.route_outages
6321 .operator_action_ended_unrefused(&module_id);
6322 }
6323
6324 self.capability_evaluator.wake_deadline_loop();
6325 self.refresh_capability_requirements();
6326 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6327 Ok(vec![control_response_body_frame(
6328 &frame,
6329 &response,
6330 "ClientControlResponse::SupervisorAck",
6331 )?])
6332 }
6333
6334 async fn handle_supervisor_health_probe(
6335 &self,
6336 frame: Frame,
6337 module_id: String,
6338 ) -> Result<Vec<Frame>, RouterError> {
6339 self.refresh_capability_requirements();
6340 let Some(registration) = self
6341 .registry
6342 .get_module(&module_id)
6343 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6344 else {
6345 return Ok(vec![control_error_frame(
6346 &frame,
6347 "unknown_module",
6348 format!("module_id '{module_id}' is not registered"),
6349 )?]);
6350 };
6351
6352 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6365 {
6366 return Ok(vec![control_error_frame(
6367 &frame,
6368 "health_not_advertised",
6369 format!("module_id '{module_id}' did not advertise health.check"),
6370 )?]);
6371 }
6372
6373 let deadline = Instant::now() + self.health_probe_timeout;
6374 let pending = match self.forwarding.begin_module_control_rpc_for(
6375 &module_id,
6376 MODULE_CONTROL_OP_HEALTH_CHECK,
6377 deadline,
6378 ) {
6379 Ok(pending) => pending,
6380 Err(err) => {
6381 return Ok(vec![control_error_frame(
6382 &frame,
6383 forwarding_error_code(&err),
6384 err.to_string(),
6385 )?])
6386 }
6387 };
6388
6389 let PendingModuleControlRpc {
6390 endpoint,
6391 module_sink,
6392 negotiated_ver,
6393 corr: probe_corr,
6394 receiver,
6395 } = pending;
6396 let mut guard =
6397 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6398 let probe_body =
6399 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6400 RouterError::backend(
6401 0,
6402 frame.header.corr,
6403 format!("failed to encode health.check request: {err}"),
6404 )
6405 })?;
6406 let probe_frame = Frame::build_with_version(
6407 negotiated_ver,
6408 FrameType::Request,
6409 control_flags(),
6410 0,
6411 0,
6412 probe_corr,
6413 probe_body,
6414 )
6415 .map_err(RouterError::FrameBuild)?;
6416
6417 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6420 Ok(Ok(())) => {}
6421 Ok(Err(err)) => {
6422 return Ok(vec![control_error_frame(
6423 &frame,
6424 "target_unavailable",
6425 err.to_string(),
6426 )?]);
6427 }
6428 Err(_) => {
6429 return Ok(vec![control_error_frame(
6430 &frame,
6431 "module_timeout",
6432 format!(
6433 "module_id '{module_id}' did not answer health.check within {:?}",
6434 self.health_probe_timeout
6435 ),
6436 )?]);
6437 }
6438 }
6439
6440 match timeout_at(deadline, receiver).await {
6441 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6442 guard.disarm();
6443 let Some(report) = response.health_report() else {
6444 return Ok(vec![control_error_frame(
6445 &frame,
6446 "invalid_control_body",
6447 "health.check RPC returned a non-health response",
6448 )?]);
6449 };
6450 let HealthReport {
6455 status,
6456 detail,
6457 metrics,
6458 } = report;
6459 let capability_detail = self
6460 .capability_evaluator
6461 .required_problem_detail(&module_id);
6462 let response = ClientControlResponse::SupervisorHealthProbe {
6463 module_id,
6464 status,
6465 detail: append_capability_problem_detail(detail, capability_detail),
6466 metrics,
6467 };
6468 Ok(vec![control_response_body_frame(
6469 &frame,
6470 &response,
6471 "ClientControlResponse::SupervisorHealthProbe",
6472 )?])
6473 }
6474 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6475 guard.disarm();
6476 Ok(vec![control_error_body_frame(&frame, body)?])
6477 }
6478 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6479 guard.disarm();
6480 Ok(vec![control_error_frame(
6481 &frame,
6482 "target_unavailable",
6483 message,
6484 )?])
6485 }
6486 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6487 guard.disarm();
6488 Ok(vec![control_error_frame(
6489 &frame,
6490 "invalid_control_body",
6491 message,
6492 )?])
6493 }
6494 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6495 guard.disarm();
6496 Ok(vec![control_error_frame(
6497 &frame,
6498 "invalid_control_body",
6499 format!("expected module-control op '{expected}', got '{actual}'"),
6500 )?])
6501 }
6502 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6503 guard.disarm();
6504 Ok(vec![control_error_frame(
6505 &frame,
6506 "module_timeout",
6507 format!(
6508 "module_id '{module_id}' answered health.check after {:?}",
6509 self.health_probe_timeout
6510 ),
6511 )?])
6512 }
6513 Ok(Err(_)) => Ok(vec![control_error_frame(
6514 &frame,
6515 "target_unavailable",
6516 "health.check waiter was canceled before the module responded",
6517 )?]),
6518 Err(_) => Ok(vec![control_error_frame(
6519 &frame,
6520 "module_timeout",
6521 format!(
6522 "module_id '{module_id}' did not answer health.check within {:?}",
6523 self.health_probe_timeout
6524 ),
6525 )?]),
6526 }
6527 }
6528
6529 fn supervisor_status(
6530 &self,
6531 module_id: &str,
6532 corr: u64,
6533 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6534 self.supervisor
6535 .get(module_id)
6536 .map(|module| {
6537 let warming = module.is_warming_for_control("status").map_err(|err| {
6538 RouterError::backend(
6539 0,
6540 corr,
6541 format!(
6542 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6543 ),
6544 )
6545 })?;
6546 module.status_for_control("status").map_err(|err| {
6547 RouterError::backend(
6548 0,
6549 corr,
6550 format!(
6551 "failed to read supervisor status for module_id '{module_id}': {err}"
6552 ),
6553 )
6554 }).map(|status| (status, warming))
6555 })
6556 .transpose()
6557 }
6558
6559 fn guard_module_control_op(
6560 &self,
6561 frame: &Frame,
6562 module_id: &str,
6563 op: &str,
6564 ) -> Result<Option<Frame>, RouterError> {
6565 if self.module_grants_op(module_id, op, frame.header.corr)? {
6566 return Ok(None);
6567 }
6568
6569 Ok(Some(control_error_frame(
6570 frame,
6571 "op_not_allowed",
6572 format!("module_id '{module_id}' did not grant control op '{op}'"),
6573 )?))
6574 }
6575
6576 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6577 let Some(registration) = self
6578 .registry
6579 .get_module(module_id)
6580 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6581 else {
6582 return Ok(false);
6583 };
6584 Ok(module_registration_grants_op(®istration.control_ops, op))
6585 }
6586
6587 fn handle_status_update(
6588 &self,
6589 endpoint: ModuleEndpointId,
6590 frame: Frame,
6591 ) -> Result<Vec<Frame>, RouterError> {
6592 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6593 Ok(update) => update,
6594 Err(err) => {
6595 if is_known_module_push_op(&frame.body) {
6600 return Ok(vec![control_error_frame(
6601 &frame,
6602 "invalid_control_body",
6603 format!("malformed module control push body: {err}"),
6604 )?]);
6605 }
6606 return Ok(Vec::new());
6607 }
6608 };
6609
6610 match update {
6611 ModuleControlPush::RouteStatus {
6612 route_channel,
6613 route_epoch,
6614 status,
6615 } => {
6616 self.forwarding
6617 .cache_status(endpoint, route_channel, route_epoch, status)
6618 .map_err(RouterError::Forwarding)?;
6619 }
6620 }
6621 Ok(Vec::new())
6622 }
6623
6624 fn handle_route_poll(
6625 &self,
6626 ctx: &RouteCtx,
6627 frame: Frame,
6628 route_channel: u16,
6629 route_epoch: u32,
6630 kind: PollKind,
6631 ) -> Result<Vec<Frame>, RouterError> {
6632 let snapshot = self
6633 .forwarding
6634 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6635 .map_err(RouterError::Forwarding)?;
6636 let response = match (kind, snapshot) {
6637 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6638 ClientControlResponse::RoutePoll {
6639 route_channel,
6640 route_epoch,
6641 status,
6642 live: None,
6643 }
6644 }
6645 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6646 route_channel,
6647 route_epoch,
6648 status: None,
6649 live: None,
6650 },
6651 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6652 let live = self
6678 .process_liveness
6679 .as_ref()
6680 .and_then(|source| source.process_live(&module_id))
6681 .unwrap_or(true);
6682 ClientControlResponse::RoutePoll {
6683 route_channel,
6684 route_epoch,
6685 status: None,
6686 live: Some(live),
6687 }
6688 }
6689 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6690 route_channel,
6691 route_epoch,
6692 status: None,
6693 live: Some(false),
6694 },
6695 };
6696
6697 Ok(vec![control_response_body_frame(
6698 &frame,
6699 &response,
6700 "ClientControlResponse::RoutePoll",
6701 )?])
6702 }
6703
6704 pub(crate) fn observe_module_control_completion(
6705 &self,
6706 completion: ModuleControlRpcCompletion,
6707 ) -> bool {
6708 match completion {
6709 ModuleControlRpcCompletion::Unknown => false,
6710 ModuleControlRpcCompletion::Settled => true,
6711 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6712 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6713 info!(
6714 module_id = %module_id,
6715 latency_ms,
6716 "late health.check answer proves the module is alive"
6717 );
6718 match self
6719 .supervisor
6720 .record_late_health_answer(&module_id, latency_ms)
6721 {
6722 Ok(true) => {}
6723 Ok(false) => debug!(
6724 module_id = %module_id,
6725 latency_ms,
6726 "late health.check answer has no active supervisor snapshot"
6727 ),
6728 Err(err) => warn!(
6729 module_id = %module_id,
6730 latency_ms,
6731 error = %err,
6732 "failed to record late health.check answer"
6733 ),
6734 }
6735 true
6736 }
6737 }
6738 }
6739
6740 fn refuse_to_end_module_connection_for_a_client(
6761 &self,
6762 module_connection_id: ConnectionId,
6763 corr: u64,
6764 err: ForwardingError,
6765 ) -> Result<(), RouterError> {
6766 if let ForwardingError::ConnectionClosing { connection_id } = err {
6767 if connection_id != module_connection_id {
6768 warn!(
6769 module_connection_id = module_connection_id.get(),
6770 client_connection_id = connection_id.get(),
6771 corr,
6772 "dropping a route.bind response for a closing client; the module connection keeps serving"
6773 );
6774 return Ok(());
6775 }
6776 }
6777 Err(RouterError::Forwarding(err))
6778 }
6779
6780 fn handle_module_relay_response(
6781 &self,
6782 connection_id: ConnectionId,
6783 frame: Frame,
6784 ) -> Result<Vec<Frame>, RouterError> {
6785 let mut secondary_error = None;
6786 let outcome = match frame.header.ty {
6787 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6788 Ok(probe) if probe.op == "route.bind" => {
6789 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6790 Ok(ModuleControlResponse::RouteBindAck {}) => {
6791 RouteBindRelayOutcome::Accepted
6792 }
6793 Ok(other) => {
6794 let message =
6795 format!("route.bind response carried unexpected body: {other:?}");
6796 secondary_error = Some(control_error_frame(
6797 &frame,
6798 "invalid_control_body",
6799 message.clone(),
6800 )?);
6801 RouteBindRelayOutcome::ModuleGone(message)
6802 }
6803 Err(err) => {
6804 let message = format!("malformed route.bind response body: {err}");
6805 secondary_error = Some(control_error_frame(
6806 &frame,
6807 "invalid_control_body",
6808 message.clone(),
6809 )?);
6810 RouteBindRelayOutcome::ModuleGone(message)
6811 }
6812 }
6813 }
6814 Ok(probe) => {
6815 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6816 {
6817 Ok(response) => ModuleControlRpcOutcome::Response(response),
6818 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6819 "malformed {} response body: {err}",
6820 probe.op
6821 )),
6822 };
6823 let completion = self
6824 .forwarding
6825 .complete_module_control_rpc(
6826 connection_id,
6827 frame.header.corr,
6828 Some(&probe.op),
6829 outcome,
6830 )
6831 .map_err(RouterError::Forwarding)?;
6832 if !self.observe_module_control_completion(completion) {
6833 debug!(
6834 connection_id = connection_id.get(),
6835 corr = frame.header.corr,
6836 op = %probe.op,
6837 "dropping late or unknown module-control RPC response"
6838 );
6839 }
6840 return Ok(Vec::new());
6841 }
6842 Err(err) => {
6843 if let Some(expected_op) = self
6844 .forwarding
6845 .pending_module_control_op(connection_id, frame.header.corr)
6846 .map_err(RouterError::Forwarding)?
6847 {
6848 let completion = self
6849 .forwarding
6850 .complete_module_control_rpc(
6851 connection_id,
6852 frame.header.corr,
6853 None,
6854 ModuleControlRpcOutcome::MalformedResponse(format!(
6855 "malformed {expected_op} response body: {err}"
6856 )),
6857 )
6858 .map_err(RouterError::Forwarding)?;
6859 if !self.observe_module_control_completion(completion) {
6860 debug!(
6861 connection_id = connection_id.get(),
6862 corr = frame.header.corr,
6863 "dropping late malformed module-control RPC response"
6864 );
6865 }
6866 return Ok(Vec::new());
6867 }
6868 let message = format!("malformed route.bind response body: {err}");
6869 secondary_error = Some(control_error_frame(
6870 &frame,
6871 "invalid_control_body",
6872 message.clone(),
6873 )?);
6874 RouteBindRelayOutcome::ModuleGone(message)
6875 }
6876 },
6877 FrameType::Error => {
6878 if self
6879 .forwarding
6880 .pending_module_control_op(connection_id, frame.header.corr)
6881 .map_err(RouterError::Forwarding)?
6882 .is_some()
6883 {
6884 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6885 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6886 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6887 "malformed module-control ERROR body: {err}"
6888 )),
6889 };
6890 let completion = self
6891 .forwarding
6892 .complete_module_control_rpc(
6893 connection_id,
6894 frame.header.corr,
6895 None,
6896 outcome,
6897 )
6898 .map_err(RouterError::Forwarding)?;
6899 if !self.observe_module_control_completion(completion) {
6900 debug!(
6901 connection_id = connection_id.get(),
6902 corr = frame.header.corr,
6903 "dropping late or unknown module-control RPC error"
6904 );
6905 }
6906 return Ok(Vec::new());
6907 }
6908 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6909 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6910 Err(err) => {
6911 let message = format!("malformed route.bind ERROR body: {err}");
6912 secondary_error = Some(control_error_frame(
6913 &frame,
6914 "invalid_control_body",
6915 message.clone(),
6916 )?);
6917 RouteBindRelayOutcome::ModuleGone(message)
6918 }
6919 }
6920 }
6921 ty => {
6922 return Ok(vec![control_error_frame(
6923 &frame,
6924 "unsupported_control_frame",
6925 format!("unsupported module channel-0 frame {ty:?}"),
6926 )?])
6927 }
6928 };
6929
6930 let settled =
6931 self.forwarding
6932 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6933 let completion = match settled {
6934 Ok(completion) => completion,
6935 Err(err) => {
6936 self.refuse_to_end_module_connection_for_a_client(
6937 connection_id,
6938 frame.header.corr,
6939 err,
6940 )?;
6941 return Ok(secondary_error.into_iter().collect());
6942 }
6943 };
6944 if let Some(target) = completion.abandoned.as_ref() {
6945 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6946 }
6947 if !completion.settled {
6948 debug!(
6949 connection_id = connection_id.get(),
6950 corr = frame.header.corr,
6951 frame_type = ?frame.header.ty,
6952 "dropping late or unknown route.bind relay response"
6953 );
6954 }
6955 Ok(secondary_error.into_iter().collect())
6956 }
6957
6958 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6959 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6960 self.cleanup_connection_with_end_reason(
6964 connection_id,
6965 RegistrationEndReason::ExplicitGoodbye,
6966 )
6967 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6968 Ok(Vec::new())
6969 }
6970}
6971
6972impl Default for ControlHandler {
6973 fn default() -> Self {
6974 Self::new(Arc::new(Registry::default()))
6975 }
6976}
6977
6978impl crate::supervise::SwapPromotionObserver for ControlHandler {
6979 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6980 self.apply_registration_capabilities(registration);
6981 }
6982}
6983
6984fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6985 CapabilityRequirementStatus {
6986 consumer: status.consumer,
6987 capability: status.capability,
6988 need: match status.need {
6989 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6990 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6991 },
6992 verdict: status.verdict.as_str().to_string(),
6993 episode_seq: status.episode_seq,
6994 config_satisfiable: status.config_satisfiable,
6995 runtime_available: status.runtime_available,
6996 detail: status.detail,
6997 }
6998}
6999
7000fn append_capability_problem_detail(
7001 detail: Option<String>,
7002 capability_detail: Option<String>,
7003) -> Option<String> {
7004 match (detail, capability_detail) {
7005 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7006 (Some(detail), None) => Some(detail),
7007 (None, Some(capability_detail)) => Some(capability_detail),
7008 (None, None) => None,
7009 }
7010}
7011
7012fn subc_ops() -> Vec<String> {
7013 SUBC_CONTROL_OPS
7014 .iter()
7015 .map(|op| (*op).to_string())
7016 .collect()
7017}
7018
7019fn module_subc_ops() -> Vec<String> {
7020 SUBC_CONTROL_OPS
7021 .iter()
7022 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7023 .map(|op| (*op).to_string())
7024 .collect()
7025}
7026
7027#[cfg(test)]
7028fn module_baseline_control_ops() -> Vec<String> {
7029 MODULE_BASELINE_CONTROL_OPS
7030 .iter()
7031 .map(|op| (*op).to_string())
7032 .collect()
7033}
7034
7035fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7036 let mut seen = HashSet::new();
7037 let mut effective = Vec::new();
7038 for op in MODULE_BASELINE_CONTROL_OPS {
7039 if seen.insert((*op).to_string()) {
7040 effective.push((*op).to_string());
7041 }
7042 }
7043 for op in declared.unwrap_or_default() {
7044 if seen.insert(op.clone()) {
7045 effective.push(op);
7046 }
7047 }
7048 effective
7049}
7050
7051fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7052 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7053}
7054
7055fn target_module_id(target: &RouteTarget) -> &str {
7056 match target {
7057 RouteTarget::ToolProvider { module_id }
7058 | RouteTarget::ManagementSurface { module_id }
7059 | RouteTarget::InternalService { module_id, .. } => module_id,
7060 }
7061}
7062
7063fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7064 roles.iter().any(|role| match (target, role) {
7065 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7066 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7067 (
7068 RouteTarget::InternalService { service_id, .. },
7069 ProviderRole::InternalService {
7070 service_id: provided,
7071 ..
7072 },
7073 ) => service_id == provided,
7074 _ => false,
7075 })
7076}
7077
7078fn is_routable_role(role: &ProviderRole) -> bool {
7079 matches!(
7080 role,
7081 ProviderRole::ToolProvider { .. }
7082 | ProviderRole::ManagementSurface { .. }
7083 | ProviderRole::InternalService { .. }
7084 )
7085}
7086
7087#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7088enum ControlRequestBodyError {
7089 UnknownOp,
7090 InvalidBody,
7091}
7092
7093#[derive(Debug, Deserialize)]
7094struct ControlOpProbe {
7095 op: String,
7096}
7097
7098const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7101
7102fn is_known_module_push_op(body: &[u8]) -> bool {
7103 serde_json::from_slice::<ControlOpProbe>(body)
7104 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7105 .unwrap_or(false)
7106}
7107
7108fn is_known_module_request_op(body: &[u8]) -> bool {
7109 serde_json::from_slice::<ControlOpProbe>(body)
7110 .map(|probe| is_module_to_subc_op(&probe.op))
7111 .unwrap_or(false)
7112}
7113
7114fn is_module_to_subc_op(op: &str) -> bool {
7115 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7116}
7117
7118fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7119 debug!(
7120 op = %op,
7121 connection_id = connection_id.get(),
7122 corr,
7123 "control dispatch"
7124 );
7125}
7126
7127fn log_slow_control_dispatch(
7128 dispatch_started_at: Option<StdInstant>,
7129 op: &'static str,
7130 connection_id: ConnectionId,
7131 corr: u64,
7132 route_open_timings: Option<&RouteOpenTimings>,
7133) {
7134 let Some(dispatch_started_at) = dispatch_started_at else {
7135 return;
7136 };
7137 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7139 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7140 if let Some(timings) = route_open_timings {
7141 warn!(
7142 op = %op,
7143 connection_id = connection_id.get(),
7144 corr,
7145 elapsed_ms = elapsed.as_millis() as u64,
7146 module_id = ?timings.module_id,
7147 principal = %timings.principal,
7148 admission_ms = timings.admission.as_millis() as u64,
7149 project_root_ms = timings.project_root.as_millis() as u64,
7150 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7151 bind_send_ms = timings.bind_send.as_millis() as u64,
7152 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7153 "slow control dispatch"
7154 );
7155 return;
7156 }
7157 warn!(
7158 op = %op,
7159 connection_id = connection_id.get(),
7160 corr,
7161 elapsed_ms = elapsed.as_millis() as u64,
7162 "slow control dispatch"
7163 );
7164 }
7165}
7166
7167#[cfg(test)]
7169type TestCallerLookup =
7170 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7171
7172fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7174 match request {
7175 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7176 Some(PrivilegedOperation::Restart { module_id })
7177 }
7178 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7179 Some(PrivilegedOperation::Swap { module_id })
7180 }
7181 ClientControlRequest::SupervisorReload { module_id } => {
7182 Some(PrivilegedOperation::Reload { module_id })
7183 }
7184 ClientControlRequest::SupervisorRescan { preview } => {
7185 if *preview {
7186 None
7187 } else {
7188 Some(PrivilegedOperation::Rescan)
7189 }
7190 }
7191 ClientControlRequest::SupervisorRescanChecked {
7196 preview,
7197 candidate,
7198 expect,
7199 } => (!*preview && candidate.is_none() && expect.is_some())
7200 .then_some(PrivilegedOperation::Rescan),
7201 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7202 Some(PrivilegedOperation::SetEnabled {
7203 module_id,
7204 enabled: *enabled,
7205 })
7206 }
7207 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7208 Some(PrivilegedOperation::ReleaseReserved { module_id })
7209 }
7210 ClientControlRequest::ServerDescribe {}
7211 | ClientControlRequest::CatalogList { .. }
7212 | ClientControlRequest::RouteOpen { .. }
7213 | ClientControlRequest::RoutePoll { .. }
7214 | ClientControlRequest::SupervisorList {}
7215 | ClientControlRequest::SupervisorHealth {}
7216 | ClientControlRequest::SupervisorHealthProbe { .. }
7217 | ClientControlRequest::SupervisorRoutes { .. }
7218 | ClientControlRequest::SupervisorProvenance { .. }
7219 | ClientControlRequest::SupervisorStderrTail { .. }
7220 | ClientControlRequest::SupervisorTerminals { .. }
7221 | ClientControlRequest::SupervisorSpawnSnapshot {}
7222 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7223 }
7224}
7225
7226fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7227 match request {
7228 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7229 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7230 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7231 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7232 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7233 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7234 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7235 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7236 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7237 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7238 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7239 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7240 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7241 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7242 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7243 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7244 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7245 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7246 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7247 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7248 }
7249}
7250
7251fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7257 use subc_protocol::scope::ScopeKind;
7258 const MAX_REF_CHARS: usize = 64;
7259 let kind = match stamp.kind {
7260 ScopeKind::Head => "head",
7261 ScopeKind::Worker => "worker",
7262 ScopeKind::Ephemeral => "ephemeral",
7263 };
7264 let mut scope_ref: String = stamp.scope_ref.chars().take(MAX_REF_CHARS).collect();
7265 if stamp.scope_ref.chars().nth(MAX_REF_CHARS).is_some() {
7266 scope_ref.push('…');
7267 }
7268 format!(
7269 "{kind}/{}/{scope_ref:?}@{}",
7270 principal_label(&stamp.owner),
7271 stamp.scope_epoch
7272 )
7273}
7274
7275fn principal_label(principal: &Principal) -> String {
7276 match principal {
7277 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7278 Principal::Direct => "direct".to_string(),
7279 other => format!("{other:?}"),
7280 }
7281}
7282
7283fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7284 match request {
7285 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7286 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7287 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7288 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7289 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7290 }
7291}
7292
7293fn parse_client_control_request(
7294 body: &[u8],
7295) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7296 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7297 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7298 Ok(probe)
7299 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7300 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7301 {
7302 ControlRequestBodyError::InvalidBody
7303 }
7304 Ok(_) => ControlRequestBodyError::UnknownOp,
7305 Err(_) => ControlRequestBodyError::InvalidBody,
7306 };
7307 (err, classification)
7308 })
7309}
7310
7311fn parse_module_control_request_from_module(
7312 body: &[u8],
7313) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7314 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7315 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7316 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7317 Ok(_) => ControlRequestBodyError::UnknownOp,
7318 Err(_) => ControlRequestBodyError::InvalidBody,
7319 };
7320 (err, classification)
7321 })
7322}
7323
7324#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7325enum ProviderRoleKind {
7326 ToolProvider,
7327 PipelineStage,
7328 ManagementSurface,
7329 InternalService,
7330}
7331
7332fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7333 match role {
7334 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7335 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7336 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7337 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7338 }
7339}
7340
7341fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7342 roles.iter().map(provider_role_kind).collect()
7343}
7344
7345const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7348
7349#[derive(Debug, Default, PartialEq, Eq)]
7351struct ScopeOutcomeCounts {
7352 created: usize,
7353 replaced: usize,
7354 updated: usize,
7355 unchanged: usize,
7356 refused: usize,
7357}
7358
7359impl ScopeOutcomeCounts {
7360 fn of(results: &[ScopeRecordResult]) -> Self {
7361 let mut counts = Self::default();
7362 for result in results {
7363 let slot = match result.outcome {
7364 ScopeRecordOutcome::Created => &mut counts.created,
7365 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7366 ScopeRecordOutcome::Updated => &mut counts.updated,
7367 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7368 ScopeRecordOutcome::Refused => &mut counts.refused,
7369 };
7370 *slot += 1;
7371 }
7372 counts
7373 }
7374}
7375
7376#[cfg(test)]
7377mod scope_outcome_count_tests {
7378 use super::*;
7379
7380 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7381 ScopeRecordResult {
7382 scope_ref: "r".to_string(),
7383 scope_epoch: 1,
7384 outcome,
7385 code: None,
7386 message: None,
7387 version: None,
7388 parent_state: None,
7389 }
7390 }
7391
7392 #[test]
7395 fn every_outcome_is_counted_in_its_own_field() {
7396 let results = [
7397 result(ScopeRecordOutcome::Created),
7398 result(ScopeRecordOutcome::Created),
7399 result(ScopeRecordOutcome::Replaced),
7400 result(ScopeRecordOutcome::Updated),
7401 result(ScopeRecordOutcome::Unchanged),
7402 result(ScopeRecordOutcome::Refused),
7403 result(ScopeRecordOutcome::Refused),
7404 result(ScopeRecordOutcome::Refused),
7405 ];
7406 assert_eq!(
7407 ScopeOutcomeCounts::of(&results),
7408 ScopeOutcomeCounts {
7409 created: 2,
7410 replaced: 1,
7411 updated: 1,
7412 unchanged: 1,
7413 refused: 3,
7414 }
7415 );
7416 }
7417}
7418
7419fn capability_census_trigger(
7423 old: Option<&CapabilityDeclarations>,
7424 new: Option<&CapabilityDeclarations>,
7425) -> bool {
7426 let old_provides = old
7427 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7428 .unwrap_or_default();
7429 let old_denies = old
7430 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7431 .unwrap_or_default();
7432 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7433 provides: Vec::new(),
7434 requires: Vec::new(),
7435 must_never_reach: Vec::new(),
7436 });
7437
7438 new.provides
7439 .iter()
7440 .any(|capability| !old_provides.contains(capability))
7441 || new
7442 .must_never_reach
7443 .iter()
7444 .any(|capability| !old_denies.contains(capability))
7445}
7446
7447fn denied_capability<'a>(
7450 opening_manifest: &'a ModuleManifest,
7451 target_manifest: &ModuleManifest,
7452) -> Option<&'a str> {
7453 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7454 let target_capabilities = target_manifest.capabilities.as_ref()?;
7455 opening_capabilities
7456 .must_never_reach
7457 .iter()
7458 .find(|denied| {
7459 target_capabilities
7460 .provides
7461 .iter()
7462 .any(|provided| provided == *denied)
7463 })
7464 .map(String::as_str)
7465}
7466
7467fn catalog_update_frozen_field_message(
7468 registered: &ModuleManifest,
7469 provides: &[ProviderRole],
7470) -> Option<String> {
7471 let old_has_provides = !registered.provides.is_empty();
7472 let new_has_provides = !provides.is_empty();
7473 if old_has_provides != new_has_provides {
7474 return Some(format!(
7475 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7476 registered.module_id
7477 ));
7478 }
7479
7480 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7481 return Some(format!(
7482 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7483 registered.module_id
7484 ));
7485 }
7486
7487 let registered_concurrency = manifest_concurrency(registered);
7488 let mut candidate = registered.clone();
7489 candidate.provides = provides.to_vec();
7490 let candidate_concurrency = manifest_concurrency(&candidate);
7491 if candidate_concurrency != registered_concurrency {
7492 return Some(format!(
7493 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7494 registered.module_id, registered_concurrency, candidate_concurrency
7495 ));
7496 }
7497
7498 None
7501}
7502
7503fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7504 manifest.provides.iter().any(is_routable_role)
7505}
7506
7507fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7513 manifest
7514 .provides
7515 .iter()
7516 .find_map(|provider| match provider {
7517 ProviderRole::ToolProvider { concurrency, .. }
7518 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7519 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7520 })
7521 .unwrap_or(Concurrency::ModuleManaged)
7522}
7523
7524fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7531 let has_management_surface = manifest
7532 .provides
7533 .iter()
7534 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7535 if !has_management_surface {
7536 return false;
7537 }
7538 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7539 return false;
7540 };
7541 let Some(provides) = raw
7542 .get("manifest")
7543 .and_then(|manifest| manifest.get("provides"))
7544 .and_then(serde_json::Value::as_array)
7545 else {
7546 return false;
7547 };
7548 provides.iter().any(|role| {
7553 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7554 && role.get("concurrency").is_none()
7555 })
7556}
7557
7558fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7559 if peer_version != PROTOCOL_VERSION {
7560 return Err(format!(
7561 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7562 ));
7563 }
7564 Ok(PROTOCOL_VERSION)
7565}
7566
7567fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7568 Frame::build_with_version(
7569 response_version(frame),
7570 FrameType::Pong,
7571 frame.header.flags,
7572 0,
7573 0,
7574 frame.header.corr,
7575 Vec::new(),
7576 )
7577 .map_err(RouterError::FrameBuild)
7578}
7579
7580pub(crate) fn control_error_frame(
7581 frame: &Frame,
7582 code: &'static str,
7583 message: impl Into<String>,
7584) -> Result<Frame, RouterError> {
7585 control_error_body_frame(
7586 frame,
7587 ErrorBody {
7588 code: code.to_string(),
7589 message: message.into(),
7590 detail: None,
7591 },
7592 )
7593}
7594
7595fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7596 let body = serde_json::to_vec(&error).map_err(|err| {
7597 RouterError::backend(
7598 0,
7599 frame.header.corr,
7600 format!("failed to encode control ERROR: {err}"),
7601 )
7602 })?;
7603
7604 Frame::build_with_version(
7605 response_version(frame),
7606 FrameType::Error,
7607 control_flags(),
7608 0,
7609 0,
7610 frame.header.corr,
7611 body,
7612 )
7613 .map_err(RouterError::FrameBuild)
7614}
7615
7616fn control_response_body_frame<T: Serialize>(
7617 frame: &Frame,
7618 reply: &T,
7619 label: &'static str,
7620) -> Result<Frame, RouterError> {
7621 let body = serde_json::to_vec(reply).map_err(|err| {
7622 RouterError::backend(
7623 0,
7624 frame.header.corr,
7625 format!("failed to encode {label}: {err}"),
7626 )
7627 })?;
7628
7629 Frame::build_with_version(
7630 response_version(frame),
7631 FrameType::Response,
7632 control_flags(),
7633 0,
7634 0,
7635 frame.header.corr,
7636 body,
7637 )
7638 .map_err(RouterError::FrameBuild)
7639}
7640
7641fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7671 match err {
7672 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7673 ForwardingError::NoModuleConnection => "target_unavailable",
7674 ForwardingError::ModuleReloading { .. } => "module_reloading",
7675 ForwardingError::ClientRouteChannelExhausted { .. }
7676 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7677 ForwardingError::StaleModuleEndpoint
7678 | ForwardingError::UnknownReservation { .. }
7679 | ForwardingError::ConnectionClosing { .. }
7680 | ForwardingError::ClientEgressClosed { .. }
7681 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7682 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7685 ForwardingError::RelayCorrelationExhausted
7686 | ForwardingError::RouteOpenBuild(_)
7687 | ForwardingError::Poisoned => "forwarding_error",
7688 }
7689}
7690
7691fn response_version(frame: &Frame) -> u8 {
7692 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7693 frame.header.ver
7694 } else {
7695 PROTOCOL_VERSION
7696 }
7697}
7698
7699fn control_flags() -> Flags {
7700 Flags::new(false, Priority::Passive, false)
7701}
7702
7703fn send_goodbye_target_best_effort(
7708 counters: &DaemonCounters,
7709 target: &GoodbyeTarget,
7710 context: &'static str,
7711) {
7712 let Ok(frame) = Frame::build_with_version(
7713 target.negotiated_ver,
7714 FrameType::Goodbye,
7715 control_flags(),
7716 target.channel,
7717 target.epoch,
7718 0,
7719 Vec::new(),
7720 ) else {
7721 return;
7722 };
7723 crate::forwarding::send_module_route_goodbye(
7724 counters,
7725 &target.sink,
7726 frame,
7727 target.module_id.as_deref(),
7728 context,
7729 );
7730}
7731
7732pub(crate) fn send_route_control_pushes(
7733 forwarding: &ForwardingTable,
7734 routes: Vec<EndpointRoute>,
7735 push: ClientControlPush,
7736) {
7737 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7738 for route in routes {
7739 let target = route.goodbye_target;
7740 if let Some((existing, channels)) = targets
7741 .iter_mut()
7742 .find(|(existing, _)| existing.connection_id == target.connection_id)
7743 {
7744 debug_assert_eq!(
7745 existing.negotiated_ver, target.negotiated_ver,
7746 "one connection cannot negotiate multiple frame versions"
7747 );
7748 if !channels.contains(&target.channel) {
7749 channels.push(target.channel);
7750 }
7751 continue;
7752 }
7753 let channel = target.channel;
7754 targets.push((target, vec![channel]));
7755 }
7756 for (target, mut channels) in targets {
7757 channels.sort_unstable();
7758 let mut push = push.clone();
7759 match &mut push {
7760 ClientControlPush::RouteClosing {
7761 channels: covered, ..
7762 }
7763 | ClientControlPush::RouteClosed {
7764 channels: covered, ..
7765 } => *covered = channels,
7766 }
7767 let body = match serde_json::to_vec(&push) {
7768 Ok(body) => body,
7769 Err(err) => {
7770 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7771 continue;
7772 }
7773 };
7774 let frame = match Frame::build_with_version(
7775 target.negotiated_ver,
7776 FrameType::Push,
7777 control_flags(),
7778 0,
7779 0,
7780 0,
7781 body.clone(),
7782 ) {
7783 Ok(frame) => frame,
7784 Err(err) => {
7785 warn!(
7786 route_channel = target.channel,
7787 error = %err,
7788 "failed to build route lifecycle control PUSH frame"
7789 );
7790 continue;
7791 }
7792 };
7793 if let Err(err) = target.sink.try_send(frame) {
7794 if target.close_on_delivery_failure() {
7795 warn!(
7796 target_connection_id = target.connection_id.get(),
7797 route_channel = target.channel,
7798 error = %err,
7799 "route lifecycle control PUSH was not delivered to client; closing target connection"
7800 );
7801 let _ = forwarding.escalate_client_delivery_failure(
7802 target.connection_id,
7803 target.channel,
7804 target.epoch,
7805 CloseReason::new(
7806 "route_lifecycle_push_delivery_failed",
7807 format!(
7808 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7809 target.channel
7810 ),
7811 ),
7812 crate::forwarding::UndeliveredFrame {
7813 module_id: target.module_id.as_deref(),
7814 sink: &target.sink,
7815 },
7816 );
7817 }
7818 }
7819 }
7820}
7821
7822#[cfg(test)]
7823mod tests {
7824 mod activity_read {
7825 include!("activity_read_tests.rs");
7826 }
7827
7828 mod lifecycle_activity {
7829 include!("control_lifecycle_tests.rs");
7830 }
7831 #[cfg(unix)]
7832 #[tokio::test]
7833 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7834 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7835 let path = dir.join("subc.jsonc");
7836 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7837 "program":"/bin/sleep","args":["60"],"protocol":"none",
7838 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7839 }}}).to_string()).unwrap();
7840 let mut configured = crate::daemon_config::load(&path)
7841 .unwrap()
7842 .unwrap()
7843 .modules
7844 .pop()
7845 .unwrap();
7846 let registry = std::sync::Arc::new(crate::Registry::default());
7847 let handle = crate::SupervisorHandle::new();
7848 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7849 .with_handle(handle.clone());
7850 let module = supervisor
7851 .supervise_configured_with_health(
7852 configured.module_spec(),
7853 true,
7854 configured.health.clone(),
7855 None,
7856 configured.restart,
7857 )
7858 .unwrap();
7859 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7860 let before = module.status().unwrap().pid;
7861 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7862 configured.health.cadence = std::time::Duration::from_secs(3600);
7863 let health_only = handler
7864 .reconcile_supervised_modules(
7865 &supervisor,
7866 vec![configured.clone()],
7867 false,
7868 Principal::Direct,
7869 )
7870 .await
7871 .unwrap();
7872 assert!(
7873 health_only.0.changed_pending_reload.is_empty(),
7874 "health policy is already applied live"
7875 );
7876 assert_eq!(module.status().unwrap().pid, before);
7877 assert_eq!(
7878 module.configuration().unwrap().1.http,
7879 configured.health.http
7880 );
7881 configured.args = vec!["61".into()];
7882 let launch = handler
7883 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7884 .await
7885 .unwrap();
7886 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
7887 assert_eq!(
7888 module.status().unwrap().pid,
7889 before,
7890 "a launch edit is stored until reload"
7891 );
7892 module.drain().await.unwrap();
7893 }
7894 use cortexkit_test_support::ScratchDir;
7895 use std::{
7896 collections::BTreeMap,
7897 fmt,
7898 path::PathBuf,
7899 sync::{Arc, Mutex},
7900 time::Duration,
7901 };
7902
7903 use serde_json::{json, Value};
7904 use subc_protocol::{
7905 manifest::{
7906 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7907 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7908 },
7909 session::HealthStatus,
7910 FrameType,
7911 };
7912
7913 use super::*;
7914 use crate::{
7915 forwarding::{DataRoute, DataRouteState},
7916 registry::ChannelState,
7917 router::FrameSink,
7918 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7919 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7920 RouteCtx, Router,
7921 };
7922 use tokio::{
7923 sync::mpsc,
7924 time::{sleep, Instant},
7925 };
7926 use tracing::{
7927 field::{Field, Visit},
7928 Event, Subscriber,
7929 };
7930 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7931
7932 fn fake_aft_stub_path() -> PathBuf {
7951 let mut path = std::env::current_exe().expect("current_exe available in tests");
7952 path.pop(); path.pop(); path.push(if cfg!(windows) {
7955 "fake-aft-stub.exe"
7956 } else {
7957 "fake-aft-stub"
7958 });
7959 assert!(
7960 path.exists(),
7961 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7962 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7963 path.display()
7964 );
7965 path
7966 }
7967
7968 fn client_retries(code: &str) -> bool {
7973 subc_protocol::error_codes::is_retryable_route_open(code)
7974 }
7975
7976 #[test]
7987 fn retryability_of_forwarding_codes_matches_the_failure() {
7988 let transient = [
7991 ForwardingError::NoModuleConnection,
7992 ForwardingError::ModuleReloading {
7993 module_id: "m".into(),
7994 },
7995 ForwardingError::StaleModuleEndpoint,
7996 ForwardingError::UnknownReservation {
7997 client_channel: 1,
7998 module_channel: 1,
7999 },
8000 ForwardingError::ConnectionClosing {
8001 connection_id: ConnectionId::new(1),
8002 },
8003 ForwardingError::ClientEgressClosed {
8004 connection_id: ConnectionId::new(1),
8005 },
8006 ForwardingError::ModuleEgressUnavailable {
8007 connection_id: ConnectionId::new(1),
8008 },
8009 ];
8010 for err in transient {
8011 let code = forwarding_error_code(&err);
8012 assert!(
8013 client_retries(code),
8014 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8015 );
8016 }
8017
8018 let permanent = [
8023 ForwardingError::ConnectionRoleConflict {
8024 connection_id: ConnectionId::new(1),
8025 },
8026 ForwardingError::ClientRouteChannelExhausted {
8027 connection_id: ConnectionId::new(1),
8028 },
8029 ForwardingError::ModuleRouteChannelExhausted {
8030 endpoint: ModuleEndpointId {
8031 connection_id: ConnectionId::new(1),
8032 generation: 1,
8033 },
8034 },
8035 ForwardingError::RelayCorrelationExhausted,
8036 ForwardingError::RouteOpenBuild("x".into()),
8037 ForwardingError::Poisoned,
8038 ];
8039 for err in permanent {
8040 let code = forwarding_error_code(&err);
8041 assert!(
8042 !client_retries(code),
8043 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8044 );
8045 }
8046 }
8047
8048 #[tokio::test]
8076 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8077 let handler = ControlHandler::default();
8078 let frame =
8079 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8080
8081 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8085 assert_eq!(
8086 stamped,
8087 Principal::Direct,
8088 "a caller that proved nothing must not be stamped as a supervised module"
8089 );
8090
8091 let forged = handler
8096 .route_open_principal(
8097 &frame,
8098 Some(ConsumerIdentity {
8099 module_id: "aft".to_string(),
8100 launch_nonce: "not-a-real-nonce".to_string(),
8101 }),
8102 )
8103 .unwrap();
8104 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8105 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8106 }
8107
8108 #[test]
8125 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8126 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8127 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8128 let ClientControlRequest::RouteOpen {
8129 consumer_identity, ..
8130 } = parsed
8131 else {
8132 panic!("route.open body must parse as RouteOpen");
8133 };
8134 assert_eq!(
8135 consumer_identity,
8136 Some(ConsumerIdentity {
8137 module_id: "aft".to_string(),
8138 launch_nonce: "n".to_string(),
8139 }),
8140 "the wire field name must reach the value route_open_principal reads"
8141 );
8142 }
8143
8144 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8145 ModuleManifest::builder(module_id, "0.1.0")
8146 .protocol_ver(protocol_ver)
8147 .provides(vec![ProviderRole::ToolProvider {
8148 tools: vec![Tool {
8149 name: "read".to_string(),
8150 description: None,
8151 execution_mode: ExecutionMode::Pure,
8152 schema: json!({"type": "object"}),
8153 }],
8154 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8155 concurrency: Concurrency::ModuleManaged,
8156 emits_push: true,
8157 sub_supervises: true,
8158 }])
8159 .build()
8160 }
8161
8162 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8163 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8164 }
8165
8166 fn hello_frame_with_control_ops(
8167 module_id: &str,
8168 protocol_ver: u8,
8169 corr: u64,
8170 control_ops: Option<Vec<String>>,
8171 ) -> Frame {
8172 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8173 }
8174
8175 fn hello_frame_with_nonce(
8176 module_id: &str,
8177 protocol_ver: u8,
8178 corr: u64,
8179 launch_nonce: Option<&str>,
8180 ) -> Frame {
8181 hello_frame_full(
8182 module_id,
8183 protocol_ver,
8184 corr,
8185 None,
8186 launch_nonce.map(ToOwned::to_owned),
8187 )
8188 }
8189
8190 fn hello_frame_full(
8191 module_id: &str,
8192 protocol_ver: u8,
8193 corr: u64,
8194 control_ops: Option<Vec<String>>,
8195 launch_nonce: Option<String>,
8196 ) -> Frame {
8197 let body = serde_json::to_vec(&ModuleHelloBody {
8198 manifest: manifest(module_id, protocol_ver),
8199 protocol_ver,
8200 control_ops,
8201 launch_nonce,
8202 })
8203 .unwrap();
8204 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8205 }
8206
8207 fn hello_frame_with_rich_ui(
8208 module_id: &str,
8209 protocol_ver: u8,
8210 corr: u64,
8211 launch_nonce: Option<&str>,
8212 rich_ui: Value,
8213 ) -> Frame {
8214 let mut body = serde_json::to_value(ModuleHelloBody {
8215 manifest: manifest(module_id, protocol_ver),
8216 protocol_ver,
8217 control_ops: None,
8218 launch_nonce: launch_nonce.map(str::to_string),
8219 })
8220 .unwrap();
8221 body["manifest"]["rich_ui"] = rich_ui;
8222 Frame::build(
8223 FrameType::Hello,
8224 control_flags(),
8225 0,
8226 0,
8227 corr,
8228 serde_json::to_vec(&body).unwrap(),
8229 )
8230 .unwrap()
8231 }
8232
8233 fn non_routable_hello_frame_with_control_ops(
8234 module_id: &str,
8235 corr: u64,
8236 control_ops: Option<Vec<String>>,
8237 ) -> Frame {
8238 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8239 manifest.provides.clear();
8240 let body = serde_json::to_vec(&ModuleHelloBody {
8241 manifest,
8242 protocol_ver: PROTOCOL_VERSION,
8243 control_ops,
8244 launch_nonce: None,
8245 })
8246 .unwrap();
8247 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8248 }
8249
8250 fn capability_grammar_hello_frame(
8251 capabilities: Value,
8252 runtime_computed: Option<Value>,
8253 corr: u64,
8254 ) -> Frame {
8255 let mut body = serde_json::to_value(ModuleHelloBody {
8256 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8257 protocol_ver: PROTOCOL_VERSION,
8258 control_ops: None,
8259 launch_nonce: None,
8260 })
8261 .expect("HELLO body serializes");
8262 body["manifest"]["capabilities"] = capabilities;
8263 if let Some(runtime_computed) = runtime_computed {
8264 body["runtime_computed"] = runtime_computed;
8265 }
8266 Frame::build(
8267 FrameType::Hello,
8268 control_flags(),
8269 0,
8270 0,
8271 corr,
8272 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8273 )
8274 .expect("HELLO frame builds")
8275 }
8276
8277 fn channel_request(channel: u16, corr: u64) -> Frame {
8278 Frame::build(
8279 FrameType::Request,
8280 Flags::new(true, Priority::Interactive, false),
8281 channel,
8282 0,
8283 corr,
8284 b"opaque".to_vec(),
8285 )
8286 .unwrap()
8287 }
8288
8289 fn route_ctx(
8290 connection_id: ConnectionId,
8291 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8292 let (tx, rx) = mpsc::channel(8);
8293 (
8294 RouteCtx {
8295 connection_id,
8296 egress: FrameSink::new(tx),
8297 },
8298 rx,
8299 )
8300 }
8301
8302 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8303 serde_json::from_slice(&frame.body).unwrap()
8304 }
8305
8306 async fn hello_via_sink(
8311 handler: &ControlHandler,
8312 ctx: &RouteCtx,
8313 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8314 hello: Frame,
8315 ) -> Frame {
8316 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8317 assert!(
8318 replies.is_empty(),
8319 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8320 );
8321 let ack = rx
8322 .try_recv()
8323 .expect("HELLO_ACK is queued on the module sink")
8324 .frame;
8325 assert_eq!(ack.header.ty, FrameType::HelloAck);
8326 ack
8327 }
8328
8329 fn parse_error(frame: &Frame) -> Value {
8330 serde_json::from_slice(&frame.body).unwrap()
8331 }
8332
8333 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8334 serde_json::from_slice(&frame.body).unwrap()
8335 }
8336
8337 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8338 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8339 route_channel,
8340 route_epoch: 0,
8341 kind,
8342 })
8343 .unwrap();
8344 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8345 }
8346
8347 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8348 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8349 module_id: module_id.to_string(),
8350 })
8351 .unwrap();
8352 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8353 }
8354
8355 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8356 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8357 }
8358
8359 fn route_open_frame_with_consumer_capabilities(
8360 corr: u64,
8361 module_id: &str,
8362 project_root: ScratchDir,
8363 consumer_capabilities: Option<Vec<String>>,
8364 ) -> Frame {
8365 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8366 target: RouteTarget::ToolProvider {
8367 module_id: module_id.to_string(),
8368 },
8369 identity: BindIdentity::new(
8370 project_root.path().to_path_buf(),
8371 "unit".to_string(),
8372 "session".to_string(),
8373 ),
8374 consumer_identity: None,
8375 consumer_capabilities,
8376 role_versions: None,
8377 admission_facts: None,
8378 scope: None,
8379 })
8380 .unwrap();
8381 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8382 }
8383
8384 fn route_open_frame_with_role_versions(
8385 corr: u64,
8386 module_id: &str,
8387 project_root: ScratchDir,
8388 role_versions: Option<BTreeMap<String, String>>,
8389 ) -> Frame {
8390 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8391 target: RouteTarget::ToolProvider {
8392 module_id: module_id.to_string(),
8393 },
8394 identity: BindIdentity::new(
8395 project_root.path().to_path_buf(),
8396 "unit".to_string(),
8397 format!("session-{corr}"),
8398 ),
8399 consumer_identity: None,
8400 consumer_capabilities: None,
8401 role_versions,
8402 admission_facts: None,
8403 scope: None,
8404 })
8405 .unwrap();
8406 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8407 }
8408
8409 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8410 entries
8411 .iter()
8412 .map(|(role, version)| (role.to_string(), version.to_string()))
8413 .collect()
8414 }
8415
8416 fn route_open_frame_with_admission_facts(
8417 corr: u64,
8418 module_id: &str,
8419 project_root: ScratchDir,
8420 consumer_identity: Option<subc_control::ConsumerIdentity>,
8421 facts: Option<Value>,
8422 ) -> Frame {
8423 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8424 target: RouteTarget::ToolProvider {
8425 module_id: module_id.to_string(),
8426 },
8427 identity: BindIdentity::new(
8428 project_root.path().to_path_buf(),
8429 "unit".to_string(),
8430 format!("session-{corr}"),
8431 ),
8432 consumer_identity,
8433 consumer_capabilities: None,
8434 role_versions: None,
8435 admission_facts: facts,
8436 scope: None,
8437 })
8438 .unwrap();
8439 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8440 }
8441
8442 #[derive(Clone, Default)]
8443 struct EventCapture {
8444 events: Arc<Mutex<Vec<CapturedEvent>>>,
8445 }
8446
8447 #[derive(Clone, Debug)]
8448 struct CapturedEvent {
8449 target: String,
8450 level: tracing::Level,
8451 fields: BTreeMap<String, String>,
8452 }
8453
8454 impl EventCapture {
8455 fn events(&self) -> Vec<CapturedEvent> {
8456 self.events.lock().unwrap().clone()
8457 }
8458 }
8459
8460 impl<S> Layer<S> for EventCapture
8461 where
8462 S: Subscriber,
8463 {
8464 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8465 let mut visitor = EventFieldVisitor::default();
8466 event.record(&mut visitor);
8467 self.events.lock().unwrap().push(CapturedEvent {
8468 target: event.metadata().target().to_string(),
8469 level: *event.metadata().level(),
8470 fields: visitor.fields,
8471 });
8472 }
8473 }
8474
8475 #[derive(Default)]
8476 struct EventFieldVisitor {
8477 fields: BTreeMap<String, String>,
8478 }
8479
8480 impl Visit for EventFieldVisitor {
8481 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8482 self.fields
8483 .insert(field.name().to_string(), format!("{value:?}"));
8484 }
8485 }
8486
8487 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8488 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8489 status,
8490 detail: Some("warming".to_string()),
8491 metrics: Some(json!({"queue_depth": 3})),
8492 })
8493 .unwrap();
8494 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8495 }
8496
8497 fn route_bind_ack(corr: u64) -> Frame {
8498 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8499 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8500 }
8501
8502 fn unique_project_root(label: &str) -> ScratchDir {
8503 ScratchDir::new(label)
8504 }
8505
8506 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8507 match parse_route_poll(frame) {
8508 ClientControlResponse::RoutePoll {
8509 status: None,
8510 live: Some(live),
8511 ..
8512 } => assert_eq!(live, expected_live),
8513 other => panic!("unexpected route.poll response: {other:?}"),
8514 }
8515 }
8516
8517 fn bind_liveness_route(
8518 registry: &Registry,
8519 forwarding: &ForwardingTable,
8520 module_id: &str,
8521 ) -> (RouteCtx, u16, u32) {
8522 let module_connection = ConnectionId::new(101);
8523 let client_connection = ConnectionId::new(202);
8524 let registration = registry
8525 .register_with_control_ops(
8526 manifest(module_id, PROTOCOL_VERSION),
8527 PROTOCOL_VERSION,
8528 module_connection,
8529 module_baseline_control_ops(),
8530 )
8531 .unwrap();
8532 let (module_tx, _module_rx) = mpsc::channel(8);
8533 let endpoint = forwarding
8534 .register_module_connection(
8535 module_connection,
8536 module_id.to_string(),
8537 PROTOCOL_VERSION,
8538 manifest_concurrency(®istration.manifest),
8539 FrameSink::new(module_tx),
8540 )
8541 .unwrap();
8542 let (client_ctx, _client_rx) = route_ctx(client_connection);
8543 let pending = forwarding
8544 .begin_route_bind_relay_for_test(
8545 client_connection,
8546 client_ctx.egress.clone(),
8547 1,
8548 module_id,
8549 )
8550 .unwrap();
8551 assert_eq!(pending.endpoint, endpoint);
8552 let route_channel = pending.client_channel;
8553 let route_epoch = pending.client_epoch;
8554 forwarding
8555 .complete_pending_relay(
8556 module_connection,
8557 pending.corr,
8558 RouteBindRelayOutcome::Accepted,
8559 )
8560 .unwrap();
8561 (client_ctx, route_channel, route_epoch)
8562 }
8563
8564 struct FakeProcessLiveness {
8565 live: Option<bool>,
8566 }
8567
8568 impl ModuleProcessLiveness for FakeProcessLiveness {
8569 fn process_live(&self, _module_id: &str) -> Option<bool> {
8570 self.live
8571 }
8572 }
8573
8574 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8575 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8576 {
8577 let registry = Arc::new(Registry::default());
8578 let supervisor_handle = SupervisorHandle::new();
8579 let supervisor = Supervisor::new_for_test(
8580 Arc::clone(®istry),
8581 RestartPolicy::new(1, Duration::from_millis(10)),
8582 )
8583 .with_handle(supervisor_handle.clone());
8584 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8585 let module = supervisor
8586 .spawn(ModuleSpec {
8587 module_id: "stderr-tail-wire".to_string(),
8588 program: fake_aft_stub_path(),
8589 args: Vec::new(),
8590 env: vec![
8591 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8592 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8593 ],
8594 reserved: false,
8595 reserved_prefixes: Vec::new(),
8596 protocol: ModuleProtocol::Subc,
8597 overlap: Default::default(),
8598 })
8599 .unwrap();
8600
8601 let deadline = Instant::now() + Duration::from_secs(5);
8602 loop {
8603 let tail = module.stderr_tail(None, None);
8604 if tail
8605 .entries
8606 .iter()
8607 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8608 && tail.entries.iter().any(|entry| {
8609 matches!(
8610 entry,
8611 TailEntry::Line {
8612 truncated: true,
8613 ..
8614 }
8615 )
8616 })
8617 {
8618 break;
8619 }
8620 assert!(
8621 Instant::now() < deadline,
8622 "module did not produce a truncated line and restart boundary: {tail:?}"
8623 );
8624 sleep(Duration::from_millis(10)).await;
8625 }
8626
8627 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8628 let request = ClientControlRequest::SupervisorStderrTail {
8629 module_id: "stderr-tail-wire".to_string(),
8630 max_lines: None,
8631 max_bytes: None,
8632 };
8633 let frame = Frame::build(
8634 FrameType::Request,
8635 control_flags(),
8636 0,
8637 0,
8638 1,
8639 serde_json::to_vec(&request).unwrap(),
8640 )
8641 .unwrap();
8642 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8643 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8644 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8645 serde_json::from_slice(&responses[0].body).unwrap()
8646 else {
8647 panic!("expected supervisor.stderr_tail response");
8648 };
8649
8650 assert!(
8651 tail.entries
8652 .iter()
8653 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8654 "the control response lost the restart boundary"
8655 );
8656 let Some(StderrTailEntry::Line {
8657 text,
8658 truncated,
8659 at_ms,
8660 }) = tail.entries.iter().find(|entry| {
8661 matches!(
8662 entry,
8663 StderrTailEntry::Line {
8664 truncated: true,
8665 ..
8666 }
8667 )
8668 })
8669 else {
8670 panic!("the control response lost the truncated line");
8671 };
8672 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8673 assert!(*truncated);
8674 assert!(
8675 at_ms.is_some(),
8676 "the control response lost the line's capture time"
8677 );
8678 }
8679
8680 #[tokio::test(flavor = "current_thread")]
8685 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8686 let dir = ScratchDir::new("terminals-off-worker");
8687 let journal_path = dir.join("terminals.jsonl");
8688 let registry = Arc::new(Registry::default());
8689 let supervisor_handle = SupervisorHandle::new();
8690 let supervisor =
8691 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8692 .with_handle(supervisor_handle.clone())
8693 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8694 let module = supervisor
8695 .spawn(ModuleSpec {
8696 module_id: "terminal-off-worker".to_string(),
8697 program: fake_aft_stub_path(),
8698 args: Vec::new(),
8699 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8700 reserved: false,
8701 reserved_prefixes: Vec::new(),
8702 protocol: ModuleProtocol::Subc,
8703 overlap: Default::default(),
8704 })
8705 .unwrap();
8706 let deadline = Instant::now() + Duration::from_secs(5);
8707 while module.terminal_history().entries.len() != 2 {
8708 assert!(Instant::now() < deadline, "module did not record two exits");
8709 sleep(Duration::from_millis(10)).await;
8710 }
8711
8712 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8713 let handler =
8714 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8715 let frame = Frame::build(
8716 FrameType::Request,
8717 control_flags(),
8718 0,
8719 0,
8720 1,
8721 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8722 module_id: "terminal-off-worker".to_string(),
8723 })
8724 .unwrap(),
8725 )
8726 .unwrap();
8727 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8728 let spawned_at = std::time::Instant::now();
8729 let read = tokio::spawn({
8730 let handler = Arc::clone(&handler);
8731 async move { handler.handle_control_frame(&ctx, frame).await }
8732 });
8733 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8736 .await
8737 .unwrap()
8738 .expect("the history read reached its pause");
8739 let elapsed = spawned_at.elapsed();
8740 assert!(
8741 elapsed < Duration::from_secs(2) && !read.is_finished(),
8742 "this task could not run while the history read was paused \
8743 (resumed after {elapsed:?}, read finished: {})",
8744 read.is_finished()
8745 );
8746
8747 drop(release);
8748 let responses = read.await.unwrap().unwrap();
8749 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8750 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8751 panic!("expected supervisor.terminals response");
8752 };
8753 assert_eq!(terminals.entries.len(), 2);
8754 assert_eq!(terminals.journal_skipped_lines, 0);
8755 assert_eq!(terminals.journal_read_errors, 0);
8756 }
8757
8758 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8759 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8760 let registry = Arc::new(Registry::default());
8761 let supervisor_handle = SupervisorHandle::new();
8762 let supervisor =
8763 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8764 .with_handle(supervisor_handle.clone());
8765 let module = supervisor
8766 .spawn(ModuleSpec {
8767 module_id: "terminal-golden".to_string(),
8768 program: fake_aft_stub_path(),
8769 args: Vec::new(),
8770 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8771 reserved: false,
8772 reserved_prefixes: Vec::new(),
8773 protocol: ModuleProtocol::Subc,
8774 overlap: Default::default(),
8775 })
8776 .unwrap();
8777
8778 let deadline = Instant::now() + Duration::from_secs(5);
8779 while module.terminal_history().entries.len() != 2 {
8780 assert!(
8781 Instant::now() < deadline,
8782 "module did not retain two terminal exits: {:?}",
8783 module.terminal_history()
8784 );
8785 sleep(Duration::from_millis(10)).await;
8786 }
8787
8788 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8789 let request = ClientControlRequest::SupervisorTerminals {
8790 module_id: "terminal-golden".to_string(),
8791 };
8792 let frame = Frame::build(
8793 FrameType::Request,
8794 control_flags(),
8795 0,
8796 0,
8797 1,
8798 serde_json::to_vec(&request).unwrap(),
8799 )
8800 .unwrap();
8801 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8802 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8803 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8804 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8805 panic!("expected supervisor.terminals response");
8806 };
8807 assert_eq!(terminals.entries.len(), 2);
8808 assert_eq!(terminals.dropped, 0);
8809
8810 let mut rendered = serde_json::to_value(response).unwrap();
8811 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8814 for (index, entry) in rendered["entries"]
8815 .as_array_mut()
8816 .expect("terminal response entries array")
8817 .iter_mut()
8818 .enumerate()
8819 {
8820 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8821 }
8822
8823 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8824 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8825 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8826 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8827 std::fs::write(&golden_path, &serialized).unwrap();
8828 }
8829 let expected: Value =
8830 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8831 assert_eq!(rendered, expected);
8832 }
8833
8834 #[test]
8835 fn hello_registers_manifest_and_returns_ack() {
8836 let registry = Arc::new(Registry::default());
8837 let handler = ControlHandler::new(Arc::clone(®istry));
8838 let conn = ConnectionId::new(1);
8839
8840 let responses = handler
8841 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8842 .unwrap();
8843
8844 assert_eq!(responses.len(), 1);
8845 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8846 assert_eq!(responses[0].header.channel, 0);
8847 assert_eq!(responses[0].header.corr, 7);
8848 let ack = parse_ack(&responses[0]);
8849 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8850 assert!(ack
8851 .subc_capabilities
8852 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8853 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8854 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8855 assert!(ack
8856 .subc_ops
8857 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8858 assert!(ack
8859 .subc_ops
8860 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8861
8862 let registration = registry.get_module("aft").unwrap().unwrap();
8863 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8864 assert_eq!(registration.state, ChannelState::Active);
8865 assert_eq!(registration.connection_id, conn);
8866 assert_eq!(registration.control_ops, module_baseline_control_ops());
8867 }
8868
8869 #[test]
8870 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8871 let invalid_identifiers = [
8872 ("case_change", "credentials-Provider/v1"),
8873 ("leading_zero", "credentials-provider/v01"),
8874 ("trailing_hyphen", "credentials-provider-/v1"),
8875 ("consecutive_hyphens", "credentials--provider/v1"),
8876 ("uppercase", "Credentials-provider/v1"),
8877 ("missing_v", "credentials-provider/1"),
8878 ("whitespace", "credentials provider/v1"),
8879 ("zero_version", "credentials-provider/v0"),
8880 ("out_of_range_version", "credentials-provider/v4294967296"),
8881 (
8882 "overlength_name",
8883 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8884 ),
8885 ];
8886 let mut cases = invalid_identifiers
8887 .into_iter()
8888 .map(|(name, identifier)| {
8889 (
8890 format!("identifier_{name}"),
8891 "capabilities.provides[0]".to_string(),
8892 identifier.to_string(),
8893 json!({ "provides": [identifier] }),
8894 None,
8895 )
8896 })
8897 .collect::<Vec<_>>();
8898 cases.extend([
8899 (
8900 "unknown_need".to_string(),
8901 "capabilities.requires[0].need".to_string(),
8902 "deferred".to_string(),
8903 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8904 None,
8905 ),
8906 (
8907 "duplicate_provides".to_string(),
8908 "capabilities.provides[1]".to_string(),
8909 "credentials-provider/v1".to_string(),
8910 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8911 None,
8912 ),
8913 (
8914 "duplicate_must_never_reach".to_string(),
8915 "capabilities.must_never_reach[1]".to_string(),
8916 "credentials-provider/v1".to_string(),
8917 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8918 None,
8919 ),
8920 (
8921 "duplicate_requires_same_need".to_string(),
8922 "capabilities.requires[1]".to_string(),
8923 "credentials-provider/v1".to_string(),
8924 json!({ "requires": [
8925 { "capability": "credentials-provider/v1", "need": "required" },
8926 { "capability": "credentials-provider/v1", "need": "required" }
8927 ] }),
8928 None,
8929 ),
8930 (
8931 "duplicate_requires_conflicting_need".to_string(),
8932 "capabilities.requires[1]".to_string(),
8933 "credentials-provider/v1".to_string(),
8934 json!({ "requires": [
8935 { "capability": "credentials-provider/v1", "need": "required" },
8936 { "capability": "credentials-provider/v1", "need": "optional" }
8937 ] }),
8938 None,
8939 ),
8940 (
8941 "capabilities_root_pointer".to_string(),
8942 "runtime_computed[0]".to_string(),
8943 "/capabilities".to_string(),
8944 json!({}),
8945 Some(json!(["/capabilities"])),
8946 ),
8947 (
8948 "capabilities_descendant_pointer".to_string(),
8949 "runtime_computed[0]".to_string(),
8950 "/capabilities/provides".to_string(),
8951 json!({}),
8952 Some(json!(["/capabilities/provides"])),
8953 ),
8954 (
8955 "malformed_pointer_without_leading_slash".to_string(),
8956 "runtime_computed[0]".to_string(),
8957 "capabilities".to_string(),
8958 json!({}),
8959 Some(json!(["capabilities"])),
8960 ),
8961 (
8962 "malformed_pointer_escape".to_string(),
8963 "runtime_computed[0]".to_string(),
8964 "/roles/~2/tools".to_string(),
8965 json!({}),
8966 Some(json!(["/roles/~2/tools"])),
8967 ),
8968 (
8969 "unknown_capabilities_field".to_string(),
8970 "capabilities.future".to_string(),
8971 "<array>".to_string(),
8972 json!({ "future": [] }),
8973 None,
8974 ),
8975 ]);
8976
8977 for (index, (name, field, value, capabilities, runtime_computed)) in
8978 cases.into_iter().enumerate()
8979 {
8980 let registry = Arc::new(Registry::default());
8981 let handler = ControlHandler::new(Arc::clone(®istry));
8982 let response = handler
8983 .handle_control(
8984 ConnectionId::new((index + 1) as u64),
8985 capability_grammar_hello_frame(
8986 capabilities,
8987 runtime_computed,
8988 index as u64 + 1,
8989 ),
8990 )
8991 .expect("invalid HELLO returns a refusal");
8992
8993 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8994 let error = parse_error(&response[0]);
8995 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8996 let message = error["message"]
8997 .as_str()
8998 .expect("error message is a string");
8999 assert!(
9000 message.contains(&field),
9001 "{name}: field missing from {message}"
9002 );
9003 assert!(
9004 message.contains(&value),
9005 "{name}: value missing from {message}"
9006 );
9007 assert_eq!(
9008 registry
9009 .active_registration_count()
9010 .expect("registry reads"),
9011 0,
9012 "{name}: refused HELLO must not create a catalog entry"
9013 );
9014 }
9015 }
9016
9017 #[test]
9018 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9019 let registry = Arc::new(Registry::default());
9020 let handler = ControlHandler::new(Arc::clone(®istry));
9021 let capabilities = json!({
9022 "provides": ["credentials-provider/v1"],
9023 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9024 "must_never_reach": ["federation-transport/v1"]
9025 });
9026 let response = handler
9027 .handle_control(
9028 ConnectionId::new(99),
9029 capability_grammar_hello_frame(
9030 capabilities.clone(),
9031 Some(json!(["/roles/0/tools"])),
9032 99,
9033 ),
9034 )
9035 .expect("valid HELLO registers");
9036 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9037
9038 let request = Frame::build(
9039 FrameType::Request,
9040 control_flags(),
9041 0,
9042 0,
9043 100,
9044 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9045 .expect("catalog request serializes"),
9046 )
9047 .expect("catalog request frame builds");
9048 let response = handler
9049 .handle_catalog_list(request, None)
9050 .expect("catalog list succeeds");
9051 let ClientControlResponse::CatalogList { modules, .. } =
9052 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9053 else {
9054 panic!("catalog request must return catalog.list");
9055 };
9056 assert_eq!(modules.len(), 1);
9057 assert_eq!(
9058 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9059 capabilities
9060 );
9061 }
9062
9063 #[test]
9064 fn catalog_list_mirrors_management_operation_description() {
9065 let registry = Arc::new(Registry::default());
9066 let handler = ControlHandler::new(Arc::clone(®istry));
9067 let description = "List managed records and return their identifiers and metadata.";
9068 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9069 manifest.provides = vec![ProviderRole::ManagementSurface {
9070 operations: vec![ManagementOperation {
9071 name: "records.list".to_string(),
9072 kind: ManagementOperationKind::Query,
9073 description: Some(description.to_string()),
9074 }],
9075 config_schema: json!({"type": "object"}),
9076 observability: vec![ObservabilitySurface {
9077 name: "records.stats".to_string(),
9078 kind: ObservabilityKind::Snapshot,
9079 }],
9080 identity_scope: vec![IdentityScope::Project],
9081 concurrency: Concurrency::ModuleManaged,
9082 }];
9083 registry
9084 .register_with_control_ops(
9085 manifest,
9086 PROTOCOL_VERSION,
9087 ConnectionId::new(99),
9088 Vec::new(),
9089 )
9090 .expect("described management manifest registers");
9091
9092 let request = Frame::build(
9093 FrameType::Request,
9094 control_flags(),
9095 0,
9096 0,
9097 100,
9098 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9099 .expect("catalog request serializes"),
9100 )
9101 .expect("catalog request frame builds");
9102 let response = handler
9103 .handle_catalog_list(request, None)
9104 .expect("catalog list succeeds");
9105 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9106 assert_eq!(
9107 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9108 "catalog.list must preserve the declared operation description verbatim"
9109 );
9110 }
9111
9112 #[test]
9113 fn catalog_list_mirrors_registered_operation_delivery() {
9114 let registry = Arc::new(Registry::default());
9115 let handler = ControlHandler::new(Arc::clone(®istry));
9116 let delivery =
9117 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9118 let mut body = serde_json::to_value(ModuleHelloBody {
9119 manifest: manifest("watch-management", PROTOCOL_VERSION),
9120 protocol_ver: PROTOCOL_VERSION,
9121 control_ops: None,
9122 launch_nonce: None,
9123 })
9124 .unwrap();
9125 body["manifest"]["provides"] = json!([{
9126 "role": "management_surface",
9127 "operations": [
9128 {"name": "status", "kind": "query"},
9129 {"name": "future", "kind": "query"},
9130 {"name": "reset", "kind": "mutate"}
9131 ],
9132 "config_schema": {}, "observability": [], "identity_scope": []
9133 }]);
9134 body["manifest"]["operation_delivery"] = delivery.clone();
9135 let hello = Frame::build(
9136 FrameType::Hello,
9137 control_flags(),
9138 0,
9139 0,
9140 99,
9141 serde_json::to_vec(&body).unwrap(),
9142 )
9143 .unwrap();
9144 let response = handler
9145 .handle_control(ConnectionId::new(99), hello)
9146 .unwrap();
9147 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9148
9149 let (_, registrations) = registry.list_modules().unwrap();
9150 let registered = ®istrations[0].manifest;
9151 assert!(registered.is_watch_operation("status"));
9152 assert!(!registered.is_watch_operation("future"));
9153 assert!(!registered.is_watch_operation("reset"));
9154 assert!(!registered.is_watch_operation("missing"));
9155
9156 let request = Frame::build(
9157 FrameType::Request,
9158 control_flags(),
9159 0,
9160 0,
9161 100,
9162 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9163 )
9164 .unwrap();
9165 let response = handler.handle_catalog_list(request, None).unwrap();
9166 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9167 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9168 let ClientControlResponse::CatalogList { modules, .. } =
9169 serde_json::from_slice(&response[0].body).unwrap()
9170 else {
9171 panic!("expected catalog.list");
9172 };
9173 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9174 }
9175
9176 #[test]
9177 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9178 let registry = Arc::new(Registry::default());
9179 let supervisor = SupervisorHandle::new();
9180 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9181 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9182 let descriptor = json!({
9183 "schema": "ckios.richui.manifest/1",
9184 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9185 "bytes": 4096,
9186 "chunks": 4
9187 });
9188 let connection = ConnectionId::new(201);
9189 let hello = handler
9190 .handle_control(
9191 connection,
9192 hello_frame_with_rich_ui(
9193 "vault",
9194 PROTOCOL_VERSION,
9195 201,
9196 Some("the-real-nonce"),
9197 descriptor.clone(),
9198 ),
9199 )
9200 .unwrap();
9201 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9202
9203 let request =
9204 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9205 let provides = registry
9206 .get_module_by_connection(connection)
9207 .unwrap()
9208 .unwrap()
9209 .manifest
9210 .provides;
9211 handler
9212 .handle_catalog_update(connection, request, provides, None, Some(false))
9213 .unwrap();
9214
9215 let request = Frame::build(
9216 FrameType::Request,
9217 control_flags(),
9218 0,
9219 0,
9220 203,
9221 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9222 )
9223 .unwrap();
9224 let response = handler.handle_catalog_list(request, None).unwrap();
9225 let ClientControlResponse::CatalogList { modules, .. } =
9226 serde_json::from_slice(&response[0].body).unwrap()
9227 else {
9228 panic!("expected catalog.list");
9229 };
9230 assert_eq!(
9231 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9232 descriptor
9233 );
9234 assert!(modules[0].nonce_verified);
9235 }
9236
9237 #[test]
9238 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9239 let registry = Arc::new(Registry::default());
9240 let handler = ControlHandler::new(registry.clone());
9241 let descriptor = json!({
9242 "schema": "ckios.richui.manifest/1",
9243 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9244 "bytes": 128,
9245 "chunks": 2
9246 });
9247 let malformed = [
9248 ("schema", json!("")),
9249 ("schema", json!("a".repeat(65))),
9250 ("schema", json!("Bad schema")),
9251 (
9252 "manifest_digest",
9253 json!(format!("sha256:{}", "A".repeat(64))),
9254 ),
9255 ("manifest_digest", json!("sha256:abc")),
9256 ("bytes", json!(0)),
9257 ("bytes", json!(16_777_217)),
9258 ("bytes", json!("128")),
9259 ("chunks", json!(0)),
9260 ("chunks", json!(1025)),
9261 ("chunks", json!("2")),
9262 ];
9263 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9264 let mut invalid = descriptor.clone();
9265 invalid[field] = invalid_value;
9266 let response = handler
9267 .handle_control(
9268 ConnectionId::new(210 + index as u64),
9269 hello_frame_with_rich_ui(
9270 &format!("bad-rich-ui-{index}"),
9271 PROTOCOL_VERSION,
9272 210 + index as u64,
9273 None,
9274 invalid,
9275 ),
9276 )
9277 .unwrap();
9278 assert_eq!(response[0].header.ty, FrameType::Error);
9279 let error = parse_error(&response[0]);
9280 assert_eq!(error["code"], "invalid_manifest");
9281 assert!(error["message"]
9282 .as_str()
9283 .unwrap()
9284 .contains(&format!("rich_ui.{field}")));
9285 assert!(registry
9286 .get_module(&format!("bad-rich-ui-{index}"))
9287 .unwrap()
9288 .is_none());
9289 }
9290 }
9291
9292 #[test]
9293 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9294 let registry = Arc::new(Registry::default());
9295 let handler = ControlHandler::new(Arc::clone(®istry));
9296 let connection = ConnectionId::new(219);
9297 registry
9298 .register_with_control_ops(
9299 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9300 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9301 "bad schema",
9302 format!("sha256:{}", "a".repeat(64)),
9303 128,
9304 2,
9305 )))
9306 .build(),
9307 PROTOCOL_VERSION,
9308 connection,
9309 Vec::new(),
9310 )
9311 .unwrap();
9312 let request =
9313 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9314 let response = handler
9315 .handle_catalog_update(connection, request, Vec::new(), None, None)
9316 .unwrap();
9317 assert_eq!(response[0].header.ty, FrameType::Error);
9318 let error = parse_error(&response[0]);
9319 assert_eq!(error["code"], "invalid_manifest");
9320 assert!(error["message"]
9321 .as_str()
9322 .unwrap()
9323 .contains("rich_ui.schema"));
9324 }
9325
9326 #[test]
9327 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9328 let registry = Arc::new(Registry::default());
9329 let supervisor = SupervisorHandle::new();
9330 supervisor.open_swap("provider", "candidate-nonce".to_string());
9331 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9332 let connection = ConnectionId::new(218);
9333 let hello = handler
9334 .handle_control(
9335 connection,
9336 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9337 )
9338 .unwrap();
9339 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9340 assert!(registry.list_modules().unwrap().1.is_empty());
9341
9342 registry.promote_candidate("provider").unwrap().unwrap();
9343 let request = Frame::build(
9344 FrameType::Request,
9345 control_flags(),
9346 0,
9347 0,
9348 219,
9349 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9350 )
9351 .unwrap();
9352 let response = handler.handle_catalog_list(request, None).unwrap();
9353 let ClientControlResponse::CatalogList { modules, .. } =
9354 serde_json::from_slice(&response[0].body).unwrap()
9355 else {
9356 panic!("expected catalog.list");
9357 };
9358 assert_eq!(modules.len(), 1);
9359 assert!(modules[0].nonce_verified);
9360 }
9361
9362 #[test]
9363 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9364 let registry = Arc::new(Registry::default());
9365 let handler = ControlHandler::new(Arc::clone(®istry));
9366 let hello = Frame::build(
9367 FrameType::Hello,
9368 control_flags(),
9369 0,
9370 0,
9371 220,
9372 serde_json::to_vec(&ModuleHelloBody {
9373 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9374 protocol_ver: PROTOCOL_VERSION,
9375 control_ops: None,
9376 launch_nonce: Some("unverified".to_string()),
9377 })
9378 .unwrap(),
9379 )
9380 .unwrap();
9381 assert_eq!(
9382 handler
9383 .handle_control(ConnectionId::new(220), hello)
9384 .unwrap()[0]
9385 .header
9386 .ty,
9387 FrameType::HelloAck
9388 );
9389 let request = Frame::build(
9390 FrameType::Request,
9391 control_flags(),
9392 0,
9393 0,
9394 221,
9395 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9396 )
9397 .unwrap();
9398 let response = handler.handle_catalog_list(request, None).unwrap();
9399 let ClientControlResponse::CatalogList { modules, .. } =
9400 serde_json::from_slice(&response[0].body).unwrap()
9401 else {
9402 panic!("expected catalog.list");
9403 };
9404 assert_eq!(
9405 serde_json::to_vec(&modules[0]).unwrap(),
9406 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9407 );
9408 assert!(!modules[0].nonce_verified);
9409 }
9410
9411 #[test]
9412 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9413 let registry = Arc::new(Registry::default());
9414 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9415 [("vault".to_string(), true), ("squatter".to_string(), true)],
9416 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9417 );
9418 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9419 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9420 provides: vec!["credentials-provider/v1".to_string()],
9421 requires: Vec::new(),
9422 must_never_reach: Vec::new(),
9423 });
9424 let frame = Frame::build(
9425 FrameType::Hello,
9426 control_flags(),
9427 0,
9428 0,
9429 77,
9430 serde_json::to_vec(&ModuleHelloBody {
9431 manifest: squatter,
9432 protocol_ver: PROTOCOL_VERSION,
9433 control_ops: None,
9434 launch_nonce: None,
9435 })
9436 .expect("HELLO serializes"),
9437 )
9438 .expect("HELLO frame builds");
9439 let response = handler
9440 .handle_control(ConnectionId::new(77), frame)
9441 .expect("reserved claim receives a typed refusal");
9442 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9443 assert_eq!(
9444 registry
9445 .active_registration_count()
9446 .expect("registry reads"),
9447 0,
9448 "a reserved capability refusal must not leave a catalog entry"
9449 );
9450 }
9451
9452 #[test]
9453 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9454 for settlement in ["timeout", "inconclusive", "drop"] {
9455 let breakers = RouteBindBreakers::default();
9456 let RouteBindAdmission::Admitted {
9457 guard: mut old,
9458 probe: false,
9459 } = breakers.admit("prov")
9460 else {
9461 panic!("ordinary relay admitted")
9462 };
9463 let RouteBindAdmission::Admitted {
9464 guard: mut opener, ..
9465 } = breakers.admit("prov")
9466 else {
9467 panic!("second relay admitted")
9468 };
9469 assert!(
9470 !opener
9471 .record_timeout(1, Duration::ZERO)
9472 .unwrap()
9473 .reopened_after_probe
9474 );
9475 let RouteBindAdmission::Admitted {
9476 guard: mut probe,
9477 probe: true,
9478 } = breakers.admit("prov")
9479 else {
9480 panic!("one cooldown probe admitted")
9481 };
9482 match settlement {
9483 "timeout" => assert!(
9484 !old.record_timeout(1, Duration::ZERO)
9485 .unwrap()
9486 .reopened_after_probe
9487 ),
9488 "inconclusive" => old.record_inconclusive(),
9489 "drop" => drop(old),
9490 _ => unreachable!(),
9491 }
9492 assert!(
9493 matches!(
9494 breakers.admit("prov"),
9495 RouteBindAdmission::Refused {
9496 probe_in_flight: true,
9497 ..
9498 }
9499 ),
9500 "{settlement} of a pre-open relay cannot release the real probe"
9501 );
9502 assert!(
9503 probe
9504 .record_timeout(1, Duration::ZERO)
9505 .unwrap()
9506 .reopened_after_probe
9507 );
9508 assert!(matches!(
9509 breakers.admit("prov"),
9510 RouteBindAdmission::Admitted { probe: true, .. }
9511 ));
9512 }
9513 let breakers = RouteBindBreakers::default();
9514 let admit = || match breakers.admit("prov") {
9515 RouteBindAdmission::Admitted { guard, .. } => guard,
9516 _ => panic!("relay admitted"),
9517 };
9518 admit().record_timeout(1, Duration::ZERO);
9519 let mut old_probe = admit();
9520 breakers.reset_for_new_module_connection("prov");
9521 admit().record_timeout(1, Duration::ZERO);
9522 let _new_probe = admit();
9523 old_probe.record_inconclusive();
9524 assert!(matches!(
9525 breakers.admit("prov"),
9526 RouteBindAdmission::Refused {
9527 probe_in_flight: true,
9528 ..
9529 }
9530 ));
9531 }
9532
9533 #[tokio::test]
9534 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9535 for candidate in [false, true] {
9536 let registry = Arc::new(Registry::default());
9537 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9538 [("vault".to_string(), true), ("squatter".to_string(), true)],
9539 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9540 );
9541 let conn = ConnectionId::new(77);
9542 let (ctx, mut rx) = route_ctx(conn);
9543 let initial = capability_manifest("squatter", &[], &[]);
9544 if candidate {
9545 registry
9546 .register_candidate_with_control_ops(
9547 initial.clone(),
9548 PROTOCOL_VERSION,
9549 conn,
9550 module_baseline_control_ops(),
9551 )
9552 .unwrap();
9553 handler
9554 .forwarding
9555 .register_candidate_module_connection(
9556 conn,
9557 "squatter".to_string(),
9558 PROTOCOL_VERSION,
9559 manifest_concurrency(&initial),
9560 ctx.egress.clone(),
9561 )
9562 .unwrap();
9563 } else {
9564 hello_via_sink(
9565 &handler,
9566 &ctx,
9567 &mut rx,
9568 hello_frame_with_manifest(initial.clone(), 1),
9569 )
9570 .await;
9571 }
9572 let response = handler
9573 .handle_control_frame(
9574 &ctx,
9575 catalog_update_with_capabilities_frame(
9576 2,
9577 capability_manifest("squatter", &["credentials-provider/v1"], &[])
9578 .capabilities
9579 .unwrap(),
9580 ),
9581 )
9582 .await
9583 .unwrap();
9584 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9585 assert_eq!(
9586 registry
9587 .get_module_by_connection(conn)
9588 .unwrap()
9589 .unwrap()
9590 .manifest,
9591 initial
9592 );
9593 }
9594 }
9595
9596 #[test]
9597 fn server_describe_surfaces_required_capability_verdict_fields() {
9598 let registry = Arc::new(Registry::default());
9599 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9600 [
9601 ("consumer".to_string(), true),
9602 ("provider".to_string(), false),
9603 ],
9604 BTreeMap::new(),
9605 );
9606 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9607 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9608 provides: Vec::new(),
9609 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9610 capability: "credentials-provider/v1".to_string(),
9611 need: subc_protocol::manifest::CapabilityNeed::Required,
9612 }],
9613 must_never_reach: Vec::new(),
9614 });
9615 let hello = Frame::build(
9616 FrameType::Hello,
9617 control_flags(),
9618 0,
9619 0,
9620 78,
9621 serde_json::to_vec(&ModuleHelloBody {
9622 manifest: consumer,
9623 protocol_ver: PROTOCOL_VERSION,
9624 control_ops: None,
9625 launch_nonce: None,
9626 })
9627 .expect("HELLO serializes"),
9628 )
9629 .expect("HELLO frame builds");
9630 handler
9631 .handle_control(ConnectionId::new(78), hello)
9632 .expect("consumer registers");
9633 let describe = Frame::build(
9634 FrameType::Request,
9635 control_flags(),
9636 0,
9637 0,
9638 79,
9639 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9640 .expect("request serializes"),
9641 )
9642 .expect("describe frame builds");
9643 let response = handler
9644 .handle_server_describe(describe)
9645 .expect("server.describe succeeds");
9646 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9647 let requirement = &rendered["capability_requirements"][0];
9648 assert_eq!(requirement["consumer"], "consumer");
9649 assert_eq!(requirement["verdict"], "never_provided");
9650 assert_eq!(requirement["episode_seq"], 1);
9651 assert_eq!(requirement["config_satisfiable"], false);
9652 assert_eq!(requirement["runtime_available"], false);
9653 assert!(requirement["detail"]
9654 .as_str()
9655 .expect("detail string")
9656 .contains("credentials-provider/v1"));
9657 }
9658
9659 fn deregister_required_provider(
9665 daemon_shutdown_begun: bool,
9666 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
9667 const CAPABILITY: &str = "project-identity/v1";
9668 let registry = Arc::new(Registry::default());
9669 let daemon_shutdown = DaemonShutdownFlag::default();
9670 let handler = ControlHandler::new(Arc::clone(®istry))
9671 .with_capability_config(
9672 [
9673 ("consumer".to_string(), true),
9674 ("provider".to_string(), true),
9675 ],
9676 BTreeMap::new(),
9677 )
9678 .with_daemon_shutdown_flag(daemon_shutdown.clone());
9679 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9680 consumer.capabilities = Some(CapabilityDeclarations {
9681 provides: Vec::new(),
9682 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9683 capability: CAPABILITY.to_string(),
9684 need: subc_protocol::manifest::CapabilityNeed::Required,
9685 }],
9686 must_never_reach: Vec::new(),
9687 });
9688 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
9689 let provider_conn = ConnectionId::new(182);
9690 for (conn, manifest, corr) in [
9691 (ConnectionId::new(181), consumer, 181),
9692 (provider_conn, provider, 182),
9693 ] {
9694 let hello = handler
9695 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
9696 .expect("module registers");
9697 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9698 }
9699 assert_eq!(
9700 handler.capability_evaluator.verdict("consumer", CAPABILITY),
9701 Some(CapabilityVerdict::Provided),
9702 "the registered provider must satisfy the requirement first"
9703 );
9704 if daemon_shutdown_begun {
9705 daemon_shutdown.set_for_test();
9706 }
9707
9708 let capture = EventCapture::default();
9709 let _subscriber =
9710 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9711 handler
9712 .cleanup_connection(provider_conn)
9713 .expect("provider deregisters");
9714 let events = capture
9715 .events()
9716 .into_iter()
9717 .filter(|event| {
9718 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
9719 })
9720 .collect();
9721 (
9722 events,
9723 handler.capability_evaluator.verdict("consumer", CAPABILITY),
9724 )
9725 }
9726
9727 #[test]
9728 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
9729 let (events, verdict) = deregister_required_provider(true);
9730 assert_eq!(
9731 verdict,
9732 Some(CapabilityVerdict::NeverProvided),
9733 "shutdown must not change the verdict route.open refusals read"
9734 );
9735 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9736 let event = &events[0];
9737 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
9738 assert_eq!(
9739 event.fields.get("cause").map(String::as_str),
9740 Some("\"daemon_shutdown\""),
9741 "{event:?}"
9742 );
9743 assert_eq!(
9744 event.fields.get("verdict").map(String::as_str),
9745 Some("\"never_provided\""),
9746 "{event:?}"
9747 );
9748 }
9749
9750 #[test]
9751 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
9752 let (events, verdict) = deregister_required_provider(false);
9753 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
9754 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9755 let event = &events[0];
9756 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
9757 assert_eq!(event.fields.get("cause"), None, "{event:?}");
9758 assert_eq!(
9759 event.fields.get("verdict").map(String::as_str),
9760 Some("\"never_provided\""),
9761 "{event:?}"
9762 );
9763 }
9764
9765 #[test]
9766 fn catalog_list_omits_capabilities_for_legacy_manifest() {
9767 let registry = Arc::new(Registry::default());
9768 let handler = ControlHandler::new(Arc::clone(®istry));
9769 let hello = handler
9770 .handle_control(
9771 ConnectionId::new(101),
9772 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9773 )
9774 .expect("legacy HELLO registers");
9775 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9776
9777 let request = Frame::build(
9778 FrameType::Request,
9779 control_flags(),
9780 0,
9781 0,
9782 102,
9783 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9784 .expect("catalog request serializes"),
9785 )
9786 .expect("catalog request frame builds");
9787 let response = handler
9788 .handle_catalog_list(request, None)
9789 .expect("catalog list succeeds");
9790 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9791 assert!(
9792 body["modules"][0].get("capabilities").is_none(),
9793 "legacy manifest must retain an absent capabilities field on catalog.list"
9794 );
9795 }
9796
9797 #[test]
9798 fn hello_ack_omits_storage_when_no_storage_config() {
9799 let registry = Arc::new(Registry::default());
9800 let handler = ControlHandler::new(Arc::clone(®istry));
9801 let responses = handler
9802 .handle_control(
9803 ConnectionId::new(1),
9804 hello_frame("aft", PROTOCOL_VERSION, 7),
9805 )
9806 .unwrap();
9807 let ack = parse_ack(&responses[0]);
9808 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9809 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9810 }
9811
9812 #[tokio::test]
9813 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9814 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9815 let registry = Arc::new(Registry::default());
9816 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
9817 let responses = handler
9818 .handle_control(
9819 ConnectionId::new(1),
9820 hello_frame("aft", PROTOCOL_VERSION, 7),
9821 )
9822 .unwrap();
9823 let ack = parse_ack(&responses[0]);
9824 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9825
9826 let described = handler
9827 .handle_control_frame(
9828 &route_ctx(ConnectionId::new(2)).0,
9829 Frame::build(
9830 FrameType::Request,
9831 control_flags(),
9832 0,
9833 0,
9834 9,
9835 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9836 )
9837 .unwrap(),
9838 )
9839 .await
9840 .unwrap();
9841 let ClientControlResponse::ServerDescribe { machine_id, .. } =
9842 serde_json::from_slice(&described[0].body).unwrap()
9843 else {
9844 panic!("server.describe answered with another shape");
9845 };
9846 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9847 }
9848
9849 #[test]
9850 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9851 let registry = Arc::new(Registry::default());
9854 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
9855 crate::daemon_config::StorageConfig::Sqlite {
9856 data_home: std::path::PathBuf::from("/data"),
9857 },
9858 ));
9859
9860 let responses = handler
9861 .handle_control(
9862 ConnectionId::new(1),
9863 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9864 )
9865 .unwrap();
9866 let ack = parse_ack(&responses[0]);
9867 assert_eq!(
9868 ack.storage,
9869 Some(serde_json::json!({
9870 "module_id": "alfonso-routing",
9871 "storage_namespace": "default",
9872 "isolation": { "kind": "module" },
9873 "backend": {
9874 "backend": "sqlite",
9875 "path": "/data/cortexkit/alfonso-routing/store.db"
9876 }
9877 })),
9878 "the delivered descriptor is the module's own sqlite store path"
9879 );
9880 }
9881
9882 #[test]
9883 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9884 let registry = Arc::new(Registry::default());
9885 let handler = ControlHandler::new(Arc::clone(®istry));
9886 let conn = ConnectionId::new(1);
9887 let responses = handler
9888 .handle_control(
9889 conn,
9890 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9891 )
9892 .unwrap();
9893 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9894 let registration = registry.get_module("aft").unwrap().unwrap();
9895 assert_eq!(registration.control_ops, module_baseline_control_ops());
9896
9897 let frame =
9898 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9899 assert!(handler
9900 .guard_module_control_op(&frame, "aft", "route.bind")
9901 .unwrap()
9902 .is_none());
9903 let error = handler
9904 .guard_module_control_op(&frame, "aft", "test.synthetic")
9905 .unwrap()
9906 .expect("synthetic ungranted op should be rejected");
9907 assert_eq!(error.header.ty, FrameType::Error);
9908 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9909 }
9910
9911 #[test]
9912 fn hello_control_ops_some_adds_optional_grants() {
9913 let registry = Arc::new(Registry::default());
9914 let handler = ControlHandler::new(Arc::clone(®istry));
9915 handler
9916 .handle_control(
9917 ConnectionId::new(1),
9918 hello_frame_with_control_ops(
9919 "aft",
9920 PROTOCOL_VERSION,
9921 7,
9922 Some(vec![
9923 "future.synthetic".to_string(),
9924 "route.bind".to_string(),
9925 ]),
9926 ),
9927 )
9928 .unwrap();
9929 let registration = registry.get_module("aft").unwrap().unwrap();
9930 assert_eq!(
9931 registration.control_ops,
9932 vec![
9933 "route.bind".to_string(),
9934 "route.status".to_string(),
9935 "future.synthetic".to_string(),
9936 ]
9937 );
9938 let frame =
9939 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9940 assert!(handler
9941 .guard_module_control_op(&frame, "aft", "future.synthetic")
9942 .unwrap()
9943 .is_none());
9944 }
9945
9946 #[tokio::test]
9947 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9948 let registry = Arc::new(Registry::default());
9949 let forwarding = Arc::new(ForwardingTable::default());
9950 let handler =
9951 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9952 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9953 hello_via_sink(
9954 &handler,
9955 &module_ctx,
9956 &mut module_rx,
9957 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9958 )
9959 .await;
9960
9961 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9962 let responses = handler
9963 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9964 .await
9965 .unwrap();
9966 assert_eq!(responses.len(), 1);
9967 assert_eq!(responses[0].header.ty, FrameType::Error);
9968 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9969 assert!(module_rx.try_recv().is_err());
9970 }
9971
9972 #[tokio::test]
9973 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9974 let registry = Arc::new(Registry::default());
9975 let forwarding = Arc::new(ForwardingTable::default());
9976 let handler =
9977 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9978 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9979 hello_via_sink(
9980 &handler,
9981 &module_ctx,
9982 &mut module_rx,
9983 hello_frame_with_control_ops(
9984 "aft",
9985 PROTOCOL_VERSION,
9986 7,
9987 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9988 ),
9989 )
9990 .await;
9991
9992 let project_root = unique_project_root("demux");
9993 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9994 let route_handler = handler.clone();
9995 let route_task = tokio::spawn(async move {
9996 route_handler
9997 .handle_control_frame(
9998 &route_client_ctx,
9999 route_open_frame(100, "aft", project_root),
10000 )
10001 .await
10002 .unwrap()
10003 });
10004 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10005 .await
10006 .unwrap()
10007 .unwrap();
10008 assert!(matches!(
10009 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10010 ModuleControlRequest::RouteBind { .. }
10011 ));
10012
10013 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10014 let health_handler = handler.clone();
10015 let health_task = tokio::spawn(async move {
10016 health_handler
10017 .handle_control_frame(
10018 &health_client_ctx,
10019 supervisor_health_probe_frame(101, "aft"),
10020 )
10021 .await
10022 .unwrap()
10023 });
10024 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10025 .await
10026 .unwrap()
10027 .unwrap();
10028 assert_eq!(
10029 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10030 ModuleControlRequest::HealthCheck {}
10031 );
10032
10033 handler
10034 .handle_control_frame(
10035 &module_ctx,
10036 health_response(health_frame.header.corr, HealthStatus::Degraded),
10037 )
10038 .await
10039 .unwrap();
10040 let health_response = health_task.await.unwrap();
10041 assert_eq!(health_response.len(), 1);
10042 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10043 ClientControlResponse::SupervisorHealthProbe {
10044 module_id,
10045 status,
10046 detail,
10047 metrics,
10048 } => {
10049 assert_eq!(module_id, "aft");
10050 assert_eq!(status, HealthStatus::Degraded);
10051 assert_eq!(detail.as_deref(), Some("warming"));
10052 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10053 }
10054 other => panic!("unexpected health response: {other:?}"),
10055 }
10056
10057 handler
10058 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10059 .await
10060 .unwrap();
10061 let route_response = route_task.await.unwrap();
10062 assert!(route_response.is_empty());
10063 let published = route_client_rx.recv().await.unwrap();
10064 assert!(matches!(
10065 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10066 ClientControlResponse::RouteOpen { .. }
10067 ));
10068 }
10069
10070 async fn relay_route_open(
10075 handler: &ControlHandler,
10076 client_connection: ConnectionId,
10077 client_egress: &FrameSink,
10078 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10079 corr: u64,
10080 module_id: &str,
10081 project_root_label: &str,
10082 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10083 let ctx = RouteCtx {
10084 connection_id: client_connection,
10085 egress: client_egress.clone(),
10086 };
10087 let handler = handler.clone();
10088 let project_root = unique_project_root(project_root_label);
10089 let module_id = module_id.to_string();
10090 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10091 let task = tokio::spawn(async move {
10092 let _guard = tracing::dispatcher::set_default(&dispatch);
10093 handler
10094 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10095 .await
10096 .unwrap()
10097 });
10098 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10099 .await
10100 .expect("module receives the relayed route.bind")
10101 .expect("module egress is open");
10102 (task, bind.frame)
10103 }
10104
10105 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10106 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10107 ModuleControlRequest::RouteBind {
10108 route_channel,
10109 epoch,
10110 ..
10111 } => (route_channel, epoch),
10112 other => panic!("expected a route.bind request, got {other:?}"),
10113 }
10114 }
10115
10116 fn published_route(frame: &Frame) -> (u16, u32) {
10117 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10118 ClientControlResponse::RouteOpen {
10119 route_channel,
10120 route_epoch,
10121 } => (route_channel, route_epoch),
10122 other => panic!("expected a route.open response, got {other:?}"),
10123 }
10124 }
10125
10126 #[tokio::test]
10146 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10147 let registry = Arc::new(Registry::default());
10148 let forwarding = Arc::new(ForwardingTable::default());
10149 let handler =
10150 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10151
10152 let module_connection = ConnectionId::new(30);
10153 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10154 hello_via_sink(
10155 &handler,
10156 &module_ctx,
10157 &mut module_rx,
10158 hello_frame("aft", PROTOCOL_VERSION, 7),
10159 )
10160 .await;
10161
10162 let dying_client = ConnectionId::new(31);
10163 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10164
10165 let (first_task, first_bind) = relay_route_open(
10168 &handler,
10169 dying_client,
10170 &dying_ctx.egress,
10171 &mut module_rx,
10172 100,
10173 "aft",
10174 "closing-first",
10175 )
10176 .await;
10177 handler
10178 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10179 .await
10180 .unwrap();
10181 assert!(first_task.await.unwrap().is_empty());
10182 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10183
10184 let (second_task, second_bind) = relay_route_open(
10186 &handler,
10187 dying_client,
10188 &dying_ctx.egress,
10189 &mut module_rx,
10190 101,
10191 "aft",
10192 "closing-second",
10193 )
10194 .await;
10195 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10196
10197 assert!(forwarding
10200 .escalate_client_delivery_failure(
10201 dying_client,
10202 first_channel,
10203 first_epoch,
10204 CloseReason::new(
10205 "module_to_client_delivery_failed",
10206 "client egress refused a module frame",
10207 ),
10208 crate::forwarding::UndeliveredFrame {
10209 module_id: None,
10210 sink: &dying_ctx.egress,
10211 },
10212 )
10213 .unwrap());
10214 assert!(!dying_ctx.egress.is_closed());
10215
10216 let ack = handler
10218 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10219 .await;
10220 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10221 if module_loop_error.is_some() {
10222 handler.cleanup_connection(module_connection).unwrap();
10226 }
10227 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10231 .await
10232 .ok()
10233 .flatten();
10234
10235 assert!(
10237 registry
10238 .get_module_by_connection(module_connection)
10239 .unwrap()
10240 .is_some(),
10241 "one client's closing connection ended the shared module connection: \
10242 {module_loop_error:?}"
10243 );
10244 let cotenant = ConnectionId::new(32);
10246 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10247 let (cotenant_task, cotenant_bind) = relay_route_open(
10248 &handler,
10249 cotenant,
10250 &cotenant_ctx.egress,
10251 &mut module_rx,
10252 102,
10253 "aft",
10254 "closing-cotenant",
10255 )
10256 .await;
10257 handler
10258 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10259 .await
10260 .unwrap();
10261 assert!(cotenant_task.await.unwrap().is_empty());
10262 let (cotenant_channel, cotenant_epoch) =
10263 published_route(&cotenant_rx.recv().await.unwrap());
10264 assert!(matches!(
10265 forwarding
10266 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10267 .unwrap(),
10268 DataRoute::Client(DataRouteState::Bound(_))
10269 ));
10270
10271 let goodbye = post_ack_module_frame
10274 .expect("module receives a GOODBYE for the abandoned route channel");
10275 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10276 assert_eq!(goodbye.header.channel, abandoned_channel);
10277 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10278
10279 assert!(dying_rx.try_recv().is_err());
10283 let second_response = second_task.await.unwrap();
10284 assert_eq!(second_response.len(), 1);
10285 assert_eq!(
10286 parse_error(&second_response[0])["code"],
10287 "target_unavailable"
10288 );
10289 }
10290
10291 async fn catalog_snapshot(
10294 handler: &ControlHandler,
10295 client_ctx: &RouteCtx,
10296 corr: u64,
10297 ) -> (u64, Vec<(String, bool)>) {
10298 let body =
10299 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10300 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10301 let reply = handler
10302 .handle_control_frame(client_ctx, request)
10303 .await
10304 .unwrap()
10305 .pop()
10306 .unwrap();
10307 let ClientControlResponse::CatalogList {
10308 generation,
10309 modules,
10310 ..
10311 } = serde_json::from_slice(&reply.body).unwrap()
10312 else {
10313 panic!("not a catalog.list reply");
10314 };
10315 (
10316 generation,
10317 modules
10318 .into_iter()
10319 .map(|entry| (entry.module_id, entry.ready))
10320 .collect(),
10321 )
10322 }
10323
10324 fn hello_held_before_forwarding() -> (
10330 ControlHandler,
10331 Arc<std::sync::Barrier>,
10332 std::thread::JoinHandle<Vec<Frame>>,
10333 RouteCtx,
10334 mpsc::Receiver<crate::router::OutboundFrame>,
10335 ) {
10336 let hold = Arc::new(std::sync::Barrier::new(2));
10337 let handler = ControlHandler::with_forwarding(
10338 Arc::new(Registry::default()),
10339 Arc::new(ForwardingTable::default()),
10340 )
10341 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10342 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10343 let hello = {
10344 let handler = handler.clone();
10345 let module_ctx = module_ctx.clone();
10346 std::thread::spawn(move || {
10347 handler
10348 .handle_hello(
10349 module_ctx.connection_id,
10350 Some(module_ctx.egress.clone()),
10351 hello_frame("aft", PROTOCOL_VERSION, 7),
10352 )
10353 .unwrap()
10354 })
10355 };
10356 (handler, hold, hello, module_ctx, module_rx)
10357 }
10358
10359 #[tokio::test]
10368 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10369 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10370 let client = ConnectionId::new(41);
10371 let (client_ctx, mut client_rx) = route_ctx(client);
10372
10373 hold.wait();
10374 assert!(handler.registry.get_module("aft").unwrap().is_some());
10376 assert!(!handler
10377 .forwarding
10378 .has_live_module_connection("aft")
10379 .unwrap());
10380 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10381 assert!(
10382 listed.is_empty(),
10383 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10384 );
10385 let refused = handler
10386 .handle_control_frame(
10387 &client_ctx,
10388 route_open_frame(301, "aft", unique_project_root("listed-window")),
10389 )
10390 .await
10391 .unwrap();
10392 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10393 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10394 let duplicate = handler
10395 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10396 .await
10397 .unwrap();
10398 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10399
10400 hold.wait();
10401 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10402 let ack = module_rx.try_recv().unwrap().frame;
10403 assert_eq!(ack.header.ty, FrameType::HelloAck);
10404 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10405 assert_eq!(listed, vec![("aft".to_string(), true)]);
10406
10407 let (open_task, bind) = relay_route_open(
10408 &handler,
10409 client,
10410 &client_ctx.egress,
10411 &mut module_rx,
10412 303,
10413 "aft",
10414 "listed-after",
10415 )
10416 .await;
10417 assert!(handler
10418 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10419 .await
10420 .unwrap()
10421 .is_empty());
10422 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10423 published_route(&client_rx.try_recv().unwrap().frame);
10424 }
10425
10426 #[tokio::test]
10433 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10434 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10435 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10436 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10437
10438 hold.wait();
10439 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10440 assert!(listed.is_empty());
10441 assert_eq!(
10442 during, before,
10443 "the generation moved while the module was not listed"
10444 );
10445
10446 hold.wait();
10447 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10448 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10449 assert_eq!(listed, vec![("aft".to_string(), true)]);
10450 assert_eq!(
10451 after,
10452 before + 1,
10453 "the module appeared without exactly one generation bump"
10454 );
10455 }
10456
10457 #[tokio::test]
10462 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10463 let registry = Arc::new(Registry::default());
10464 let handler = ControlHandler::with_forwarding(
10465 Arc::clone(®istry),
10466 Arc::new(ForwardingTable::default()),
10467 );
10468 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10469 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10470 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10471 drop(module_rx);
10472
10473 let replies = handler
10474 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10475 .await
10476 .unwrap();
10477 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10478 assert!(registry.get_module("aft").unwrap().is_none());
10479 assert!(registry
10480 .get_module_by_connection(module_ctx.connection_id)
10481 .unwrap()
10482 .is_none());
10483 assert_eq!(
10484 catalog_snapshot(&handler, &client_ctx, 301).await,
10485 (before, Vec::new())
10486 );
10487 }
10488
10489 async fn route_open_held_after_bind_send() -> (
10493 ControlHandler,
10494 Arc<tokio::sync::Barrier>,
10495 RouteCtx,
10496 mpsc::Receiver<crate::router::OutboundFrame>,
10497 ) {
10498 let hold = Arc::new(tokio::sync::Barrier::new(2));
10499 let handler = ControlHandler::with_forwarding(
10500 Arc::new(Registry::default()),
10501 Arc::new(ForwardingTable::default()),
10502 )
10503 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10504 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10505 hello_via_sink(
10506 &handler,
10507 &module_ctx,
10508 &mut module_rx,
10509 hello_frame("aft", PROTOCOL_VERSION, 7),
10510 )
10511 .await;
10512 (handler, hold, module_ctx, module_rx)
10513 }
10514
10515 #[tokio::test]
10522 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10523 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10524 let client = ConnectionId::new(41);
10525 let (client_ctx, mut client_rx) = route_ctx(client);
10526
10527 let (open_task, bind) = relay_route_open(
10528 &handler,
10529 client,
10530 &client_ctx.egress,
10531 &mut module_rx,
10532 200,
10533 "aft",
10534 "acked-before-mark",
10535 )
10536 .await;
10537 let (module_channel, module_epoch) = route_bind_channel(&bind);
10538 hold.wait().await;
10540
10541 assert!(handler
10543 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10544 .await
10545 .unwrap()
10546 .is_empty());
10547 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10548
10549 hold.wait().await;
10550 assert!(
10551 open_task.await.unwrap().is_empty(),
10552 "route.open answers accepted"
10553 );
10554
10555 match module_rx.try_recv() {
10558 Err(mpsc::error::TryRecvError::Empty) => {}
10559 Ok(outbound) => panic!(
10560 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10561 outbound.frame.header.ty,
10562 outbound.frame.header.channel,
10563 outbound.frame.header.epoch
10564 ),
10565 Err(err) => panic!("module egress closed: {err}"),
10566 }
10567
10568 let router = Router::with_control_handler(Arc::new(handler.clone()));
10571 let request = Frame::build(
10572 FrameType::Request,
10573 Flags::new(true, Priority::Interactive, false),
10574 client_channel,
10575 client_epoch,
10576 201,
10577 b"first request".to_vec(),
10578 )
10579 .unwrap();
10580 router
10581 .route_for_connection(&client_ctx, request)
10582 .await
10583 .unwrap();
10584 let delivered = module_rx.try_recv().unwrap().frame;
10585 assert_eq!(delivered.header.ty, FrameType::Request);
10586 assert_eq!(
10587 (delivered.header.channel, delivered.header.epoch),
10588 (module_channel, module_epoch)
10589 );
10590 assert_eq!(delivered.body, b"first request");
10591 }
10592
10593 #[tokio::test]
10599 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10600 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10601 let client = ConnectionId::new(42);
10602 let (client_ctx, _client_rx) = route_ctx(client);
10603
10604 let (open_task, bind) = relay_route_open(
10605 &handler,
10606 client,
10607 &client_ctx.egress,
10608 &mut module_rx,
10609 300,
10610 "aft",
10611 "released-before-mark",
10612 )
10613 .await;
10614 let (module_channel, module_epoch) = route_bind_channel(&bind);
10615 hold.wait().await;
10616
10617 handler.cleanup_connection(client).unwrap();
10618 assert!(
10619 module_rx.try_recv().is_err(),
10620 "client teardown must not GOODBYE a bind route.open is still queueing"
10621 );
10622
10623 hold.wait().await;
10624 let response = open_task.await.unwrap();
10625 assert_eq!(response.len(), 1);
10626 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10627
10628 let goodbye = module_rx
10629 .try_recv()
10630 .expect("module is told to drop the binding the daemon abandoned")
10631 .frame;
10632 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10633 assert_eq!(
10634 (goodbye.header.channel, goodbye.header.epoch),
10635 (module_channel, module_epoch)
10636 );
10637 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10638 }
10639
10640 #[tokio::test]
10645 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10646 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10647 let client = ConnectionId::new(43);
10648 let (client_ctx, client_rx) = route_ctx(client);
10649
10650 let (open_task, bind) = relay_route_open(
10651 &handler,
10652 client,
10653 &client_ctx.egress,
10654 &mut module_rx,
10655 400,
10656 "aft",
10657 "gone-client-before-mark",
10658 )
10659 .await;
10660 let (module_channel, module_epoch) = route_bind_channel(&bind);
10661 hold.wait().await;
10662
10663 drop(client_rx);
10664 assert!(client_ctx.egress.is_closed());
10665 handler
10666 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10667 .await
10668 .unwrap();
10669
10670 hold.wait().await;
10671 let response = open_task.await.unwrap();
10672 assert_eq!(response.len(), 1);
10673 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10674
10675 let goodbye = module_rx
10676 .try_recv()
10677 .expect("module is told to drop the binding it acked")
10678 .frame;
10679 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10680 assert_eq!(
10681 (goodbye.header.channel, goodbye.header.epoch),
10682 (module_channel, module_epoch)
10683 );
10684 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10685 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10686 }
10687
10688 #[test]
10695 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10696 let handler = ControlHandler::default();
10697 let module_connection = ConnectionId::new(30);
10698 let client_connection = ConnectionId::new(31);
10699
10700 handler
10701 .refuse_to_end_module_connection_for_a_client(
10702 module_connection,
10703 77,
10704 ForwardingError::ConnectionClosing {
10705 connection_id: client_connection,
10706 },
10707 )
10708 .expect("a closing client must never end the module connection");
10709
10710 assert!(matches!(
10711 handler.refuse_to_end_module_connection_for_a_client(
10712 module_connection,
10713 78,
10714 ForwardingError::ConnectionClosing {
10715 connection_id: module_connection,
10716 },
10717 ),
10718 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10719 connection_id
10720 })) if connection_id == module_connection
10721 ));
10722 assert!(matches!(
10723 handler.refuse_to_end_module_connection_for_a_client(
10724 module_connection,
10725 79,
10726 ForwardingError::Poisoned,
10727 ),
10728 Err(RouterError::Forwarding(ForwardingError::Poisoned))
10729 ));
10730 assert!(matches!(
10731 handler.refuse_to_end_module_connection_for_a_client(
10732 module_connection,
10733 80,
10734 ForwardingError::StaleModuleEndpoint,
10735 ),
10736 Err(RouterError::Forwarding(
10737 ForwardingError::StaleModuleEndpoint
10738 ))
10739 ));
10740 }
10741
10742 #[tokio::test]
10755 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10756 let registry = Arc::new(Registry::default());
10757 let forwarding = Arc::new(ForwardingTable::default());
10758 let supervisor = SupervisorHandle::new();
10759 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10760 let handler =
10761 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10762 .with_supervisor(supervisor);
10763
10764 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10765 hello_via_sink(
10766 &handler,
10767 &target_ctx,
10768 &mut target_rx,
10769 hello_frame("target", PROTOCOL_VERSION, 1),
10770 )
10771 .await;
10772
10773 let wrong_nonce = handler
10777 .handle_control_frame(
10778 &route_ctx(ConnectionId::new(91)).0,
10779 route_open_frame_with_admission_facts(
10780 20,
10781 "target",
10782 unique_project_root("admission-facts"),
10783 Some(subc_control::ConsumerIdentity {
10784 module_id: "fed".to_string(),
10785 launch_nonce: "not-the-real-nonce".to_string(),
10786 }),
10787 None,
10788 ),
10789 )
10790 .await
10791 .unwrap();
10792 assert_eq!(
10793 parse_error(&wrong_nonce[0])["code"],
10794 "bad_consumer_identity",
10795 "a mismatched launch nonce must be refused, not stamped Reserved"
10796 );
10797
10798 let never_spawned = handler
10802 .handle_control_frame(
10803 &route_ctx(ConnectionId::new(92)).0,
10804 route_open_frame_with_admission_facts(
10805 21,
10806 "target",
10807 unique_project_root("admission-facts"),
10808 Some(subc_control::ConsumerIdentity {
10809 module_id: "never-spawned".to_string(),
10810 launch_nonce: "any-nonce".to_string(),
10811 }),
10812 None,
10813 ),
10814 )
10815 .await
10816 .unwrap();
10817 assert_eq!(
10818 parse_error(&never_spawned[0])["code"],
10819 "bad_consumer_identity",
10820 "an unspawned module_id must be refused rather than accepted for lack of a record"
10821 );
10822 }
10823
10824 #[tokio::test]
10847 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10848 let registry = Arc::new(Registry::default());
10849 let forwarding = Arc::new(ForwardingTable::default());
10850 let supervisor = SupervisorHandle::new();
10851 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10852 let handler =
10853 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10854 .with_supervisor(supervisor);
10855
10856 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10857 hello_via_sink(
10858 &handler,
10859 &target_ctx,
10860 &mut target_rx,
10861 hello_frame("target", PROTOCOL_VERSION, 1),
10862 )
10863 .await;
10864
10865 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10866 let route_handler = handler.clone();
10867 let route_task = tokio::spawn(async move {
10868 route_handler
10869 .handle_control_frame(
10870 &client_ctx,
10871 route_open_frame_with_admission_facts(
10872 30,
10873 "target",
10874 unique_project_root("admission-facts"),
10875 Some(subc_control::ConsumerIdentity {
10876 module_id: "fed".to_string(),
10877 launch_nonce: "fed-nonce".to_string(),
10878 }),
10879 None,
10880 ),
10881 )
10882 .await
10883 .unwrap()
10884 });
10885
10886 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10893 .await
10894 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10895 .expect("module control channel closed before route.bind");
10896 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10897 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10898 panic!("expected route.bind")
10899 };
10900 assert_eq!(
10901 principal,
10902 Some(Principal::Reserved {
10903 module_id: "fed".to_string()
10904 }),
10905 "a correctly attested consumer must be stamped Reserved for its own id"
10906 );
10907
10908 handler
10909 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10910 .await
10911 .unwrap();
10912 assert!(route_task.await.unwrap().is_empty());
10913 assert!(
10914 matches!(
10915 serde_json::from_slice::<ClientControlResponse>(
10916 &client_rx.recv().await.unwrap().body
10917 )
10918 .unwrap(),
10919 ClientControlResponse::RouteOpen { .. }
10920 ),
10921 "the route must actually open, not merely avoid an error"
10922 );
10923 }
10924
10925 #[tokio::test(start_paused = true)]
10926 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10927 let registry = Arc::new(Registry::default());
10928 let forwarding = Arc::new(ForwardingTable::default());
10929 let supervisor = SupervisorHandle::new();
10930 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10931 let handler =
10932 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10933 .with_supervisor(supervisor);
10934
10935 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10936 hello_via_sink(
10937 &handler,
10938 &target_ctx,
10939 &mut target_rx,
10940 hello_frame("target", PROTOCOL_VERSION, 1),
10941 )
10942 .await;
10943
10944 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10945 let direct_handler = handler.clone();
10946 let direct_open = tokio::spawn(async move {
10947 direct_handler
10948 .handle_control_frame(
10949 &direct_ctx,
10950 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10951 )
10952 .await
10953 .unwrap()
10954 });
10955 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10956 .await
10957 .expect("no direct route.bind within 5s")
10958 .expect("target control channel closed before direct route.bind");
10959 handler
10960 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10961 .await
10962 .unwrap();
10963 assert!(direct_open.await.unwrap().is_empty());
10964 let _ = direct_rx.recv().await.unwrap();
10965
10966 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10967 let reserved_handler = handler.clone();
10968 let reserved_open = tokio::spawn(async move {
10969 reserved_handler
10970 .handle_control_frame(
10971 &reserved_ctx,
10972 route_open_frame_with_admission_facts(
10973 3,
10974 "target",
10975 unique_project_root("admission-facts"),
10976 Some(ConsumerIdentity {
10977 module_id: "fed".to_string(),
10978 launch_nonce: "fed-nonce".to_string(),
10979 }),
10980 None,
10981 ),
10982 )
10983 .await
10984 .unwrap()
10985 });
10986 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10987 .await
10988 .expect("no reserved route.bind within 5s")
10989 .expect("target control channel closed before reserved route.bind");
10990 handler
10991 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10992 .await
10993 .unwrap();
10994 assert!(reserved_open.await.unwrap().is_empty());
10995 let _ = reserved_rx.recv().await.unwrap();
10996
10997 forwarding
10998 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10999 .unwrap();
11000 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11001 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11002 module_id: Some("target".to_string()),
11003 })
11004 .unwrap();
11005 let census_frame =
11006 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11007 let response = handler
11008 .handle_control_frame(&census_ctx, census_frame)
11009 .await
11010 .unwrap()
11011 .pop()
11012 .unwrap();
11013 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11014 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11015 assert!(matches!(
11016 decoded,
11017 ClientControlResponse::SupervisorRoutes { .. }
11018 ));
11019 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11020 assert_eq!(routes.len(), 2);
11021 assert!(routes.iter().all(|route| route["draining"] == true));
11022 assert!(
11025 routes.iter().all(|route| route["drain_reason"] == "reload"),
11026 "draining routes must name the drain's reason: {routes:?}"
11027 );
11028 assert!(routes.iter().any(|route| {
11029 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11030 }));
11031 assert!(routes.iter().any(|route| {
11032 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11033 }));
11034
11035 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11036 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11037 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11038 std::fs::write(
11039 &golden_path,
11040 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11041 )
11042 .unwrap();
11043 }
11044 let expected: Value =
11045 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11046 assert_eq!(actual, expected);
11047 }
11048
11049 async fn query_live_roots(
11050 handler: &ControlHandler,
11051 module_ctx: &RouteCtx,
11052 ) -> ModuleControlResponseToModule {
11053 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11054 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11055 let response = handler
11056 .handle_control_frame(module_ctx, frame)
11057 .await
11058 .unwrap()
11059 .pop()
11060 .unwrap();
11061 serde_json::from_slice(&response.body).unwrap()
11062 }
11063
11064 #[tokio::test(start_paused = true)]
11065 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11066 let registry = Arc::new(Registry::default());
11067 let forwarding = Arc::new(ForwardingTable::default());
11068 let handler = ControlHandler::with_forwarding(registry, forwarding);
11069 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11070 hello_via_sink(
11071 &handler,
11072 &target_ctx,
11073 &mut target_rx,
11074 hello_frame("target", PROTOCOL_VERSION, 1),
11075 )
11076 .await;
11077 let root = unique_project_root("live-roots-known");
11078 let path = ProjectRootId::from_path_allowing_missing(root.path())
11079 .unwrap()
11080 .as_path()
11081 .to_path_buf();
11082 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11083 let open_handler = handler.clone();
11084 let opened = tokio::spawn(async move {
11085 open_handler
11086 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11087 .await
11088 .unwrap()
11089 });
11090 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11091 .await
11092 .unwrap()
11093 .unwrap();
11094 handler
11095 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11096 .await
11097 .unwrap();
11098 assert!(opened.await.unwrap().is_empty());
11099 let _ = client_rx.recv().await.unwrap();
11100
11101 let root = unique_project_root("live-roots-pending");
11102 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11103 .unwrap()
11104 .as_path()
11105 .to_path_buf();
11106 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11107 let open_handler = handler.clone();
11108 let pending = tokio::spawn(async move {
11109 open_handler
11110 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11111 .await
11112 .unwrap()
11113 });
11114 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11115 .await
11116 .unwrap()
11117 .unwrap();
11118 let actual = query_live_roots(&handler, &target_ctx).await;
11119 let ModuleControlResponseToModule::LiveRoots {
11120 roots,
11121 unknown_root_bindings,
11122 total_bindings,
11123 } = actual
11124 else {
11125 panic!("expected live roots")
11126 };
11127 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11128 assert_eq!(unknown_root_bindings, 0);
11129 assert_eq!(
11130 roots.len(),
11131 2,
11132 "root-known arm must retain each canonical root"
11133 );
11134 assert_eq!(
11135 total_bindings,
11136 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11137 );
11138 let counts = roots
11139 .iter()
11140 .map(|root| (root.project_root.clone(), root.bound, root.pending))
11141 .collect::<Vec<_>>();
11142 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11143 expected.sort_by(|a, b| a.0.cmp(&b.0));
11144 assert_eq!(
11145 counts, expected,
11146 "roots must sort by path and count pending separately"
11147 );
11148 handler
11149 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11150 .await
11151 .unwrap();
11152 assert!(pending.await.unwrap().is_empty());
11153 }
11154
11155 #[tokio::test(start_paused = true)]
11156 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11157 let forwarding = Arc::new(ForwardingTable::default());
11158 let handler =
11159 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11160 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11161 hello_via_sink(
11162 &handler,
11163 &target_ctx,
11164 &mut target_rx,
11165 hello_frame("target", PROTOCOL_VERSION, 1),
11166 )
11167 .await;
11168 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11169 let pending = forwarding
11170 .begin_route_bind_relay_for_test(
11171 client_ctx.connection_id,
11172 client_ctx.egress.clone(),
11173 2,
11174 "target",
11175 )
11176 .unwrap();
11177 forwarding
11178 .complete_pending_relay(
11179 target_ctx.connection_id,
11180 pending.corr,
11181 RouteBindRelayOutcome::Accepted,
11182 )
11183 .unwrap();
11184 let actual = query_live_roots(&handler, &target_ctx).await;
11185 let ModuleControlResponseToModule::LiveRoots {
11186 roots,
11187 unknown_root_bindings,
11188 total_bindings,
11189 } = actual
11190 else {
11191 panic!("expected live roots")
11192 };
11193 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11194 assert_eq!(
11195 unknown_root_bindings, 1,
11196 "unknown-root arm must not read as no bindings"
11197 );
11198 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11199 assert_eq!(
11200 total_bindings,
11201 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11202 );
11203 }
11204
11205 #[tokio::test(start_paused = true)]
11213 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11214 let forwarding = Arc::new(ForwardingTable::default());
11215 let handler =
11216 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11217 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11218 let replies = handler
11219 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11220 .await
11221 .unwrap();
11222 let queued_by_hello = module_rx.len();
11223
11224 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11225 let open_handler = handler.clone();
11226 let open = tokio::spawn(async move {
11227 open_handler
11228 .handle_control_frame(
11229 &client_ctx,
11230 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11231 )
11232 .await
11233 .unwrap()
11234 });
11235 let mut spins = 0;
11237 while module_rx.len() == queued_by_hello {
11238 spins += 1;
11239 assert!(spins < 10_000, "route.open never queued a route.bind");
11240 tokio::task::yield_now().await;
11241 }
11242
11243 for reply in replies {
11245 module_ctx.egress.send(reply).await.unwrap();
11246 }
11247
11248 let first = module_rx.recv().await.unwrap().frame;
11249 assert_eq!(
11250 first.header.ty,
11251 FrameType::HelloAck,
11252 "the first frame a registering module reads must be its HELLO_ACK"
11253 );
11254 assert_eq!(first.header.corr, 7);
11255 let second = module_rx.recv().await.unwrap().frame;
11256 assert_eq!(second.header.ty, FrameType::Request);
11257 assert!(
11258 matches!(
11259 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11260 ModuleControlRequest::RouteBind { .. }
11261 ),
11262 "the route.bind follows the ack"
11263 );
11264 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11265
11266 handler
11267 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11268 .await
11269 .unwrap();
11270 assert!(open.await.unwrap().is_empty());
11271 let _ = client_rx.recv().await.unwrap();
11272 }
11273
11274 #[tokio::test(start_paused = true)]
11275 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11276 let handler = ControlHandler::with_forwarding(
11277 Arc::new(Registry::default()),
11278 Arc::new(ForwardingTable::default()),
11279 );
11280 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11281 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11282 hello_via_sink(
11283 &handler,
11284 &first_ctx,
11285 &mut first_rx,
11286 hello_frame("first", PROTOCOL_VERSION, 1),
11287 )
11288 .await;
11289 hello_via_sink(
11290 &handler,
11291 &second_ctx,
11292 &mut second_rx,
11293 hello_frame("second", PROTOCOL_VERSION, 2),
11294 )
11295 .await;
11296 let root = unique_project_root("second-only");
11297 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11298 let cloned = handler.clone();
11299 let open = tokio::spawn(async move {
11300 cloned
11301 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11302 .await
11303 .unwrap()
11304 });
11305 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11306 .await
11307 .unwrap()
11308 .unwrap();
11309 let first = query_live_roots(&handler, &first_ctx).await;
11310 let second = query_live_roots(&handler, &second_ctx).await;
11311 assert!(
11312 matches!(
11313 first,
11314 ModuleControlResponseToModule::LiveRoots {
11315 total_bindings: 0,
11316 ..
11317 }
11318 ),
11319 "cross-module scope must not expose another module's roots"
11320 );
11321 assert!(
11322 matches!(
11323 second,
11324 ModuleControlResponseToModule::LiveRoots {
11325 total_bindings: 1,
11326 ..
11327 }
11328 ),
11329 "second module must see its pending route"
11330 );
11331 handler
11332 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11333 .await
11334 .unwrap();
11335 assert!(open.await.unwrap().is_empty());
11336 }
11337
11338 #[tokio::test(start_paused = true)]
11339 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11340 let handler = ControlHandler::with_forwarding(
11341 Arc::new(Registry::default()),
11342 Arc::new(ForwardingTable::default()),
11343 );
11344 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11345 hello_via_sink(
11346 &handler,
11347 &target_ctx,
11348 &mut target_rx,
11349 hello_frame("target", PROTOCOL_VERSION, 1),
11350 )
11351 .await;
11352 let actual = query_live_roots(&handler, &target_ctx).await;
11353 let ModuleControlResponseToModule::LiveRoots {
11354 roots,
11355 unknown_root_bindings,
11356 total_bindings,
11357 } = actual
11358 else {
11359 panic!("expected live roots")
11360 };
11361 assert!(roots.is_empty());
11362 assert_eq!(unknown_root_bindings, 0);
11363 assert_eq!(total_bindings, 0);
11364 assert_eq!(
11365 total_bindings,
11366 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11367 );
11368 }
11369
11370 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11377 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11378 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11379 let text = std::fs::read_to_string(&path)
11380 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11381 let vectors: Vec<(String, Value)> = text
11382 .lines()
11383 .filter(|line| !line.trim().is_empty())
11384 .map(|line| {
11385 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11386 let id = entry["corpus_id"]
11387 .as_str()
11388 .expect("every vector carries a corpus_id")
11389 .to_string();
11390 (id, entry["package"].clone())
11391 })
11392 .collect();
11393 assert_eq!(
11397 vectors.len(),
11398 3,
11399 "vendored fed corpus changed size; re-sync from subc-federation"
11400 );
11401
11402 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11417 let richest = vectors
11418 .iter()
11419 .filter_map(|(_, package)| package.as_object())
11420 .map(|object| {
11421 object
11422 .keys()
11423 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11424 .count()
11425 })
11426 .max()
11427 .unwrap_or(0);
11428 assert!(
11429 richest >= 2,
11430 "vendored corpus no longer carries a package with unmodelled fields, \
11431 so the relay test can no longer distinguish a verbatim relay from a lossy one"
11432 );
11433
11434 vectors
11435 }
11436
11437 #[tokio::test]
11445 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11446 for (corpus_id, package) in fed_admission_facts_vectors() {
11447 let registry = Arc::new(Registry::default());
11448 let forwarding = Arc::new(ForwardingTable::default());
11449 let supervisor = SupervisorHandle::new();
11450 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11451 let handler =
11452 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11453 .with_supervisor(supervisor)
11454 .with_admission_facts_config(
11455 Some("fed".to_string()),
11456 Some(vec!["target".to_string()]),
11457 );
11458
11459 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11460 hello_via_sink(
11461 &handler,
11462 &target_ctx,
11463 &mut target_rx,
11464 hello_frame("target", PROTOCOL_VERSION, 1),
11465 )
11466 .await;
11467
11468 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11469 let route_handler = handler.clone();
11470 let expected = package.clone();
11471 let route_task = tokio::spawn(async move {
11472 route_handler
11473 .handle_control_frame(
11474 &client_ctx,
11475 route_open_frame_with_admission_facts(
11476 20,
11477 "target",
11478 unique_project_root("admission-facts"),
11479 Some(subc_control::ConsumerIdentity {
11480 module_id: "fed".to_string(),
11481 launch_nonce: "fed-nonce".to_string(),
11482 }),
11483 Some(package),
11484 ),
11485 )
11486 .await
11487 .unwrap()
11488 });
11489
11490 let bind_frame = target_rx.recv().await.unwrap();
11491 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11492 let ModuleControlRequest::RouteBind {
11493 admission_facts, ..
11494 } = bind
11495 else {
11496 panic!("{corpus_id}: expected route.bind")
11497 };
11498 assert_eq!(
11499 admission_facts,
11500 Some(expected),
11501 "{corpus_id}: relay must not add, drop or reshape any field"
11502 );
11503
11504 handler
11505 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11506 .await
11507 .unwrap();
11508 route_task.await.unwrap();
11509 }
11510 }
11511
11512 #[tokio::test]
11513 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11514 let registry = Arc::new(Registry::default());
11515 let forwarding = Arc::new(ForwardingTable::default());
11516 let supervisor = SupervisorHandle::new();
11517 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11518 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11519 let handler =
11520 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11521 .with_supervisor(supervisor)
11522 .with_admission_facts_config(
11523 Some("fed".to_string()),
11524 Some(vec!["target".to_string()]),
11525 );
11526
11527 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11528 hello_via_sink(
11529 &handler,
11530 &target_ctx,
11531 &mut target_rx,
11532 hello_frame("target", PROTOCOL_VERSION, 1),
11533 )
11534 .await;
11535 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11536 hello_via_sink(
11537 &handler,
11538 &other_ctx,
11539 &mut other_rx,
11540 hello_frame("other", PROTOCOL_VERSION, 2),
11541 )
11542 .await;
11543
11544 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11545 let expected_facts = facts.clone();
11546 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11547 let route_handler = handler.clone();
11548 let route_task = tokio::spawn(async move {
11549 route_handler
11550 .handle_control_frame(
11551 &client_ctx,
11552 route_open_frame_with_admission_facts(
11553 10,
11554 "target",
11555 unique_project_root("admission-facts"),
11556 Some(subc_control::ConsumerIdentity {
11557 module_id: "fed".to_string(),
11558 launch_nonce: "fed-nonce".to_string(),
11559 }),
11560 Some(facts.clone()),
11561 ),
11562 )
11563 .await
11564 .unwrap()
11565 });
11566 let bind_frame = target_rx.recv().await.unwrap();
11567 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11568 let ModuleControlRequest::RouteBind {
11569 admission_facts, ..
11570 } = bind
11571 else {
11572 panic!("expected route.bind")
11573 };
11574 assert_eq!(admission_facts, Some(expected_facts));
11575 handler
11576 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11577 .await
11578 .unwrap();
11579 assert!(route_task.await.unwrap().is_empty());
11580 assert!(matches!(
11581 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11582 .unwrap(),
11583 ClientControlResponse::RouteOpen { .. }
11584 ));
11585
11586 let direct = handler
11587 .handle_control_frame(
11588 &route_ctx(ConnectionId::new(73)).0,
11589 route_open_frame_with_admission_facts(
11590 11,
11591 "target",
11592 unique_project_root("admission-facts"),
11593 None,
11594 Some(json!({"x": 1})),
11595 ),
11596 )
11597 .await
11598 .unwrap();
11599 assert_eq!(
11600 parse_error(&direct[0])["code"],
11601 "admission_facts_not_permitted"
11602 );
11603
11604 let different_reserved = handler
11605 .handle_control_frame(
11606 &route_ctx(ConnectionId::new(77)).0,
11607 route_open_frame_with_admission_facts(
11608 15,
11609 "target",
11610 unique_project_root("admission-facts"),
11611 Some(subc_control::ConsumerIdentity {
11612 module_id: "other".to_string(),
11613 launch_nonce: "other-nonce".to_string(),
11614 }),
11615 Some(json!({"x": 1})),
11616 ),
11617 )
11618 .await
11619 .unwrap();
11620 assert_eq!(
11621 parse_error(&different_reserved[0])["code"],
11622 "admission_facts_not_permitted"
11623 );
11624
11625 let other_target = handler
11626 .handle_control_frame(
11627 &route_ctx(ConnectionId::new(74)).0,
11628 route_open_frame_with_admission_facts(
11629 12,
11630 "other",
11631 unique_project_root("admission-facts"),
11632 Some(subc_control::ConsumerIdentity {
11633 module_id: "fed".to_string(),
11634 launch_nonce: "fed-nonce".to_string(),
11635 }),
11636 Some(json!({"x": 1})),
11637 ),
11638 )
11639 .await
11640 .unwrap();
11641 assert_eq!(
11642 parse_error(&other_target[0])["code"],
11643 "admission_facts_target_not_allowed"
11644 );
11645
11646 let nonexistent = handler
11647 .handle_control_frame(
11648 &route_ctx(ConnectionId::new(75)).0,
11649 route_open_frame_with_admission_facts(
11650 13,
11651 "missing",
11652 unique_project_root("admission-facts"),
11653 None,
11654 Some(json!({"x": 1})),
11655 ),
11656 )
11657 .await
11658 .unwrap();
11659 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
11660
11661 let described = handler
11662 .handle_control_frame(
11663 &route_ctx(ConnectionId::new(76)).0,
11664 Frame::build(
11665 FrameType::Request,
11666 control_flags(),
11667 0,
11668 0,
11669 14,
11670 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11671 )
11672 .unwrap(),
11673 )
11674 .await
11675 .unwrap();
11676 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11677 serde_json::from_slice(&described[0].body).unwrap()
11678 else {
11679 panic!("expected server.describe response")
11680 };
11681 assert!(capabilities
11682 .iter()
11683 .any(|cap| cap == "admission_facts_relay_v1"));
11684 }
11685
11686 #[tokio::test]
11687 async fn admission_facts_without_configured_carrier_are_rejected() {
11688 let registry = Arc::new(Registry::default());
11689 let forwarding = Arc::new(ForwardingTable::default());
11690 let handler = ControlHandler::with_forwarding(registry, forwarding);
11691 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11692 hello_via_sink(
11693 &handler,
11694 &target_ctx,
11695 &mut target_rx,
11696 hello_frame("target", PROTOCOL_VERSION, 1),
11697 )
11698 .await;
11699
11700 let responses = handler
11701 .handle_control_frame(
11702 &route_ctx(ConnectionId::new(79)).0,
11703 route_open_frame_with_admission_facts(
11704 16,
11705 "target",
11706 unique_project_root("admission-facts"),
11707 None,
11708 Some(json!({"x": 1})),
11709 ),
11710 )
11711 .await
11712 .unwrap();
11713 assert_eq!(
11714 parse_error(&responses[0])["code"],
11715 "admission_facts_not_permitted"
11716 );
11717 }
11718
11719 #[tokio::test]
11720 async fn route_open_relays_consumer_capabilities_verbatim() {
11721 let registry = Arc::new(Registry::default());
11722 let forwarding = Arc::new(ForwardingTable::default());
11723 let handler =
11724 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11725 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11726 hello_via_sink(
11727 &handler,
11728 &module_ctx,
11729 &mut module_rx,
11730 hello_frame("aft", PROTOCOL_VERSION, 7),
11731 )
11732 .await;
11733
11734 let expected = vec!["elicitation".to_string(), "roots".to_string()];
11735 let expected_for_request = expected.clone();
11736 let project_root = unique_project_root("consumer-capabilities-present");
11737 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11738 let route_handler = handler.clone();
11739 let route_task = tokio::spawn(async move {
11740 route_handler
11741 .handle_control_frame(
11742 &client_ctx,
11743 route_open_frame_with_consumer_capabilities(
11744 401,
11745 "aft",
11746 project_root,
11747 Some(expected_for_request),
11748 ),
11749 )
11750 .await
11751 .unwrap()
11752 });
11753 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11754 .await
11755 .unwrap()
11756 .unwrap();
11757 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11758 let ModuleControlRequest::RouteBind {
11759 consumer_capabilities,
11760 ..
11761 } = bind
11762 else {
11763 panic!("expected route.bind request, got {bind:?}");
11764 };
11765 assert_eq!(consumer_capabilities, Some(expected.clone()));
11766
11767 handler
11768 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11769 .await
11770 .unwrap();
11771 let route_response = route_task.await.unwrap();
11772 assert!(route_response.is_empty());
11773 let published = client_rx.recv().await.unwrap();
11774 assert!(matches!(
11775 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11776 ClientControlResponse::RouteOpen { .. }
11777 ));
11778 }
11779
11780 #[tokio::test]
11781 async fn route_open_without_consumer_capabilities_relays_none() {
11782 let registry = Arc::new(Registry::default());
11783 let forwarding = Arc::new(ForwardingTable::default());
11784 let handler =
11785 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11786 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11787 hello_via_sink(
11788 &handler,
11789 &module_ctx,
11790 &mut module_rx,
11791 hello_frame("aft", PROTOCOL_VERSION, 7),
11792 )
11793 .await;
11794
11795 let project_root = unique_project_root("consumer-capabilities-absent");
11796 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11797 let route_handler = handler.clone();
11798 let route_task = tokio::spawn(async move {
11799 route_handler
11800 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11801 .await
11802 .unwrap()
11803 });
11804 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11805 .await
11806 .unwrap()
11807 .unwrap();
11808 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11809 let ModuleControlRequest::RouteBind {
11810 consumer_capabilities,
11811 ..
11812 } = bind
11813 else {
11814 panic!("expected route.bind request, got {bind:?}");
11815 };
11816 assert_eq!(consumer_capabilities, None);
11817
11818 handler
11819 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11820 .await
11821 .unwrap();
11822 let route_response = route_task.await.unwrap();
11823 assert!(route_response.is_empty());
11824 let published = client_rx.recv().await.unwrap();
11825 assert!(matches!(
11826 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11827 ClientControlResponse::RouteOpen { .. }
11828 ));
11829 }
11830
11831 async fn bind_role_versions_for(
11835 sent: Option<BTreeMap<String, String>>,
11836 connection: u64,
11837 ) -> Option<BTreeMap<String, String>> {
11838 let registry = Arc::new(Registry::default());
11839 let forwarding = Arc::new(ForwardingTable::default());
11840 let handler =
11841 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11842 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11843 hello_via_sink(
11844 &handler,
11845 &module_ctx,
11846 &mut module_rx,
11847 hello_frame("aft", PROTOCOL_VERSION, 7),
11848 )
11849 .await;
11850 let project_root = unique_project_root("role-versions");
11851 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11852 let route_handler = handler.clone();
11853 let route_task = tokio::spawn(async move {
11854 route_handler
11855 .handle_control_frame(
11856 &client_ctx,
11857 route_open_frame_with_role_versions(403, "aft", project_root, sent),
11858 )
11859 .await
11860 .unwrap()
11861 });
11862 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11863 .await
11864 .expect("a well-formed route.open reaches the module as a bind")
11865 .unwrap();
11866 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11867 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11868 panic!("expected route.bind request, got {bind:?}");
11869 };
11870 handler
11871 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11872 .await
11873 .unwrap();
11874 assert!(route_task.await.unwrap().is_empty());
11875 let published = client_rx.recv().await.unwrap();
11876 assert!(matches!(
11877 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11878 ClientControlResponse::RouteOpen { .. }
11879 ));
11880 role_versions
11881 }
11882
11883 #[tokio::test]
11884 async fn route_open_relays_role_versions_verbatim() {
11885 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11886 assert_eq!(
11887 bind_role_versions_for(Some(sent.clone()), 141).await,
11888 Some(sent)
11889 );
11890 }
11891
11892 #[tokio::test]
11895 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11896 assert_eq!(bind_role_versions_for(None, 143).await, None);
11897 assert_eq!(
11898 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11899 None
11900 );
11901 }
11902
11903 #[tokio::test]
11904 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11905 let registry = Arc::new(Registry::default());
11906 let forwarding = Arc::new(ForwardingTable::default());
11907 let handler =
11908 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11909 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11910 hello_via_sink(
11911 &handler,
11912 &module_ctx,
11913 &mut module_rx,
11914 hello_frame("aft", PROTOCOL_VERSION, 7),
11915 )
11916 .await;
11917
11918 let nine: BTreeMap<String, String> = (0..9)
11919 .map(|index| (format!("role-{index}"), "v1".to_string()))
11920 .collect();
11921 for (label, malformed) in [
11922 (
11923 "invalid role name",
11924 role_versions(&[("Tool_Provider", "v1")]),
11925 ),
11926 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11927 ("nine entries", nine),
11928 ] {
11929 let responses = tokio::time::timeout(
11932 Duration::from_secs(1),
11933 handler.handle_control_frame(
11934 &route_ctx(ConnectionId::new(148)).0,
11935 route_open_frame_with_role_versions(
11936 404,
11937 "aft",
11938 unique_project_root("role-versions-malformed"),
11939 Some(malformed),
11940 ),
11941 ),
11942 )
11943 .await
11944 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11945 .unwrap();
11946 assert_eq!(responses.len(), 1, "{label}");
11947 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11948 let error = parse_error(&responses[0]);
11949 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11950 assert_eq!(
11951 error["detail"]["field"], "role_versions",
11952 "{label}: {error}"
11953 );
11954 assert!(
11955 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11956 "{label}: a malformed declaration is terminal"
11957 );
11958 assert!(
11959 module_rx.try_recv().is_err(),
11960 "{label}: the module must never see a bind"
11961 );
11962 }
11963 }
11964
11965 #[tokio::test]
11969 async fn route_role_versions_capability_is_advertised() {
11970 let handler = ControlHandler::new(Arc::new(Registry::default()));
11971 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11972 let ack = hello_via_sink(
11973 &handler,
11974 &ctx,
11975 &mut rx,
11976 hello_frame("m", PROTOCOL_VERSION, 1),
11977 )
11978 .await;
11979 let ack = parse_ack(&ack);
11980 assert!(
11981 ack.subc_capabilities
11982 .iter()
11983 .any(|c| c == "route-role-versions/v1"),
11984 "{:?}",
11985 ack.subc_capabilities
11986 );
11987
11988 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11989 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11990 let reply = handler
11991 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11992 .await
11993 .unwrap()
11994 .pop()
11995 .unwrap();
11996 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11997 serde_json::from_slice(&reply.body).unwrap()
11998 else {
11999 panic!("not a server.describe reply");
12000 };
12001 assert!(
12002 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12003 "{capabilities:?}"
12004 );
12005 }
12006
12007 #[tokio::test]
12011 async fn catalog_events_capability_is_advertised() {
12012 let handler = ControlHandler::new(Arc::new(Registry::default()));
12013 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12014 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12015 let reply = handler
12016 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12017 .await
12018 .unwrap()
12019 .pop()
12020 .unwrap();
12021 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12022 serde_json::from_slice(&reply.body).unwrap()
12023 else {
12024 panic!("not a server.describe reply");
12025 };
12026 assert!(
12027 capabilities
12028 .iter()
12029 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12030 "{capabilities:?}"
12031 );
12032 }
12033
12034 #[tokio::test]
12035 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12036 let handler = ControlHandler::new(Arc::new(Registry::default()));
12037 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12038 let ack = hello_via_sink(
12039 &handler,
12040 &module,
12041 &mut module_rx,
12042 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12043 )
12044 .await;
12045 let ack = parse_ack(&ack);
12046 assert!(
12047 ack.subc_capabilities
12048 .iter()
12049 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12050 "{:?}",
12051 ack.subc_capabilities
12052 );
12053 assert!(ack
12054 .subc_capabilities
12055 .iter()
12056 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12057
12058 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12059 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12060 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12061 let reply = handler
12062 .handle_control_frame(&client, frame)
12063 .await
12064 .unwrap()
12065 .pop()
12066 .unwrap();
12067 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12068 serde_json::from_slice(&reply.body).unwrap()
12069 else {
12070 panic!("not a server.describe reply");
12071 };
12072 assert!(
12073 capabilities
12074 .iter()
12075 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12076 "{capabilities:?}"
12077 );
12078 assert!(capabilities
12079 .iter()
12080 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12081 }
12082
12083 #[tokio::test]
12084 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12085 let handler = ControlHandler::new(Arc::new(Registry::default()));
12086 let (module, mut rx) = route_ctx(ConnectionId::new(156));
12087 let ack = hello_via_sink(
12088 &handler,
12089 &module,
12090 &mut rx,
12091 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12092 )
12093 .await;
12094 assert!(parse_ack(&ack)
12095 .subc_capabilities
12096 .iter()
12097 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12098 assert!(parse_ack(&ack)
12099 .subc_capabilities
12100 .iter()
12101 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12102 let (client, _rx) = route_ctx(ConnectionId::new(157));
12103 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12104 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12105 let reply = handler
12106 .handle_control_frame(&client, frame)
12107 .await
12108 .unwrap()
12109 .pop()
12110 .unwrap();
12111 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12112 serde_json::from_slice(&reply.body).unwrap()
12113 else {
12114 panic!("server.describe reply")
12115 };
12116 assert!(capabilities
12117 .iter()
12118 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12119 assert!(capabilities
12120 .iter()
12121 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12122 }
12123
12124 #[tokio::test]
12125 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12126 let handler = ControlHandler::new(Arc::new(Registry::default()));
12127 let (module, mut rx) = route_ctx(ConnectionId::new(158));
12128 let ack = hello_via_sink(
12129 &handler,
12130 &module,
12131 &mut rx,
12132 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12133 )
12134 .await;
12135 assert!(parse_ack(&ack)
12136 .subc_capabilities
12137 .iter()
12138 .any(|cap| cap == "scope-workdir/v1"));
12139 let (client, _rx) = route_ctx(ConnectionId::new(159));
12140 let frame = Frame::build(
12141 FrameType::Request,
12142 control_flags(),
12143 0,
12144 0,
12145 5,
12146 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12147 )
12148 .unwrap();
12149 let reply = handler
12150 .handle_control_frame(&client, frame)
12151 .await
12152 .unwrap()
12153 .remove(0);
12154 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12155 serde_json::from_slice(&reply.body).unwrap()
12156 else {
12157 panic!("server.describe reply")
12158 };
12159 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12160 }
12161
12162 #[tokio::test]
12163 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12164 let handler = ControlHandler::new(Arc::new(Registry::default()));
12165 let (module, mut rx) = route_ctx(ConnectionId::new(154));
12166 let ack = hello_via_sink(
12167 &handler,
12168 &module,
12169 &mut rx,
12170 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12171 )
12172 .await;
12173 assert!(parse_ack(&ack)
12174 .subc_capabilities
12175 .iter()
12176 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12177 let (client, _rx) = route_ctx(ConnectionId::new(155));
12178 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12179 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12180 let reply = handler
12181 .handle_control_frame(&client, frame)
12182 .await
12183 .unwrap()
12184 .pop()
12185 .unwrap();
12186 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12187 serde_json::from_slice(&reply.body).unwrap()
12188 else {
12189 panic!("not a server.describe reply")
12190 };
12191 assert!(capabilities
12192 .iter()
12193 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12194 }
12195
12196 #[tokio::test]
12197 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12198 let registry = Arc::new(Registry::default());
12199 let forwarding = Arc::new(ForwardingTable::default());
12200 let handler =
12201 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12202 .with_health_probe_timeout(Duration::from_secs(5));
12203 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12204 hello_via_sink(
12205 &handler,
12206 &module_ctx,
12207 &mut module_rx,
12208 non_routable_hello_frame_with_control_ops(
12209 "mcp",
12210 300,
12211 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12212 ),
12213 )
12214 .await;
12215 assert!(registry
12216 .get_module("mcp")
12217 .unwrap()
12218 .unwrap()
12219 .manifest
12220 .provides
12221 .is_empty());
12222
12223 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12224 let route_response = handler
12225 .handle_control_frame(
12226 &route_client_ctx,
12227 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12228 )
12229 .await
12230 .unwrap();
12231 assert_eq!(route_response[0].header.ty, FrameType::Error);
12232 assert_eq!(
12233 parse_error(&route_response[0])["code"],
12234 "target_unavailable"
12235 );
12236 assert!(parse_error(&route_response[0])["message"]
12237 .as_str()
12238 .unwrap()
12239 .contains("does not provide the requested target"));
12240 assert!(module_rx.try_recv().is_err());
12241
12242 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12243 let health_handler = handler.clone();
12244 let health_task = tokio::spawn(async move {
12245 health_handler
12246 .handle_control_frame(
12247 &health_client_ctx,
12248 supervisor_health_probe_frame(302, "mcp"),
12249 )
12250 .await
12251 .unwrap()
12252 });
12253 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12254 .await
12255 .unwrap()
12256 .unwrap();
12257 assert_eq!(
12258 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12259 ModuleControlRequest::HealthCheck {}
12260 );
12261 handler
12262 .handle_control_frame(
12263 &module_ctx,
12264 health_response(health_frame.header.corr, HealthStatus::Ok),
12265 )
12266 .await
12267 .unwrap();
12268 let health_response = health_task.await.unwrap();
12269 assert_eq!(health_response[0].header.ty, FrameType::Response);
12270 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12271 ClientControlResponse::SupervisorHealthProbe {
12272 module_id, status, ..
12273 } => {
12274 assert_eq!(module_id, "mcp");
12275 assert_eq!(status, HealthStatus::Ok);
12276 }
12277 other => panic!("unexpected health response: {other:?}"),
12278 }
12279
12280 forwarding
12285 .cleanup_connection(module_ctx.connection_id)
12286 .unwrap();
12287 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12288 let cleanup_response = tokio::time::timeout(
12289 Duration::from_millis(200),
12290 handler.handle_control_frame(
12291 &cleanup_probe_ctx,
12292 supervisor_health_probe_frame(303, "mcp"),
12293 ),
12294 )
12295 .await
12296 .expect("probe should fail immediately when the control lane is gone")
12297 .unwrap();
12298 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12299 assert_eq!(
12300 parse_error(&cleanup_response[0])["code"],
12301 "target_unavailable"
12302 );
12303 assert!(parse_error(&cleanup_response[0])["message"]
12304 .as_str()
12305 .unwrap()
12306 .contains("no module connection"));
12307
12308 handler
12309 .cleanup_connection(module_ctx.connection_id)
12310 .unwrap();
12311 }
12312
12313 #[tokio::test]
12314 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12315 let registry = Arc::new(Registry::default());
12316 let supervisor_handle = SupervisorHandle::new();
12317 let supervisor =
12318 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12319 .with_handle(supervisor_handle.clone())
12320 .with_connection_file_path(
12321 std::env::temp_dir()
12322 .join(format!("subc-route-open-warming-{}", std::process::id())),
12323 );
12324 let module = supervisor
12325 .supervise_configured(
12326 ModuleSpec {
12327 module_id: "warming".to_string(),
12328 program: fake_aft_stub_path(),
12329 args: Vec::new(),
12330 env: Vec::new(),
12331 reserved: false,
12332 reserved_prefixes: Vec::new(),
12333 protocol: ModuleProtocol::Subc,
12334 overlap: Default::default(),
12335 },
12336 true,
12337 )
12338 .unwrap();
12339 assert_eq!(module.state().unwrap(), ModuleState::Running);
12340
12341 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12342 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12343 let response = handler
12344 .handle_control_frame(
12345 &ctx,
12346 route_open_frame(304, "warming", unique_project_root("warming")),
12347 )
12348 .await
12349 .unwrap();
12350 module.stop().await.unwrap();
12351
12352 assert_eq!(response[0].header.ty, FrameType::Error);
12353 let error = parse_error(&response[0]);
12354 assert_eq!(error["code"], "module_warming");
12355 assert!(error["message"]
12356 .as_str()
12357 .unwrap()
12358 .contains("state=running, enabled=true, live=false"));
12359 }
12360
12361 #[test]
12362 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12363 let handler = ControlHandler::new(Arc::new(Registry::default()));
12364 let capture = EventCapture::default();
12365 let _subscriber =
12366 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12367 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12368 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12369 let pending = (0..limit).collect::<Vec<_>>();
12370 let response = handler
12371 .route_open_capacity_refusal(
12372 &ctx,
12373 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12374 "busy",
12375 pending.len(),
12376 limit,
12377 )
12378 .unwrap();
12379 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12380 let event = capture
12381 .events()
12382 .into_iter()
12383 .find(|event| {
12384 event.target == "control"
12385 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12386 })
12387 .expect("connection admission refusal event");
12388 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12389 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12390 }
12391
12392 #[test]
12393 fn route_open_target_cap_logs_admission_reason_and_capacity() {
12394 let handler = ControlHandler::new(Arc::new(Registry::default()));
12395 let capture = EventCapture::default();
12396 let _subscriber =
12397 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12398 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12399 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12400 let guards = (0..limit)
12401 .map(|_| {
12402 handler
12403 .route_bind_concurrency
12404 .try_admit("busy", limit)
12405 .unwrap()
12406 })
12407 .collect::<Vec<_>>();
12408 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12409 Err(in_flight) => in_flight,
12410 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12411 };
12412 let response = handler
12413 .route_open_target_capacity_refusal(
12414 &ctx,
12415 &route_open_frame(397, "busy", unique_project_root("target-cap")),
12416 "busy",
12417 in_flight,
12418 )
12419 .unwrap();
12420 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12421 let event = capture
12422 .events()
12423 .into_iter()
12424 .find(|event| {
12425 event.target == "control"
12426 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12427 })
12428 .expect("target admission refusal event");
12429 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12430 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12431 drop(guards);
12432 }
12433
12434 #[tokio::test]
12435 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
12436 let registry = Arc::new(Registry::default());
12437 let handle = SupervisorHandle::new();
12438 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
12439 .with_handle(handle.clone());
12440 supervisor
12441 .supervise_configured(
12442 crate::ModuleSpec {
12443 module_id: "drift".into(),
12444 program: PathBuf::from("disabled-module"),
12445 args: Vec::new(),
12446 env: Vec::new(),
12447 reserved: false,
12448 reserved_prefixes: Vec::new(),
12449 protocol: ModuleProtocol::Subc,
12450 overlap: Default::default(),
12451 },
12452 false,
12453 )
12454 .unwrap();
12455 let handler = ControlHandler::new(registry).with_supervisor(handle);
12456 for configured in [None, Some(false), Some(true)] {
12457 handler.capability_evaluator.configure(
12458 configured.map(|enabled| ("drift".to_string(), enabled)),
12459 BTreeMap::new(),
12460 );
12461 let frame =
12462 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12463 let replies = handler
12464 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
12465 .await
12466 .unwrap();
12467 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12468 assert!(body.get("configured_enabled").is_none());
12469 assert_eq!(body["op"], "supervisor.ack");
12470 let frame =
12471 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12472 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12473 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12474 assert_eq!(
12475 body["modules"][0].get("configured_enabled"),
12476 configured.map(Value::Bool).as_ref()
12477 );
12478 }
12479 }
12480
12481 #[tokio::test(flavor = "current_thread")]
12482 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12483 let handler = ControlHandler::new(Arc::new(Registry::default()));
12484 let capture = EventCapture::default();
12485 let _subscriber =
12486 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12487 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12488 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12489
12490 let _ = handler
12491 .handle_client_control_request(
12492 &ctx,
12493 frame,
12494 ClientControlRequest::SupervisorSetEnabled {
12495 module_id: "broca".to_string(),
12496 enabled: false,
12497 },
12498 &mut None,
12499 )
12500 .await
12501 .unwrap();
12502
12503 let events = capture
12504 .events()
12505 .into_iter()
12506 .filter(|event| {
12507 event.target == "control"
12508 && event.fields.get("message").map(String::as_str)
12509 == Some("supervisor request received")
12510 })
12511 .collect::<Vec<_>>();
12512 assert_eq!(events.len(), 1, "one supervisor request log line");
12513 let event = &events[0];
12514 assert_eq!(event.level, tracing::Level::INFO);
12515 assert_eq!(
12516 event.fields.get("op"),
12517 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12518 );
12519 assert_eq!(
12520 event.fields.get("module_id"),
12521 Some(&"\"broca\"".to_string())
12522 );
12523 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12524 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12525 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12526 }
12527
12528 #[tokio::test(flavor = "current_thread")]
12529 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12530 let registry = Arc::new(Registry::default());
12531 let module_connection = ConnectionId::new(303);
12532 registry
12533 .register_with_control_ops(
12534 manifest("aft", PROTOCOL_VERSION),
12535 PROTOCOL_VERSION,
12536 module_connection,
12537 module_baseline_control_ops(),
12538 )
12539 .unwrap();
12540 let handler = ControlHandler::new(registry);
12541 let capture = EventCapture::default();
12542 let _subscriber =
12543 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12544 let (ctx, _rx) = route_ctx(module_connection);
12545 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12546
12547 let _ = handler
12548 .handle_client_control_request(
12549 &ctx,
12550 frame,
12551 ClientControlRequest::SupervisorSetEnabled {
12552 module_id: "broca".to_string(),
12553 enabled: false,
12554 },
12555 &mut None,
12556 )
12557 .await
12558 .unwrap();
12559
12560 let callers = capture
12561 .events()
12562 .into_iter()
12563 .filter(|event| {
12564 event.fields.get("message").map(String::as_str)
12565 == Some("supervisor request received")
12566 })
12567 .map(|event| event.fields.get("caller").cloned())
12568 .collect::<Vec<_>>();
12569 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12570 }
12571
12572 #[tokio::test(flavor = "current_thread")]
12573 async fn supervisor_list_does_not_log_request_received() {
12574 let handler = ControlHandler::new(Arc::new(Registry::default()));
12575 let capture = EventCapture::default();
12576 let _subscriber =
12577 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12578 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12579 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12580
12581 handler
12582 .handle_client_control_request(
12583 &ctx,
12584 frame,
12585 ClientControlRequest::SupervisorList {},
12586 &mut None,
12587 )
12588 .await
12589 .unwrap();
12590
12591 assert!(capture.events().into_iter().all(|event| {
12592 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12593 }));
12594 }
12595
12596 #[tokio::test(flavor = "current_thread")]
12597 async fn supervisor_rescan_preview_does_not_log_request_received() {
12598 let handler = ControlHandler::new(Arc::new(Registry::default()));
12599 let capture = EventCapture::default();
12600 let _subscriber =
12601 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12602 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
12603 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12604
12605 handler
12606 .handle_client_control_request(
12607 &ctx,
12608 frame,
12609 ClientControlRequest::SupervisorRescan { preview: true },
12610 &mut None,
12611 )
12612 .await
12613 .unwrap();
12614
12615 assert!(capture.events().into_iter().all(|event| {
12616 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12617 }));
12618 }
12619
12620 struct ApproveOperator;
12621 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
12622 fn prompt(
12623 &self,
12624 _: &str,
12625 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
12626 ) -> crate::operator_confirm::ProviderResult {
12627 crate::operator_confirm::ProviderResult::Approved
12628 }
12629 }
12630
12631 #[tokio::test]
12632 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
12633 assert_eq!(
12634 ControlHandler::default().privileged_operator_prompt,
12635 PrivilegedOperatorPrompt::Log
12636 );
12637 for (initial, changed, running) in [
12638 ("log", "enforce", PrivilegedOperatorPrompt::Log),
12639 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
12640 ] {
12641 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
12642 let path = dir.join("subc.jsonc");
12643 std::fs::write(
12644 &path,
12645 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
12646 )
12647 .unwrap();
12648 let config = crate::daemon_config::load(&path).unwrap().unwrap();
12649 let registry = Arc::new(Registry::default());
12650 let handle = SupervisorHandle::new();
12651 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
12652 .with_handle(handle.clone());
12653 let handler = ControlHandler::new(registry)
12654 .with_supervisor(handle)
12655 .with_privileged_operator_prompt(config.privileged_operator_prompt)
12656 .with_supervisor_rescan(supervisor, &path, config.port);
12657 handler.forwarding.operator_confirms().configure(
12658 Arc::new(ApproveOperator),
12659 crate::operator_confirm::OperatorLimits::default(),
12660 );
12661 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
12662
12663 for (disk_mode, needs_restart) in [
12664 (Some(initial), false),
12665 (Some(changed), true),
12666 (Some(changed), true),
12667 (None, running == PrivilegedOperatorPrompt::Enforce),
12668 (Some(initial), false),
12669 ] {
12670 let mut doc = json!({"version":1});
12671 if let Some(mode) = disk_mode {
12672 doc["privileged_operator_prompt"] = json!(mode);
12673 }
12674 std::fs::write(&path, doc.to_string()).unwrap();
12675 for preview in [true, false] {
12676 let request = ClientControlRequest::SupervisorRescan { preview };
12677 let frame = Frame::build(
12678 FrameType::Request,
12679 control_flags(),
12680 0,
12681 0,
12682 1,
12683 serde_json::to_vec(&request).unwrap(),
12684 )
12685 .unwrap();
12686 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12687 assert_eq!(frames.len(), 1);
12688 let ClientControlResponse::SupervisorRescan { result } =
12689 serde_json::from_slice(&frames[0].body).unwrap()
12690 else {
12691 panic!("expected a rescan response");
12692 };
12693 let expected = if needs_restart {
12694 vec!["privileged_operator_prompt"]
12695 } else {
12696 Vec::new()
12697 };
12698 assert_eq!(result.restart_required, expected);
12699 assert_eq!(handler.privileged_operator_prompt, running);
12700 assert_eq!(
12701 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
12702 running
12703 );
12704 }
12705 }
12706 }
12707 }
12708
12709 #[tokio::test]
12710 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
12711 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
12712 let path = dir.join("subc.jsonc");
12713 let registry = Arc::new(Registry::default());
12714 let handle = SupervisorHandle::new();
12715 let supervisor =
12716 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
12717 let handler = ControlHandler::new(registry)
12718 .with_supervisor(handle)
12719 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
12720 .with_supervisor_rescan(supervisor, &path, None);
12721 handler.forwarding.operator_confirms().configure(
12722 Arc::new(ApproveOperator),
12723 crate::operator_confirm::OperatorLimits::default(),
12724 );
12725 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
12726 for preview in [true, false] {
12727 std::fs::write(
12728 &path,
12729 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
12730 )
12731 .unwrap();
12732 let request = ClientControlRequest::SupervisorRescan { preview };
12733 let frame = Frame::build(
12734 FrameType::Request,
12735 control_flags(),
12736 0,
12737 0,
12738 1,
12739 serde_json::to_vec(&request).unwrap(),
12740 )
12741 .unwrap();
12742 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12743 assert_eq!(frames.len(), 1);
12744 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
12745 assert_eq!(
12746 handler.privileged_operator_prompt,
12747 PrivilegedOperatorPrompt::Enforce
12748 );
12749 }
12750 }
12751
12752 #[tokio::test]
12757 async fn route_open_refusal_names_the_check_that_refused() {
12758 let handler = ControlHandler::new(Arc::new(Registry::default()));
12759 let capture = EventCapture::default();
12760 let _subscriber =
12761 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12762 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12763 let response = handler
12764 .handle_control_frame(
12765 &ctx,
12766 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
12767 )
12768 .await
12769 .unwrap();
12770
12771 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12772 let event = capture
12773 .events()
12774 .into_iter()
12775 .find(|event| {
12776 event.target == "control"
12777 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12778 })
12779 .expect("route.open refusal event");
12780 assert_eq!(
12781 event.fields.get("reason"),
12782 Some(&"\"not_registered\"".to_string())
12783 );
12784 }
12785
12786 #[tokio::test]
12787 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12788 let registry = Arc::new(Registry::default());
12789 let supervisor_handle = SupervisorHandle::new();
12790 let supervisor =
12791 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12792 .with_handle(supervisor_handle.clone())
12793 .with_connection_file_path(std::env::temp_dir().join(format!(
12794 "subc-route-open-refusal-info-{}",
12795 std::process::id()
12796 )));
12797 let module = supervisor
12798 .supervise_configured(
12799 ModuleSpec {
12800 module_id: "warming".to_string(),
12801 program: fake_aft_stub_path(),
12802 args: Vec::new(),
12803 env: Vec::new(),
12804 reserved: false,
12805 reserved_prefixes: Vec::new(),
12806 protocol: ModuleProtocol::Subc,
12807 overlap: Default::default(),
12808 },
12809 true,
12810 )
12811 .unwrap();
12812 assert_eq!(module.state().unwrap(), ModuleState::Running);
12813
12814 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12815 assert!(handler
12816 .counters()
12817 .snapshot()
12818 .get("route_open_refused_by_code")
12819 .is_none());
12820 let capture = EventCapture::default();
12821 let _subscriber =
12822 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12823 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
12824 let response = handler
12825 .handle_control_frame(
12826 &ctx,
12827 route_open_frame(394, "warming", unique_project_root("refusal-info")),
12828 )
12829 .await
12830 .unwrap();
12831 module.stop().await.unwrap();
12832
12833 assert_eq!(parse_error(&response[0])["code"], "module_warming");
12834 let event = capture
12835 .events()
12836 .into_iter()
12837 .find(|event| {
12838 event.target == "control"
12839 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
12840 })
12841 .expect("route.open refusal event");
12842 assert_eq!(
12843 event.fields.get("module_id"),
12844 Some(&"\"warming\"".to_string())
12845 );
12846 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
12847 assert_eq!(
12848 event.fields.get("reason"),
12849 Some(&"\"supervised_not_registered\"".to_string())
12850 );
12851 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
12852 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
12853 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
12854 assert_eq!(
12855 handler.counters().snapshot()["route_open_refused_by_code"],
12856 json!({ "module_warming": 1 })
12857 );
12858 }
12859
12860 const OUTAGE_START: &str = "route.open refusing module: not serving";
12861 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
12862
12863 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
12864 capture
12865 .events()
12866 .into_iter()
12867 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
12868 .collect()
12869 }
12870
12871 fn supervise_stub(
12872 registry: &Arc<Registry>,
12873 module_id: &str,
12874 enabled: bool,
12875 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
12876 let supervisor_handle = SupervisorHandle::new();
12877 let supervisor =
12878 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
12879 .with_handle(supervisor_handle.clone())
12880 .with_connection_file_path(std::env::temp_dir().join(format!(
12881 "subc-route-outage-{module_id}-{}",
12882 std::process::id()
12883 )));
12884 let module = supervisor
12885 .supervise_configured(
12886 ModuleSpec {
12887 module_id: module_id.to_string(),
12888 program: fake_aft_stub_path(),
12889 args: Vec::new(),
12890 env: Vec::new(),
12891 reserved: false,
12892 reserved_prefixes: Vec::new(),
12893 protocol: ModuleProtocol::Subc,
12894 overlap: Default::default(),
12895 },
12896 enabled,
12897 )
12898 .unwrap();
12899 (supervisor_handle, module)
12900 }
12901
12902 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12903 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12904 module_id: module_id.to_string(),
12905 drain_timeout_ms: Some(50),
12906 })
12907 .unwrap();
12908 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12909 }
12910
12911 #[test]
12915 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12916 let registry = Arc::new(Registry::default());
12917 let forwarding = Arc::new(ForwardingTable::default());
12918 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12919 let second = ControlHandler::with_forwarding(registry, forwarding);
12920 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12921 }
12922
12923 #[tokio::test(flavor = "current_thread")]
12927 async fn route_open_unknown_module_refusals_add_no_outage_state() {
12928 let handler = ControlHandler::new(Arc::new(Registry::default()));
12929 let capture = EventCapture::default();
12930 let _subscriber =
12931 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12932 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12933 for corr in 0..8 {
12934 let response = handler
12935 .handle_control_frame(
12936 &ctx,
12937 route_open_frame(
12938 380 + corr,
12939 &format!("nobody-{corr}"),
12940 unique_project_root("outage-unknown"),
12941 ),
12942 )
12943 .await
12944 .unwrap();
12945 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12946 }
12947
12948 assert_eq!(handler.route_outages.tracked_module_count(), 0);
12949 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12950 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12951 }
12952
12953 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12958 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12959 let registry = Arc::new(Registry::default());
12960 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
12961 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12962 let capture = EventCapture::default();
12963 let _subscriber =
12964 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12965 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12966 handler.route_outages.record_accepted("outage-restart");
12969
12970 let response = handler
12971 .handle_control_frame(
12972 &ctx,
12973 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12974 )
12975 .await
12976 .unwrap();
12977 assert_eq!(response[0].header.ty, FrameType::Error);
12978 let starts = outage_lines(&capture, OUTAGE_START);
12979 assert_eq!(starts.len(), 1, "{starts:?}");
12980 assert_eq!(starts[0].level, tracing::Level::WARN);
12981 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12982 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12983 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12984 handler.route_outages.record_accepted("outage-restart");
12985 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12986
12987 let restart = handler
12988 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12989 .await
12990 .unwrap();
12991 assert_eq!(
12992 restart[0].header.ty,
12993 FrameType::Response,
12994 "{:?}",
12995 parse_error(&restart[0])
12996 );
12997 handler
12998 .handle_control_frame(
12999 &ctx,
13000 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13001 )
13002 .await
13003 .unwrap();
13004 module.stop().await.unwrap();
13005
13006 let starts = outage_lines(&capture, OUTAGE_START);
13007 assert_eq!(starts.len(), 2, "{starts:?}");
13008 assert_eq!(starts[1].level, tracing::Level::INFO);
13009 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13010 }
13011
13012 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13016 async fn failed_operator_restart_leaves_no_operator_mark() {
13017 let registry = Arc::new(Registry::default());
13018 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
13019 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13020 let capture = EventCapture::default();
13021 let _subscriber =
13022 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13023 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13024 handler.route_outages.record_accepted("outage-disabled");
13025
13026 let restart = handler
13027 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13028 .await
13029 .unwrap();
13030 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13031 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13032
13033 handler
13034 .handle_control_frame(
13035 &ctx,
13036 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13037 )
13038 .await
13039 .unwrap();
13040 let starts = outage_lines(&capture, OUTAGE_START);
13041 assert_eq!(starts.len(), 1, "{starts:?}");
13042 assert_eq!(starts[0].level, tracing::Level::WARN);
13043 }
13044
13045 #[tokio::test(flavor = "current_thread")]
13046 async fn route_open_unknown_module_escapes_target_module_id() {
13047 let handler = ControlHandler::new(Arc::new(Registry::default()));
13048 let capture = EventCapture::default();
13049 let _subscriber =
13050 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13051 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13052 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13053 let response = handler
13054 .handle_control_frame(
13055 &ctx,
13056 route_open_frame(
13057 395,
13058 hostile_module_id,
13059 unique_project_root("hostile-target-module-id"),
13060 ),
13061 )
13062 .await
13063 .unwrap();
13064
13065 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13066 let event = capture
13067 .events()
13068 .into_iter()
13069 .find(|event| {
13070 event.target == "control"
13071 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13072 })
13073 .expect("route.open unknown-module refusal event");
13074 let logged = event.fields.get("module_id").expect("module_id field");
13075 assert!(!logged.bytes().any(|byte| byte < 0x20));
13076 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13077 }
13078
13079 #[tokio::test(flavor = "current_thread")]
13080 async fn route_open_module_rejection_uses_daemon_counter_key() {
13081 let registry = Arc::new(Registry::default());
13082 let forwarding = Arc::new(ForwardingTable::default());
13083 let handler =
13084 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13085 let module_connection = ConnectionId::new(95);
13086 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13087 hello_via_sink(
13088 &handler,
13089 &module_ctx,
13090 &mut module_rx,
13091 hello_frame("aft", PROTOCOL_VERSION, 395),
13092 )
13093 .await;
13094
13095 let client_connection = ConnectionId::new(96);
13096 let (client_ctx, _client_rx) = route_ctx(client_connection);
13097 let capture = EventCapture::default();
13098 let _subscriber =
13099 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13100 let (route_task, bind) = relay_route_open(
13101 &handler,
13102 client_connection,
13103 &client_ctx.egress,
13104 &mut module_rx,
13105 396,
13106 "aft",
13107 "hostile-module-code",
13108 )
13109 .await;
13110 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13111 let rejection = Frame::build(
13112 FrameType::Error,
13113 control_flags(),
13114 0,
13115 0,
13116 bind.header.corr,
13117 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13118 )
13119 .unwrap();
13120 handler
13121 .handle_control_frame(&module_ctx, rejection)
13122 .await
13123 .unwrap();
13124
13125 let response = route_task.await.unwrap();
13126 assert_eq!(parse_error(&response[0])["code"], hostile_code);
13127 let counters = handler.counters().snapshot();
13128 assert_eq!(
13129 counters["route_open_refused_by_code"],
13130 json!({ "module_rejected": 1 })
13131 );
13132 assert!(counters["route_open_refused_by_code"]
13133 .get(hostile_code)
13134 .is_none());
13135
13136 let event = capture
13137 .events()
13138 .into_iter()
13139 .find(|event| {
13140 event.target == "control"
13141 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13142 })
13143 .expect("route.open module-rejection refusal event");
13144 let logged = event.fields.get("module_code").expect("module_code field");
13145 assert!(!logged.bytes().any(|byte| byte < 0x20));
13146 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13147 }
13148
13149 #[tokio::test]
13150 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13151 let registry = Arc::new(Registry::default());
13152 let supervisor_handle = SupervisorHandle::new();
13153 let missing_program = std::env::temp_dir().join(format!(
13154 "subc-route-open-missing-program-{}",
13155 std::process::id()
13156 ));
13157 let supervisor =
13158 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13159 .with_handle(supervisor_handle.clone());
13160 let module = supervisor
13161 .supervise_configured(
13162 ModuleSpec {
13163 module_id: "failed".to_string(),
13164 program: missing_program,
13165 args: Vec::new(),
13166 env: Vec::new(),
13167 reserved: false,
13168 reserved_prefixes: Vec::new(),
13169 protocol: ModuleProtocol::Subc,
13170 overlap: Default::default(),
13171 },
13172 true,
13173 )
13174 .unwrap();
13175 assert_eq!(module.state().unwrap(), ModuleState::Failed);
13176
13177 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13178 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13179 let response = handler
13180 .handle_control_frame(
13181 &ctx,
13182 route_open_frame(305, "failed", unique_project_root("failed")),
13183 )
13184 .await
13185 .unwrap();
13186
13187 assert_eq!(response[0].header.ty, FrameType::Error);
13188 let error = parse_error(&response[0]);
13189 assert_eq!(error["code"], "target_unavailable");
13190 assert!(error["message"]
13191 .as_str()
13192 .unwrap()
13193 .contains("state=failed, enabled=true, live=false"));
13194 }
13195
13196 #[tokio::test]
13197 async fn route_open_role_mismatch_remains_target_unavailable() {
13198 let registry = Arc::new(Registry::default());
13199 let handler = ControlHandler::new(Arc::clone(®istry));
13200 handler
13201 .handle_control(
13202 ConnectionId::new(41),
13203 non_routable_hello_frame_with_control_ops("health-only", 306, None),
13204 )
13205 .unwrap();
13206
13207 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13208 let response = handler
13209 .handle_control_frame(
13210 &ctx,
13211 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13212 )
13213 .await
13214 .unwrap();
13215
13216 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13217 assert!(parse_error(&response[0])["message"]
13218 .as_str()
13219 .unwrap()
13220 .contains("does not provide the requested target"));
13221 }
13222
13223 #[tokio::test]
13224 async fn route_open_inactive_registration_remains_target_unavailable() {
13225 let registry = Arc::new(Registry::default());
13226 let handler = ControlHandler::new(Arc::clone(®istry));
13227 handler
13228 .handle_control(
13229 ConnectionId::new(43),
13230 hello_frame("inactive", PROTOCOL_VERSION, 308),
13231 )
13232 .unwrap();
13233 assert!(registry
13234 .set_module_state_for_test("inactive", ChannelState::Closed)
13235 .unwrap());
13236
13237 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13238 let response = handler
13239 .handle_control_frame(
13240 &ctx,
13241 route_open_frame(309, "inactive", unique_project_root("inactive")),
13242 )
13243 .await
13244 .unwrap();
13245
13246 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13247 assert!(parse_error(&response[0])["message"]
13248 .as_str()
13249 .unwrap()
13250 .contains("is not active"));
13251 }
13252
13253 #[tokio::test]
13254 async fn late_health_reply_is_recorded_through_the_module_response_path() {
13255 let registry = Arc::new(Registry::default());
13256 let forwarding = Arc::new(ForwardingTable::default());
13257 let supervisor_handle = SupervisorHandle::new();
13258 let supervisor =
13259 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
13260 .with_forwarding(Arc::clone(&forwarding))
13261 .with_handle(supervisor_handle.clone());
13262 let module = supervisor
13263 .supervise_configured(
13264 crate::ModuleSpec {
13265 module_id: "late-health-response".to_string(),
13266 program: PathBuf::from("disabled-module"),
13267 args: Vec::new(),
13268 env: Vec::new(),
13269 reserved: false,
13270 reserved_prefixes: Vec::new(),
13271 protocol: ModuleProtocol::Subc,
13272 overlap: Default::default(),
13273 },
13274 false,
13275 )
13276 .unwrap();
13277 let handler =
13278 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13279 .with_supervisor(supervisor_handle);
13280 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13281 handler
13282 .handle_control_frame(
13283 &module_ctx,
13284 hello_frame_with_control_ops(
13285 "late-health-response",
13286 PROTOCOL_VERSION,
13287 7,
13288 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13289 ),
13290 )
13291 .await
13292 .unwrap();
13293 let probe_started_at = Instant::now() - Duration::from_millis(80);
13294 let pending = forwarding
13295 .begin_health_probe_rpc_for(
13296 "late-health-response",
13297 MODULE_CONTROL_OP_HEALTH_CHECK,
13298 probe_started_at,
13299 Instant::now() - Duration::from_millis(1),
13300 )
13301 .unwrap();
13302 assert!(forwarding
13303 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13304 .unwrap());
13305
13306 let responses = handler
13307 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13308 .await
13309 .unwrap();
13310
13311 assert!(responses.is_empty());
13312 let health = module.status().unwrap().health;
13313 assert_eq!(health.late_answer_count, 1);
13314 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13315 }
13316
13317 #[tokio::test]
13318 async fn health_probe_timeout_and_module_death_are_typed() {
13319 let registry = Arc::new(Registry::default());
13320 let forwarding = Arc::new(ForwardingTable::default());
13321 let handler =
13322 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13323 .with_health_probe_timeout(Duration::from_millis(50));
13324 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13325 hello_via_sink(
13326 &handler,
13327 &module_ctx,
13328 &mut module_rx,
13329 hello_frame_with_control_ops(
13330 "aft",
13331 PROTOCOL_VERSION,
13332 7,
13333 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13334 ),
13335 )
13336 .await;
13337
13338 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13339 let responses = handler
13340 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13341 .await
13342 .unwrap();
13343 assert_eq!(responses[0].header.ty, FrameType::Error);
13344 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13345 let _ = module_rx.try_recv();
13346
13347 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13348 let health_handler = handler.clone();
13349 let death_task = tokio::spawn(async move {
13350 health_handler
13351 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13352 .await
13353 .unwrap()
13354 });
13355 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13356 .await
13357 .unwrap()
13358 .unwrap();
13359 handler
13360 .cleanup_connection(module_ctx.connection_id)
13361 .unwrap();
13362 let responses = death_task.await.unwrap();
13363 assert_eq!(responses[0].header.ty, FrameType::Error);
13364 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13365 }
13366
13367 #[test]
13368 fn hello_requires_exact_protocol_version() {
13369 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13370 let registry = Arc::new(Registry::default());
13371 let handler = ControlHandler::new(Arc::clone(®istry));
13372 let responses = handler
13373 .handle_control(
13374 ConnectionId::new(connection),
13375 hello_frame("aft", offered, 9),
13376 )
13377 .unwrap();
13378
13379 assert_eq!(responses.len(), 1);
13380 assert_eq!(responses[0].header.ty, FrameType::Error);
13381 let error = parse_error(&responses[0]);
13382 assert_eq!(error["code"], "version_unsupported");
13383 assert!(registry.get_module("aft").unwrap().is_none());
13384 assert_eq!(registry.active_registration_count().unwrap(), 0);
13385 }
13386 }
13387
13388 #[test]
13389 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13390 let registry = Arc::new(Registry::default());
13391 let forwarding = Arc::new(ForwardingTable::default());
13392 let handler =
13393 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13394 let module_connection = ConnectionId::new(301);
13395 let registration = registry
13396 .register_with_control_ops(
13397 manifest("aft-push", PROTOCOL_VERSION),
13398 PROTOCOL_VERSION,
13399 module_connection,
13400 module_baseline_control_ops(),
13401 )
13402 .unwrap();
13403 let (module_tx, _module_rx) = mpsc::channel(8);
13404 let endpoint = forwarding
13405 .register_module_connection(
13406 module_connection,
13407 "aft-push".to_string(),
13408 PROTOCOL_VERSION,
13409 manifest_concurrency(®istration.manifest),
13410 FrameSink::new(module_tx),
13411 )
13412 .unwrap();
13413
13414 let unknown = Frame::build(
13416 FrameType::Push,
13417 control_flags(),
13418 0,
13419 0,
13420 5,
13421 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13422 )
13423 .unwrap();
13424 let out = handler.handle_status_update(endpoint, unknown).unwrap();
13425 assert!(
13426 out.is_empty(),
13427 "unknown push op must be ignored, got {out:?}"
13428 );
13429
13430 let malformed = Frame::build(
13432 FrameType::Push,
13433 control_flags(),
13434 0,
13435 0,
13436 6,
13437 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13438 )
13439 .unwrap();
13440 let out = handler.handle_status_update(endpoint, malformed).unwrap();
13441 assert_eq!(out.len(), 1);
13442 assert_eq!(out[0].header.ty, FrameType::Error);
13443 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13444 }
13445
13446 #[test]
13447 fn hello_rejected_when_connection_already_owns_client_routes() {
13448 let registry = Arc::new(Registry::default());
13449 let forwarding = Arc::new(ForwardingTable::default());
13450 let handler =
13451 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13452 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
13454 let client_connection = ConnectionId::new(202);
13455
13456 let responses = handler
13459 .handle_control(
13460 client_connection,
13461 hello_frame("aft-second", PROTOCOL_VERSION, 9),
13462 )
13463 .unwrap();
13464 assert_eq!(responses[0].header.ty, FrameType::Error);
13465 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13466 assert!(registry.get_module("aft-second").unwrap().is_none());
13467 }
13468
13469 #[tokio::test]
13470 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13471 let registry = Arc::new(Registry::default());
13472 let forwarding = Arc::new(ForwardingTable::default());
13473 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13474 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13475 hello_via_sink(
13476 &handler,
13477 &module_ctx,
13478 &mut module_rx,
13479 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13480 )
13481 .await;
13482 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13483 let pending = forwarding
13484 .begin_route_bind_relay_for_test(
13485 client_ctx.connection_id,
13486 client_ctx.egress.clone(),
13487 2,
13488 "alpha",
13489 )
13490 .unwrap();
13491 forwarding
13492 .complete_pending_relay(
13493 module_ctx.connection_id,
13494 pending.corr,
13495 RouteBindRelayOutcome::Accepted,
13496 )
13497 .unwrap();
13498 client_rx.try_recv().unwrap();
13499 for module_id in ["beta", "alpha"] {
13500 let replies = handler
13501 .handle_control_frame(
13502 &module_ctx,
13503 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13504 )
13505 .await
13506 .unwrap();
13507 assert_eq!(replies.len(), 1, "second HELLO must be refused");
13508 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13509 }
13510 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13511 assert!(registry.get_module("beta").unwrap().is_none());
13512 assert!(matches!(
13513 forwarding
13514 .lookup_data_route(
13515 client_ctx.connection_id,
13516 pending.client_channel,
13517 pending.client_epoch,
13518 )
13519 .unwrap(),
13520 DataRoute::Client(DataRouteState::Bound(_))
13521 ));
13522 assert!(handler
13523 .hello_launch_nonces
13524 .lock()
13525 .unwrap()
13526 .presented(module_ctx.connection_id, Some("alpha-nonce")));
13527 assert!(module_rx.try_recv().is_err());
13528 }
13529
13530 #[test]
13531 fn reserved_module_hello_requires_matching_launch_nonce() {
13532 let registry = Arc::new(Registry::default());
13533 let supervisor = SupervisorHandle::new();
13534 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13537 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13538
13539 let no_nonce = handler
13541 .handle_control(
13542 ConnectionId::new(1),
13543 hello_frame("vault", PROTOCOL_VERSION, 1),
13544 )
13545 .unwrap();
13546 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13547 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13548 assert!(registry.get_module("vault").unwrap().is_none());
13549
13550 let wrong = handler
13552 .handle_control(
13553 ConnectionId::new(2),
13554 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13555 )
13556 .unwrap();
13557 assert_eq!(wrong[0].header.ty, FrameType::Error);
13558 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13559 assert!(registry.get_module("vault").unwrap().is_none());
13560
13561 let ok = handler
13563 .handle_control(
13564 ConnectionId::new(3),
13565 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13566 )
13567 .unwrap();
13568 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13569 assert!(registry.get_module("vault").unwrap().is_some());
13570 }
13571
13572 #[test]
13573 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13574 let registry = Arc::new(Registry::default());
13575 let supervisor = SupervisorHandle::new();
13576 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13577 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13578 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13579
13580 let squat = handler
13581 .handle_control(
13582 ConnectionId::new(1),
13583 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13584 )
13585 .unwrap();
13586 assert_eq!(squat[0].header.ty, FrameType::Error);
13587 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
13588 assert!(parse_error(&squat[0])["message"]
13589 .as_str()
13590 .unwrap()
13591 .contains("fed:"));
13592
13593 let accepted_peer = handler
13594 .handle_control(
13595 ConnectionId::new(2),
13596 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
13597 )
13598 .unwrap();
13599 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
13600
13601 let accepted_short = handler
13602 .handle_control(
13603 ConnectionId::new(3),
13604 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
13605 )
13606 .unwrap();
13607 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
13608
13609 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
13610 let response = handler
13611 .handle_control(
13612 ConnectionId::new(conn),
13613 hello_frame(module_id, PROTOCOL_VERSION, conn),
13614 )
13615 .unwrap();
13616 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
13617 }
13618 }
13619
13620 #[test]
13621 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
13622 let registry = Arc::new(Registry::default());
13623 let supervisor = SupervisorHandle::new();
13624 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13625 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13626 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
13627 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13628
13629 let owner_nonce = handler
13630 .handle_control(
13631 ConnectionId::new(1),
13632 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
13633 )
13634 .unwrap();
13635 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
13636 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
13637 assert!(registry.get_module("fed:special").unwrap().is_none());
13638
13639 let exact_nonce = handler
13640 .handle_control(
13641 ConnectionId::new(2),
13642 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
13643 )
13644 .unwrap();
13645 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
13646 assert!(registry.get_module("fed:special").unwrap().is_some());
13647 }
13648
13649 #[test]
13650 fn non_reserved_module_ignores_launch_nonce() {
13651 let registry = Arc::new(Registry::default());
13652 let handler = ControlHandler::new(Arc::clone(®istry));
13655 let no_nonce = handler
13656 .handle_control(
13657 ConnectionId::new(1),
13658 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
13659 )
13660 .unwrap();
13661 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
13662 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
13663
13664 let echoed_nonce = handler
13665 .handle_control(
13666 ConnectionId::new(2),
13667 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
13668 )
13669 .unwrap();
13670 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
13671 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
13672 }
13673
13674 #[test]
13675 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
13676 let handler = ControlHandler::default();
13677 let conn = ConnectionId::new(1);
13678 let malformed = Frame::build(
13679 FrameType::Hello,
13680 control_flags(),
13681 0,
13682 0,
13683 3,
13684 b"{not json".to_vec(),
13685 )
13686 .unwrap();
13687
13688 let error = handler.handle_control(conn, malformed).unwrap();
13689 assert_eq!(error[0].header.ty, FrameType::Error);
13690 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
13691
13692 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13693 let pong = handler.handle_control(conn, ping).unwrap();
13694 assert_eq!(pong[0].header.ty, FrameType::Pong);
13695 assert_eq!(pong[0].header.corr, 4);
13696 }
13697
13698 #[test]
13699 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
13700 let registry = Arc::new(Registry::default());
13701 let handler = ControlHandler::new(Arc::clone(®istry));
13702
13703 handler
13704 .handle_control(
13705 ConnectionId::new(1),
13706 hello_frame("aft", PROTOCOL_VERSION, 1),
13707 )
13708 .unwrap();
13709 let duplicate = handler
13710 .handle_control(
13711 ConnectionId::new(2),
13712 hello_frame("aft", PROTOCOL_VERSION, 2),
13713 )
13714 .unwrap();
13715
13716 assert_eq!(duplicate[0].header.ty, FrameType::Error);
13717 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13718 let registration = registry.get_module("aft").unwrap().unwrap();
13719 assert_eq!(registration.connection_id, ConnectionId::new(1));
13720 }
13721
13722 #[test]
13723 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
13724 let registry = Arc::new(Registry::default());
13725 let forwarding = Arc::new(ForwardingTable::default());
13726 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
13727 let handler =
13728 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13729 .with_process_liveness(process_liveness);
13730 let (ctx, route_channel, route_epoch) =
13731 bind_liveness_route(®istry, &forwarding, "aft-dead");
13732 let responses = handler
13733 .handle_route_poll(
13734 &ctx,
13735 route_poll_frame(41, PollKind::Liveness, route_channel),
13736 route_channel,
13737 route_epoch,
13738 PollKind::Liveness,
13739 )
13740 .unwrap();
13741
13742 assert_eq!(responses.len(), 1);
13743 assert_eq!(responses[0].header.ty, FrameType::Response);
13744 assert_route_poll_liveness(&responses[0], false);
13745 }
13746
13747 #[test]
13748 fn liveness_poll_without_process_source_uses_bound_route() {
13749 let registry = Arc::new(Registry::default());
13750 let forwarding = Arc::new(ForwardingTable::default());
13751 let handler =
13752 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13753 let (ctx, route_channel, route_epoch) =
13754 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
13755 let responses = handler
13756 .handle_route_poll(
13757 &ctx,
13758 route_poll_frame(42, PollKind::Liveness, route_channel),
13759 route_channel,
13760 route_epoch,
13761 PollKind::Liveness,
13762 )
13763 .unwrap();
13764
13765 assert_route_poll_liveness(&responses[0], true);
13766 }
13767
13768 #[test]
13769 fn liveness_poll_untracked_process_source_uses_bound_route() {
13770 let registry = Arc::new(Registry::default());
13771 let forwarding = Arc::new(ForwardingTable::default());
13772 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13773 let handler =
13774 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13775 .with_process_liveness(process_liveness);
13776 let (ctx, route_channel, route_epoch) =
13777 bind_liveness_route(®istry, &forwarding, "aft-untracked");
13778 let responses = handler
13779 .handle_route_poll(
13780 &ctx,
13781 route_poll_frame(43, PollKind::Liveness, route_channel),
13782 route_channel,
13783 route_epoch,
13784 PollKind::Liveness,
13785 )
13786 .unwrap();
13787
13788 assert_route_poll_liveness(&responses[0], true);
13789 }
13790
13791 #[tokio::test]
13792 async fn unknown_op_returns_unknown_control_op() {
13793 let handler = ControlHandler::default();
13794 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13795 let request = Frame::build(
13796 FrameType::Request,
13797 control_flags(),
13798 0,
13799 0,
13800 55,
13801 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13802 )
13803 .unwrap();
13804
13805 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13806
13807 assert_eq!(response.len(), 1);
13808 assert_eq!(response[0].header.ty, FrameType::Error);
13809 assert_eq!(response[0].header.corr, 55);
13810 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
13811 }
13812
13813 #[tokio::test]
13814 async fn supervisor_provenance_rejects_unknown_exact_module() {
13815 let handler = ControlHandler::default();
13816 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
13817 let request = Frame::build(
13818 FrameType::Request,
13819 control_flags(),
13820 0,
13821 0,
13822 57,
13823 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
13824 )
13825 .unwrap();
13826
13827 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13828
13829 assert_eq!(response.len(), 1);
13830 assert_eq!(response[0].header.ty, FrameType::Error);
13831 assert_eq!(response[0].header.corr, 57);
13832 let error = parse_error(&response[0]);
13833 assert_eq!(error["code"], "unknown_module");
13834 assert_eq!(error["message"], "module_id 'missing' is not supervised");
13835 }
13836
13837 #[test]
13838 fn provenance_probe_override_keeps_handler_tests_deterministic() {
13839 let expected = subc_control::RunningImageAgreement::Unavailable {
13840 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13841 };
13842 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
13843 assert_eq!(handler.provenance_probe_override, Some(expected));
13844 }
13845
13846 #[test]
13847 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
13848 let verdict = reload_verdict(
13849 std::path::Path::new("/bin/new"),
13850 Some(std::path::Path::new("/bin/old")),
13851 subc_control::RunningImageAgreement::Unavailable {
13852 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13853 },
13854 );
13855 assert!(matches!(
13856 verdict.path,
13857 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
13858 if configured == std::path::Path::new("/bin/new")
13859 && spawned_from == std::path::Path::new("/bin/old")
13860 ));
13861 }
13862
13863 #[test]
13864 fn launch_spec_without_running_digest_is_unavailable_not_match() {
13865 use subc_control::{
13866 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
13867 };
13868 let configured = crate::supervise::ModuleSpec {
13869 module_id: "legacy".into(),
13870 program: "/module".into(),
13871 args: Vec::new(),
13872 env: Vec::new(),
13873 reserved: false,
13874 reserved_prefixes: Vec::new(),
13875 protocol: subc_control::ModuleProtocol::Subc,
13876 overlap: Default::default(),
13877 };
13878 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
13879 assert_eq!(
13880 launch_spec_agreement(&configured, false, Some(&digest)),
13881 Agreement::Unavailable {
13882 reason: Reason::NotRunning
13883 }
13884 );
13885 assert_eq!(
13886 launch_spec_agreement(&configured, true, None),
13887 Agreement::Unavailable {
13888 reason: Reason::SpawnedSpecUnavailable
13889 }
13890 );
13891 assert_eq!(
13892 launch_spec_agreement(&configured, true, Some(&digest)),
13893 Agreement::Match
13894 );
13895 }
13896
13897 #[cfg(windows)]
13898 #[test]
13899 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13900 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13901 let configured = dir.join("module.exe");
13902 std::fs::write(&configured, b"executable path fixture").unwrap();
13903 let spawned = std::fs::canonicalize(&configured).unwrap();
13904 assert_ne!(
13905 configured, spawned,
13906 "fixture must exercise different path spellings"
13907 );
13908 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13909 let lower_case = configured.to_string_lossy().to_lowercase();
13910 for spelling in [
13911 &configured,
13912 Path::new(&forward_slashes),
13913 Path::new(&lower_case),
13914 ] {
13915 let verdict = reload_verdict(
13916 spelling,
13917 Some(&spawned),
13918 subc_control::RunningImageAgreement::Unavailable {
13919 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13920 },
13921 );
13922 assert_eq!(
13923 verdict.path,
13924 ReloadPathAgreement::Match,
13925 "configured {spelling:?}, spawned {spawned:?}"
13926 );
13927 }
13928 }
13929
13930 #[cfg(windows)]
13931 #[test]
13932 fn windows_reload_path_keeps_different_existing_program_pending() {
13933 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13934 let old = dir.join("old.exe");
13935 let new = dir.join("new.exe");
13936 std::fs::write(&old, b"same executable content").unwrap();
13937 std::fs::copy(&old, &new).unwrap();
13938 let spawned = std::fs::canonicalize(&old).unwrap();
13939 let image = subc_control::RunningImageAgreement::Unavailable {
13940 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13941 };
13942 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13943 assert!(
13944 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13945 if configured == new && spawned_from == spawned)
13946 );
13947 let missing = dir.join("missing.exe");
13948 assert!(matches!(
13949 reload_verdict(&missing, Some(&spawned), image).path,
13950 ReloadPathAgreement::Mismatch { .. }
13951 ));
13952 }
13953
13954 #[test]
13955 fn reload_verdict_detects_replaced_image_at_same_path() {
13956 let image = subc_control::RunningImageAgreement::Mismatch {
13957 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13958 digest: "old".into(),
13959 },
13960 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13961 digest: "new".into(),
13962 },
13963 };
13964 let verdict = reload_verdict(
13965 std::path::Path::new("/bin/same"),
13966 Some(std::path::Path::new("/bin/same")),
13967 image.clone(),
13968 );
13969 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13970 assert_eq!(verdict.image, image);
13971 }
13972
13973 #[test]
13974 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13975 let image = subc_control::RunningImageAgreement::Unavailable {
13976 reason: subc_control::RunningImageUnavailableReason::NotRunning,
13977 };
13978 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13979 assert_eq!(
13980 verdict.path,
13981 subc_control::ReloadPathAgreement::Unavailable {
13982 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13983 }
13984 );
13985 assert_eq!(verdict.image, image);
13986
13987 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13988 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13989 };
13990 let verdict = reload_verdict(
13991 std::path::Path::new("/bin/same"),
13992 Some(std::path::Path::new("/bin/same")),
13993 unconfirmed.clone(),
13994 );
13995 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13996 assert_eq!(verdict.image, unconfirmed);
13997 }
13998
13999 #[test]
14000 fn reload_verdict_preserves_each_image_unavailability_reason() {
14001 use subc_control::RunningImageUnavailableReason as Reason;
14002
14003 for reason in [
14004 Reason::NotRunning,
14005 Reason::UnsupportedPlatform,
14006 Reason::RunningExecutableUnreadable,
14007 Reason::SpawnedPathUnreadable,
14008 Reason::HashFailed,
14009 Reason::ProcessIdentityUnconfirmed,
14010 Reason::Unknown("future_probe_reason".to_string()),
14011 ] {
14012 let image = subc_control::RunningImageAgreement::Unavailable {
14013 reason: reason.clone(),
14014 };
14015 let verdict = reload_verdict(
14016 std::path::Path::new("/bin/same"),
14017 Some(std::path::Path::new("/bin/same")),
14018 image.clone(),
14019 );
14020 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14021 assert_eq!(verdict.image, image, "{reason:?}");
14022 }
14023 }
14024
14025 #[tokio::test]
14026 async fn malformed_control_bodies_return_invalid_control_body() {
14027 let handler = ControlHandler::default();
14028 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14029
14030 for (corr, body) in [
14031 (56, br#"{"route_channel":1}"#.as_slice()),
14032 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14033 (
14034 58,
14035 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14036 ),
14037 ] {
14038 let request = Frame::build(
14039 FrameType::Request,
14040 control_flags(),
14041 0,
14042 0,
14043 corr,
14044 body.to_vec(),
14045 )
14046 .unwrap();
14047 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14048
14049 assert_eq!(response.len(), 1);
14050 assert_eq!(response[0].header.ty, FrameType::Error);
14051 assert_eq!(response[0].header.corr, corr);
14052 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14053 }
14054 }
14055
14056 #[tokio::test]
14057 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14058 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14059 let registry = Arc::new(Registry::default());
14060 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14061 let router = Router::with_control_handler(Arc::clone(&control));
14062 let connection = router.begin_connection();
14063 let (ctx, mut rx) = route_ctx(connection.id());
14064
14065 router
14066 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14067 .await
14068 .unwrap();
14069 let response = rx.recv().await.unwrap();
14070 let ack = parse_ack(&response);
14071 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14072 let channel = 1;
14073
14074 let goodbye =
14075 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14076 router.route_for_connection(&ctx, goodbye).await.unwrap();
14077 assert!(rx.try_recv().is_err());
14078 assert!(registry.get_module("aft").unwrap().is_none());
14079
14080 router
14081 .route_for_connection(&ctx, channel_request(channel, 13))
14082 .await
14083 .unwrap();
14084 let error_frame = rx.recv().await.unwrap();
14085 assert_eq!(error_frame.header.ty, FrameType::Error);
14086 assert_eq!(error_frame.header.channel, channel);
14087 let captured = crate::router::test_log::captured_logs(&logs);
14088 assert_eq!(
14089 captured
14090 .lines()
14091 .filter(|line| {
14092 line.contains("module_id=aft")
14093 && line.contains("reason=explicit_goodbye")
14094 && line.contains("module registration ended")
14095 })
14096 .count(),
14097 1,
14098 "unexpected GOODBYE registry log: {captured}"
14099 );
14100 }
14101
14102 #[tokio::test]
14103 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14104 let registry = Arc::new(Registry::default());
14105 let handler = ControlHandler::new(registry).with_capability_config(
14106 [("prov".to_string(), true), ("cons".to_string(), true)],
14107 BTreeMap::new(),
14108 );
14109 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14110 register_capability_manifest(
14111 &handler,
14112 &provider_ctx,
14113 &mut provider_rx,
14114 capability_manifest("prov", &["thing/v1"], &[]),
14115 1,
14116 )
14117 .await;
14118 let mut consumer = capability_manifest("cons", &[], &[]);
14119 consumer.capabilities.as_mut().unwrap().requires.push(
14120 subc_protocol::manifest::CapabilityRequirement {
14121 capability: "thing/v1".to_string(),
14122 need: subc_protocol::manifest::CapabilityNeed::Required,
14123 },
14124 );
14125 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14126 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14127 assert_eq!(
14128 handler.capability_evaluator.verdict("cons", "thing/v1"),
14129 Some(CapabilityVerdict::Provided)
14130 );
14131 let (mut client_rx, _) = open_route_for_capability_test(
14132 &handler,
14133 &provider_ctx,
14134 &mut provider_rx,
14135 703,
14136 3,
14137 "prov",
14138 None,
14139 )
14140 .await;
14141 let goodbye =
14142 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14143 handler
14144 .handle_control_frame(&provider_ctx, goodbye)
14145 .await
14146 .unwrap();
14147 assert_eq!(
14148 handler.capability_evaluator.verdict("cons", "thing/v1"),
14149 Some(CapabilityVerdict::NeverProvided)
14150 );
14151 let closed = client_rx
14152 .try_recv()
14153 .expect("GOODBYE pushes route.closed before route GOODBYE");
14154 assert!(
14155 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14156 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14157 );
14158 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14159 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14160 }
14161
14162 struct RouteClosedFixture {
14163 handler: ControlHandler,
14164 #[cfg(unix)]
14165 supervisor: Supervisor,
14166 module_ctx: RouteCtx,
14167 #[cfg(unix)]
14168 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14169 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14170 route: Arc<crate::forwarding::RouteBinding>,
14171 }
14172
14173 async fn route_closed_fixture() -> RouteClosedFixture {
14174 let registry = Arc::new(Registry::default());
14175 let handler = ControlHandler::new(Arc::clone(®istry));
14176 #[cfg(unix)]
14177 let supervisor = Supervisor::new(registry, RestartPolicy::default())
14178 .with_forwarding(Arc::clone(&handler.forwarding));
14179 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14180 register_capability_manifest(
14181 &handler,
14182 &module_ctx,
14183 &mut module_rx,
14184 capability_manifest("prov", &[], &[]),
14185 1,
14186 )
14187 .await;
14188 let (client_rx, bind) = open_route_for_capability_test(
14189 &handler,
14190 &module_ctx,
14191 &mut module_rx,
14192 702,
14193 2,
14194 "prov",
14195 None,
14196 )
14197 .await;
14198 let ModuleControlRequest::RouteBind {
14199 route_channel: channel,
14200 epoch: route_epoch,
14201 ..
14202 } = bind
14203 else {
14204 panic!("expected route.bind");
14205 };
14206 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14207 handler
14208 .forwarding
14209 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14210 .unwrap()
14211 else {
14212 panic!("the acknowledged route must be bound");
14213 };
14214 RouteClosedFixture {
14215 handler,
14216 #[cfg(unix)]
14217 supervisor,
14218 module_ctx,
14219 #[cfg(unix)]
14220 module_rx,
14221 client_rx,
14222 route,
14223 }
14224 }
14225
14226 impl RouteClosedFixture {
14227 #[cfg(unix)]
14228 async fn cut(&mut self) {
14229 let notices = async {
14230 let mut draining = self.module_rx.recv().await.unwrap();
14231 assert!(matches!(
14232 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14233 &draining.body
14234 )
14235 .unwrap(),
14236 subc_protocol::session::ModuleControlCommand::Draining {
14237 reason: RouteCloseReason::Restart,
14238 ..
14239 }
14240 ));
14241 draining.flushed.take().unwrap().send(()).unwrap();
14242 let mut closing = self.client_rx.recv().await.unwrap();
14243 assert!(matches!(
14244 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14245 ClientControlPush::RouteClosing {
14246 reason: RouteCloseReason::Restart,
14247 ..
14248 }
14249 ));
14250 closing.flushed.take().unwrap().send(()).unwrap();
14251 };
14252 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14253 result.unwrap();
14254 }
14255
14256 fn disconnect(&mut self) -> ClientControlPush {
14257 self.handler
14258 .cleanup_connection(self.module_ctx.connection_id)
14259 .unwrap();
14260 let closed = self.client_rx.try_recv().unwrap();
14261 assert_eq!(closed.header.ty, FrameType::Push);
14262 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14263 assert!(matches!(
14264 &push,
14265 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14266 if channels == &[self.route.client_channel]
14267 ));
14268 assert_eq!(
14269 self.client_rx.try_recv().unwrap().header.ty,
14270 FrameType::Goodbye
14271 );
14272 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14273 push
14274 }
14275 }
14276
14277 #[cfg(unix)]
14278 #[tokio::test(start_paused = true)]
14279 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14280 let mut fixture = route_closed_fixture().await;
14281 fixture.cut().await;
14282 assert!(matches!(
14283 fixture.disconnect(),
14284 ClientControlPush::RouteClosed {
14285 reason: RouteCloseReason::Restart,
14286 drained: true,
14287 ..
14288 }
14289 ));
14290 }
14291
14292 #[cfg(unix)]
14293 #[tokio::test(start_paused = true)]
14294 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14295 let mut fixture = route_closed_fixture().await;
14296 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14297 let started = tokio::time::Instant::now();
14298 fixture.cut().await;
14299 assert!(started.elapsed() >= Duration::from_secs(2));
14300 assert_eq!(fixture.route.flow.in_flight(), 1);
14301 assert!(matches!(
14302 fixture.disconnect(),
14303 ClientControlPush::RouteClosed {
14304 reason: RouteCloseReason::Restart,
14305 drained: false,
14306 ..
14307 }
14308 ));
14309 }
14310
14311 #[tokio::test]
14312 async fn module_connection_crash_reports_crash_and_not_drained() {
14313 let mut fixture = route_closed_fixture().await;
14314 assert!(matches!(
14318 fixture.disconnect(),
14319 ClientControlPush::RouteClosed {
14320 reason: RouteCloseReason::Crash,
14321 drained: false,
14322 ..
14323 }
14324 ));
14325 }
14326
14327 #[tokio::test]
14328 async fn dropping_router_connection_releases_registration() {
14329 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14330 let registry = Arc::new(Registry::default());
14331 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14332 let router = Router::with_control_handler(Arc::clone(&control));
14333 let connection = router.begin_connection();
14334 let connection_id = connection.id();
14335 let (ctx, mut rx) = route_ctx(connection_id);
14336
14337 router
14338 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14339 .await
14340 .unwrap();
14341 let response = rx.recv().await.unwrap();
14342 let ack = parse_ack(&response);
14343 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14344 assert!(registry.get_module("aft").unwrap().is_some());
14345
14346 drop(connection);
14347
14348 assert!(registry.get_module("aft").unwrap().is_none());
14349 assert_eq!(registry.active_registration_count().unwrap(), 0);
14350
14351 control
14352 .cleanup_connection(ConnectionId::new(u64::MAX))
14353 .unwrap();
14354 let captured = crate::router::test_log::captured_logs(&logs);
14355 println!("captured registry lifecycle logs:\n{captured}");
14356 let events: Vec<_> = captured
14357 .lines()
14358 .filter(|line| {
14359 line.contains("module registered module_id=aft ")
14360 || line.contains("module registration ended")
14361 })
14362 .collect();
14363 assert_eq!(
14364 events.len(),
14365 2,
14366 "unexpected registry lifecycle logs: {captured}"
14367 );
14368 assert!(events[0].contains("module registered module_id=aft "));
14369 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14370 assert!(events[1].contains(&format!(
14371 "module_id=aft connection_id={}",
14372 connection_id.get()
14373 )));
14374 assert!(events[1].contains("reason=connection_closed"));
14375 assert!(events[1].contains("module registration ended"));
14376 assert_eq!(
14377 captured
14378 .lines()
14379 .filter(|line| line.contains("module registered module_id=aft "))
14380 .count(),
14381 1,
14382 "legacy registration admission line must appear once: {captured}"
14383 );
14384 }
14385
14386 #[tokio::test]
14387 async fn hello_registration_keeps_legacy_module_registered_line_once() {
14388 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14389 let registry = Arc::new(Registry::default());
14390 let control = ControlHandler::new(Arc::clone(®istry));
14391 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14392 hello_via_sink(
14393 &control,
14394 &ctx,
14395 &mut rx,
14396 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14397 )
14398 .await;
14399
14400 let captured = crate::router::test_log::captured_logs(&logs);
14401 let admissions: Vec<_> = captured
14402 .lines()
14403 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14404 .collect();
14405 assert_eq!(
14406 admissions.len(),
14407 1,
14408 "legacy admission line must remain exactly once: {captured}"
14409 );
14410 assert!(admissions[0].contains("routable_provider=true"));
14411 assert!(admissions[0].contains("connection_id=777"));
14412 }
14413
14414 fn capability_manifest(
14415 module_id: &str,
14416 provides: &[&str],
14417 must_never_reach: &[&str],
14418 ) -> ModuleManifest {
14419 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14420 manifest.capabilities = Some(CapabilityDeclarations {
14421 provides: provides
14422 .iter()
14423 .map(|capability| (*capability).to_string())
14424 .collect(),
14425 requires: Vec::new(),
14426 must_never_reach: must_never_reach
14427 .iter()
14428 .map(|capability| (*capability).to_string())
14429 .collect(),
14430 });
14431 manifest
14432 }
14433
14434 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14435 Frame::build(
14436 FrameType::Hello,
14437 control_flags(),
14438 0,
14439 0,
14440 corr,
14441 serde_json::to_vec(&ModuleHelloBody {
14442 protocol_ver: manifest.protocol_ver,
14443 manifest,
14444 control_ops: None,
14445 launch_nonce: None,
14446 })
14447 .expect("capability test HELLO serializes"),
14448 )
14449 .expect("capability test HELLO frame builds")
14450 }
14451
14452 fn catalog_update_with_capabilities_frame(
14453 corr: u64,
14454 capabilities: CapabilityDeclarations,
14455 ) -> Frame {
14456 Frame::build(
14457 FrameType::Request,
14458 control_flags(),
14459 0,
14460 0,
14461 corr,
14462 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14463 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14464 capabilities: Some(capabilities),
14465 ready: None,
14466 })
14467 .expect("capability catalog.update serializes"),
14468 )
14469 .expect("capability catalog.update frame builds")
14470 }
14471
14472 async fn register_capability_manifest(
14473 handler: &ControlHandler,
14474 ctx: &RouteCtx,
14475 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14476 manifest: ModuleManifest,
14477 corr: u64,
14478 ) {
14479 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14480 }
14481
14482 async fn open_route_for_capability_test(
14483 handler: &ControlHandler,
14484 target_ctx: &RouteCtx,
14485 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14486 client_connection_id: u64,
14487 corr: u64,
14488 target_module_id: &str,
14489 consumer_identity: Option<ConsumerIdentity>,
14490 ) -> (
14491 mpsc::Receiver<crate::router::OutboundFrame>,
14492 ModuleControlRequest,
14493 ) {
14494 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14495 let route_handler = handler.clone();
14496 let target_module_id = target_module_id.to_string();
14497 let route_task = tokio::spawn(async move {
14498 route_handler
14499 .handle_control_frame(
14500 &client_ctx,
14501 route_open_frame_with_admission_facts(
14502 corr,
14503 &target_module_id,
14504 unique_project_root("admission-facts"),
14505 consumer_identity,
14506 None,
14507 ),
14508 )
14509 .await
14510 .expect("capability test route.open succeeds")
14511 });
14512 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14513 .await
14514 .expect("capability test route.open must reach route.bind")
14515 .expect("target control receiver stays open");
14516 let bind_request: ModuleControlRequest =
14517 serde_json::from_slice(&bind.body).expect("route.bind decodes");
14518 handler
14519 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14520 .await
14521 .expect("capability test route.bind ACK succeeds");
14522 assert!(route_task.await.expect("route.open task joins").is_empty());
14523 let opened = client_rx
14524 .recv()
14525 .await
14526 .expect("successful route.open publishes a response");
14527 assert!(matches!(
14528 serde_json::from_slice::<ClientControlResponse>(&opened.body),
14529 Ok(ClientControlResponse::RouteOpen { .. })
14530 ));
14531 (client_rx, bind_request)
14532 }
14533
14534 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
14535 assert_eq!(frame.header.ty, FrameType::Push);
14536 assert_eq!(frame.header.channel, 0);
14537 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14538 .expect("route.closed control push decodes");
14539 let ClientControlPush::RouteClosed { channels, .. } = &push else {
14540 panic!("expected route.closed");
14541 };
14542 assert_eq!(channels.len(), 1, "exactly one violating route closed");
14543 let channels = channels.clone();
14544 assert_eq!(
14545 push,
14546 ClientControlPush::RouteClosed {
14547 module_id: target_module_id.to_string(),
14548 channels,
14549 reason: RouteCloseReason::CapabilityDenied,
14550 drained: false,
14551 abandoned: 0,
14552 excluded_subscriptions: 0,
14553 terminal: Some(false),
14554 }
14555 );
14556 }
14557
14558 #[tokio::test]
14559 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14560 let registry = Arc::new(Registry::default());
14561 let forwarding = Arc::new(ForwardingTable::default());
14562 let supervisor = SupervisorHandle::new();
14563 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14564 let handler =
14565 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14566 .with_supervisor(supervisor);
14567 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14568 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14569 register_capability_manifest(
14570 &handler,
14571 &target_ctx,
14572 &mut target_rx,
14573 capability_manifest("target", &["credentials-provider/v1"], &[]),
14574 1,
14575 )
14576 .await;
14577 register_capability_manifest(
14578 &handler,
14579 &opener_ctx,
14580 &mut opener_rx,
14581 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14582 2,
14583 )
14584 .await;
14585
14586 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14587 let replies = handler
14588 .handle_control_frame(
14589 &client_ctx,
14590 route_open_frame_with_admission_facts(
14591 3,
14592 "target",
14593 unique_project_root("admission-facts"),
14594 Some(ConsumerIdentity {
14595 module_id: "opener".to_string(),
14596 launch_nonce: "opener-nonce".to_string(),
14597 }),
14598 None,
14599 ),
14600 )
14601 .await
14602 .expect("denied route.open returns a typed frame");
14603 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14604 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14605 assert!(
14606 target_rx.try_recv().is_err(),
14607 "forbidden route.open must not relay route.bind"
14608 );
14609 }
14610
14611 #[tokio::test]
14612 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
14613 let registry = Arc::new(Registry::default());
14614 let forwarding = Arc::new(ForwardingTable::default());
14615 let supervisor = SupervisorHandle::new();
14616 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14617 let handler =
14618 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14619 .with_supervisor(supervisor);
14620 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
14621 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
14622 register_capability_manifest(
14623 &handler,
14624 &target_ctx,
14625 &mut target_rx,
14626 capability_manifest("target", &["credentials-provider/v1"], &[]),
14627 1,
14628 )
14629 .await;
14630 register_capability_manifest(
14631 &handler,
14632 &old_opener_ctx,
14633 &mut old_opener_rx,
14634 capability_manifest("opener", &[], &[]),
14635 2,
14636 )
14637 .await;
14638 let (mut client_rx, _) = open_route_for_capability_test(
14639 &handler,
14640 &target_ctx,
14641 &mut target_rx,
14642 712,
14643 3,
14644 "target",
14645 Some(ConsumerIdentity {
14646 module_id: "opener".to_string(),
14647 launch_nonce: "opener-nonce".to_string(),
14648 }),
14649 )
14650 .await;
14651 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14652
14653 handler
14654 .cleanup_connection(old_opener_ctx.connection_id)
14655 .expect("old opener registration cleans up");
14656 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
14657 register_capability_manifest(
14658 &handler,
14659 &new_opener_ctx,
14660 &mut new_opener_rx,
14661 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14662 4,
14663 )
14664 .await;
14665
14666 assert_capability_denied_push(
14667 client_rx
14668 .try_recv()
14669 .expect("HELLO deny addition must emit route.closed")
14670 .frame,
14671 "target",
14672 );
14673 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14674 assert!(matches!(
14675 target_rx.try_recv(),
14676 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14677 ));
14678 }
14679
14680 #[tokio::test]
14681 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
14682 let registry = Arc::new(Registry::default());
14683 let forwarding = Arc::new(ForwardingTable::default());
14684 let supervisor = SupervisorHandle::new();
14685 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14686 let handler =
14687 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14688 .with_supervisor(supervisor);
14689 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
14690 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
14691 register_capability_manifest(
14692 &handler,
14693 &target_ctx,
14694 &mut target_rx,
14695 capability_manifest("target", &[], &[]),
14696 1,
14697 )
14698 .await;
14699 register_capability_manifest(
14700 &handler,
14701 &opener_ctx,
14702 &mut opener_rx,
14703 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14704 2,
14705 )
14706 .await;
14707 let (mut client_rx, _) = open_route_for_capability_test(
14708 &handler,
14709 &target_ctx,
14710 &mut target_rx,
14711 722,
14712 3,
14713 "target",
14714 Some(ConsumerIdentity {
14715 module_id: "opener".to_string(),
14716 launch_nonce: "opener-nonce".to_string(),
14717 }),
14718 )
14719 .await;
14720 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14721
14722 let replies = handler
14723 .handle_control_frame(
14724 &target_ctx,
14725 catalog_update_with_capabilities_frame(
14726 4,
14727 CapabilityDeclarations {
14728 provides: vec!["credentials-provider/v1".to_string()],
14729 requires: Vec::new(),
14730 must_never_reach: Vec::new(),
14731 },
14732 ),
14733 )
14734 .await
14735 .expect("claim catalog.update succeeds");
14736 assert!(matches!(
14737 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
14738 Ok(ModuleControlResponseToModule::CatalogUpdate {})
14739 ));
14740 assert_capability_denied_push(
14741 client_rx
14742 .try_recv()
14743 .expect("claim addition must emit route.closed")
14744 .frame,
14745 "target",
14746 );
14747 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14748 assert!(matches!(
14749 target_rx.try_recv(),
14750 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14751 ));
14752 }
14753
14754 #[tokio::test]
14755 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
14756 let registry = Arc::new(Registry::default());
14757 let forwarding = Arc::new(ForwardingTable::default());
14758 let supervisor = SupervisorHandle::new();
14759 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14760 let handler =
14761 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14762 .with_supervisor(supervisor);
14763 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
14764 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
14765 register_capability_manifest(
14766 &handler,
14767 &target_ctx,
14768 &mut target_rx,
14769 capability_manifest("target", &["credentials-provider/v1"], &[]),
14770 1,
14771 )
14772 .await;
14773 register_capability_manifest(
14774 &handler,
14775 &opener_ctx,
14776 &mut opener_rx,
14777 capability_manifest("opener", &[], &[]),
14778 2,
14779 )
14780 .await;
14781 let (mut client_rx, _) = open_route_for_capability_test(
14782 &handler,
14783 &target_ctx,
14784 &mut target_rx,
14785 732,
14786 3,
14787 "target",
14788 Some(ConsumerIdentity {
14789 module_id: "opener".to_string(),
14790 launch_nonce: "opener-nonce".to_string(),
14791 }),
14792 )
14793 .await;
14794 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14795
14796 handler
14797 .handle_control_frame(
14798 &target_ctx,
14799 catalog_update_with_capabilities_frame(
14800 4,
14801 CapabilityDeclarations {
14802 provides: Vec::new(),
14803 requires: Vec::new(),
14804 must_never_reach: Vec::new(),
14805 },
14806 ),
14807 )
14808 .await
14809 .expect("claim removal catalog.update succeeds");
14810 assert_eq!(
14811 forwarding.active_binding_count().unwrap(),
14812 1,
14813 "removing an attested target claim must leave the route census unchanged"
14814 );
14815 assert!(
14816 client_rx.try_recv().is_err(),
14817 "claim removal must not emit route.closed capability_denied"
14818 );
14819 assert!(
14820 target_rx.try_recv().is_err(),
14821 "claim removal must not send the target a route GOODBYE"
14822 );
14823 }
14824
14825 #[tokio::test]
14828 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
14829 let registry = Arc::new(Registry::default());
14830 let forwarding = Arc::new(ForwardingTable::default());
14831 let supervisor = SupervisorHandle::new();
14832 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14833 let handler =
14834 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14835 .with_supervisor(supervisor);
14836 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
14837 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
14838 register_capability_manifest(
14839 &handler,
14840 &target_ctx,
14841 &mut target_rx,
14842 capability_manifest("target", &["credentials-provider/v1"], &[]),
14843 1,
14844 )
14845 .await;
14846 register_capability_manifest(
14847 &handler,
14848 &opener_ctx,
14849 &mut opener_rx,
14850 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14851 2,
14852 )
14853 .await;
14854
14855 let (_client_rx, bind) = open_route_for_capability_test(
14856 &handler,
14857 &target_ctx,
14858 &mut target_rx,
14859 742,
14860 3,
14861 "target",
14862 None,
14863 )
14864 .await;
14865 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14866 panic!("direct scope-honesty route must bind");
14867 };
14868 assert_eq!(principal, Some(Principal::Direct));
14869 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14870 }
14871
14872 #[tokio::test]
14875 async fn must_never_reach_self_route_is_capability_forbidden() {
14876 let registry = Arc::new(Registry::default());
14877 let forwarding = Arc::new(ForwardingTable::default());
14878 let supervisor = SupervisorHandle::new();
14879 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
14880 let handler =
14881 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14882 .with_supervisor(supervisor);
14883 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
14884 register_capability_manifest(
14885 &handler,
14886 &self_ctx,
14887 &mut self_rx,
14888 capability_manifest(
14889 "self-provider",
14890 &["credentials-provider/v1"],
14891 &["credentials-provider/v1"],
14892 ),
14893 1,
14894 )
14895 .await;
14896
14897 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14898 let replies = handler
14899 .handle_control_frame(
14900 &client_ctx,
14901 route_open_frame_with_admission_facts(
14902 2,
14903 "self-provider",
14904 unique_project_root("admission-facts"),
14905 Some(ConsumerIdentity {
14906 module_id: "self-provider".to_string(),
14907 launch_nonce: "self-nonce".to_string(),
14908 }),
14909 None,
14910 ),
14911 )
14912 .await
14913 .expect("self-route refusal returns a typed frame");
14914 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14915 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14916 assert!(
14917 self_rx.try_recv().is_err(),
14918 "self denial must not relay route.bind"
14919 );
14920 }
14921
14922 #[test]
14923 fn unsupported_channel_zero_frame_returns_error() {
14924 let handler = ControlHandler::default();
14925 let request = Frame::build(
14926 FrameType::Request,
14927 control_flags(),
14928 0,
14929 0,
14930 21,
14931 b"opaque".to_vec(),
14932 )
14933 .unwrap();
14934
14935 let response = handler
14936 .handle_control(ConnectionId::new(1), request)
14937 .unwrap();
14938
14939 assert_eq!(response[0].header.ty, FrameType::Error);
14940 assert_eq!(
14941 parse_error(&response[0])["code"],
14942 "unsupported_control_frame"
14943 );
14944 }
14945
14946 mod swap {
14951 use super::*;
14952
14953 const INCUMBENT: ConnectionId = ConnectionId::new(30);
14954 const CANDIDATE: ConnectionId = ConnectionId::new(40);
14955
14956 struct Swap {
14957 registry: Arc<Registry>,
14958 forwarding: Arc<ForwardingTable>,
14959 handler: ControlHandler,
14960 incumbent_ctx: RouteCtx,
14961 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14962 candidate_ctx: RouteCtx,
14963 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14964 }
14965
14966 async fn swap_with_incumbent() -> Swap {
14967 let registry = Arc::new(Registry::default());
14968 let forwarding = Arc::new(ForwardingTable::default());
14969 let handler =
14970 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14971 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14972 hello_via_sink(
14973 &handler,
14974 &incumbent_ctx,
14975 &mut incumbent_rx,
14976 hello_frame("aft", PROTOCOL_VERSION, 7),
14977 )
14978 .await;
14979 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14980 Swap {
14981 registry,
14982 forwarding,
14983 handler,
14984 incumbent_ctx,
14985 incumbent_rx,
14986 candidate_ctx,
14987 candidate_rx,
14988 }
14989 }
14990
14991 fn register_candidate(swap: &Swap, ready: Option<bool>) {
14992 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14993 candidate_manifest.ready = ready;
14994 let registration = swap
14995 .registry
14996 .register_candidate_with_control_ops(
14997 candidate_manifest,
14998 PROTOCOL_VERSION,
14999 CANDIDATE,
15000 module_baseline_control_ops(),
15001 )
15002 .unwrap();
15003 swap.forwarding
15004 .register_candidate_module_connection(
15005 CANDIDATE,
15006 "aft".to_string(),
15007 PROTOCOL_VERSION,
15008 manifest_concurrency(®istration.manifest),
15009 swap.candidate_ctx.egress.clone(),
15010 )
15011 .unwrap();
15012 }
15013
15014 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15015 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15016 swap.registry.promote_candidate("aft").unwrap().unwrap();
15017 cutover.incumbent.unwrap()
15018 }
15019
15020 #[tokio::test]
15029 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15030 let mut swap = swap_with_incumbent().await;
15031 let hold = Arc::new(std::sync::Barrier::new(2));
15032 let handler = swap
15033 .handler
15034 .clone()
15035 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15036 handler
15037 .supervisor
15038 .open_swap("aft", "candidate-nonce".to_string());
15039 let hello = {
15040 let handler = handler.clone();
15041 let candidate_ctx = swap.candidate_ctx.clone();
15042 std::thread::spawn(move || {
15043 handler
15044 .handle_hello(
15045 candidate_ctx.connection_id,
15046 Some(candidate_ctx.egress.clone()),
15047 hello_frame_with_nonce(
15048 "aft",
15049 PROTOCOL_VERSION,
15050 8,
15051 Some("candidate-nonce"),
15052 ),
15053 )
15054 .unwrap()
15055 })
15056 };
15057
15058 hold.wait();
15059 assert!(swap
15062 .registry
15063 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15064 .unwrap()
15065 .is_some());
15066 assert!(swap
15067 .registry
15068 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15069 .unwrap()
15070 .is_none());
15071 assert!(swap
15072 .forwarding
15073 .module_endpoint_for_connection(CANDIDATE)
15074 .unwrap()
15075 .is_none());
15076
15077 hold.wait();
15078 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15079 assert_eq!(
15080 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15081 FrameType::HelloAck
15082 );
15083 let published = swap
15084 .registry
15085 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15086 .unwrap()
15087 .expect("candidate is published once its HELLO completes");
15088 assert_eq!(published.connection_id, CANDIDATE);
15089 assert!(swap
15090 .forwarding
15091 .module_endpoint_for_connection(CANDIDATE)
15092 .unwrap()
15093 .is_some());
15094 }
15095
15096 fn keyed_total(counters: &Value, key: &str) -> u64 {
15097 counters[key]
15098 .as_object()
15099 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15100 .unwrap_or(0)
15101 }
15102
15103 #[tokio::test]
15104 async fn replacement_logs_old_end_and_new_admission_once() {
15105 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15106 let mut swap = swap_with_incumbent().await;
15107 swap.handler
15108 .supervisor
15109 .open_swap("aft", "candidate-nonce".to_string());
15110 hello_via_sink(
15111 &swap.handler,
15112 &swap.candidate_ctx,
15113 &mut swap.candidate_rx,
15114 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15115 )
15116 .await;
15117 cutover(&swap);
15118 swap.handler.cleanup_connection(INCUMBENT).unwrap();
15119
15120 let captured = crate::router::test_log::captured_logs(&logs);
15121 println!("captured replacement registry logs:\n{captured}");
15122 let new_admissions: Vec<_> = captured
15123 .lines()
15124 .filter(|line| {
15125 line.contains("connection_id=40")
15126 && line.contains("swap candidate registered; not routable until cutover")
15127 })
15128 .collect();
15129 assert_eq!(
15130 new_admissions.len(),
15131 1,
15132 "unexpected admission logs: {captured}"
15133 );
15134 assert!(new_admissions[0].contains("module_id=aft"));
15135 assert!(new_admissions[0].contains("connection_id=40"));
15136 assert!(
15137 new_admissions[0].contains("swap candidate registered; not routable until cutover")
15138 );
15139 assert!(new_admissions[0].contains("ready=true"));
15140
15141 let old_admissions: Vec<_> = captured
15142 .lines()
15143 .filter(|line| {
15144 line.contains("module registered module_id=aft ")
15145 && line.contains("connection_id=30")
15146 })
15147 .collect();
15148 assert_eq!(
15149 old_admissions.len(),
15150 1,
15151 "unexpected incumbent admission: {captured}"
15152 );
15153
15154 let old_ends: Vec<_> = captured
15155 .lines()
15156 .filter(|line| {
15157 line.contains("connection_id=30") && line.contains("module registration ended")
15158 })
15159 .collect();
15160 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15161 assert!(old_ends[0].contains("module_id=aft"));
15162 assert!(old_ends[0].contains("connection_id=30"));
15163 assert!(old_ends[0].contains("reason=replaced"));
15164 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15165 assert_eq!(
15166 captured
15167 .lines()
15168 .filter(|line| line.contains("module registration promoted"))
15169 .count(),
15170 1,
15171 "unexpected promotion logs: {captured}"
15172 );
15173 }
15174
15175 #[tokio::test]
15181 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15182 let mut swap = swap_with_incumbent().await;
15183 let handler = swap.handler.clone();
15184
15185 let cotenant = ConnectionId::new(31);
15187 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15188 let (cotenant_task, cotenant_bind) = relay_route_open(
15189 &handler,
15190 cotenant,
15191 &cotenant_ctx.egress,
15192 &mut swap.incumbent_rx,
15193 100,
15194 "aft",
15195 "swap-cotenant",
15196 )
15197 .await;
15198 handler
15199 .handle_control_frame(
15200 &swap.incumbent_ctx,
15201 route_bind_ack(cotenant_bind.header.corr),
15202 )
15203 .await
15204 .unwrap();
15205 assert!(cotenant_task.await.unwrap().is_empty());
15206 let (cotenant_channel, cotenant_epoch) =
15207 published_route(&cotenant_rx.recv().await.unwrap());
15208
15209 let caller = ConnectionId::new(32);
15211 let (caller_ctx, mut caller_rx) = route_ctx(caller);
15212 let (caller_task, caller_bind) = relay_route_open(
15213 &handler,
15214 caller,
15215 &caller_ctx.egress,
15216 &mut swap.incumbent_rx,
15217 101,
15218 "aft",
15219 "swap-caller",
15220 )
15221 .await;
15222 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15223
15224 register_candidate(&swap, None);
15225 cutover(&swap);
15226
15227 let ack = handler
15229 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15230 .await;
15231 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15232 if module_loop_error.is_some() {
15233 handler.cleanup_connection(INCUMBENT).unwrap();
15236 }
15237
15238 assert!(
15240 cotenant_rx.try_recv().is_err(),
15241 "the co-tenant route on the incumbent was torn down by one late ack: \
15242 {module_loop_error:?}"
15243 );
15244 assert!(matches!(
15245 swap.forwarding
15246 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15247 .unwrap(),
15248 DataRoute::Client(DataRouteState::Bound(_))
15249 ));
15250 assert_eq!(module_loop_error, None);
15251 assert!(swap
15252 .registry
15253 .get_module_by_connection(INCUMBENT)
15254 .unwrap()
15255 .is_some());
15256
15257 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15259 .await
15260 .expect("the incumbent is told to drop the abandoned binding")
15261 .unwrap()
15262 .frame;
15263 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15264 assert_eq!(goodbye.header.channel, abandoned_channel);
15265 assert_eq!(goodbye.header.epoch, abandoned_epoch);
15266 assert!(swap.incumbent_rx.try_recv().is_err());
15267
15268 let response = caller_task.await.unwrap();
15270 assert_eq!(response.len(), 1);
15271 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15272 assert!(caller_rx.try_recv().is_err());
15273
15274 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15278 let counters = handler.counters().snapshot();
15279 assert_eq!(
15280 keyed_total(&counters, "route_open_accepted_by_principal"),
15281 1
15282 );
15283 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15284 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15285 }
15286
15287 #[tokio::test]
15291 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15292 let mut swap = swap_with_incumbent().await;
15293 register_candidate(&swap, None);
15294 let incumbent = cutover(&swap);
15295 swap.forwarding
15296 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15297 .unwrap()
15298 .expect("the incumbent is still registered");
15299
15300 let client = ConnectionId::new(33);
15301 let (client_ctx, mut client_rx) = route_ctx(client);
15302 let route_handler = swap.handler.clone();
15303 let open_ctx = RouteCtx {
15304 connection_id: client,
15305 egress: client_ctx.egress.clone(),
15306 };
15307 let mut route_task = tokio::spawn(async move {
15308 route_handler
15309 .handle_control_frame(
15310 &open_ctx,
15311 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15312 )
15313 .await
15314 .unwrap()
15315 });
15316 let bind = tokio::select! {
15317 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15318 response = &mut route_task => {
15319 let response = response.unwrap();
15320 panic!(
15321 "post-cutover route.open was refused instead of relayed to the candidate: {}",
15322 parse_error(&response[0])["code"]
15323 );
15324 }
15325 };
15326 swap.handler
15327 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15328 .await
15329 .unwrap();
15330 assert!(route_task.await.unwrap().is_empty());
15331 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15332 match swap
15333 .forwarding
15334 .lookup_data_route(client, channel, epoch)
15335 .unwrap()
15336 {
15337 DataRoute::Client(DataRouteState::Bound(route)) => {
15338 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15339 }
15340 other => panic!("expected a bound route on the candidate, got {other:?}"),
15341 }
15342 assert!(swap.incumbent_rx.try_recv().is_err());
15343 }
15344
15345 #[tokio::test]
15350 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15351 let swap = swap_with_incumbent().await;
15352 register_candidate(&swap, Some(false));
15353 let update = Frame::build(
15354 FrameType::Request,
15355 control_flags(),
15356 0,
15357 0,
15358 55,
15359 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15360 provides: manifest("aft", PROTOCOL_VERSION).provides,
15361 capabilities: None,
15362 ready: Some(true),
15363 })
15364 .unwrap(),
15365 )
15366 .unwrap();
15367
15368 let replies = swap
15369 .handler
15370 .handle_control_frame(&swap.candidate_ctx, update)
15371 .await
15372 .unwrap();
15373
15374 assert_eq!(replies.len(), 1);
15375 assert_eq!(
15376 replies[0].header.ty,
15377 FrameType::Response,
15378 "candidate catalog.update was refused: {:?}",
15379 serde_json::from_slice::<Value>(&replies[0].body).ok()
15380 );
15381 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15382 assert_eq!(
15383 swap.registry
15384 .get_module("aft")
15385 .unwrap()
15386 .unwrap()
15387 .connection_id,
15388 INCUMBENT
15389 );
15390 }
15391 }
15392
15393 mod swap_admission {
15397 use super::*;
15398
15399 const INCUMBENT_NONCE: &str = "incumbent-nonce";
15400 const CANDIDATE_NONCE: &str = "candidate-nonce";
15401
15402 fn handler_with_incumbent(
15403 module_id: &str,
15404 reserved: bool,
15405 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15406 let registry = Arc::new(Registry::default());
15407 let supervisor = SupervisorHandle::new();
15408 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15409 if reserved {
15410 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15411 }
15412 let handler =
15413 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
15414 let incumbent = handler
15415 .handle_control(
15416 ConnectionId::new(1),
15417 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15418 )
15419 .unwrap();
15420 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15421 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15422 (registry, supervisor, handler)
15423 }
15424
15425 #[test]
15431 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15432 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15433
15434 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15435 let replies = handler
15436 .handle_control(
15437 ConnectionId::new(connection),
15438 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15439 )
15440 .unwrap();
15441 assert_eq!(replies[0].header.ty, FrameType::Error);
15442 assert_eq!(
15443 parse_error(&replies[0])["code"],
15444 "swap_token_invalid",
15445 "nonce {nonce:?}"
15446 );
15447 }
15448 assert!(registry.get_candidate("aft").unwrap().is_none());
15449 assert_eq!(
15450 registry.get_module("aft").unwrap().unwrap().connection_id,
15451 ConnectionId::new(1)
15452 );
15453
15454 let admitted = handler
15457 .handle_control(
15458 ConnectionId::new(4),
15459 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15460 )
15461 .unwrap();
15462 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15463 assert_eq!(
15464 registry
15465 .get_candidate("aft")
15466 .unwrap()
15467 .unwrap()
15468 .connection_id,
15469 ConnectionId::new(4)
15470 );
15471 assert_eq!(
15472 registry.get_module("aft").unwrap().unwrap().connection_id,
15473 ConnectionId::new(1),
15474 "the candidate must not take the active slot"
15475 );
15476 let replayed = handler
15477 .handle_control(
15478 ConnectionId::new(5),
15479 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15480 )
15481 .unwrap();
15482 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15483
15484 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15488 let squatter = handler
15489 .handle_control(
15490 ConnectionId::new(6),
15491 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15492 )
15493 .unwrap();
15494 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15495 assert!(
15496 registry.get_module("aft").unwrap().is_none(),
15497 "a squatter took the active slot of an id being swapped"
15498 );
15499 }
15500
15501 #[test]
15506 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15507 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15508
15509 let replies = handler
15510 .handle_control(
15511 ConnectionId::new(2),
15512 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15513 )
15514 .unwrap();
15515
15516 assert_eq!(
15517 replies[0].header.ty,
15518 FrameType::HelloAck,
15519 "reserved candidate refused: {:?}",
15520 serde_json::from_slice::<Value>(&replies[0].body).ok()
15521 );
15522 assert_eq!(
15523 registry
15524 .get_candidate("vault")
15525 .unwrap()
15526 .unwrap()
15527 .connection_id,
15528 ConnectionId::new(2)
15529 );
15530 }
15531
15532 #[test]
15535 fn without_an_open_swap_the_ordinary_gates_decide() {
15536 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15537 supervisor.close_swap("vault");
15538
15539 let candidate = handler
15540 .handle_control(
15541 ConnectionId::new(2),
15542 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15543 )
15544 .unwrap();
15545 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15546 let duplicate = handler
15547 .handle_control(
15548 ConnectionId::new(3),
15549 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15550 )
15551 .unwrap();
15552 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15553 assert!(registry.get_candidate("vault").unwrap().is_none());
15554 }
15555 }
15556
15557 mod scopes {
15561 mod idle_tests {
15562 use super::*;
15563 include!("scope_idle_tests.rs");
15564 }
15565 mod route_open_timings {
15566 use super::*;
15567 include!("route_open_timing_tests.rs");
15568 }
15569 mod activity {
15570 include!("activity_scope_tests.rs");
15571 }
15572 mod delta {
15573 include!("scope_handler_delta_tests.rs");
15574 }
15575 mod call_modes {
15576 include!("scope_call_mode_tests.rs");
15577 }
15578 #[tokio::test]
15579 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15580 let rig = rig().await;
15581 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15582 "s".into(),
15583 1,
15584 "tool".into(),
15585 ))
15586 .unwrap();
15587 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
15588 let (direct, _rx) = wide_ctx(999);
15589 for (ctx, expected) in [
15590 (&rig.owner, FrameType::Response),
15591 (&direct, FrameType::Error),
15592 ] {
15593 let frame =
15594 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
15595 .unwrap();
15596 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
15597 assert_eq!(replies[0].header.ty, expected);
15598 if expected == FrameType::Response {
15599 let reply: subc_control::ScopeServedByReply =
15600 serde_json::from_slice(&replies[0].body).unwrap();
15601 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
15602 } else {
15603 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
15604 }
15605 }
15606 let reply = call(
15607 &rig.handler,
15608 &rig.owner,
15609 &ModuleControlRequestFromModule::ScopeDescribe {
15610 owner: Principal::Reserved {
15611 module_id: OWNER.into(),
15612 },
15613 scope_ref: "s".into(),
15614 },
15615 )
15616 .await;
15617 assert!(matches!(
15618 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
15619 ModuleControlResponseToModule::ScopeDescribe { .. }
15620 ));
15621 }
15622
15623 use subc_protocol::scope::{
15624 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
15625 ScopeStatus,
15626 };
15627
15628 use super::*;
15629
15630 const OWNER: &str = "prefrontal-core";
15631
15632 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
15633 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
15634 }
15635
15636 async fn call(
15637 handler: &ControlHandler,
15638 ctx: &RouteCtx,
15639 request: &ModuleControlRequestFromModule,
15640 ) -> Frame {
15641 let body = serde_json::to_vec(request).unwrap();
15642 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
15643 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
15644 assert_eq!(replies.len(), 1, "{replies:?}");
15645 replies.pop().unwrap()
15646 }
15647
15648 async fn sync(
15649 handler: &ControlHandler,
15650 ctx: &RouteCtx,
15651 generation: u64,
15652 scopes: Vec<ScopeRecord>,
15653 ) -> Result<ModuleControlResponseToModule, String> {
15654 let reply = call(
15655 handler,
15656 ctx,
15657 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
15658 )
15659 .await;
15660 match reply.header.ty {
15661 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
15662 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
15663 }
15664 }
15665
15666 async fn describe(
15667 handler: &ControlHandler,
15668 ctx: &RouteCtx,
15669 owner: &str,
15670 scope_ref: &str,
15671 ) -> ModuleControlResponseToModule {
15672 let reply = call(
15673 handler,
15674 ctx,
15675 &ModuleControlRequestFromModule::ScopeDescribe {
15676 owner: Principal::Reserved {
15677 module_id: owner.to_string(),
15678 },
15679 scope_ref: scope_ref.to_string(),
15680 },
15681 )
15682 .await;
15683 assert_eq!(
15684 reply.header.ty,
15685 FrameType::Response,
15686 "{:?}",
15687 parse_error(&reply)
15688 );
15689 serde_json::from_slice(&reply.body).unwrap()
15690 }
15691
15692 #[tokio::test]
15693 async fn scope_describe_never_echoes_bind_proof_or_hash() {
15694 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15695 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15696 let mut rig = rig().await;
15697 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15698 .await;
15699 let _bound = rig
15700 .bound(
15701 Some(AFT),
15702 PLEXUS,
15703 Some(
15704 rig.selector("s", Some(1))
15705 .with_bind_proof(Some(PROOF.into())),
15706 ),
15707 )
15708 .await;
15709 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
15710 let output = serde_json::to_string(&reply).unwrap();
15711 assert!(output.contains("\"status\":\"live\""));
15712 assert!(!output.contains(PROOF));
15713 assert!(!output.contains(HASH));
15714 }
15715
15716 async fn module(
15718 handler: &ControlHandler,
15719 connection: u64,
15720 module_id: &str,
15721 nonce: Option<&str>,
15722 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15723 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
15724 hello_via_sink(
15725 handler,
15726 &ctx,
15727 &mut rx,
15728 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
15729 )
15730 .await;
15731 (ctx, rx)
15732 }
15733
15734 #[tokio::test]
15737 async fn a_client_connection_cannot_sync_or_describe() {
15738 let handler = ControlHandler::new(Arc::new(Registry::default()));
15739 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
15740 for request in [
15741 ModuleControlRequestFromModule::ScopeSync {
15742 generation: 1,
15743 scopes: vec![head("s", 1)],
15744 },
15745 ModuleControlRequestFromModule::ScopeDescribe {
15746 owner: Principal::Direct,
15747 scope_ref: "s".to_string(),
15748 },
15749 ] {
15750 let reply = call(&handler, &ctx, &request).await;
15751 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
15752 }
15753 assert!(
15754 !handler
15755 .scopes
15756 .read()
15757 .unwrap()
15758 .describe(
15759 &Principal::Reserved {
15760 module_id: OWNER.to_string()
15761 },
15762 "s"
15763 )
15764 .owner_synced
15765 );
15766 }
15767
15768 #[tokio::test]
15771 async fn a_module_without_a_supervised_launch_cannot_sync() {
15772 let handler = ControlHandler::new(Arc::new(Registry::default()));
15773 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
15774 assert_eq!(
15775 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
15776 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15777 );
15778 }
15779
15780 #[tokio::test]
15781 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
15782 let supervisor = SupervisorHandle::new();
15783 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15784 let handler = ControlHandler::new(Arc::new(Registry::default()))
15785 .with_supervisor(supervisor.clone());
15786 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15787 sync(&handler, &incumbent, 1, vec![head("s", 1)])
15788 .await
15789 .expect("the current launch syncs");
15790
15791 supervisor.open_swap(OWNER, "n2".to_string());
15794 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
15795 assert_eq!(
15796 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15797 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15798 );
15799 sync(&handler, &incumbent, 2, vec![head("s", 1)])
15800 .await
15801 .expect("the serving owner syncs during the swap");
15802
15803 supervisor.close_swap(OWNER);
15806 assert_eq!(
15807 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15808 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15809 );
15810 sync(&handler, &incumbent, 3, vec![head("s", 1)])
15811 .await
15812 .expect("the serving owner syncs after the rollback");
15813 handler.cleanup_connection(candidate.connection_id).unwrap();
15814
15815 supervisor.open_swap(OWNER, "n3".to_string());
15820 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
15821 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
15822 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
15823 .await
15824 .expect("the promoted launch takes authority");
15825 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15826 panic!("unexpected reply {reply:?}");
15827 };
15828 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
15829 assert_eq!(
15830 sync(&handler, &incumbent, 4, Vec::new()).await,
15831 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15832 );
15833 }
15834
15835 #[tokio::test]
15838 async fn closing_the_authority_connection_frees_sync_authority() {
15839 let supervisor = SupervisorHandle::new();
15840 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15841 let handler = ControlHandler::new(Arc::new(Registry::default()))
15842 .with_supervisor(supervisor.clone());
15843 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15844 sync(&handler, &first, 10, vec![head("s", 1)])
15845 .await
15846 .unwrap();
15847 handler.cleanup_connection(first.connection_id).unwrap();
15848
15849 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
15850 sync(&handler, &second, 1, vec![head("s", 1)])
15851 .await
15852 .expect("the next connection takes the released authority");
15853 }
15854
15855 #[tokio::test]
15856 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
15857 let supervisor = SupervisorHandle::new();
15858 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15859 let handler =
15860 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
15861 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
15862 sync(&handler, &first, 10, vec![head("s", 1)])
15863 .await
15864 .unwrap();
15865 handler
15866 .handle_control_frame(
15867 &first,
15868 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
15869 )
15870 .await
15871 .unwrap();
15872 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
15873 sync(&handler, &second, 1, vec![head("s", 1)])
15874 .await
15875 .expect("GOODBYE releases authority even if the old socket remains open");
15876 }
15877
15878 #[tokio::test]
15879 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
15880 let registry = Arc::new(Registry::default());
15881 let supervisor_handle = SupervisorHandle::new();
15882 let supervisor =
15883 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15884 .with_handle(supervisor_handle.clone())
15885 .with_daemon_incarnation("incarnation-7".to_string());
15886 supervisor
15889 .supervise_configured(
15890 ModuleSpec {
15891 module_id: OWNER.to_string(),
15892 program: PathBuf::from("/nonexistent/prefrontal-core"),
15893 args: Vec::new(),
15894 env: Vec::new(),
15895 reserved: false,
15896 reserved_prefixes: Vec::new(),
15897 protocol: ModuleProtocol::Subc,
15898 overlap: Default::default(),
15899 },
15900 false,
15901 )
15902 .unwrap();
15903 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
15904 let handler =
15905 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15906 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
15907
15908 let ModuleControlResponseToModule::ScopeDescribe {
15910 status,
15911 daemon_incarnation,
15912 owner_synced,
15913 owner_configured,
15914 scope,
15915 ..
15916 } = describe(&handler, &reader, OWNER, "s").await
15917 else {
15918 panic!("not a describe reply");
15919 };
15920 assert_eq!(status, ScopeStatus::NotLive);
15921 assert_eq!(daemon_incarnation, "incarnation-7");
15922 assert!(!owner_synced);
15923 assert!(owner_configured);
15924 assert!(scope.is_none());
15925
15926 let ModuleControlResponseToModule::ScopeDescribe {
15929 status,
15930 owner_configured,
15931 ..
15932 } = describe(&handler, &reader, "ghost", "s").await
15933 else {
15934 panic!("not a describe reply");
15935 };
15936 assert_eq!(status, ScopeStatus::NotLive);
15937 assert!(!owner_configured);
15938
15939 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15941 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15942 let ModuleControlResponseToModule::ScopeDescribe {
15943 status,
15944 scope_epoch,
15945 owner_synced,
15946 scope,
15947 ..
15948 } = describe(&handler, &reader, OWNER, "s").await
15949 else {
15950 panic!("not a describe reply");
15951 };
15952 assert_eq!(status, ScopeStatus::Live);
15953 assert_eq!(scope_epoch, Some(4));
15954 assert!(owner_synced);
15955 let stamp = scope.expect("a live scope carries its stamp");
15956 assert!(
15957 stamp.owner_authorized,
15958 "prefrontal-core is the default authority"
15959 );
15960 assert_eq!(stamp.kind, ScopeKind::Head);
15961 }
15962
15963 #[tokio::test]
15964 async fn scope_authority_owners_decides_owner_authorized() {
15965 let supervisor = SupervisorHandle::new();
15966 supervisor.set_spawn_nonce("broca", "b1".to_string());
15967 let handler = ControlHandler::new(Arc::new(Registry::default()))
15968 .with_supervisor(supervisor)
15969 .with_scope_authority_owners(vec!["broca".to_string()]);
15970 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15971 let mut gated = head("s", 1);
15972 gated.attributes.agent_id = Some("agent".to_string());
15973 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15974 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15975 describe(&handler, &broca, "broca", "s").await
15976 else {
15977 panic!("not a describe reply");
15978 };
15979 assert!(scope.unwrap().owner_authorized);
15980 }
15981
15982 #[tokio::test]
15986 async fn scope_ops_and_the_scopes_capability_are_advertised() {
15987 let handler = ControlHandler::new(Arc::new(Registry::default()));
15988 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15989 let ack = hello_via_sink(
15990 &handler,
15991 &ctx,
15992 &mut rx,
15993 hello_frame("m", PROTOCOL_VERSION, 1),
15994 )
15995 .await;
15996 let ack = parse_ack(&ack);
15997 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15998 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15999 }
16000 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16001
16002 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16003 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16004 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16005 let reply = handler
16006 .handle_control_frame(&client, frame)
16007 .await
16008 .unwrap()
16009 .pop()
16010 .unwrap();
16011 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16012 serde_json::from_slice(&reply.body).unwrap()
16013 else {
16014 panic!("not a server.describe reply");
16015 };
16016 assert!(
16017 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16018 "{capabilities:?}"
16019 );
16020 }
16021
16022 const PLEXUS: &str = "plexus";
16025 const OTHER: &str = "other";
16026 const AFT: &str = "aft";
16027 const BROCA: &str = "broca";
16028 const MAGIC: &str = "magic-context";
16029
16030 fn nonce(module_id: &str) -> String {
16031 format!("nonce-{module_id}")
16032 }
16033
16034 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16035 let (tx, rx) = mpsc::channel(64);
16036 (
16037 RouteCtx {
16038 connection_id: ConnectionId::new(connection),
16039 egress: FrameSink::new(tx),
16040 },
16041 rx,
16042 )
16043 }
16044
16045 struct Rig {
16049 handler: ControlHandler,
16050 forwarding: Arc<ForwardingTable>,
16051 owner: RouteCtx,
16052 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16053 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16054 generation: u64,
16055 next_connection: u64,
16056 _supervisor: Supervisor,
16057 }
16058
16059 async fn rig() -> Rig {
16060 rig_with_flow_support(true).await
16061 }
16062
16063 async fn rig_with_flow_support(flow_support: bool) -> Rig {
16064 rig_with_scope_capabilities(flow_support, false).await
16065 }
16066
16067 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16068 let mut provides = Vec::new();
16069 if flow_support {
16070 provides.push("flow-scopes/v1");
16071 }
16072 if run_support {
16073 provides.push("agent-run-scopes/v1");
16074 }
16075 rig_with_target_capabilities(&provides).await
16076 }
16077
16078 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16079 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16080 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16081 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16084 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16085 Frame::build(
16086 FrameType::Hello,
16087 control_flags(),
16088 0,
16089 0,
16090 connection,
16091 serde_json::to_vec(&body).unwrap(),
16092 )
16093 .unwrap()
16094 }
16095
16096 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16097 let registry = Arc::new(Registry::default());
16098 let forwarding = Arc::new(ForwardingTable::default());
16099 let supervisor_handle = SupervisorHandle::new();
16100 let supervisor =
16101 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16102 .with_handle(supervisor_handle.clone());
16103 supervisor
16104 .supervise_configured(
16105 ModuleSpec {
16106 module_id: OWNER.to_string(),
16107 program: PathBuf::from("/nonexistent/prefrontal-core"),
16108 args: Vec::new(),
16109 env: Vec::new(),
16110 reserved: false,
16111 reserved_prefixes: Vec::new(),
16112 protocol: ModuleProtocol::Subc,
16113 overlap: Default::default(),
16114 },
16115 false,
16116 )
16117 .unwrap();
16118 for module_id in [OWNER, AFT, BROCA, MAGIC] {
16119 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16120 }
16121 let handler =
16122 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16123 .with_supervisor(supervisor_handle);
16124 let (owner, mut owner_rx) = wide_ctx(1);
16125 hello_via_sink(
16126 &handler,
16127 &owner,
16128 &mut owner_rx,
16129 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16130 )
16131 .await;
16132 let mut modules = BTreeMap::new();
16133 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16134 let (ctx, mut rx) = wide_ctx(connection);
16135 let hello = hello_with_capabilities(module_id, connection, provides);
16136 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16137 modules.insert(module_id.to_string(), (ctx, rx));
16138 }
16139 Rig {
16140 handler,
16141 forwarding,
16142 owner,
16143 _owner_rx: owner_rx,
16144 modules,
16145 generation: 0,
16146 next_connection: 100,
16147 _supervisor: supervisor,
16148 }
16149 }
16150
16151 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16152 ScopeCarrier::new(Principal::Reserved {
16153 module_id: module_id.to_string(),
16154 })
16155 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16156 }
16157
16158 fn session(scope_epoch: u64) -> ScopeRecord {
16161 let mut record = head("s", scope_epoch);
16162 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16163 record.attributes.agent_id = Some("agent-1".to_string());
16164 record.attributes.delegates = true;
16165 record
16166 }
16167
16168 impl Rig {
16169 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16170 self.generation += 1;
16171 sync(&self.handler, &self.owner, self.generation, scopes)
16172 .await
16173 .expect("the owner's sync is accepted");
16174 }
16175
16176 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16177 ScopeSelector::new(
16178 Principal::Reserved {
16179 module_id: OWNER.to_string(),
16180 },
16181 scope_ref.to_string(),
16182 scope_epoch,
16183 )
16184 }
16185
16186 fn open_frame(
16187 &mut self,
16188 opener: Option<&str>,
16189 target: &str,
16190 scope: Option<ScopeSelector>,
16191 ) -> (
16192 RouteCtx,
16193 mpsc::Receiver<crate::router::OutboundFrame>,
16194 Frame,
16195 ) {
16196 self.next_connection += 1;
16197 let (ctx, rx) = wide_ctx(self.next_connection);
16198 let root = unique_project_root("scoped-open");
16199 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16200 target: RouteTarget::ToolProvider {
16201 module_id: target.to_string(),
16202 },
16203 identity: BindIdentity::new(
16204 root.path().to_path_buf(),
16205 "unit".to_string(),
16206 "session".to_string(),
16207 ),
16208 consumer_identity: opener.map(|module_id| ConsumerIdentity {
16209 module_id: module_id.to_string(),
16210 launch_nonce: nonce(module_id),
16211 }),
16212 consumer_capabilities: None,
16213 role_versions: None,
16214 admission_facts: None,
16215 scope,
16216 })
16217 .unwrap();
16218 let frame = Frame::build(
16219 FrameType::Request,
16220 control_flags(),
16221 0,
16222 0,
16223 self.next_connection,
16224 body,
16225 )
16226 .unwrap();
16227 (ctx, rx, frame)
16228 }
16229
16230 async fn refused(
16232 &mut self,
16233 opener: Option<&str>,
16234 target: &str,
16235 scope: Option<ScopeSelector>,
16236 ) -> String {
16237 self.refusal_body(opener, target, scope).await["code"]
16238 .as_str()
16239 .unwrap()
16240 .to_string()
16241 }
16242
16243 async fn refusal_body(
16244 &mut self,
16245 opener: Option<&str>,
16246 target: &str,
16247 scope: Option<ScopeSelector>,
16248 ) -> Value {
16249 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16250 let replies = tokio::time::timeout(
16251 Duration::from_secs(2),
16252 self.handler.handle_control_frame(&ctx, frame),
16253 )
16254 .await
16255 .expect("the open must be refused before waiting for a bind ack")
16256 .unwrap();
16257 assert_eq!(replies.len(), 1, "{replies:?}");
16258 assert_eq!(replies[0].header.ty, FrameType::Error);
16259 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16260 assert!(
16261 module_rx.try_recv().is_err(),
16262 "a refused open relays nothing"
16263 );
16264 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16265 parse_error(&replies[0])
16266 }
16267
16268 async fn relayed(
16270 &mut self,
16271 opener: Option<&str>,
16272 target: &str,
16273 scope: Option<ScopeSelector>,
16274 ) -> Relayed {
16275 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16276 let handler = self.handler.clone();
16277 let task_ctx = ctx.clone();
16278 let task = tokio::spawn(async move {
16279 handler
16280 .handle_control_frame(&task_ctx, frame)
16281 .await
16282 .unwrap()
16283 });
16284 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16285 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16286 .await
16287 .expect("the target receives the relayed route.bind")
16288 .unwrap()
16289 .frame;
16290 Relayed {
16291 target: target.to_string(),
16292 client: ctx,
16293 client_rx: rx,
16294 task,
16295 bind,
16296 }
16297 }
16298
16299 async fn ack(&self, relayed: &Relayed) {
16300 let (module, _) = &self.modules[&relayed.target];
16301 self.handler
16302 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16303 .await
16304 .unwrap();
16305 }
16306
16307 async fn bound(
16309 &mut self,
16310 opener: Option<&str>,
16311 target: &str,
16312 scope: Option<ScopeSelector>,
16313 ) -> Bound {
16314 let relayed = self.relayed(opener, target, scope).await;
16315 self.ack(&relayed).await;
16316 let Relayed {
16317 target,
16318 client,
16319 mut client_rx,
16320 task,
16321 bind,
16322 } = relayed;
16323 assert!(
16324 task.await.unwrap().is_empty(),
16325 "the open is answered by commit"
16326 );
16327 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16328 Bound {
16329 target,
16330 client,
16331 client_rx,
16332 channel,
16333 epoch,
16334 bind,
16335 }
16336 }
16337
16338 fn live(&self, route: &Bound) -> bool {
16339 matches!(
16340 self.forwarding
16341 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16342 .unwrap(),
16343 DataRoute::Client(DataRouteState::Bound(_))
16344 )
16345 }
16346 }
16347
16348 struct Relayed {
16349 target: String,
16350 client: RouteCtx,
16351 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16352 task: tokio::task::JoinHandle<Vec<Frame>>,
16353 bind: Frame,
16354 }
16355
16356 struct Bound {
16357 target: String,
16358 client: RouteCtx,
16359 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16360 channel: u16,
16361 epoch: u32,
16362 bind: Frame,
16363 }
16364
16365 impl Bound {
16366 fn closed_reason(&mut self) -> RouteCloseReason {
16369 let mut reason = None;
16370 let mut goodbye = false;
16371 while let Ok(outbound) = self.client_rx.try_recv() {
16372 let frame = outbound.frame;
16373 match frame.header.ty {
16374 FrameType::Goodbye => {
16375 assert_eq!(
16376 (frame.header.channel, frame.header.epoch),
16377 (self.channel, self.epoch)
16378 );
16379 goodbye = true;
16380 }
16381 FrameType::Push => {
16382 let ClientControlPush::RouteClosed {
16383 reason: r,
16384 module_id,
16385 ..
16386 } = serde_json::from_slice(&frame.body).unwrap()
16387 else {
16388 panic!("unexpected push");
16389 };
16390 assert_eq!(module_id, self.target);
16391 reason = Some(r);
16392 }
16393 other => panic!("unexpected frame {other:?}"),
16394 }
16395 }
16396 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16397 reason.expect("the client is told why the route closed")
16398 }
16399
16400 fn untouched(&mut self) -> bool {
16401 self.client_rx.try_recv().is_err()
16402 }
16403
16404 fn stamp(&self) -> Option<ScopeStamp> {
16405 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16406 ModuleControlRequest::RouteBind { scope, .. } => scope,
16407 other => panic!("expected a route.bind, got {other:?}"),
16408 }
16409 }
16410 }
16411
16412 #[tokio::test]
16413 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16414 ) {
16415 let mut rig = rig().await;
16416 let mut record = session(1);
16417 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16418 record.child_owners = vec![Principal::Reserved {
16419 module_id: MAGIC.to_string(),
16420 }];
16421 rig.sync(vec![record]).await;
16422 let scope = || Some(rig_selector("s", Some(1)));
16423
16424 rig.bound(Some(OWNER), PLEXUS, scope()).await;
16427 rig.bound(Some(AFT), OTHER, scope()).await;
16428 rig.bound(Some(BROCA), PLEXUS, scope()).await;
16429
16430 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16434 assert_eq!(
16435 rig.refused(opener, target, scope()).await,
16436 error_codes::SCOPE_NOT_CARRIER,
16437 "{opener:?} -> {target}"
16438 );
16439 }
16440 }
16441
16442 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16443 ScopeSelector::new(
16444 Principal::Reserved {
16445 module_id: OWNER.to_string(),
16446 },
16447 scope_ref.to_string(),
16448 scope_epoch,
16449 )
16450 }
16451
16452 #[tokio::test]
16453 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16454 let mut rig = rig().await;
16455 rig.sync(vec![session(1)]).await;
16456 for opener in [OWNER, AFT] {
16457 assert_eq!(
16458 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16459 .await,
16460 error_codes::SCOPE_EPOCH_REQUIRED,
16461 "{opener}"
16462 );
16463 }
16464 }
16465
16466 #[tokio::test]
16467 async fn admission_separates_not_synced_not_live_and_ended() {
16468 let mut rig = rig().await;
16469 let code = rig
16471 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16472 .await;
16473 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16474 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16475
16476 let ghost = ScopeSelector::new(
16478 Principal::Reserved {
16479 module_id: "ghost".to_string(),
16480 },
16481 "s".to_string(),
16482 Some(1),
16483 );
16484 assert_eq!(
16485 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16486 error_codes::SCOPE_NOT_LIVE
16487 );
16488
16489 rig.sync(vec![session(2)]).await;
16490 assert_eq!(
16491 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16492 .await,
16493 error_codes::SCOPE_NOT_LIVE
16494 );
16495 for epoch in [1, 3] {
16496 assert_eq!(
16497 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16498 .await,
16499 error_codes::SCOPE_ENDED,
16500 "epoch {epoch}"
16501 );
16502 }
16503 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16505 .await;
16506 }
16507
16508 #[tokio::test]
16509 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16510 let mut rig = rig().await;
16511 rig.sync(vec![session(1)]).await;
16512 let route = rig
16513 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16514 .await;
16515 let stamp = route.stamp().expect("a scoped bind carries the stamp");
16516 assert_eq!(stamp.scope_ref, "s");
16517 assert_eq!(stamp.scope_epoch, 1);
16518 assert_eq!(stamp.kind, ScopeKind::Head);
16519 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16520 assert!(stamp.attributes.delegates);
16521 assert!(stamp.owner_authorized);
16522 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16523 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16524
16525 let (broca, mut broca_rx) = wide_ctx(50);
16528 hello_via_sink(
16529 &rig.handler,
16530 &broca,
16531 &mut broca_rx,
16532 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16533 )
16534 .await;
16535 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16536 .await
16537 .unwrap();
16538 let own = ScopeSelector::new(
16539 Principal::Reserved {
16540 module_id: BROCA.to_string(),
16541 },
16542 "b".to_string(),
16543 Some(1),
16544 );
16545 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16546 assert!(!route.stamp().unwrap().owner_authorized);
16547 }
16548
16549 #[tokio::test]
16550 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16551 let mut rig = rig().await;
16552 let mut record = head("s", 1);
16553 record.carriers = vec![carrier(AFT, None)];
16554 let flow_id = "Flow:run-7/step_2!~";
16555 record.attributes.flow_id = Some(flow_id.to_string());
16556 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16557 .await
16558 .unwrap();
16559 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16560 panic!("not a sync reply");
16561 };
16562 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16563 let route = rig
16564 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16565 .await;
16566 assert!(rig.live(&route), "the stamped bind committed");
16567 let stamp = route.stamp().expect("a flow scope carries a stamp");
16568 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16569 assert_eq!(stamp.attributes.agent_id, None);
16570 assert!(!stamp.attributes.delegates);
16571 assert!(stamp.owner_authorized);
16572 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16573 assert_eq!(unscoped.stamp(), None);
16574 }
16575
16576 #[tokio::test]
16577 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16578 let mut rig = rig_with_flow_support(false).await;
16579 let mut record = session(1);
16580 record.attributes.flow_id = Some("flow:7".to_string());
16581 rig.sync(vec![record]).await;
16582 for opener in [OWNER, AFT] {
16583 let body = rig
16584 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16585 .await;
16586 assert_eq!(body["code"], "target_flow_unsupported");
16587 let message = body["message"].as_str().unwrap();
16588 for required in [PLEXUS, "flow-scopes/v1"] {
16589 assert!(message.contains(required), "{message}");
16590 }
16591 }
16592 }
16593
16594 #[tokio::test]
16595 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
16596 let mut rig = rig_with_flow_support(true).await;
16597 let mut record = session(1);
16598 record.attributes.flow_id = Some("flow:7".to_string());
16599 rig.sync(vec![record]).await;
16600 for opener in [OWNER, AFT] {
16601 let route = rig
16602 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16603 .await;
16604 assert!(rig.live(&route));
16605 assert_eq!(
16606 route.stamp().unwrap().attributes.flow_id.as_deref(),
16607 Some("flow:7")
16608 );
16609 }
16610 }
16611
16612 #[tokio::test]
16613 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
16614 use std::future::Future;
16615
16616 let mut rig = rig_with_flow_support(true).await;
16617 let mut record = session(1);
16618 record.attributes.flow_id = Some("flow:7".to_string());
16619 rig.sync(vec![record]).await;
16620 let (client, mut client_rx, frame) =
16621 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
16622 for _ in 0..64 {
16625 client.egress.try_send(route_bind_ack(1)).unwrap();
16626 }
16627 let handler = rig.handler.clone();
16628 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16629 std::future::poll_fn(|cx| {
16630 assert!(open.as_mut().poll(cx).is_pending());
16631 std::task::Poll::Ready(())
16632 })
16633 .await;
16634 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16635
16636 let old_connection = rig.modules[PLEXUS].0.connection_id;
16637 rig.handler.cleanup_connection(old_connection).unwrap();
16638 let (replacement, mut replacement_rx) = wide_ctx(200);
16639 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
16640 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16641 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16642 let hello = Frame::build(
16643 FrameType::Hello,
16644 control_flags(),
16645 0,
16646 0,
16647 200,
16648 serde_json::to_vec(&body).unwrap(),
16649 )
16650 .unwrap();
16651 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
16652
16653 client_rx.try_recv().unwrap();
16654 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16655 .await
16656 .expect("the replacement is refused without waiting for a bind ack")
16657 .unwrap();
16658 assert_eq!(replies.len(), 1);
16659 let body = parse_error(&replies[0]);
16660 assert_eq!(body["code"], "target_flow_unsupported");
16661 for required in [PLEXUS, "flow-scopes/v1"] {
16662 assert!(body["message"].as_str().unwrap().contains(required));
16663 }
16664 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
16665 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16666 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16667 assert!(rig
16668 .handler
16669 .registry
16670 .get_module_by_connection(replacement.connection_id)
16671 .unwrap()
16672 .is_some());
16673 }
16674
16675 #[tokio::test]
16676 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
16677 use std::future::Future;
16678 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16679 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16680 const NEXT_HASH: &str =
16681 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16682 let mut rig = rig().await;
16683 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
16684 record.carriers.push(ScopeCarrier::new(Principal::Direct));
16685 rig.sync(vec![record.clone()]).await;
16686 let (client, mut rx, frame) = rig.open_frame(
16687 None,
16688 PLEXUS,
16689 Some(
16690 rig.selector("s", Some(1))
16691 .with_bind_proof(Some(PROOF.into())),
16692 ),
16693 );
16694 for _ in 0..64 {
16697 client.egress.try_send(route_bind_ack(1)).unwrap();
16698 }
16699 let handler = rig.handler.clone();
16700 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16701 std::future::poll_fn(|cx| {
16702 assert!(open.as_mut().poll(cx).is_pending());
16703 std::task::Poll::Ready(())
16704 })
16705 .await;
16706 record.bind_proof_sha256 = Some(NEXT_HASH.into());
16707 rig.sync(vec![record]).await;
16708 rx.try_recv().unwrap();
16709 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16710 .await
16711 .unwrap()
16712 .unwrap();
16713 assert_eq!(
16714 parse_error(&replies[0])["code"],
16715 error_codes::SCOPE_BIND_PROOF_MISMATCH
16716 );
16717 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
16718 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16719 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16720 }
16721
16722 #[tokio::test]
16723 async fn scope_bind_proof_reserved_openers_ignore_proof() {
16724 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16725 let mut rig = rig().await;
16726 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16727 .await;
16728 for opener in [OWNER, AFT] {
16729 for proof in [None, Some("malformed+proof=")] {
16730 let route = rig
16731 .bound(
16732 Some(opener),
16733 PLEXUS,
16734 Some(
16735 rig.selector("s", Some(1))
16736 .with_bind_proof(proof.map(str::to_string)),
16737 ),
16738 )
16739 .await;
16740 assert!(rig.live(&route));
16741 }
16742 }
16743 }
16744
16745 #[tokio::test]
16746 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
16747 use std::future::Future;
16748 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16749 const NEXT_HASH: &str =
16750 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16751 let mut rig = rig().await;
16752 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16753 .await;
16754 let (client, mut rx, frame) =
16755 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
16756 for _ in 0..64 {
16757 client.egress.try_send(route_bind_ack(1)).unwrap();
16758 }
16759 let handler = rig.handler.clone();
16760 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16761 std::future::poll_fn(|cx| {
16762 assert!(open.as_mut().poll(cx).is_pending());
16763 std::task::Poll::Ready(())
16764 })
16765 .await;
16766 rig.sync(vec![
16767 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
16768 ])
16769 .await;
16770 rx.try_recv().unwrap();
16771 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16772 let bind = std::future::poll_fn(|cx| {
16773 assert!(open.as_mut().poll(cx).is_pending());
16774 module_rx.poll_recv(cx)
16775 })
16776 .await
16777 .unwrap()
16778 .frame;
16779 handler
16780 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
16781 .await
16782 .unwrap();
16783 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16784 .await
16785 .unwrap()
16786 .unwrap();
16787 assert_eq!(replies.len(), 1);
16788 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
16791 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16792 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
16793 let route = rig
16794 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
16795 .await;
16796 assert!(rig.live(&route));
16797 }
16798
16799 #[tokio::test]
16800 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
16801 {
16802 let mut rig = rig_with_flow_support(false).await;
16803 rig.sync(vec![session(1)]).await;
16804 let route = rig
16805 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16806 .await;
16807 assert!(rig.live(&route));
16808 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
16809 let mut record = session(1);
16810 record.attributes.flow_id = Some("flow:7".to_string());
16811 rig.sync(vec![record]).await;
16812 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
16813 assert!(rig.live(&unscoped));
16814 assert_eq!(unscoped.stamp(), None);
16815 }
16816
16817 #[tokio::test]
16818 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
16819 let mut rig = rig().await;
16820 let mut record = session(1);
16821 record.attributes.flow_id = Some("flow:7".to_string());
16822 rig.sync(vec![record.clone()]).await;
16823 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16824 let mut owner_route = rig
16825 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16826 .await;
16827 let mut carrier_route = rig
16828 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
16829 .await;
16830 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16831 rig.sync(vec![record.clone()]).await;
16832 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16833 assert!(rig.live(&owner_route) && owner_route.untouched());
16834 assert!(rig.live(&carrier_route) && carrier_route.untouched());
16835
16836 record.attributes.flow_id = Some("flow:8".to_string());
16837 rig.sync(vec![record]).await;
16838 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
16839 let before = before.unwrap();
16840 assert_eq!(after.scope_epoch, before.scope_epoch);
16841 assert!(after.version > before.version);
16842 for route in [&mut owner_route, &mut carrier_route] {
16843 assert!(!rig.live(route));
16844 assert_eq!(
16845 route.closed_reason(),
16846 RouteCloseReason::ScopeDelegationChanged
16847 );
16848 }
16849 assert!(rig.live(&unscoped) && unscoped.untouched());
16850 for target in [PLEXUS, OTHER] {
16853 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
16854 let goodbye = module_rx
16855 .try_recv()
16856 .expect("the provider sees the drain")
16857 .frame;
16858 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16859 assert!(module_rx.try_recv().is_err());
16860 }
16861 let rebound = rig
16862 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16863 .await;
16864 assert_eq!(
16865 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
16866 Some("flow:8")
16867 );
16868 }
16869
16870 #[tokio::test]
16875 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
16876 let mut rig = rig().await;
16877 rig.sync(vec![session(1)]).await;
16878 let mut cotenant = rig
16879 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16880 .await;
16881
16882 let mut changed = session(1);
16883 changed.child_owners.push(Principal::Reserved {
16884 module_id: MAGIC.to_string(),
16885 });
16886 let mut ended = None;
16887 for (code, next) in [
16888 (error_codes::SCOPE_CHANGED, vec![changed]),
16889 (error_codes::SCOPE_ENDED, Vec::new()),
16890 ] {
16891 let relayed = rig
16892 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16893 .await;
16894 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
16895 ended = Some(next.is_empty());
16896 rig.sync(next).await;
16897 rig.ack(&relayed).await;
16898 let replies = relayed.task.await.unwrap();
16899 assert_eq!(replies.len(), 1, "{replies:?}");
16900 assert_eq!(parse_error(&replies[0])["code"], code);
16901 assert_eq!(
16902 subc_protocol::error_codes::is_retryable_route_open(code),
16903 code == error_codes::SCOPE_CHANGED
16904 );
16905 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16906 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16908 let mut goodbyes = Vec::new();
16911 while let Ok(outbound) = plexus_rx.try_recv() {
16912 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
16913 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
16914 }
16915 assert!(
16916 goodbyes.contains(&(bind_channel, bind_epoch)),
16917 "{goodbyes:?}"
16918 );
16919 assert!(rig
16920 .handler
16921 .registry
16922 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
16923 .unwrap()
16924 .is_some());
16925 }
16926 assert_eq!(ended, Some(true));
16927 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
16930 }
16931
16932 #[tokio::test]
16935 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
16936 struct Case {
16937 name: &'static str,
16938 change: fn(&mut ScopeRecord),
16939 closed: [Option<RouteCloseReason>; 4],
16942 }
16943 use RouteCloseReason::*;
16944 let cases = [
16945 Case {
16946 name: "a carrier entry removed",
16947 change: |r| {
16948 r.carriers.retain(|c| {
16949 c.principal
16950 != Principal::Reserved {
16951 module_id: AFT.to_string(),
16952 }
16953 })
16954 },
16955 closed: [None, Some(ScopeCarrierRemoved), None, None],
16956 },
16957 Case {
16958 name: "a target removed from a carrier",
16959 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16960 closed: [None, None, None, Some(ScopeCarrierRemoved)],
16961 },
16962 Case {
16963 name: "a bare carrier narrowed to targets",
16964 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16965 closed: [None, Some(ScopeCarrierRemoved), None, None],
16966 },
16967 Case {
16968 name: "delegates turned off",
16969 change: |r| r.attributes.delegates = false,
16970 closed: [Some(ScopeDelegationChanged); 4],
16971 },
16972 Case {
16973 name: "agent_id changed",
16974 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16975 closed: [Some(ScopeDelegationChanged); 4],
16976 },
16977 Case {
16978 name: "a carrier added, child owners changed, the record re-sent",
16979 change: |r| {
16980 r.carriers.push(carrier(MAGIC, None));
16981 r.child_owners.push(Principal::Reserved {
16982 module_id: MAGIC.to_string(),
16983 });
16984 },
16985 closed: [None; 4],
16986 },
16987 Case {
16988 name: "a target added",
16989 change: |r| {
16990 r.carriers[1]
16991 .targets
16992 .as_mut()
16993 .unwrap()
16994 .push("third".to_string())
16995 },
16996 closed: [None; 4],
16997 },
16998 Case {
16999 name: "delegates turned on",
17000 change: |r| r.attributes.delegates = true,
17001 closed: [None; 4],
17002 },
17003 ];
17004 for case in cases {
17005 let mut rig = rig().await;
17006 rig.sync(vec![session(1)]).await;
17007 let scope = || Some(rig_selector("s", Some(1)));
17008 let mut routes = [
17009 rig.bound(Some(OWNER), PLEXUS, scope()).await,
17010 rig.bound(Some(AFT), PLEXUS, scope()).await,
17011 rig.bound(Some(BROCA), PLEXUS, scope()).await,
17012 rig.bound(Some(BROCA), OTHER, scope()).await,
17013 ];
17014 let mut record = session(1);
17015 (case.change)(&mut record);
17016 rig.sync(vec![record]).await;
17017 for (index, expected) in case.closed.iter().enumerate() {
17018 let route = &mut routes[index];
17019 match expected {
17020 Some(reason) => {
17021 assert!(!rig.live(route), "{}: route {index} still live", case.name);
17022 assert_eq!(
17023 route.closed_reason(),
17024 *reason,
17025 "{}: route {index}",
17026 case.name
17027 );
17028 }
17029 None => {
17030 assert!(rig.live(route), "{}: route {index} closed", case.name);
17031 assert!(
17032 route.untouched(),
17033 "{}: route {index} was told something",
17034 case.name
17035 );
17036 }
17037 }
17038 }
17039 }
17040 }
17041
17042 #[tokio::test]
17043 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17044 for next in [Vec::new(), vec![session(2)]] {
17046 let mut rig = rig().await;
17047 rig.sync(vec![session(1)]).await;
17048 let mut route = rig
17049 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17050 .await;
17051 rig.sync(next).await;
17052 assert!(!rig.live(&route));
17053 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17054 }
17055
17056 let mut rig = rig().await;
17059 let mut child = session(1);
17060 child.scope_ref = "child".to_string();
17061 child.kind = ScopeKind::Worker;
17062 child.parent = Some(ScopeParent::new(
17063 Principal::Reserved {
17064 module_id: OWNER.to_string(),
17065 },
17066 "s".to_string(),
17067 1,
17068 ));
17069 rig.sync(vec![session(1), child.clone()]).await;
17070 let mut child_route = rig
17071 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17072 .await;
17073 assert_eq!(
17074 child_route.stamp().unwrap().parent_state,
17075 Some(ParentState::Linked)
17076 );
17077 rig.sync(vec![child]).await;
17078 assert!(!rig.live(&child_route));
17079 assert_eq!(
17080 child_route.closed_reason(),
17081 RouteCloseReason::ScopeParentEnded
17082 );
17083 }
17084
17085 #[tokio::test]
17086 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17087 ) {
17088 let mut rig = rig().await;
17089 rig.sync(vec![session(1)]).await;
17090 let mut route = rig
17091 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17092 .await;
17093 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17094 rig.sync(vec![session(1)]).await;
17095 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17096 assert!(rig.live(&route) && route.untouched());
17097
17098 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17102 .forwarding
17103 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17104 .unwrap()
17105 else {
17106 panic!("the route is bound");
17107 };
17108 binding.flow.acquire_tagged(9, false).await.unwrap();
17109 let mut widened = session(1);
17110 widened.carriers.push(carrier(MAGIC, None));
17111 rig.sync(vec![widened]).await;
17112 assert!(rig.live(&route) && route.untouched());
17113 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17114 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17115 assert_eq!(binding.flow.in_flight(), 1);
17118 binding
17119 .flow
17120 .acquire_tagged(10, false)
17121 .await
17122 .expect("the flow is still open");
17123 }
17124
17125 #[tokio::test]
17128 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17129 let mut rig = rig().await;
17130 rig.sync(vec![session(1)]).await;
17131 let mut on_incumbent = rig
17132 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17133 .await;
17134
17135 let (candidate, _candidate_rx) = wide_ctx(9);
17138 let registration = rig
17139 .handler
17140 .registry
17141 .register_candidate_with_control_ops(
17142 manifest(PLEXUS, PROTOCOL_VERSION),
17143 PROTOCOL_VERSION,
17144 candidate.connection_id,
17145 module_baseline_control_ops(),
17146 )
17147 .unwrap();
17148 rig.forwarding
17149 .register_candidate_module_connection(
17150 candidate.connection_id,
17151 PLEXUS.to_string(),
17152 PROTOCOL_VERSION,
17153 manifest_concurrency(®istration.manifest),
17154 candidate.egress.clone(),
17155 )
17156 .unwrap();
17157 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17158 rig.handler
17159 .registry
17160 .promote_candidate(PLEXUS)
17161 .unwrap()
17162 .unwrap();
17163 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17164
17165 rig.sync(Vec::new()).await;
17166 assert!(!rig.live(&on_incumbent));
17167 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17168 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17169 let goodbye = incumbent_rx
17170 .try_recv()
17171 .expect("the superseded endpoint is told")
17172 .frame;
17173 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17174 }
17175 }
17176}
17177
17178#[cfg(test)]
17179mod concurrency_default_exposure_tests {
17180 use super::*;
17181
17182 fn hello_body(role_json: &str) -> Vec<u8> {
17183 format!(
17184 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17185 )
17186 .into_bytes()
17187 }
17188
17189 fn manifest_from(body: &[u8]) -> ModuleManifest {
17190 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17191 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17192 .expect("manifest parses")
17193 }
17194
17195 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17196
17197 #[test]
17198 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17199 let body = hello_body(&format!(
17200 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17201 ));
17202 let manifest = manifest_from(&body);
17203 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17206 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17207 }
17208
17209 #[test]
17210 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17211 let body = hello_body(&format!(
17212 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17213 ));
17214 let manifest = manifest_from(&body);
17215 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17216 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17217 }
17218
17219 #[test]
17220 fn non_management_roles_are_never_reported() {
17221 let body = hello_body(
17222 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17223 );
17224 let manifest = manifest_from(&body);
17225 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17226 }
17227}