1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27 },
28 scope::{
29 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31 SCOPE_SYNC_OP,
32 },
33 session::{
34 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37 ROLE_VERSIONS_FIELD,
38 },
39 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40 Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46 caller_identity::CallerIdentity,
47 capability_requirements::{
48 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50 RuntimeModule,
51 },
52 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
53 forwarding::{
54 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
55 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
56 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
57 },
58 observability::{
59 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
60 },
61 privileged_prompt::{prompt_text, PrivilegedOperation},
62 provenance::{
63 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
64 },
65 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
66 router::{RouteCtx, RouterError},
67 scopes::{HelloLaunchNonces, ScopeTable},
68 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
69 stderr_tail::{CaptureState, TailEntry},
70 supervise::{
71 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
72 SupervisorHandle, SwapHelloAdmission,
73 },
74 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
75};
76
77pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
83
84const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
85const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
86const CAP_PING_PONG: &str = "ping_pong_v1";
87const CAP_SESSION_ATTACH: &str = "session_attach_v1";
88const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
89
90const SUBC_CONTROL_OPS: &[&str] = &[
91 ops::ACTIVITY_RECORD,
92 ops::ACTIVITY_QUERY,
93 ops::ACTIVITY_SUBSCRIBE,
94 ops::SERVER_DESCRIBE,
95 ops::CATALOG_LIST,
96 ops::ROUTE_OPEN,
97 ops::ROUTE_POLL,
98 ops::ROUTE_CLOSING,
99 ops::ROUTE_CLOSED,
100 ops::SUPERVISOR_LIST,
101 ops::SUPERVISOR_RESTART,
102 ops::SUPERVISOR_SWAP,
103 ops::SUPERVISOR_RELOAD,
104 ops::SUPERVISOR_RESCAN,
105 ops::SUPERVISOR_RELEASE_RESERVED,
106 ops::SUPERVISOR_SET_ENABLED,
107 ops::SUPERVISOR_HEALTH_PROBE,
108 ops::SUPERVISOR_HEALTH,
109 ops::SUPERVISOR_STDERR_TAIL,
110 ops::SUPERVISOR_TERMINALS,
111 ops::SUPERVISOR_ROUTES,
112 ops::SUPERVISOR_PROVENANCE,
113 ops::SUPERVISOR_SPAWN_SNAPSHOT,
114 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
115];
116
117const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
118 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
119 "supervisor.live_roots",
120 SCOPE_SYNC_OP,
121 SCOPE_APPLY_OP,
122 SCOPE_DESCRIBE_OP,
123 "operator.confirm",
124];
125
126const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
130
131const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
132
133pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
141
142pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
157
158pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
176
177const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
178const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
179
180fn reload_verdict(
181 configured: &Path,
182 spawned_from: Option<&Path>,
183 image: subc_control::RunningImageAgreement,
184) -> PendingReloadVerdict {
185 let path = match spawned_from {
186 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
187 ReloadPathAgreement::Match
188 }
189 Some(spawned_from) => ReloadPathAgreement::Mismatch {
190 configured: configured.to_path_buf(),
191 spawned_from: spawned_from.to_path_buf(),
192 },
193 None => ReloadPathAgreement::Unavailable {
194 reason: if matches!(
195 image,
196 subc_control::RunningImageAgreement::Unavailable {
197 reason: subc_control::RunningImageUnavailableReason::NotRunning
198 }
199 ) {
200 ReloadPathUnavailableReason::NotRunning
201 } else {
202 ReloadPathUnavailableReason::SpawnedPathUnavailable
203 },
204 },
205 };
206 PendingReloadVerdict { path, image }
207}
208
209fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
210 if configured == spawned_from {
211 return true;
212 }
213 #[cfg(windows)]
214 {
215 match (
221 std::fs::canonicalize(configured),
222 std::fs::canonicalize(spawned_from),
223 ) {
224 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
225 _ => false,
226 }
227 }
228 #[cfg(not(windows))]
229 {
230 false
231 }
232}
233
234#[derive(Clone)]
235struct DaemonProvenanceFacts {
236 build: DaemonBuildProvenance,
237 pid: Option<u32>,
238 started_at_ms: Option<u64>,
239 start_clock: Option<crate::clock::StartClock>,
240 executable_path: Option<PathBuf>,
241 executable_identity: Option<SpawnedFileIdentity>,
242 process_start_time: Option<u64>,
243 probe: ExecutableIdentityProbe,
244}
245
246impl Default for DaemonProvenanceFacts {
247 fn default() -> Self {
248 Self {
249 build: DaemonBuildProvenance {
250 build_git_sha: None,
251 build_lock_digest: None,
252 },
253 pid: None,
254 started_at_ms: None,
255 start_clock: None,
256 executable_path: None,
257 executable_identity: None,
258 process_start_time: None,
259 probe: ExecutableIdentityProbe::default(),
260 }
261 }
262}
263
264#[derive(Debug, Clone)]
265struct SupervisorRescanContext {
266 supervisor: Supervisor,
267 config_path: PathBuf,
268 configured_port: Option<u16>,
269 storage_config: Option<crate::daemon_config::StorageConfig>,
270 admission_facts_carrier_module_id: Option<String>,
271 admission_facts_targets: Option<Vec<String>>,
272 scope_authority_owners: Vec<String>,
273 privileged_operator_prompt: PrivilegedOperatorPrompt,
274}
275
276const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
297 "reloading",
298 "supervisor_not_live",
299 "registration_not_active",
300 "no_forwarding_connection",
301 "relay_send_failed",
302];
303
304#[derive(Clone)]
306pub struct ControlHandler {
307 registry: Arc<Registry>,
308 activity: Option<crate::activity::ActivityLedger>,
309 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
310 forwarding: Arc<ForwardingTable>,
311 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
312 supervisor: SupervisorHandle,
313 subc_capabilities: Arc<[String]>,
314 route_bind_relay_timeout: Duration,
318 route_bind_relay_timeouts: BTreeMap<String, Duration>,
322 route_bind_breakers: RouteBindBreakers,
325 route_bind_concurrency: RouteBindConcurrency,
328 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
333 route_bind_breaker_threshold: u32,
335 route_bind_breaker_cooldown: Duration,
337 health_probe_timeout: Duration,
338 storage_config: Option<crate::daemon_config::StorageConfig>,
341 machine_id: Option<crate::machine_id::MachineId>,
345 admission_facts_carrier_module_id: Option<String>,
346 admission_facts_targets: Option<Vec<String>>,
347 scopes: Arc<RwLock<ScopeTable>>,
351 scope_deadline_added: Arc<tokio::sync::Notify>,
355 scope_authority_owners: Vec<String>,
358 privileged_operator_prompt: PrivilegedOperatorPrompt,
360 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
363 #[cfg(test)]
364 caller_lookup: Option<TestCallerLookup>,
365 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
368 rescan: Option<SupervisorRescanContext>,
369 connected_clients: ConnectedClients,
370 counters: DaemonCounters,
371 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
372 daemon_provenance: DaemonProvenanceFacts,
373 #[cfg(test)]
374 control_dispatch_delay: Option<Duration>,
375 #[cfg(test)]
376 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
377 #[cfg(test)]
381 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
382 #[cfg(test)]
388 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
389}
390
391impl fmt::Debug for ControlHandler {
392 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
393 f.debug_struct("ControlHandler")
394 .field("registry", &self.registry)
395 .field("forwarding", &self.forwarding)
396 .field("process_liveness", &self.process_liveness.is_some())
397 .field("supervisor", &self.supervisor)
398 .field("subc_capabilities", &self.subc_capabilities)
399 .finish()
400 }
401}
402
403struct RouteOpenRequest {
404 target: RouteTarget,
405 identity: BindIdentity,
406 consumer_identity: Option<ConsumerIdentity>,
407 consumer_capabilities: Option<Vec<String>>,
408 role_versions: Option<BTreeMap<String, String>>,
409 admission_facts: Option<serde_json::Value>,
410 scope: Option<ScopeSelector>,
411}
412
413struct RouteBindReservationGuard {
414 forwarding: Arc<ForwardingTable>,
415 endpoint: ModuleEndpointId,
416 relay_corr: u64,
417 armed: bool,
418}
419
420struct ModuleControlRpcGuard {
421 forwarding: Arc<ForwardingTable>,
422 endpoint: ModuleEndpointId,
423 corr: u64,
424 armed: bool,
425}
426
427impl ModuleControlRpcGuard {
428 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
429 Self {
430 forwarding,
431 endpoint,
432 corr,
433 armed: true,
434 }
435 }
436
437 fn disarm(&mut self) {
438 self.armed = false;
439 }
440}
441
442impl Drop for ModuleControlRpcGuard {
443 fn drop(&mut self) {
444 if self.armed {
445 let _ = self
446 .forwarding
447 .cancel_module_control_rpc(self.endpoint, self.corr);
448 }
449 }
450}
451
452impl RouteBindReservationGuard {
453 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
454 Self {
455 forwarding,
456 endpoint,
457 relay_corr,
458 armed: true,
459 }
460 }
461
462 fn release_and_disarm(&mut self) {
463 if !self.armed {
464 return;
465 }
466 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
467 self.endpoint,
468 self.relay_corr,
469 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
470 ) {
471 send_goodbye_target_best_effort(
472 &self.forwarding.counters(),
473 &target,
474 "canceled route.bind",
475 );
476 }
477 self.armed = false;
478 }
479
480 fn disarm(&mut self) {
481 self.armed = false;
482 }
483}
484
485impl Drop for RouteBindReservationGuard {
486 fn drop(&mut self) {
487 self.release_and_disarm();
488 }
489}
490
491#[derive(Debug, Clone, Default)]
521pub(crate) struct RouteBindBreakers {
522 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
523}
524
525#[derive(Debug, Clone, Default)]
526pub(crate) struct RouteBindConcurrency {
527 modules: Arc<Mutex<HashMap<String, usize>>>,
528}
529
530struct RouteBindConcurrencyGuard {
531 concurrency: RouteBindConcurrency,
532 module_id: String,
533}
534
535impl RouteBindConcurrency {
536 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
539 let mut modules = self
540 .modules
541 .lock()
542 .expect("route.bind concurrency mutex poisoned");
543 let in_flight = modules.entry(module_id.to_string()).or_default();
544 if *in_flight >= limit {
545 return Err(*in_flight);
546 }
547 *in_flight += 1;
548 Ok(RouteBindConcurrencyGuard {
549 concurrency: self.clone(),
550 module_id: module_id.to_string(),
551 })
552 }
553}
554
555impl Drop for RouteBindConcurrencyGuard {
556 fn drop(&mut self) {
557 let mut modules = self
558 .concurrency
559 .modules
560 .lock()
561 .expect("route.bind concurrency mutex poisoned");
562 let remove = {
563 let in_flight = modules
564 .get_mut(&self.module_id)
565 .expect("admitted route.bind has a concurrency entry");
566 *in_flight -= 1;
567 *in_flight == 0
568 };
569 if remove {
570 modules.remove(&self.module_id);
571 }
572 }
573}
574
575#[derive(Debug, Default)]
576struct ModuleBreakerState {
577 consecutive_timeouts: u32,
579 cooldown_until: Option<Instant>,
582 probe_in_flight: Option<Arc<()>>,
587}
588
589enum RouteBindAdmission<'a> {
591 Admitted {
592 guard: RouteBindBreakerGuard<'a>,
593 probe: bool,
596 },
597 Refused {
598 consecutive_timeouts: u32,
599 retry_in: Duration,
603 probe_in_flight: bool,
604 },
605}
606
607struct RouteBindBreakerGuard<'a> {
614 breakers: RouteBindBreakers,
615 module_id: &'a str,
616 probe_token: Option<Arc<()>>,
617 settled: bool,
618}
619
620impl RouteBindBreakerGuard<'_> {
621 fn record_accepted(&mut self) -> bool {
625 self.settled = true;
626 self.breakers.record_accepted(self.module_id)
627 }
628
629 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
632 self.settled = true;
633 self.breakers.record_timeout(
634 self.module_id,
635 self.probe_token.as_ref(),
636 threshold,
637 cooldown,
638 )
639 }
640
641 fn record_inconclusive(&mut self) {
650 self.settled = true;
651 self.breakers
652 .record_inconclusive(self.module_id, self.probe_token.as_ref());
653 }
654}
655
656impl Drop for RouteBindBreakerGuard<'_> {
657 fn drop(&mut self) {
658 if !self.settled {
659 self.breakers
660 .record_inconclusive(self.module_id, self.probe_token.as_ref());
661 }
662 }
663}
664
665struct BreakerOpened {
669 consecutive_timeouts: u32,
670 reopened_after_probe: bool,
673}
674
675impl RouteBindBreakers {
676 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
677 self.modules
678 .lock()
679 .expect("route.bind breaker mutex poisoned")
680 }
681
682 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
685 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
686 probe: probe_token.is_some(),
687 guard: RouteBindBreakerGuard {
688 breakers: self.clone(),
689 module_id,
690 probe_token,
691 settled: false,
692 },
693 };
694
695 let mut modules = self.lock();
696 let Some(state) = modules.get_mut(module_id) else {
697 return admitted(None);
698 };
699 let Some(cooldown_until) = state.cooldown_until else {
700 return admitted(None);
701 };
702 if state.probe_in_flight.is_some() {
703 return RouteBindAdmission::Refused {
704 consecutive_timeouts: state.consecutive_timeouts,
705 retry_in: Duration::ZERO,
706 probe_in_flight: true,
707 };
708 }
709 let now = Instant::now();
710 if now < cooldown_until {
711 return RouteBindAdmission::Refused {
712 consecutive_timeouts: state.consecutive_timeouts,
713 retry_in: cooldown_until - now,
714 probe_in_flight: false,
715 };
716 }
717 let token = Arc::new(());
718 state.probe_in_flight = Some(Arc::clone(&token));
719 admitted(Some(token))
720 }
721
722 fn record_accepted(&self, module_id: &str) -> bool {
723 self.lock()
724 .remove(module_id)
725 .is_some_and(|state| state.cooldown_until.is_some())
726 }
727
728 fn record_timeout(
729 &self,
730 module_id: &str,
731 probe_token: Option<&Arc<()>>,
732 threshold: u32,
733 cooldown: Duration,
734 ) -> Option<BreakerOpened> {
735 let mut modules = self.lock();
736 let state = modules.entry(module_id.to_string()).or_default();
737 let was_open = state.cooldown_until.is_some();
738 let was_probe = Self::owns_probe(state, probe_token);
739 if was_probe {
740 state.probe_in_flight = None;
741 }
742 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
743 if state.consecutive_timeouts < threshold {
744 return None;
745 }
746 state.cooldown_until = Some(Instant::now() + cooldown);
747 Some(BreakerOpened {
748 consecutive_timeouts: state.consecutive_timeouts,
749 reopened_after_probe: was_open && was_probe,
750 })
751 }
752
753 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
754 state
758 .probe_in_flight
759 .as_ref()
760 .zip(token)
761 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
762 }
763
764 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
765 if let Some(state) = self.lock().get_mut(module_id) {
766 if Self::owns_probe(state, probe_token) {
767 state.probe_in_flight = None;
768 }
769 }
770 }
771
772 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
787 self.lock()
788 .remove(module_id)
789 .map(|state| state.consecutive_timeouts)
790 .filter(|discarded| *discarded > 0)
791 }
792
793 fn open_snapshot(&self) -> Option<serde_json::Value> {
800 let now = Instant::now();
801 let modules = self.lock();
802 let open = modules
803 .iter()
804 .filter_map(|(module_id, state)| {
805 let cooldown_until = state.cooldown_until?;
806 Some((
807 module_id.clone(),
808 serde_json::json!({
809 "consecutive_timeouts": state.consecutive_timeouts,
810 "cooldown_remaining_ms":
811 cooldown_until.saturating_duration_since(now).as_millis() as u64,
812 "probe_in_flight": state.probe_in_flight.is_some(),
813 }),
814 ))
815 })
816 .collect::<serde_json::Map<String, serde_json::Value>>();
817 (!open.is_empty()).then_some(serde_json::Value::Object(open))
818 }
819}
820
821impl ControlHandler {
822 pub fn new(registry: Arc<Registry>) -> Self {
823 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
824 }
825
826 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
827 let counters = forwarding.counters();
828 let route_bind_breakers = forwarding.route_bind_breakers();
832 let route_bind_concurrency = forwarding.route_bind_concurrency();
833 let route_outages = forwarding.route_outages();
834 Self {
835 registry,
836 activity: None,
837 activity_streams: Arc::new(Mutex::new(HashMap::new())),
838 forwarding,
839 process_liveness: None,
840 supervisor: SupervisorHandle::new(),
841 subc_capabilities: Arc::from([
842 CAP_MANIFEST_REGISTRATION.to_string(),
843 CAP_CHANNEL_LIFECYCLE.to_string(),
844 CAP_PING_PONG.to_string(),
845 CAP_SESSION_ATTACH.to_string(),
846 CAP_ADMISSION_FACTS_RELAY.to_string(),
847 CAP_SCOPES_V1.to_string(),
848 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
849 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
850 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
851 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
852 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
853 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
854 ]),
855 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
856 route_bind_relay_timeouts: BTreeMap::new(),
857 route_bind_breakers,
858 route_bind_concurrency,
859 route_outages,
860 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
861 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
862 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
863 storage_config: None,
864 machine_id: None,
865 admission_facts_carrier_module_id: None,
866 admission_facts_targets: None,
867 scopes: Arc::new(RwLock::new(ScopeTable::new(
868 crate::daemon_config::default_scope_authority_owners(),
869 ))),
870 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
871 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
872 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
873 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
874 #[cfg(test)]
875 caller_lookup: None,
876 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
877 rescan: None,
878 connected_clients: ConnectedClients::new(),
879 counters,
880 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
881 daemon_provenance: DaemonProvenanceFacts::default(),
882 #[cfg(test)]
883 control_dispatch_delay: None,
884 #[cfg(test)]
885 provenance_probe_override: None,
886 #[cfg(test)]
887 route_bind_relay_sent_hold: None,
888 #[cfg(test)]
889 hello_registry_to_forwarding_hold: None,
890 }
891 }
892
893 pub fn with_storage_config(
896 mut self,
897 storage_config: Option<crate::daemon_config::StorageConfig>,
898 ) -> Self {
899 self.storage_config = storage_config;
900 self
901 }
902
903 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
906 self.activity = Some(crate::activity::ActivityLedger::open(
907 run_dir.join("activity.jsonl"),
908 incarnation,
909 ));
910 self.supervisor.set_activity(self.activity.clone());
911 self
912 }
913
914 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
917 self.machine_id = machine_id;
918 self
919 }
920
921 pub fn with_admission_facts_config(
925 mut self,
926 carrier_module_id: Option<String>,
927 targets: Option<Vec<String>>,
928 ) -> Self {
929 self.admission_facts_carrier_module_id = carrier_module_id;
930 self.admission_facts_targets = targets;
931 self
932 }
933
934 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
938 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
939 self.scope_authority_owners = owners;
940 self
941 }
942
943 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
945 self.privileged_operator_prompt = mode;
946 self
947 }
948
949 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
952 self.route_bind_relay_timeout = timeout;
953 self
954 }
955
956 pub fn with_route_bind_relay_timeouts(
962 mut self,
963 timeouts: impl IntoIterator<Item = (String, Duration)>,
964 ) -> Self {
965 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
966 self
967 }
968
969 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
975 self.route_bind_relay_timeouts
976 .get(module_id)
977 .copied()
978 .unwrap_or(self.route_bind_relay_timeout)
979 }
980
981 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
989 self.route_bind_breaker_threshold = threshold.max(1);
990 self.route_bind_breaker_cooldown = cooldown;
991 self
992 }
993
994 #[cfg(test)]
995 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
996 self.health_probe_timeout = timeout;
997 self
998 }
999
1000 #[cfg(test)]
1001 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1002 self.control_dispatch_delay = Some(delay);
1003 self
1004 }
1005
1006 #[cfg(test)]
1010 pub(crate) fn with_route_bind_relay_sent_hold(
1011 mut self,
1012 hold: Arc<tokio::sync::Barrier>,
1013 ) -> Self {
1014 self.route_bind_relay_sent_hold = Some(hold);
1015 self
1016 }
1017
1018 #[cfg(test)]
1023 pub(crate) fn with_hello_registry_to_forwarding_hold(
1024 mut self,
1025 hold: Arc<std::sync::Barrier>,
1026 ) -> Self {
1027 self.hello_registry_to_forwarding_hold = Some(hold);
1028 self
1029 }
1030
1031 #[cfg(test)]
1032 fn hold_between_registry_and_forwarding(&self) {
1033 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1034 hold.wait();
1035 hold.wait();
1036 }
1037 }
1038
1039 pub fn with_process_liveness(
1040 mut self,
1041 process_liveness: Arc<dyn ModuleProcessLiveness>,
1042 ) -> Self {
1043 self.process_liveness = Some(process_liveness);
1044 self
1045 }
1046
1047 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1048 self.supervisor = supervisor;
1049 self.supervisor.set_activity(self.activity.clone());
1050 self
1051 }
1052
1053 pub fn with_daemon_provenance(
1054 mut self,
1055 pid: u32,
1056 started_at_ms: u64,
1057 executable_path: Option<PathBuf>,
1058 build_git_sha: Option<String>,
1059 build_lock_digest: Option<String>,
1060 ) -> Self {
1061 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1062 let process_start_time = process_start_time(pid);
1063 self.daemon_provenance = DaemonProvenanceFacts {
1064 build: DaemonBuildProvenance {
1065 build_git_sha,
1066 build_lock_digest,
1067 },
1068 pid: Some(pid),
1069 started_at_ms: Some(started_at_ms),
1070 start_clock: None,
1071 executable_path,
1072 executable_identity,
1073 process_start_time,
1074 probe: ExecutableIdentityProbe::default(),
1075 };
1076 self
1077 }
1078
1079 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1080 self.daemon_provenance.start_clock = Some(clock);
1081 self
1082 }
1083
1084 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1086 self.scopes
1087 .write()
1088 .unwrap_or_else(|poisoned| poisoned.into_inner())
1089 .set_wall_clock(Arc::new(clock));
1090 self
1091 }
1092
1093 #[cfg(test)]
1094 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1095 self.provenance_probe_override = Some(result);
1096 self
1097 }
1098
1099 pub fn with_capability_config(
1103 self,
1104 modules: impl IntoIterator<Item = (String, bool)>,
1105 reserved_capabilities: BTreeMap<String, String>,
1106 ) -> Self {
1107 self.capability_evaluator
1108 .configure(modules, reserved_capabilities);
1109 self
1110 }
1111
1112 pub fn with_supervisor_rescan(
1113 mut self,
1114 supervisor: Supervisor,
1115 config_path: impl Into<PathBuf>,
1116 configured_port: Option<u16>,
1117 ) -> Self {
1118 self.rescan = Some(SupervisorRescanContext {
1119 supervisor,
1120 config_path: config_path.into(),
1121 configured_port,
1122 storage_config: self.storage_config.clone(),
1123 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1124 admission_facts_targets: self.admission_facts_targets.clone(),
1125 scope_authority_owners: self.scope_authority_owners.clone(),
1126 privileged_operator_prompt: self.privileged_operator_prompt,
1127 });
1128 self
1129 }
1130
1131 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1132 self.connected_clients = connected_clients;
1133 self
1134 }
1135
1136 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1137 Arc::clone(&self.forwarding)
1138 }
1139
1140 pub(crate) fn counters(&self) -> DaemonCounters {
1141 self.counters.clone()
1142 }
1143
1144 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1147 tokio::spawn(async move {
1148 loop {
1149 self.capability_evaluator
1150 .wait_for_change_or_deadline()
1151 .await;
1152 self.refresh_capability_requirements();
1153 }
1154 });
1155 }
1156
1157 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1163 tokio::spawn(async move {
1164 loop {
1165 let has_deadlines = self
1166 .scopes
1167 .read()
1168 .unwrap_or_else(|poisoned| poisoned.into_inner())
1169 .has_deadlines();
1170 if !has_deadlines {
1171 self.scope_deadline_added.notified().await;
1172 continue;
1173 }
1174 tokio::time::sleep(Duration::from_secs(1)).await;
1175 if let Err(error) = self.sweep_expired_scopes() {
1176 warn!(%error, "scope expiry sweep failed");
1177 }
1178 }
1179 });
1180 }
1181
1182 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1183 let mut table = self
1184 .scopes
1185 .write()
1186 .unwrap_or_else(|poisoned| poisoned.into_inner());
1187 let swept = table.sweep_expired();
1188 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1189 drop(table);
1190 self.close_scope_drained_routes(drained);
1191 Ok(())
1192 }
1193
1194 fn publish_scope_changes(
1195 &self,
1196 changes: &[crate::scopes::ScopeTagChange],
1197 expired: &[crate::scopes::ScopeExpired],
1198 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1199 if changes.is_empty() {
1200 return Ok(Vec::new());
1201 }
1202 let drained = self
1203 .forwarding
1204 .publish_scope_changes(changes)
1205 .map_err(RouterError::Forwarding)?;
1206 let mut counts = HashMap::new();
1207 for route in &drained {
1208 *counts
1209 .entry((
1210 route.scope.owner.as_str(),
1211 route.scope.scope_ref.as_str(),
1212 route.scope.tag.scope_epoch,
1213 ))
1214 .or_insert(0usize) += 1;
1215 }
1216 for scope in expired {
1217 let routes_closed = counts
1218 .get(&(
1219 scope.owner.as_str(),
1220 scope.scope_ref.as_str(),
1221 scope.scope_epoch,
1222 ))
1223 .copied()
1224 .unwrap_or(0);
1225 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1226 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1227 routes_closed, "scope expired");
1228 }
1229 Ok(drained)
1230 }
1231
1232 fn runtime_capability_snapshot(
1233 &self,
1234 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1235 let runtime = self
1236 .supervisor
1237 .list()
1238 .into_iter()
1239 .map(|module| {
1240 let status = module.status().map_err(|err| {
1241 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1242 })?;
1243 Ok(RuntimeModule {
1244 module_id: status.module_id,
1245 state: status.state,
1246 enabled: status.enabled,
1247 })
1248 })
1249 .collect::<Result<Vec<_>, RouterError>>()?;
1250 let registrations = self.registry.list_registered_modules().map_err(|err| {
1256 RouterError::backend(
1257 0,
1258 0,
1259 format!("failed to list capability registrations: {err}"),
1260 )
1261 })?;
1262 let registrations = registrations
1263 .into_iter()
1264 .map(|registration| RegisteredModule {
1265 module_id: registration.manifest.module_id,
1266 module_version: registration.manifest.module_version,
1267 capabilities: registration.manifest.capabilities,
1268 })
1269 .collect();
1270 Ok((runtime, registrations))
1271 }
1272
1273 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1280 let cached_registration = RegisteredModule {
1281 module_id: registration.manifest.module_id.clone(),
1282 module_version: registration.manifest.module_version.clone(),
1283 capabilities: registration.manifest.capabilities.clone(),
1284 };
1285 if self.capability_evaluator.record_hello(&cached_registration) {
1286 warn!(
1287 module_id = %cached_registration.module_id,
1288 "capability claims drifted from the cached manifest"
1289 );
1290 }
1291 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1292 self.enforce_capability_denies();
1293 }
1294 self.refresh_capability_requirements();
1295 }
1296
1297 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1301 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1302 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1303 self.supervisor.set_swap_promotion_observer(observer);
1304 }
1305
1306 pub fn refresh_capability_requirements(&self) {
1307 match self.runtime_capability_snapshot() {
1308 Ok((runtime, registrations)) => {
1309 log_requirement_events(
1310 self.capability_evaluator
1311 .evaluate_now(&runtime, ®istrations),
1312 );
1313 }
1314 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1315 }
1316 }
1317
1318 fn enforce_capability_denies(&self) {
1322 let registrations = match self.registry.list_registered_modules() {
1323 Ok(snapshot) => snapshot,
1324 Err(err) => {
1325 warn!(error = %err, "failed to read registrations for capability deny census");
1326 return;
1327 }
1328 };
1329 let manifests = registrations
1330 .into_iter()
1331 .map(|registration| {
1332 (
1333 registration.manifest.module_id.clone(),
1334 registration.manifest,
1335 )
1336 })
1337 .collect::<BTreeMap<_, _>>();
1338 let census = match self.forwarding.route_census(None) {
1339 Ok(census) => census,
1340 Err(err) => {
1341 warn!(error = %err, "failed to read route census for capability deny enforcement");
1342 return;
1343 }
1344 };
1345
1346 for (target_module_id, routes) in census {
1347 let Some(target_manifest) = manifests.get(&target_module_id) else {
1348 continue;
1349 };
1350 let mut closed_routes = Vec::new();
1351 let mut module_goodbyes = Vec::new();
1352 for route in routes {
1353 let Principal::Reserved {
1354 module_id: opening_module_id,
1355 } = &route.principal
1356 else {
1357 continue;
1358 };
1359 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1360 continue;
1361 };
1362 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1363 continue;
1364 };
1365
1366 match self.forwarding.release_client_route(
1367 route.goodbye_target.connection_id,
1368 route.goodbye_target.channel,
1369 route.goodbye_target.epoch,
1370 ) {
1371 Ok(RouteRelease::Removed(module_goodbye)) => {
1372 warn!(
1373 opening_module_id,
1374 target_module_id,
1375 capability,
1376 "force-closing route because an attested capability deny edge now matches"
1377 );
1378 closed_routes.push(route);
1379 module_goodbyes.push(module_goodbye);
1380 }
1381 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1382 Err(err) => warn!(
1383 opening_module_id,
1384 target_module_id,
1385 capability,
1386 error = %err,
1387 "failed to force-close capability-denied route"
1388 ),
1389 }
1390 }
1391
1392 if closed_routes.is_empty() {
1393 continue;
1394 }
1395 send_route_control_pushes(
1396 &self.forwarding,
1397 closed_routes,
1398 ClientControlPush::RouteClosed {
1399 module_id: target_module_id,
1400 channels: Vec::new(),
1401 reason: RouteCloseReason::CapabilityDenied,
1402 drained: false,
1403 abandoned: 0,
1404 excluded_subscriptions: 0,
1405 terminal: Some(false),
1406 },
1407 );
1408 self.emit_route_goodbyes(module_goodbyes);
1409 }
1410 }
1411
1412 fn not_ready_reason(
1418 &self,
1419 registration: &crate::registry::ModuleRegistration,
1420 ) -> Option<NotReadyReason> {
1421 if !registration.ready {
1422 return Some(NotReadyReason {
1423 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1424 capability: None,
1425 });
1426 }
1427 self.first_unprovided_required_capability(registration)
1428 .map(|capability| NotReadyReason {
1429 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1430 capability: Some(capability),
1431 })
1432 }
1433
1434 fn first_unprovided_required_capability(
1452 &self,
1453 registration: &crate::registry::ModuleRegistration,
1454 ) -> Option<String> {
1455 let required = registration
1456 .manifest
1457 .capabilities
1458 .iter()
1459 .flat_map(|declarations| declarations.requires.iter())
1460 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1461 .map(|requirement| requirement.capability.as_str())
1462 .collect::<BTreeSet<_>>();
1463 if required.is_empty() {
1464 return None;
1465 }
1466 let module_id = registration.manifest.module_id.as_str();
1467 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1468 if required
1469 .iter()
1470 .any(|capability| verdict(capability).is_none())
1471 {
1472 self.refresh_capability_requirements();
1473 }
1474 required
1475 .into_iter()
1476 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1477 .map(str::to_string)
1478 }
1479
1480 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1481 self.capability_evaluator
1482 .statuses()
1483 .into_iter()
1484 .map(capability_requirement_status)
1485 .collect()
1486 }
1487
1488 fn deregister_connection(
1495 &self,
1496 connection_id: ConnectionId,
1497 reason: RegistrationEndReason,
1498 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1499 self.registry
1500 .deregister_connection_with_reason(connection_id, reason)
1501 }
1502
1503 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1504 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1505 return None;
1506 }
1507 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1508 parse_client_control_request(&frame.body)
1509 else {
1510 return None;
1511 };
1512 Some(target_module_id(&target).to_string())
1513 }
1514
1515 pub(crate) fn route_open_capacity_refusal(
1516 &self,
1517 ctx: &RouteCtx,
1518 frame: &Frame,
1519 target_module_id: &str,
1520 in_flight: usize,
1521 limit: usize,
1522 ) -> Result<Frame, RouterError> {
1523 self.route_open_admission_refusal_frame(
1524 ctx,
1525 frame,
1526 target_module_id,
1527 "open_admission_full",
1528 (in_flight, limit),
1529 format!(
1530 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1531 ),
1532 )
1533 }
1534
1535 fn route_open_target_capacity_refusal(
1536 &self,
1537 ctx: &RouteCtx,
1538 frame: &Frame,
1539 target_module_id: &str,
1540 in_flight: usize,
1541 ) -> Result<Frame, RouterError> {
1542 self.route_open_admission_refusal_frame(
1543 ctx,
1544 frame,
1545 target_module_id,
1546 "target_binds_full",
1547 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1548 format!(
1549 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1550 ),
1551 )
1552 }
1553
1554 fn route_open_admission_refusal_frame(
1562 &self,
1563 ctx: &RouteCtx,
1564 frame: &Frame,
1565 target_module_id: &str,
1566 reason: &'static str,
1567 (in_flight, limit): (usize, usize),
1568 message: impl Into<String>,
1569 ) -> Result<Frame, RouterError> {
1570 let code = error_codes::TARGET_UNAVAILABLE;
1571 self.counters.increment_route_open_refused(code);
1572 info!(
1573 target: "control",
1574 code,
1575 reason,
1576 module_id = ?target_module_id,
1577 connection_id = ctx.connection_id.get(),
1578 in_flight,
1579 limit,
1580 "route.open refused"
1581 );
1582 control_error_frame(frame, code, message.into())
1583 }
1584
1585 #[cfg(test)]
1591 pub fn handle_control(
1592 &self,
1593 connection_id: ConnectionId,
1594 frame: Frame,
1595 ) -> Result<Vec<Frame>, RouterError> {
1596 match frame.header.ty {
1597 FrameType::Ping => Ok(vec![pong(&frame)?]),
1598 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1599 FrameType::Goodbye => self.handle_goodbye(connection_id),
1600 ty => Ok(vec![control_error_frame(
1601 &frame,
1602 "unsupported_control_frame",
1603 format!("unsupported channel-0 frame {ty:?}"),
1604 )?]),
1605 }
1606 }
1607
1608 pub async fn handle_control_frame(
1609 &self,
1610 ctx: &RouteCtx,
1611 frame: Frame,
1612 ) -> Result<Vec<Frame>, RouterError> {
1613 self.handle_control_frame_timed(ctx, frame, None).await
1614 }
1615
1616 pub(crate) fn record_caller_endpoints(
1617 &self,
1618 connection: ConnectionId,
1619 endpoints: Option<(SocketAddr, SocketAddr)>,
1620 ) {
1621 let mut callers = self
1622 .caller_endpoints
1623 .lock()
1624 .unwrap_or_else(|p| p.into_inner());
1625 callers.remove(&connection);
1626 if let Some(endpoints) = endpoints {
1627 callers.insert(connection, endpoints);
1628 }
1629 }
1630
1631 #[cfg(test)]
1632 pub(crate) fn with_caller_lookup_for_test(
1633 mut self,
1634 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1635 ) -> Self {
1636 self.caller_lookup = Some(Arc::new(lookup));
1637 self
1638 }
1639
1640 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1641 let endpoints = self
1642 .caller_endpoints
1643 .lock()
1644 .unwrap_or_else(|p| p.into_inner())
1645 .get(&connection)
1646 .copied();
1647 #[cfg(test)]
1648 let lookup = self.caller_lookup.clone();
1649 tokio::task::spawn_blocking(move || {
1650 #[cfg(test)]
1651 if let Some(lookup) = lookup {
1652 return lookup(endpoints);
1653 }
1654 endpoints
1655 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1656 .unwrap_or_default()
1657 })
1658 .await
1659 .unwrap_or_default()
1660 }
1661
1662 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1663 frame.header.channel == 0
1664 && frame.header.ty == FrameType::Request
1665 && self
1666 .forwarding
1667 .module_endpoint_for_connection(ctx.connection_id)
1668 .is_ok_and(|endpoint| endpoint.is_none())
1669 && parse_client_control_request(&frame.body)
1670 .ok()
1671 .is_some_and(|request| privileged_operation(&request).is_some())
1672 }
1673
1674 pub(crate) async fn handle_control_frame_timed(
1675 &self,
1676 ctx: &RouteCtx,
1677 frame: Frame,
1678 dispatch_started_at: Option<StdInstant>,
1679 ) -> Result<Vec<Frame>, RouterError> {
1680 match frame.header.ty {
1681 FrameType::Ping => Ok(vec![pong(&frame)?]),
1682 FrameType::Hello => {
1683 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1684 }
1685 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1686 FrameType::Cancel => {
1687 if self
1690 .forwarding
1691 .operator_confirms()
1692 .cancel(ctx.connection_id, frame.header.corr)
1693 || self
1694 .supervisor
1695 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1696 || self.cancel_activity_subscription(ctx, &frame)?
1697 {
1698 Ok(Vec::new())
1699 } else {
1700 Ok(vec![control_error_frame(
1701 &frame,
1702 "unknown_subscription",
1703 "no subscription has this correlation id",
1704 )?])
1705 }
1706 }
1707 FrameType::Request => {
1708 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1711 if op
1712 .as_ref()
1713 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1714 {
1715 return self.handle_activity_record(ctx, frame).await;
1716 }
1717 if op.as_ref().is_some_and(|probe| {
1718 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1719 }) {
1720 return self.handle_activity_read(ctx, frame).await;
1721 }
1722 if op
1723 .as_ref()
1724 .is_some_and(|probe| probe.op == "operator.confirm")
1725 {
1726 return self.handle_operator_confirm(ctx, frame);
1727 }
1728 if self
1729 .forwarding
1730 .module_endpoint_for_connection(ctx.connection_id)
1731 .map_err(RouterError::Forwarding)?
1732 .is_some()
1733 {
1734 if !is_known_module_request_op(&frame.body) {
1735 return Ok(vec![control_error_frame(
1736 &frame,
1737 "unsupported_control_frame",
1738 "module-originated channel-0 REQUEST is not supported",
1739 )?]);
1740 }
1741 let request = match parse_module_control_request_from_module(&frame.body) {
1742 Ok(request) => request,
1743 Err((err, ControlRequestBodyError::UnknownOp)) => {
1744 return Ok(vec![control_error_frame(
1745 &frame,
1746 "unsupported_control_frame",
1747 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1748 )?])
1749 }
1750 Err((err, ControlRequestBodyError::InvalidBody)) => {
1751 return Ok(vec![control_error_frame(
1752 &frame,
1753 "invalid_control_body",
1754 format!("malformed module control body: {err}"),
1755 )?])
1756 }
1757 };
1758 let op = module_control_request_op(&request);
1759 let corr = frame.header.corr;
1760 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1761 let result =
1762 self.handle_module_control_request(ctx.connection_id, frame, request);
1763 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1764 return result;
1765 }
1766
1767 if is_known_module_request_op(&frame.body) {
1768 return Ok(vec![control_error_frame(
1769 &frame,
1770 "not_registered",
1771 "catalog.update requires an active module registration owned by this connection",
1772 )?]);
1773 }
1774
1775 let request = match parse_client_control_request(&frame.body) {
1776 Ok(request) => request,
1777 Err((err, ControlRequestBodyError::UnknownOp)) => {
1778 return Ok(vec![control_error_frame(
1779 &frame,
1780 "unknown_control_op",
1781 format!("unknown client control op: {err}"),
1782 )?])
1783 }
1784 Err((err, ControlRequestBodyError::InvalidBody)) => {
1785 return Ok(vec![control_error_frame(
1786 &frame,
1787 "invalid_control_body",
1788 format!("malformed client control body: {err}"),
1789 )?])
1790 }
1791 };
1792 let op = client_control_request_op(&request);
1793 let corr = frame.header.corr;
1794 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1795 #[cfg(test)]
1796 if let Some(delay) = self.control_dispatch_delay {
1797 tokio::time::sleep(delay).await;
1798 }
1799 let result = self
1800 .handle_client_control_request(ctx, frame, request)
1801 .await;
1802 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1803 result
1804 }
1805 FrameType::Push => {
1806 let Some(endpoint) = self
1807 .forwarding
1808 .module_endpoint_for_connection(ctx.connection_id)
1809 .map_err(RouterError::Forwarding)?
1810 else {
1811 return Ok(vec![control_error_frame(
1812 &frame,
1813 "unsupported_control_frame",
1814 "client-originated channel-0 PUSH is not supported",
1815 )?]);
1816 };
1817 self.handle_status_update(endpoint, frame)
1818 }
1819 FrameType::Response | FrameType::Error
1820 if self
1821 .forwarding
1822 .module_endpoint_for_connection(ctx.connection_id)
1823 .map_err(RouterError::Forwarding)?
1824 .is_some() =>
1825 {
1826 self.handle_module_relay_response(ctx.connection_id, frame)
1827 }
1828 ty => Ok(vec![control_error_frame(
1829 &frame,
1830 "unsupported_control_frame",
1831 format!("unsupported channel-0 frame {ty:?}"),
1832 )?]),
1833 }
1834 }
1835
1836 pub fn cleanup_connection(
1837 &self,
1838 connection_id: ConnectionId,
1839 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1840 self.caller_endpoints
1841 .lock()
1842 .unwrap_or_else(|p| p.into_inner())
1843 .remove(&connection_id);
1844 self.forwarding
1845 .operator_confirms()
1846 .connection_closed(connection_id);
1847 self.cleanup_connection_with_end_reason(
1848 connection_id,
1849 RegistrationEndReason::ConnectionClosed,
1850 )
1851 }
1852
1853 fn cleanup_connection_with_end_reason(
1854 &self,
1855 connection_id: ConnectionId,
1856 requested_reason: RegistrationEndReason,
1857 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1858 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1859 self.registry
1860 .get_module_by_connection(connection_id)?
1861 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1862 .and_then(|module| module.registration_end_reason().ok().flatten())
1863 .unwrap_or(requested_reason)
1864 } else {
1865 requested_reason
1866 };
1867 let crash_closed = self
1868 .registry
1869 .get_module_by_connection(connection_id)?
1870 .and_then(|registration| {
1871 self.forwarding
1872 .module_endpoint_for_connection(connection_id)
1873 .ok()
1874 .flatten()
1875 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1876 .map(|routes| (registration.manifest.module_id, routes))
1877 });
1878 let crash_closed = crash_closed.map(|(module_id, routes)| {
1879 let terminal = match self.supervisor.get(&module_id) {
1880 None => false,
1881 Some(module) => match module.will_recover_after_connection_loss() {
1882 Ok(will_recover) => !will_recover,
1883 Err(err) => {
1884 warn!(
1885 %module_id,
1886 error = %err,
1887 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1888 );
1889 false
1890 }
1891 },
1892 };
1893 let (reason, drained) = match self
1897 .forwarding
1898 .daemon_drain_result_for_connection(connection_id)
1899 {
1900 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
1901 Ok(None) => (RouteCloseReason::Crash, false),
1902 Err(err) => {
1903 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1904 (RouteCloseReason::Crash, false)
1905 }
1906 };
1907 (module_id, routes, reason, drained, terminal)
1908 });
1909 let registrations = self.deregister_connection(connection_id, end_reason);
1910 let cleanup = if crash_closed.is_some() {
1911 self.forwarding.cleanup_connection_counted(connection_id)
1912 } else {
1913 self.forwarding
1919 .cleanup_connection(connection_id)
1920 .map(|released| crate::forwarding::ConnectionCleanup {
1921 released,
1922 abandoned_relays: 0,
1923 })
1924 };
1925 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
1930 let abandoned = cleanup
1931 .as_ref()
1932 .map_or(0, |cleanup| cleanup.abandoned_relays);
1933 send_route_control_pushes(
1934 &self.forwarding,
1935 routes,
1936 ClientControlPush::RouteClosed {
1937 module_id,
1938 channels: Vec::new(),
1939 reason,
1940 drained,
1941 abandoned,
1942 excluded_subscriptions: 0,
1943 terminal: Some(terminal),
1944 },
1945 );
1946 }
1947 if let Ok(cleanup) = cleanup {
1948 self.emit_route_goodbyes(cleanup.released);
1949 }
1950 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1954 crate::supervise::notify_registration_release();
1955 self.capability_evaluator.wake_deadline_loop();
1956 self.refresh_capability_requirements();
1957 }
1958 self.supervisor.remove_spawn_subscribers(connection_id);
1959 self.activity_streams
1960 .lock()
1961 .unwrap_or_else(|p| p.into_inner())
1962 .retain(|(id, _), task| {
1963 if *id == connection_id {
1964 task.abort();
1965 false
1966 } else {
1967 true
1968 }
1969 });
1970 self.hello_launch_nonces
1973 .lock()
1974 .unwrap_or_else(|poisoned| poisoned.into_inner())
1975 .forget(connection_id);
1976 self.scopes
1977 .write()
1978 .unwrap_or_else(|poisoned| poisoned.into_inner())
1979 .release_connection(connection_id);
1980 registrations
1981 }
1982
1983 pub(crate) fn handle_route_goodbye(
1984 &self,
1985 connection_id: ConnectionId,
1986 route_channel: u16,
1987 route_epoch: u32,
1988 ) -> Result<bool, RouterError> {
1989 debug!(
1990 connection_id = connection_id.get(),
1991 route_channel, route_epoch, "handling route GOODBYE"
1992 );
1993 let RouteRelease::Removed(released_route) = self
1994 .forwarding
1995 .release_client_route(connection_id, route_channel, route_epoch)
1996 .map_err(RouterError::Forwarding)?
1997 else {
1998 return Ok(false);
1999 };
2000 self.emit_route_goodbyes(vec![released_route]);
2001 Ok(true)
2002 }
2003
2004 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2005 for released in released_routes {
2006 let frame = match Frame::build_with_version(
2007 released.negotiated_ver,
2008 FrameType::Goodbye,
2009 control_flags(),
2010 released.channel,
2011 released.epoch,
2012 0,
2013 Vec::new(),
2014 ) {
2015 Ok(frame) => frame,
2016 Err(err) => {
2017 warn!(
2018 route_channel = released.channel,
2019 error = %err,
2020 "failed to build route GOODBYE frame"
2021 );
2022 continue;
2023 }
2024 };
2025 if !released.close_on_delivery_failure() {
2026 crate::forwarding::send_module_route_goodbye(
2027 &self.counters,
2028 &released.sink,
2029 frame,
2030 released.module_id.as_deref(),
2031 "client route released",
2032 );
2033 continue;
2034 }
2035 if let Err(err) = released.sink.try_send(frame) {
2036 warn!(
2037 target_connection_id = released.connection_id.get(),
2038 route_channel = released.channel,
2039 error = %err,
2040 "route GOODBYE was not delivered to client; closing target connection"
2041 );
2042 if self
2043 .forwarding
2044 .escalate_client_delivery_failure(
2045 released.connection_id,
2046 released.channel,
2047 released.epoch,
2048 CloseReason::new(
2049 "route_goodbye_delivery_failed",
2050 format!(
2051 "failed to enqueue route GOODBYE for channel {}: {err}",
2052 released.channel
2053 ),
2054 ),
2055 crate::forwarding::UndeliveredFrame {
2056 module_id: released.module_id.as_deref(),
2057 sink: &released.sink,
2058 },
2059 )
2060 .unwrap_or(false)
2061 {
2062 self.counters.increment_goodbye_relay_client_failed();
2063 }
2064 }
2065 }
2066 }
2067
2068 fn send_abandoned_route_bind_goodbye(
2080 &self,
2081 module_sink: &crate::FrameSink,
2082 negotiated_ver: u8,
2083 module_channel: u16,
2084 module_epoch: u32,
2085 ) {
2086 let frame = match Frame::build_with_version(
2087 negotiated_ver,
2088 FrameType::Goodbye,
2089 control_flags(),
2090 module_channel,
2091 module_epoch,
2092 0,
2093 Vec::new(),
2094 ) {
2095 Ok(frame) => frame,
2096 Err(err) => {
2097 warn!(
2098 route_channel = module_channel,
2099 error = %err,
2100 "failed to build GOODBYE for abandoned route.bind"
2101 );
2102 return;
2103 }
2104 };
2105 crate::forwarding::send_module_route_goodbye(
2106 &self.counters,
2107 module_sink,
2108 frame,
2109 None,
2110 "abandoned route.bind",
2111 );
2112 }
2113
2114 fn handle_hello(
2115 &self,
2116 connection_id: ConnectionId,
2117 sink: Option<crate::FrameSink>,
2118 frame: Frame,
2119 ) -> Result<Vec<Frame>, RouterError> {
2120 debug!(
2121 connection_id = connection_id.get(),
2122 corr = frame.header.corr,
2123 "handling HELLO"
2124 );
2125 if self
2129 .registry
2130 .get_module_by_connection(connection_id)
2131 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2132 .is_some()
2133 {
2134 return Ok(vec![control_error_frame(
2135 &frame,
2136 "invalid_hello",
2137 "connection is already registered as a module",
2138 )?]);
2139 }
2140 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2141 Ok(value) => value,
2142 Err(err) => {
2143 return Ok(vec![control_error_frame(
2144 &frame,
2145 "invalid_hello",
2146 format!("malformed HELLO body: {err}"),
2147 )?])
2148 }
2149 };
2150 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2151 return Ok(vec![control_error_frame(
2152 &frame,
2153 "invalid_capability_grammar",
2154 err.to_string(),
2155 )?]);
2156 }
2157 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2158 return Ok(vec![control_error_frame(
2159 &frame,
2160 "invalid_manifest",
2161 err.to_string(),
2162 )?]);
2163 }
2164 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2165 return Ok(vec![control_error_frame(
2166 &frame,
2167 "invalid_event_declaration",
2168 err.to_string(),
2169 )?]);
2170 }
2171 if let Some(provenance) = hello_value
2172 .get("manifest")
2173 .and_then(|manifest| manifest.get("provenance"))
2174 {
2175 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2176 return Ok(vec![control_error_frame(
2177 &frame,
2178 "invalid_manifest",
2179 format!("malformed manifest provenance: {err}"),
2180 )?]);
2181 }
2182 }
2183 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2184 Ok(hello) => hello,
2185 Err(err) => {
2186 return Ok(vec![control_error_frame(
2187 &frame,
2188 "invalid_hello",
2189 format!("malformed HELLO body: {err}"),
2190 )?])
2191 }
2192 };
2193
2194 if hello.protocol_ver != hello.manifest.protocol_ver {
2195 return Ok(vec![control_error_frame(
2196 &frame,
2197 "invalid_manifest",
2198 format!(
2199 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2200 hello.protocol_ver, hello.manifest.protocol_ver
2201 ),
2202 )?]);
2203 }
2204
2205 if hello.manifest.module_id.trim().is_empty() {
2206 return Ok(vec![control_error_frame(
2207 &frame,
2208 "invalid_manifest",
2209 "manifest module_id must not be empty",
2210 )?]);
2211 }
2212
2213 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2214 Ok(negotiated_ver) => negotiated_ver,
2215 Err(message) => {
2216 return Ok(vec![control_error_frame(
2217 &frame,
2218 "version_unsupported",
2219 message,
2220 )?])
2221 }
2222 };
2223
2224 let swap_admission = self
2232 .supervisor
2233 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2234 if swap_admission == SwapHelloAdmission::Refused {
2235 warn!(
2236 module_id = %hello.manifest.module_id,
2237 connection_id = connection_id.get(),
2238 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2239 );
2240 return Ok(vec![control_error_frame(
2241 &frame,
2242 "swap_token_invalid",
2243 format!(
2244 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2245 hello.manifest.module_id
2246 ),
2247 )?]);
2248 }
2249 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2250
2251 if let Some(rejection) = (!swap_candidate)
2259 .then(|| {
2260 self.supervisor.reserved_hello_rejection(
2261 &hello.manifest.module_id,
2262 hello.launch_nonce.as_deref(),
2263 )
2264 })
2265 .flatten()
2266 {
2267 let message = match rejection {
2268 ReservedHelloRejection::Exact { module_id } => format!(
2269 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2270 ),
2271 ReservedHelloRejection::Prefix {
2272 prefix,
2273 owner_module_id,
2274 } => format!(
2275 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2276 hello.manifest.module_id
2277 ),
2278 };
2279 return Ok(vec![control_error_frame(
2280 &frame,
2281 "reserved_module",
2282 message,
2283 )?]);
2284 }
2285
2286 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2287 &hello.manifest.module_id,
2288 hello.manifest.capabilities.as_ref(),
2289 );
2290 if let Some(refusal) = reserved_capability_refusals.first() {
2291 let capability = refusal.capability.clone();
2292 let bound_module = refusal.claimants[0].clone();
2293 log_duplicate_claim_events(reserved_capability_refusals);
2294 return Ok(vec![control_error_frame(
2295 &frame,
2296 "reserved_capability",
2297 format!(
2298 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2299 capability, bound_module, hello.manifest.module_id
2300 ),
2301 )?]);
2302 }
2303
2304 if self
2307 .forwarding
2308 .connection_has_client_routes(connection_id)
2309 .map_err(RouterError::Forwarding)?
2310 {
2311 return Ok(vec![control_error_frame(
2312 &frame,
2313 "invalid_hello",
2314 "connection has open client routes and cannot also register as a module",
2315 )?]);
2316 }
2317
2318 self.hello_launch_nonces
2323 .lock()
2324 .unwrap_or_else(|poisoned| poisoned.into_inner())
2325 .record(connection_id, hello.launch_nonce.as_deref());
2326 let control_ops = effective_module_control_ops(hello.control_ops);
2327 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2330 if swap_candidate {
2331 return self.register_swap_candidate(
2332 connection_id,
2333 sink,
2334 &frame,
2335 hello.manifest,
2336 negotiated_ver,
2337 control_ops,
2338 hello_ack,
2339 );
2340 }
2341 let registration = match self.registry.register_unpublished_with_control_ops(
2348 hello.manifest,
2349 negotiated_ver,
2350 connection_id,
2351 control_ops,
2352 ) {
2353 Ok(registration) => registration,
2354 Err(RegistryError::DuplicateModuleId { module_id }) => {
2355 return Ok(vec![control_error_frame(
2356 &frame,
2357 "duplicate_module_id",
2358 format!(
2359 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2360 ),
2361 )?])
2362 }
2363 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2364 return Ok(vec![control_error_frame(
2365 &frame,
2366 "invalid_module_id",
2367 err.to_string(),
2368 )?])
2369 }
2370 Err(err) => {
2371 return Ok(vec![control_error_frame(
2372 &frame,
2373 "registry_error",
2374 err.to_string(),
2375 )?])
2376 }
2377 };
2378 #[cfg(test)]
2379 self.hold_between_registry_and_forwarding();
2380
2381 let reply = if let Some(sink) = sink {
2382 let concurrency = manifest_concurrency(®istration.manifest);
2398 if let Err(err) = self.forwarding.register_module_connection_acked(
2399 connection_id,
2400 registration.manifest.module_id.clone(),
2401 negotiated_ver,
2402 concurrency,
2403 sink,
2404 hello_ack,
2405 ) {
2406 if matches!(
2411 self.deregister_connection(
2412 connection_id,
2413 RegistrationEndReason::RegistrationFailed,
2414 ),
2415 Ok(r) if !r.is_empty()
2416 ) {
2417 crate::supervise::notify_registration_release();
2418 }
2419 return Ok(vec![control_error_frame(
2420 &frame,
2421 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2422 "invalid_hello"
2423 } else {
2424 forwarding_error_code(&err)
2425 },
2426 err.to_string(),
2427 )?]);
2428 }
2429 Vec::new()
2430 } else {
2431 vec![hello_ack]
2434 };
2435
2436 if !self
2441 .registry
2442 .publish_registration(connection_id)
2443 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2444 {
2445 debug!(
2446 module_id = %registration.manifest.module_id,
2447 connection_id = connection_id.get(),
2448 "registration was removed before it could be published"
2449 );
2450 }
2451
2452 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2461 info!(
2462 module_id = %registration.manifest.module_id,
2463 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2464 );
2465 }
2466
2467 self.apply_registration_capabilities(®istration);
2468
2469 self.supervisor
2470 .record_activity_registration(®istration.manifest.module_id);
2471
2472 info!(
2473 module_id = %registration.manifest.module_id,
2474 module_version = %registration.manifest.module_version,
2475 negotiated_ver,
2476 routable_provider = manifest_provides_routable_role(®istration.manifest),
2477 connection_id = connection_id.get(),
2478 "module registered"
2479 );
2480
2481 Ok(reply)
2482 }
2483
2484 #[allow(clippy::too_many_arguments)]
2501 fn register_swap_candidate(
2502 &self,
2503 connection_id: ConnectionId,
2504 sink: Option<crate::FrameSink>,
2505 frame: &Frame,
2506 manifest: ModuleManifest,
2507 negotiated_ver: u8,
2508 control_ops: Vec<String>,
2509 hello_ack: Frame,
2510 ) -> Result<Vec<Frame>, RouterError> {
2511 let module_id = manifest.module_id.clone();
2512 let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2513 manifest,
2514 negotiated_ver,
2515 connection_id,
2516 control_ops,
2517 ) {
2518 Ok(registration) => registration,
2519 Err(RegistryError::DuplicateModuleId { module_id }) => {
2520 return Ok(vec![control_error_frame(
2521 frame,
2522 "duplicate_module_id",
2523 format!(
2524 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2525 ),
2526 )?])
2527 }
2528 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2529 return Ok(vec![control_error_frame(
2530 frame,
2531 "invalid_module_id",
2532 err.to_string(),
2533 )?])
2534 }
2535 Err(err) => {
2536 return Ok(vec![control_error_frame(
2537 frame,
2538 "registry_error",
2539 err.to_string(),
2540 )?])
2541 }
2542 };
2543 #[cfg(test)]
2544 self.hold_between_registry_and_forwarding();
2545 let reply = if let Some(sink) = sink {
2546 let concurrency = manifest_concurrency(®istration.manifest);
2550 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2551 connection_id,
2552 module_id.clone(),
2553 negotiated_ver,
2554 concurrency,
2555 sink,
2556 hello_ack,
2557 ) {
2558 if matches!(
2559 self.deregister_connection(
2560 connection_id,
2561 RegistrationEndReason::RegistrationFailed,
2562 ),
2563 Ok(r) if !r.is_empty()
2564 ) {
2565 crate::supervise::notify_registration_release();
2566 }
2567 return Ok(vec![control_error_frame(
2568 frame,
2569 forwarding_error_code(&err),
2570 err.to_string(),
2571 )?]);
2572 }
2573 Vec::new()
2574 } else {
2575 vec![hello_ack]
2576 };
2577 if !self
2578 .registry
2579 .publish_registration(connection_id)
2580 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2581 {
2582 debug!(
2583 module_id = %module_id,
2584 connection_id = connection_id.get(),
2585 "swap candidate registration was removed before it could be published"
2586 );
2587 }
2588 self.supervisor.mark_swap_candidate_admitted(&module_id);
2589 info!(
2590 module_id = %module_id,
2591 module_version = %registration.manifest.module_version,
2592 negotiated_ver,
2593 ready = registration.ready,
2594 connection_id = connection_id.get(),
2595 "swap candidate registered; not routable until cutover"
2596 );
2597 Ok(reply)
2598 }
2599
2600 fn build_hello_ack(
2601 &self,
2602 frame: &Frame,
2603 negotiated_ver: u8,
2604 module_id: &str,
2605 ) -> Result<Frame, RouterError> {
2606 let ack = ModuleHelloAckBody {
2607 negotiated_ver,
2608 subc_ops: module_subc_ops(),
2609 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2610 storage: self
2611 .storage_config
2612 .as_ref()
2613 .map(|cfg| cfg.descriptor_for(module_id)),
2614 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2615 };
2616 let body = serde_json::to_vec(&ack).map_err(|err| {
2617 RouterError::backend(
2618 0,
2619 frame.header.corr,
2620 format!("failed to encode HELLO_ACK: {err}"),
2621 )
2622 })?;
2623
2624 Frame::build_with_version(
2625 negotiated_ver,
2626 FrameType::HelloAck,
2627 control_flags(),
2628 0,
2629 0,
2630 frame.header.corr,
2631 body,
2632 )
2633 .map_err(RouterError::FrameBuild)
2634 }
2635
2636 async fn handle_client_control_request(
2637 &self,
2638 ctx: &RouteCtx,
2639 frame: Frame,
2640 request: ClientControlRequest,
2641 ) -> Result<Vec<Frame>, RouterError> {
2642 if let Some(operation) = privileged_operation(&request) {
2643 let caller = self.resolve_caller(ctx.connection_id).await;
2644 let op = client_control_request_op(&request);
2645 let text = prompt_text(operation, &caller);
2646 match self.privileged_operator_prompt {
2647 PrivilegedOperatorPrompt::Log => {
2648 info!(target: "subc_daemon::privileged_prompt", operation = op,
2649 target = ?operation, caller = %caller.description(), summary = %text,
2650 "privileged request would prompt the operator");
2651 }
2652 PrivilegedOperatorPrompt::Enforce => {
2653 let key =
2654 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2655 let outcome = self
2656 .forwarding
2657 .operator_confirms()
2658 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2659 .await;
2660 if outcome.code != "confirmed" {
2661 return Ok(vec![outcome.frame(&frame)]);
2662 }
2663 }
2664 }
2665 }
2666 match request {
2667 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2668 ClientControlRequest::CatalogList { module_id } => {
2669 self.handle_catalog_list(frame, module_id)
2670 }
2671 ClientControlRequest::RouteOpen {
2672 target,
2673 identity,
2674 consumer_identity,
2675 consumer_capabilities,
2676 role_versions,
2677 admission_facts,
2678 scope,
2679 } => {
2680 self.handle_route_open(
2681 ctx,
2682 frame,
2683 RouteOpenRequest {
2684 target,
2685 identity,
2686 consumer_identity,
2687 consumer_capabilities,
2688 role_versions,
2689 admission_facts,
2690 scope,
2691 },
2692 )
2693 .await
2694 }
2695 ClientControlRequest::RoutePoll {
2696 route_channel,
2697 route_epoch,
2698 kind,
2699 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2700 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2701 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2702 self.handle_supervisor_spawn_snapshot(frame)
2703 }
2704 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2705 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2706 }
2707 ClientControlRequest::SupervisorRestart {
2708 module_id,
2709 drain_timeout_ms,
2710 } => {
2711 self.log_supervisor_request_received(
2712 ctx,
2713 frame.header.corr,
2714 ops::SUPERVISOR_RESTART,
2715 Some(&module_id),
2716 None,
2717 )?;
2718 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2719 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2720 .await
2721 }
2722 ClientControlRequest::SupervisorSwap {
2723 module_id,
2724 ready_timeout_ms,
2725 } => {
2726 self.log_supervisor_request_received(
2727 ctx,
2728 frame.header.corr,
2729 ops::SUPERVISOR_SWAP,
2730 Some(&module_id),
2731 None,
2732 )?;
2733 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2734 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2735 .await
2736 }
2737 ClientControlRequest::SupervisorReload { module_id } => {
2738 self.log_supervisor_request_received(
2739 ctx,
2740 frame.header.corr,
2741 ops::SUPERVISOR_RELOAD,
2742 Some(&module_id),
2743 None,
2744 )?;
2745 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2746 self.handle_supervisor_reload(frame, module_id, requester)
2747 .await
2748 }
2749 ClientControlRequest::SupervisorRescan { preview } => {
2750 if !preview {
2751 self.log_supervisor_request_received(
2752 ctx,
2753 frame.header.corr,
2754 ops::SUPERVISOR_RESCAN,
2755 None,
2756 None,
2757 )?;
2758 }
2759 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2760 self.handle_supervisor_rescan(frame, preview, requester)
2761 .await
2762 }
2763 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2764 self.log_supervisor_request_received(
2765 ctx,
2766 frame.header.corr,
2767 ops::SUPERVISOR_RELEASE_RESERVED,
2768 Some(&module_id),
2769 None,
2770 )?;
2771 self.handle_supervisor_release_reserved(frame, module_id)
2772 .await
2773 }
2774 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2775 self.log_supervisor_request_received(
2776 ctx,
2777 frame.header.corr,
2778 ops::SUPERVISOR_SET_ENABLED,
2779 Some(&module_id),
2780 Some(enabled),
2781 )?;
2782 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2783 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2784 .await
2785 }
2786 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2787 self.handle_supervisor_health_probe(frame, module_id).await
2788 }
2789 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2790 ClientControlRequest::SupervisorRoutes { module_id } => {
2791 self.handle_supervisor_routes(frame, module_id)
2792 }
2793 ClientControlRequest::SupervisorProvenance { module_id } => {
2794 self.handle_supervisor_provenance(frame, module_id).await
2795 }
2796 ClientControlRequest::SupervisorStderrTail {
2797 module_id,
2798 max_lines,
2799 max_bytes,
2800 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2801 ClientControlRequest::SupervisorTerminals { module_id } => {
2802 self.handle_supervisor_terminals(frame, module_id).await
2803 }
2804 }
2805 }
2806
2807 fn log_supervisor_request_received(
2808 &self,
2809 ctx: &RouteCtx,
2810 corr: u64,
2811 op: &'static str,
2812 module_id: Option<&str>,
2813 enabled: Option<bool>,
2814 ) -> Result<(), RouterError> {
2815 let caller = self.supervisor_requester(ctx, corr)?;
2816 let caller = principal_label(&caller);
2817 info!(
2820 target: "control",
2821 op,
2822 module_id,
2823 enabled,
2824 connection_id = ctx.connection_id.get(),
2825 caller = %caller,
2826 "supervisor request received"
2827 );
2828 Ok(())
2829 }
2830
2831 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2832 Ok(self
2833 .registry
2834 .get_module_by_connection(ctx.connection_id)
2835 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2836 .map(|registration| Principal::Reserved {
2837 module_id: registration.manifest.module_id,
2838 })
2839 .unwrap_or(Principal::Direct))
2840 }
2841
2842 fn handle_module_control_request(
2843 &self,
2844 connection_id: ConnectionId,
2845 frame: Frame,
2846 request: ModuleControlRequestFromModule,
2847 ) -> Result<Vec<Frame>, RouterError> {
2848 match request {
2849 ModuleControlRequestFromModule::CatalogUpdate {
2850 provides,
2851 capabilities,
2852 ready,
2853 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2854 ModuleControlRequestFromModule::LiveRoots {} => {
2855 let registered = self
2856 .registry
2857 .get_module_by_connection(connection_id)
2858 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2859 let Some(registration) = registered else {
2860 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2861 };
2862 let response = self
2863 .forwarding
2864 .live_roots(®istration.manifest.module_id)
2865 .map_err(RouterError::Forwarding)?;
2866 Ok(vec![control_response_body_frame(
2867 &frame,
2868 &response,
2869 "ModuleControlResponseToModule::LiveRoots",
2870 )?])
2871 }
2872 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2873 self.handle_scope_sync(connection_id, frame, generation, scopes)
2874 }
2875 ModuleControlRequestFromModule::ScopeApply {
2876 generation,
2877 upsert,
2878 end,
2879 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2880 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2881 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2882 }
2883 }
2884 }
2885
2886 fn handle_operator_confirm(
2887 &self,
2888 ctx: &RouteCtx,
2889 frame: Frame,
2890 ) -> Result<Vec<Frame>, RouterError> {
2891 use crate::operator_confirm::{audit, Outcome};
2892 let registration = self
2893 .registry
2894 .get_module_by_connection(ctx.connection_id)
2895 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2896 let Some(registration) = registration else {
2897 let outcome = Outcome::refusal("not_registered");
2898 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2899 return Ok(vec![outcome.frame(&frame)]);
2900 };
2901 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2902 Ok(request) => request,
2903 Err(_) => {
2904 let outcome = Outcome::refusal("invalid_control_body");
2905 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2906 return Ok(vec![outcome.frame(&frame)]);
2907 }
2908 };
2909 let module_id = registration.manifest.module_id;
2910 let nonce = self
2914 .hello_launch_nonces
2915 .lock()
2916 .unwrap_or_else(|p| p.into_inner())
2917 .nonce(ctx.connection_id)
2918 .map(str::to_owned);
2919 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2920 self.supervisor
2921 .spawned_consumer_authorized(&module_id, nonce)
2922 });
2923 let confirms = self.forwarding.operator_confirms();
2924 self.forwarding
2925 .with_operator_route(
2926 ctx.connection_id,
2927 request.route_channel,
2928 request.route_epoch,
2929 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2930 )
2931 .map_err(RouterError::Forwarding)
2932 }
2933
2934 fn handle_scope_sync(
2938 &self,
2939 connection_id: ConnectionId,
2940 frame: Frame,
2941 generation: u64,
2942 scopes: Vec<ScopeRecord>,
2943 ) -> Result<Vec<Frame>, RouterError> {
2944 self.handle_scope_change(connection_id, frame, generation, scopes, None)
2945 }
2946
2947 fn handle_scope_change(
2948 &self,
2949 connection_id: ConnectionId,
2950 frame: Frame,
2951 generation: u64,
2952 scopes: Vec<ScopeRecord>,
2953 end: Option<Vec<ScopeEnd>>,
2954 ) -> Result<Vec<Frame>, RouterError> {
2955 let is_apply = end.is_some();
2956 let Some(registration) = self
2957 .registry
2958 .get_module_by_connection(connection_id)
2959 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2960 else {
2961 return Ok(vec![control_error_frame(
2962 &frame,
2963 "not_registered",
2964 "scope.sync and scope.apply require an active module registration owned by this connection",
2965 )?]);
2966 };
2967 let owner = registration.manifest.module_id;
2968 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2969 let is_current_launch = |connection: ConnectionId| {
2970 self.hello_launch_nonces
2971 .lock()
2972 .unwrap_or_else(|poisoned| poisoned.into_inner())
2973 .presented(connection, current_nonce.as_deref())
2974 };
2975 let mut table = self
2980 .scopes
2981 .write()
2982 .unwrap_or_else(|poisoned| poisoned.into_inner());
2983 let outcome = match end {
2984 Some(end) => table.apply(
2985 &owner,
2986 connection_id,
2987 is_current_launch,
2988 generation,
2989 scopes,
2990 end,
2991 ),
2992 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2993 };
2994 let drained = match &outcome {
2995 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2996 Err(_) => Vec::new(),
2997 };
2998 let has_deadlines = outcome.is_ok() && table.has_deadlines();
2999 drop(table);
3000 if has_deadlines {
3001 self.scope_deadline_added.notify_one();
3002 }
3003 match outcome {
3004 Ok(applied) => {
3005 let counts = ScopeOutcomeCounts::of(&applied.results);
3006 info!(
3007 owner = %owner,
3008 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3009 generation,
3010 records = applied.results.len(),
3011 created = counts.created,
3012 replaced = counts.replaced,
3013 updated = counts.updated,
3014 unchanged = counts.unchanged,
3015 refused = counts.refused,
3016 ended = applied.ended.len(),
3017 tag_changes = applied.tag_changes.len(),
3018 routes_closed = drained.len(),
3019 "scope change accepted"
3020 );
3021 for refused in applied
3027 .results
3028 .iter()
3029 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3030 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3031 {
3032 warn!(
3033 owner = %owner,
3034 generation,
3035 scope_ref = %refused.scope_ref,
3036 scope_epoch = refused.scope_epoch,
3037 code = refused.code.as_deref().unwrap_or(""),
3038 "scope record refused"
3039 );
3040 }
3041 self.close_scope_drained_routes(drained);
3042 let response = if is_apply {
3043 ModuleControlResponseToModule::ScopeApply {
3044 generation,
3045 results: applied.results,
3046 end_results: applied.end_results,
3047 ended: applied.ended,
3048 }
3049 } else {
3050 ModuleControlResponseToModule::ScopeSync {
3051 generation,
3052 results: applied.results,
3053 ended: applied.ended,
3054 }
3055 };
3056 Ok(vec![control_response_body_frame(
3057 &frame,
3058 &response,
3059 "scope change response",
3060 )?])
3061 }
3062 Err(refusal) => {
3063 info!(
3064 owner = %owner,
3065 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3066 generation,
3067 code = refusal.code,
3068 "scope change refused"
3069 );
3070 Ok(vec![control_error_frame(
3071 &frame,
3072 refusal.code,
3073 refusal.message,
3074 )?])
3075 }
3076 }
3077 }
3078
3079 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3085 if drained.is_empty() {
3086 return;
3087 }
3088 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3089 BTreeMap::new();
3090 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3091 for route in drained {
3092 warn!(
3093 module_id = %route.module_id,
3094 reason = ?route.reason,
3095 client_connection_id = route.client.connection_id.get(),
3096 route_channel = route.client.channel,
3097 "closing route because its scope changed"
3098 );
3099 pushes
3100 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3101 .or_insert_with(|| (route.reason, Vec::new()))
3102 .1
3103 .push(EndpointRoute {
3104 goodbye_target: route.client.clone(),
3105 principal: Principal::Unverified,
3106 bound_at: Instant::now(),
3107 draining: false,
3108 drain_reason: None,
3109 });
3110 goodbyes.push(route.module);
3111 goodbyes.push(route.client);
3112 }
3113 for ((module_id, _), (reason, routes)) in pushes {
3114 send_route_control_pushes(
3115 &self.forwarding,
3116 routes,
3117 ClientControlPush::RouteClosed {
3118 module_id,
3119 channels: Vec::new(),
3120 reason,
3121 drained: false,
3122 abandoned: 0,
3123 excluded_subscriptions: 0,
3124 terminal: Some(false),
3125 },
3126 );
3127 }
3128 self.emit_route_goodbyes(goodbyes);
3129 }
3130
3131 fn handle_scope_describe(
3134 &self,
3135 connection_id: ConnectionId,
3136 frame: Frame,
3137 owner: Principal,
3138 scope_ref: String,
3139 ) -> Result<Vec<Frame>, RouterError> {
3140 let registered = self
3141 .registry
3142 .get_module_by_connection(connection_id)
3143 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3144 if registered.is_none() {
3145 return Ok(vec![control_error_frame(
3146 &frame,
3147 "not_registered",
3148 "scope.describe requires an active module registration owned by this connection",
3149 )?]);
3150 }
3151 let description = self
3152 .scopes
3153 .read()
3154 .unwrap_or_else(|poisoned| poisoned.into_inner())
3155 .describe(&owner, &scope_ref);
3156 let owner_configured = match &owner {
3157 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3162 _ => false,
3163 };
3164 let response = ModuleControlResponseToModule::ScopeDescribe {
3165 status: description.status,
3166 scope_epoch: description.scope_epoch,
3167 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3168 owner_synced: description.owner_synced,
3169 owner_configured,
3170 scope: description.stamp,
3171 };
3172 Ok(vec![control_response_body_frame(
3173 &frame,
3174 &response,
3175 "ModuleControlResponseToModule::ScopeDescribe",
3176 )?])
3177 }
3178
3179 fn handle_catalog_update(
3180 &self,
3181 connection_id: ConnectionId,
3182 frame: Frame,
3183 provides: Vec<ProviderRole>,
3184 capabilities: Option<CapabilityDeclarations>,
3185 ready: Option<bool>,
3186 ) -> Result<Vec<Frame>, RouterError> {
3187 self.refresh_capability_requirements();
3188 let Some(registration) = self
3189 .registry
3190 .get_module_by_connection(connection_id)
3191 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3192 else {
3193 return Ok(vec![control_error_frame(
3194 &frame,
3195 "not_registered",
3196 "catalog.update requires an active module registration owned by this connection",
3197 )?]);
3198 };
3199
3200 if let Some(message) =
3201 catalog_update_frozen_field_message(®istration.manifest, &provides)
3202 {
3203 return Ok(vec![control_error_frame(
3204 &frame,
3205 "catalog_update_frozen_field",
3206 message,
3207 )?]);
3208 }
3209
3210 let mut candidate = registration.manifest.clone();
3211 candidate.provides = provides.clone();
3212 candidate.capabilities = capabilities
3213 .clone()
3214 .or_else(|| registration.manifest.capabilities.clone());
3215 if let Err(err) = candidate.validate_capability_grammar() {
3216 return Ok(vec![control_error_frame(
3217 &frame,
3218 "invalid_capability_grammar",
3219 err.to_string(),
3220 )?]);
3221 }
3222
3223 let mut conflicts = self
3226 .capability_evaluator
3227 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3228 if let Some(conflict) = conflicts.first() {
3229 let message = format!(
3230 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3231 conflict.capability, conflict.claimants[0], candidate.module_id
3232 );
3233 for conflict in &mut conflicts {
3234 conflict.source = DuplicateClaimSource::CatalogUpdate;
3235 }
3236 log_duplicate_claim_events(conflicts);
3237 return Ok(vec![control_error_frame(
3238 &frame,
3239 "reserved_capability",
3240 message,
3241 )?]);
3242 }
3243
3244 let updated = self
3245 .registry
3246 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3247 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3248 if updated.is_none() {
3249 return Ok(vec![control_error_frame(
3250 &frame,
3251 "not_registered",
3252 "catalog.update requires an active module registration owned by this connection",
3253 )?]);
3254 }
3255 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3256 log_duplicate_claim_events(
3257 self.capability_evaluator
3258 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3259 );
3260 }
3261 if capability_census_trigger(
3262 registration.manifest.capabilities.as_ref(),
3263 updated
3264 .as_ref()
3265 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3266 ) {
3267 self.enforce_capability_denies();
3268 }
3269 self.refresh_capability_requirements();
3270
3271 let response = ModuleControlResponseToModule::CatalogUpdate {};
3272 control_response_body_frame(
3273 &frame,
3274 &response,
3275 "ModuleControlResponseToModule::CatalogUpdate",
3276 )
3277 .map(|frame| vec![frame])
3278 }
3279
3280 pub(crate) fn with_activity_ledger_config(
3281 mut self,
3282 run_dir: &Path,
3283 incarnation: String,
3284 config: crate::activity::ActivityConfig,
3285 ) -> Self {
3286 self.activity = Some(crate::activity::ActivityLedger::open_config(
3287 run_dir.join("activity.jsonl"),
3288 incarnation,
3289 config,
3290 ));
3291 self.supervisor.set_activity(self.activity.clone());
3292 self
3293 }
3294
3295 pub(crate) async fn configure_activity(
3296 &self,
3297 config: crate::activity::ActivityConfig,
3298 ) -> Result<(), &'static str> {
3299 match &self.activity {
3300 Some(ledger) => ledger.configure(config).await,
3301 None => Ok(()),
3302 }
3303 }
3304
3305 fn cancel_activity_subscription(
3306 &self,
3307 ctx: &RouteCtx,
3308 frame: &Frame,
3309 ) -> Result<bool, RouterError> {
3310 let task = self
3311 .activity_streams
3312 .lock()
3313 .unwrap_or_else(|p| p.into_inner())
3314 .remove(&(ctx.connection_id, frame.header.corr));
3315 let Some(task) = task else {
3316 return Ok(false);
3317 };
3318 task.abort();
3319 let end = Frame::build_with_version(
3320 response_version(frame),
3321 FrameType::StreamEnd,
3322 control_flags(),
3323 0,
3324 0,
3325 frame.header.corr,
3326 Vec::new(),
3327 )
3328 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3329 let sink = ctx.egress.clone();
3330 tokio::spawn(async move {
3331 let _ = sink.send(end).await;
3332 });
3333 Ok(true)
3334 }
3335
3336 async fn handle_activity_read(
3337 &self,
3338 ctx: &RouteCtx,
3339 frame: Frame,
3340 ) -> Result<Vec<Frame>, RouterError> {
3341 use subc_control::{
3342 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3343 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3344 };
3345 #[derive(Deserialize)]
3346 struct Identity {
3347 #[serde(default)]
3348 consumer_identity: Option<ConsumerIdentity>,
3349 }
3350 let principal = match self
3354 .forwarding
3355 .module_id_for_connection(ctx.connection_id)
3356 .map_err(RouterError::Forwarding)?
3357 {
3358 Some(module_id) => Ok(Principal::Reserved { module_id }),
3359 None => {
3360 let identity: Identity = match serde_json::from_slice(&frame.body) {
3361 Ok(identity) => identity,
3362 Err(_) => {
3363 return Ok(vec![control_error_frame(
3364 &frame,
3365 ACTIVITY_MALFORMED,
3366 "malformed activity read",
3367 )?])
3368 }
3369 };
3370 self.route_open_principal(&frame, identity.consumer_identity)?
3371 }
3372 };
3373 let admitted = matches!(principal, Ok(Principal::Direct))
3374 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3375 if !admitted {
3376 return Ok(vec![control_error_frame(
3377 &frame,
3378 ACTIVITY_READ_FORBIDDEN,
3379 "activity reads require an operator or trusted mirror principal",
3380 )?]);
3381 }
3382 let Some(ledger) = &self.activity else {
3383 return Ok(vec![control_error_frame(
3384 &frame,
3385 ACTIVITY_UNAVAILABLE,
3386 "activity ledger unavailable",
3387 )?]);
3388 };
3389 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3390 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3391 {
3392 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3393 Ok(request) => ledger.query(request).await,
3394 Err(_) => Err(ACTIVITY_MALFORMED),
3395 };
3396 return match result {
3397 Ok(reply) => Ok(vec![control_response_body_frame(
3398 &frame,
3399 &reply,
3400 ops::ACTIVITY_QUERY,
3401 )?]),
3402 Err(code) => Ok(vec![control_error_frame(
3403 &frame,
3404 code,
3405 "activity query refused",
3406 )?]),
3407 };
3408 }
3409 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3410 Ok(request) => request,
3411 Err(_) => {
3412 return Ok(vec![control_error_frame(
3413 &frame,
3414 ACTIVITY_MALFORMED,
3415 "malformed activity subscription",
3416 )?])
3417 }
3418 };
3419 let mut subscription = match ledger.subscribe(request.since).await {
3420 Ok(subscription) => subscription,
3421 Err(code) => {
3422 return Ok(vec![control_error_frame(
3423 &frame,
3424 code,
3425 "activity subscription refused",
3426 )?])
3427 }
3428 };
3429 let key = (ctx.connection_id, frame.header.corr);
3430 let sink = ctx.egress.clone();
3431 let mut streams = self
3432 .activity_streams
3433 .lock()
3434 .unwrap_or_else(|p| p.into_inner());
3435 streams.retain(|_, task| !task.is_finished());
3436 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3437 return Ok(vec![control_error_frame(
3438 &frame,
3439 "duplicate_subscription",
3440 "activity subscription correlation id already in use",
3441 )?]);
3442 }
3443 let task = tokio::spawn(async move {
3444 loop {
3445 let entry = if let Some(entry) = subscription.replay.pop_front() {
3446 Some(entry)
3447 } else {
3448 subscription.receiver.recv().await
3449 };
3450 let Some(entry) = entry else {
3451 break;
3452 };
3453 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3454 return;
3455 };
3456 let Ok(next) = Frame::build_with_version(
3457 response_version(&frame),
3458 FrameType::StreamData,
3459 control_flags(),
3460 0,
3461 0,
3462 frame.header.corr,
3463 body,
3464 ) else {
3465 return;
3466 };
3467 if sink.send(next).await.is_err() {
3468 return;
3469 }
3470 }
3471 if let Some(code) = subscription.terminal_code() {
3474 if let Ok(end) =
3475 control_error_frame(&frame, code, "activity subscriber fell behind")
3476 {
3477 let _ = sink.send(end).await;
3478 }
3479 }
3480 });
3481 streams.insert(key, task.abort_handle());
3482 Ok(Vec::new())
3483 }
3484
3485 async fn handle_activity_record(
3486 &self,
3487 ctx: &RouteCtx,
3488 frame: Frame,
3489 ) -> Result<Vec<Frame>, RouterError> {
3490 use subc_control::{
3491 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3492 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3493 };
3494 let result = async {
3495 let module = self
3496 .forwarding
3497 .module_id_for_connection(ctx.connection_id)
3498 .ok()
3499 .flatten()
3500 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3501 let request: ActivityRecordRequest =
3502 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3503 crate::activity::validate(&module, &request)?;
3504 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3505 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3506 let at_ms = std::time::SystemTime::now()
3507 .duration_since(std::time::UNIX_EPOCH)
3508 .unwrap_or_default()
3509 .as_millis() as u64;
3510 ledger.record(module, request, actor, at_ms).await
3511 }
3512 .await;
3513 match result {
3514 Ok(entry) => Ok(vec![control_response_body_frame(
3515 &frame,
3516 &ActivityRecordResponse { seq: entry.seq },
3517 "activity.record",
3518 )?]),
3519 Err(code) => Ok(vec![control_error_frame(
3520 &frame,
3521 code,
3522 "activity recording refused",
3523 )?]),
3524 }
3525 }
3526
3527 fn activity_actor(
3528 &self,
3529 connection: ConnectionId,
3530 module: &str,
3531 on: Option<&subc_control::ActivityOn>,
3532 ) -> Result<subc_control::ActivityActor, &'static str> {
3533 use crate::forwarding::{DataRoute, DataRouteState};
3534 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3535 let principal = |id: &str| Principal::Reserved {
3536 module_id: id.to_owned(),
3537 };
3538 let mut actor = ActivityActor {
3539 principal: principal(module),
3540 scope_owner: None,
3541 agent_id: None,
3542 run_id: None,
3543 flow_id: None,
3544 project_root: None,
3545 };
3546 let scope = match on {
3547 None => return Ok(actor),
3548 Some(ActivityOn::Scope(scope)) => {
3549 let selector = subc_protocol::scope::ScopeSelector::new(
3550 principal(module),
3551 scope.scope_ref.clone(),
3552 Some(scope.epoch),
3553 );
3554 self.scopes
3555 .read()
3556 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3557 .attribute(&principal(module), module, &selector, true)
3558 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3559 }
3560 Some(ActivityOn::Route(route)) => {
3561 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3562 .forwarding
3563 .lookup_data_route(connection, route.channel, route.epoch)
3564 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3565 else {
3566 return Err(ACTIVITY_ACTOR_UNKNOWN);
3567 };
3568 if binding.module_id != module {
3569 return Err(ACTIVITY_ACTOR_UNKNOWN);
3570 }
3571 actor.principal = binding.principal.clone();
3572 actor.project_root = binding
3573 .project_root
3574 .as_ref()
3575 .map(|root| root.as_path().to_path_buf());
3576 let Some(bound) = &binding.scope else {
3577 return Ok(actor);
3578 };
3579 let owner = principal(&bound.owner);
3580 let selector = subc_protocol::scope::ScopeSelector::new(
3581 owner.clone(),
3582 bound.scope_ref.clone(),
3583 Some(bound.tag.scope_epoch),
3584 );
3585 let admitted = self
3586 .scopes
3587 .read()
3588 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3589 .attribute(&owner, module, &selector, true)
3590 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3591 if admitted.tag != bound.tag {
3592 return Err(ACTIVITY_ACTOR_UNKNOWN);
3593 }
3594 admitted
3595 }
3596 };
3597 actor.scope_owner = Some(scope.stamp.owner);
3598 actor.agent_id = scope.stamp.attributes.agent_id;
3599 actor.run_id = scope.stamp.attributes.run_id;
3600 actor.flow_id = scope.stamp.attributes.flow_id;
3601 Ok(actor)
3602 }
3603
3604 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3605 self.refresh_capability_requirements();
3606 let mut counters = self.counters.snapshot();
3612 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3613 obj.insert(
3614 "activity_unparseable_lines".into(),
3615 ledger.read_errors().into(),
3616 );
3617 obj.insert(
3618 "activity_lifecycle_append_failures".into(),
3619 ledger.lifecycle_failures().into(),
3620 );
3621 obj.insert(
3622 "activity_started_digest_skips".into(),
3623 ledger.lifecycle_digest_skips().into(),
3624 );
3625 }
3626 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3627 self.forwarding.client_route_concentration(),
3628 counters.as_object_mut(),
3629 ) {
3630 obj.insert(
3631 "client_connections_with_routes".into(),
3632 connections_with_routes.into(),
3633 );
3634 obj.insert("max_routes_on_one_connection".into(), max.into());
3635 }
3636 if let (Some(open_breakers), Some(obj)) = (
3642 self.route_bind_breakers.open_snapshot(),
3643 counters.as_object_mut(),
3644 ) {
3645 obj.insert("route_bind_breakers_open".into(), open_breakers);
3646 }
3647 let response = ClientControlResponse::ServerDescribe {
3648 protocol_ver: PROTOCOL_VERSION,
3649 subc_ops: subc_ops(),
3650 capabilities: self.subc_capabilities.as_ref().to_vec(),
3651 connected_clients: self.connected_clients.count(),
3652 counters: Some(counters),
3653 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3654 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3655 capability_requirements: self.capability_requirement_statuses(),
3656 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3657 };
3658 Ok(vec![control_response_body_frame(
3659 &frame,
3660 &response,
3661 "ClientControlResponse::ServerDescribe",
3662 )?])
3663 }
3664
3665 fn handle_catalog_list(
3666 &self,
3667 frame: Frame,
3668 module_id: Option<String>,
3669 ) -> Result<Vec<Frame>, RouterError> {
3670 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3671 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3672 })?;
3673 let entries = modules
3674 .into_iter()
3675 .filter(|registration| {
3676 module_id
3677 .as_deref()
3678 .map(|wanted| registration.manifest.module_id == wanted)
3679 .unwrap_or(true)
3680 })
3681 .map(|registration| {
3682 let not_ready = self.not_ready_reason(®istration);
3683 let roles = registration.manifest.provides;
3684 CatalogEntry::new(
3685 registration.manifest.module_id,
3686 roles,
3687 registration.control_ops,
3688 )
3689 .with_ready(not_ready.is_none())
3690 .with_not_ready(not_ready)
3691 .with_module_version(Some(registration.manifest.module_version))
3692 .with_capabilities(registration.manifest.capabilities)
3693 .with_self_signals(registration.manifest.self_signals)
3694 .with_events(registration.manifest.events)
3695 })
3696 .collect();
3697 let response = ClientControlResponse::CatalogList {
3698 generation,
3699 modules: entries,
3700 subc_ops: subc_ops(),
3701 };
3702 Ok(vec![control_response_body_frame(
3703 &frame,
3704 &response,
3705 "ClientControlResponse::CatalogList",
3706 )?])
3707 }
3708
3709 fn route_open_principal(
3710 &self,
3711 frame: &Frame,
3712 consumer_identity: Option<ConsumerIdentity>,
3713 ) -> Result<Result<Principal, Frame>, RouterError> {
3714 let Some(consumer_identity) = consumer_identity else {
3715 return Ok(Ok(Principal::Direct));
3716 };
3717
3718 if self.supervisor.spawned_consumer_authorized(
3719 &consumer_identity.module_id,
3720 &consumer_identity.launch_nonce,
3721 ) {
3722 return Ok(Ok(Principal::Reserved {
3723 module_id: consumer_identity.module_id,
3724 }));
3725 }
3726
3727 Ok(Err(control_error_frame(
3728 frame,
3729 "bad_consumer_identity",
3730 format!(
3731 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3732 consumer_identity.module_id
3733 ),
3734 )?))
3735 }
3736
3737 fn route_open_refusal_frame(
3744 &self,
3745 ctx: &RouteCtx,
3746 frame: &Frame,
3747 module_id: &str,
3748 reason: &'static str,
3749 code: &'static str,
3750 message: impl Into<String>,
3751 ) -> Result<Frame, RouterError> {
3752 self.observe_route_open_refusal(ctx, module_id, reason, code);
3753 control_error_frame(frame, code, message.into())
3754 }
3755
3756 fn route_open_breaker_refusal_frame(
3774 &self,
3775 ctx: &RouteCtx,
3776 frame: &Frame,
3777 module_id: &str,
3778 consecutive_timeouts: u32,
3779 retry_in: Duration,
3780 probe_in_flight: bool,
3781 ) -> Result<Frame, RouterError> {
3782 self.counters
3783 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3784 debug!(
3785 target: "control",
3786 code = "module_timeout",
3787 module_id = ?module_id,
3788 connection_id = ctx.connection_id.get(),
3789 consecutive_timeouts,
3790 retry_in_ms = retry_in.as_millis() as u64,
3791 probe_in_flight,
3792 "route.open refused by open bind-relay breaker"
3793 );
3794 let detail = if probe_in_flight {
3801 "one test route is already being tried; retry once it settles".to_string()
3802 } else {
3803 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3804 };
3805 control_error_frame(
3806 frame,
3807 "module_timeout",
3808 format!(
3809 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3810 timed out in a row); {detail}; its established routes are unaffected"
3811 ),
3812 )
3813 }
3814
3815 fn observe_route_open_refusal(
3828 &self,
3829 ctx: &RouteCtx,
3830 module_id: &str,
3831 reason: &'static str,
3832 code: &'static str,
3833 ) {
3834 self.counters.increment_route_open_refused(code);
3835 info!(
3836 target: "control",
3837 code,
3838 reason,
3839 module_id = ?module_id,
3840 connection_id = ctx.connection_id.get(),
3841 "route.open refused"
3842 );
3843 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3844 self.route_outages.record_not_serving(module_id, reason);
3845 }
3846 }
3847
3848 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3893 self.route_outages.record_accepted(module_id);
3894 self.counters.increment_route_open_accepted(principal);
3895 info!(
3896 target: "control",
3897 principal,
3898 module_id,
3899 connection_id = ctx.connection_id.get(),
3900 "route.open accepted"
3901 );
3902 }
3903
3904 fn supervised_absent_route_open_refusal_frame(
3905 &self,
3906 ctx: &RouteCtx,
3907 frame: &Frame,
3908 module_id: &str,
3909 code: &'static str,
3910 status: &crate::supervise::ModuleStatus,
3911 ) -> Result<Frame, RouterError> {
3912 self.counters.increment_route_open_refused(code);
3913 info!(
3914 target: "control",
3915 code,
3916 reason = "supervised_not_registered",
3917 module_id = ?module_id,
3918 connection_id = ctx.connection_id.get(),
3919 state = %status.state,
3920 enabled = status.enabled,
3921 live = status.live,
3922 "route.open refused"
3923 );
3924 self.route_outages
3928 .record_not_serving(module_id, "supervised_not_registered");
3929 let message = if let Some(pid) = status.waiting_for_previous_pid {
3930 format!("waiting: previous instance (pid {pid}) is still running but hidden; it cannot be identified")
3931 } else {
3932 format!(
3933 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3934 status.state, status.enabled, status.live
3935 )
3936 };
3937 control_error_frame(frame, code, message)
3938 }
3939
3940 async fn handle_route_open(
3941 &self,
3942 ctx: &RouteCtx,
3943 frame: Frame,
3944 request: RouteOpenRequest,
3945 ) -> Result<Vec<Frame>, RouterError> {
3946 let RouteOpenRequest {
3947 target,
3948 mut identity,
3949 consumer_identity,
3950 consumer_capabilities,
3951 role_versions,
3952 admission_facts,
3953 scope,
3954 } = request;
3955 let target_module_id = target_module_id(&target).to_string();
3956 if self
3957 .registry
3958 .get_module_by_connection(ctx.connection_id)
3959 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3960 .is_some()
3961 {
3962 return Ok(vec![control_error_frame(
3963 &frame,
3964 "invalid_request",
3965 "module connections cannot open client routes",
3966 )?]);
3967 }
3968 debug!(
3969 connection_id = ctx.connection_id.get(),
3970 corr = frame.header.corr,
3971 module_id = %target_module_id,
3972 "handling route.open"
3973 );
3974
3975 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3981 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3982 self.observe_route_open_refusal(
3983 ctx,
3984 &target_module_id,
3985 "invalid_role_versions",
3986 error_codes::INVALID_REQUEST,
3987 );
3988 return Ok(vec![control_error_body_frame(
3989 &frame,
3990 ErrorBody {
3991 code: error_codes::INVALID_REQUEST.to_string(),
3992 message: error.to_string(),
3993 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3994 },
3995 )?]);
3996 }
3997
3998 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
3999 if status.waiting_for_previous_pid.is_some() {
4000 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4001 ctx,
4002 &frame,
4003 &target_module_id,
4004 "module_warming",
4005 &status,
4006 )?]);
4007 }
4008 }
4009
4010 let Some(registration) = self
4029 .registry
4030 .get_module(&target_module_id)
4031 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4032 else {
4033 if let Some((status, warming)) =
4034 self.supervisor_status(&target_module_id, frame.header.corr)?
4035 {
4036 if status.protocol == ModuleProtocol::None {
4043 return Ok(vec![self.route_open_refusal_frame(
4044 ctx,
4045 &frame,
4046 &target_module_id,
4047 "protocol_none",
4048 error_codes::MODULE_NO_PROTOCOL,
4049 format!(
4050 "module_id '{target_module_id}' is declared protocol: none; \
4051 it speaks no subc wire and serves no routes"
4052 ),
4053 )?]);
4054 }
4055 let code = if warming {
4056 "module_warming"
4057 } else {
4058 "target_unavailable"
4059 };
4060 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4061 ctx,
4062 &frame,
4063 &target_module_id,
4064 code,
4065 &status,
4066 )?]);
4067 }
4068 if let Some(removed_ago_ms) =
4069 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4070 {
4071 return Ok(vec![self.route_open_refusal_frame(
4072 ctx,
4073 &frame,
4074 &target_module_id,
4075 "removed",
4076 error_codes::MODULE_REMOVED,
4077 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4078 )?]);
4079 }
4080 return Ok(vec![self.route_open_refusal_frame(
4081 ctx,
4082 &frame,
4083 &target_module_id,
4084 "not_registered",
4085 error_codes::UNKNOWN_MODULE,
4086 format!("module_id '{target_module_id}' is not registered"),
4087 )?]);
4088 };
4089
4090 if !registration.ready {
4094 self.counters
4095 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4096 info!(
4097 target: "control",
4098 code = error_codes::MODULE_WARMING,
4099 module_id = ?target_module_id,
4100 connection_id = ctx.connection_id.get(),
4101 reason = "declared_not_ready",
4102 "route.open refused"
4103 );
4104 self.route_outages
4107 .record_not_serving(&target_module_id, "declared_not_ready");
4108 return Ok(vec![control_error_body_frame(
4109 &frame,
4110 ErrorBody {
4111 code: error_codes::MODULE_WARMING.to_string(),
4112 message: format!(
4113 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4114 ),
4115 detail: Some(serde_json::json!({
4116 "reason": "declared_not_ready"
4117 })),
4118 },
4119 )?]);
4120 }
4121
4122 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4141 self.counters
4142 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4143 info!(
4144 target: "control",
4145 code = error_codes::MODULE_WARMING,
4146 module_id = ?target_module_id,
4147 connection_id = ctx.connection_id.get(),
4148 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4149 capability = %capability,
4150 "route.open refused"
4151 );
4152 return Ok(vec![control_error_body_frame(
4153 &frame,
4154 ErrorBody {
4155 code: error_codes::MODULE_WARMING.to_string(),
4156 message: format!(
4157 "module_id '{target_module_id}' requires capability '{capability}', \
4158 which no registered module provides; retry"
4159 ),
4160 detail: Some(serde_json::json!({
4161 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4162 "capability": capability,
4163 })),
4164 },
4165 )?]);
4166 }
4167
4168 if !target_has_required_role(&target, ®istration.manifest.provides) {
4169 return Ok(vec![self.route_open_refusal_frame(
4170 ctx,
4171 &frame,
4172 &target_module_id,
4173 "role_not_provided",
4174 "target_unavailable",
4175 format!("module_id '{target_module_id}' does not provide the requested target"),
4176 )?]);
4177 }
4178
4179 if registration.state != ChannelState::Active {
4180 return Ok(vec![self.route_open_refusal_frame(
4181 ctx,
4182 &frame,
4183 &target_module_id,
4184 "registration_not_active",
4185 "target_unavailable",
4186 format!("module_id '{target_module_id}' is not active"),
4187 )?]);
4188 }
4189
4190 if self
4191 .forwarding
4192 .module_is_draining(&target_module_id)
4193 .map_err(RouterError::Forwarding)?
4194 {
4195 return Ok(vec![self.route_open_refusal_frame(
4196 ctx,
4197 &frame,
4198 &target_module_id,
4199 "reloading",
4200 "module_reloading",
4201 format!("module_id '{target_module_id}' is reloading"),
4202 )?]);
4203 }
4204
4205 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4206 process_liveness.process_live(&target_module_id) == Some(false)
4207 }) {
4208 if process_liveness.process_replacing(&target_module_id) {
4215 return Ok(vec![self.route_open_refusal_frame(
4216 ctx,
4217 &frame,
4218 &target_module_id,
4219 "reloading",
4220 "module_reloading",
4221 format!("module_id '{target_module_id}' is reloading"),
4222 )?]);
4223 }
4224 return Ok(vec![self.route_open_refusal_frame(
4225 ctx,
4226 &frame,
4227 &target_module_id,
4228 "supervisor_not_live",
4229 "target_unavailable",
4230 format!("module_id '{target_module_id}' is not live"),
4231 )?]);
4232 }
4233
4234 if !self
4235 .forwarding
4236 .has_live_module_connection(&target_module_id)
4237 .map_err(RouterError::Forwarding)?
4238 {
4239 return Ok(vec![self.route_open_refusal_frame(
4240 ctx,
4241 &frame,
4242 &target_module_id,
4243 "no_forwarding_connection",
4244 "target_unavailable",
4245 format!("module_id '{target_module_id}' has no live forwarding connection"),
4246 )?]);
4247 }
4248
4249 if let Some(error) =
4250 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4251 {
4252 self.observe_route_open_refusal(
4253 ctx,
4254 &target_module_id,
4255 "op_not_allowed",
4256 "op_not_allowed",
4257 );
4258 return Ok(vec![error]);
4259 }
4260
4261 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4262 Ok(principal) => principal,
4263 Err(error) => {
4264 self.observe_route_open_refusal(
4265 ctx,
4266 &target_module_id,
4267 "bad_consumer_identity",
4268 "bad_consumer_identity",
4269 );
4270 return Ok(vec![error]);
4271 }
4272 };
4273
4274 if let Principal::Reserved {
4278 module_id: opening_module_id,
4279 } = &principal
4280 {
4281 if let Some(opening_registration) = self
4282 .registry
4283 .get_module(opening_module_id)
4284 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4285 {
4286 if let Some(capability) =
4287 denied_capability(&opening_registration.manifest, ®istration.manifest)
4288 {
4289 warn!(
4290 opening_module_id,
4291 target_module_id,
4292 capability,
4293 "refusing route.open because an attested capability deny edge matches"
4294 );
4295 return Ok(vec![self.route_open_refusal_frame(
4296 ctx,
4297 &frame,
4298 &target_module_id,
4299 "capability_deny_edge",
4300 "capability_forbidden",
4301 format!(
4302 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4303 ),
4304 )?]);
4305 }
4306 }
4307 }
4308
4309 if admission_facts.is_some() {
4310 let carrier_matches = matches!(
4311 &principal,
4312 Principal::Reserved { module_id }
4313 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4314 );
4315 if !carrier_matches {
4316 return Ok(vec![self.route_open_refusal_frame(
4317 ctx,
4318 &frame,
4319 &target_module_id,
4320 "admission_facts_carrier_not_permitted",
4321 "admission_facts_not_permitted",
4322 "admission facts may only be carried by the configured reserved module",
4323 )?]);
4324 }
4325
4326 let target_allowed = self
4327 .admission_facts_targets
4328 .as_ref()
4329 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4330 if !target_allowed {
4331 return Ok(vec![self.route_open_refusal_frame(
4332 ctx,
4333 &frame,
4334 &target_module_id,
4335 "admission_facts_target_not_listed",
4336 "admission_facts_target_not_allowed",
4337 format!(
4338 "admission facts are not permitted for target module_id '{target_module_id}'"
4339 ),
4340 )?]);
4341 }
4342
4343 }
4347
4348 let (bound_scope, mut scope_stamp) = match &scope {
4354 None => (None, None),
4355 Some(selector) => {
4356 let owner_configured = match &selector.owner {
4357 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4363 _ => false,
4364 };
4365 let mut scopes = self
4366 .scopes
4367 .write()
4368 .unwrap_or_else(|poisoned| poisoned.into_inner());
4369 let admitted = scopes
4370 .admit(&principal, &target_module_id, selector, owner_configured)
4371 .and_then(|admission| {
4372 crate::scopes::check_target_flow_support(
4373 &admission.stamp,
4374 &target_module_id,
4375 registration.manifest.capabilities.as_ref(),
4376 )?;
4377 crate::scopes::check_target_agent_run_support(
4378 &admission.stamp,
4379 &target_module_id,
4380 registration.manifest.capabilities.as_ref(),
4381 )?;
4382 let bound = scopes.reserve_bind(
4383 &admission,
4384 &principal,
4385 ctx.connection_id,
4386 selector.hold_presence,
4387 );
4388 Ok((admission, bound))
4389 });
4390 match admitted {
4391 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4392 Err(refusal) => {
4393 return Ok(vec![self.route_open_refusal_frame(
4394 ctx,
4395 &frame,
4396 &target_module_id,
4397 refusal.code,
4398 refusal.code,
4399 refusal.message,
4400 )?]);
4401 }
4402 }
4403 }
4404 };
4405
4406 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4428 Ok(project_root) => project_root,
4429 Err(err) => {
4430 return Ok(vec![control_error_frame(
4431 &frame,
4432 "invalid_project_root",
4433 err.to_string(),
4434 )?])
4435 }
4436 };
4437 identity.project_root = project_root.as_path().to_path_buf();
4438
4439 let _concurrency_guard = match self
4450 .route_bind_concurrency
4451 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4452 {
4453 Ok(guard) => guard,
4454 Err(in_flight) => {
4455 return Ok(vec![self.route_open_target_capacity_refusal(
4456 ctx,
4457 &frame,
4458 &target_module_id,
4459 in_flight,
4460 )?]);
4461 }
4462 };
4463
4464 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4467 RouteBindAdmission::Admitted { guard, probe } => {
4468 if probe {
4469 info!(
4470 module_id = %target_module_id,
4471 connection_id = ctx.connection_id.get(),
4472 "route.bind breaker half-open: admitting one probe"
4473 );
4474 }
4475 guard
4476 }
4477 RouteBindAdmission::Refused {
4478 consecutive_timeouts,
4479 retry_in,
4480 probe_in_flight,
4481 } => {
4482 return Ok(vec![self.route_open_breaker_refusal_frame(
4483 ctx,
4484 &frame,
4485 &target_module_id,
4486 consecutive_timeouts,
4487 retry_in,
4488 probe_in_flight,
4489 )?]);
4490 }
4491 };
4492
4493 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4498 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4499 let pending = match self
4500 .forwarding
4501 .begin_route_bind_relay_for(
4502 ctx.connection_id,
4503 ctx.egress.clone(),
4504 response_version(&frame),
4505 frame.header.corr,
4506 &target_module_id,
4507 principal.clone(),
4508 bound_scope,
4509 Some(project_root),
4510 relay_deadline,
4511 )
4512 .await
4513 {
4514 Ok(pending) => pending,
4515 Err(err) => {
4516 return Ok(vec![self.route_open_refusal_frame(
4517 ctx,
4518 &frame,
4519 &target_module_id,
4520 "relay_reservation_failed",
4521 forwarding_error_code(&err),
4522 err.to_string(),
4523 )?])
4524 }
4525 };
4526 let crate::forwarding::PendingRouteBindRelay {
4527 endpoint,
4528 module_sink,
4529 negotiated_ver,
4530 client_channel,
4531 client_epoch,
4532 module_channel,
4533 module_epoch,
4534 corr: relay_corr,
4535 receiver,
4536 } = pending;
4537 let mut reservation =
4538 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4539
4540 let relay_registration = self
4545 .registry
4546 .get_module_by_connection(endpoint.connection_id)
4547 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4548 let relay_capabilities = relay_registration
4549 .as_ref()
4550 .and_then(|registration| registration.manifest.capabilities.as_ref());
4551 if let Some(selector) = &scope {
4552 if let Err(refusal) = self
4555 .scopes
4556 .read()
4557 .unwrap_or_else(|poisoned| poisoned.into_inner())
4558 .check_bind_proof(&principal, selector)
4559 {
4560 reservation.release_and_disarm();
4561 return Ok(vec![self.route_open_refusal_frame(
4562 ctx,
4563 &frame,
4564 &target_module_id,
4565 refusal.code,
4566 refusal.code,
4567 refusal.message,
4568 )?]);
4569 }
4570 }
4571 if let Some(stamp) = scope_stamp
4572 .as_ref()
4573 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4574 {
4575 if let Err(refusal) = crate::scopes::check_target_flow_support(
4576 stamp,
4577 &target_module_id,
4578 relay_capabilities,
4579 )
4580 .and_then(|()| {
4581 crate::scopes::check_target_agent_run_support(
4582 stamp,
4583 &target_module_id,
4584 relay_capabilities,
4585 )
4586 }) {
4587 reservation.release_and_disarm();
4588 return Ok(vec![self.route_open_refusal_frame(
4589 ctx,
4590 &frame,
4591 &target_module_id,
4592 refusal.code,
4593 refusal.code,
4594 refusal.message,
4595 )?]);
4596 }
4597 }
4598
4599 let call_mode = if relay_capabilities
4602 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4603 {
4604 Some(derive_call_mode(scope_stamp.as_ref()))
4605 } else {
4606 if let Some(stamp) = scope_stamp.as_mut() {
4607 stamp.expires_at_ms = None;
4608 }
4609 None
4610 };
4611
4612 debug!(
4613 connection_id = ctx.connection_id.get(),
4614 client_channel,
4615 client_epoch,
4616 module_channel,
4617 module_epoch,
4618 "reserved route handle pair"
4619 );
4620 let principal_label = principal_label(&principal);
4623 let relay = ModuleControlRequest::RouteBind {
4624 route_channel: module_channel,
4625 epoch: module_epoch,
4626 target,
4627 identity,
4628 principal: Some(principal),
4629 consumer_capabilities,
4630 role_versions,
4631 admission_facts,
4632 scope: scope_stamp,
4633 call_mode,
4634 };
4635 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4636 RouterError::backend(
4637 0,
4638 frame.header.corr,
4639 format!("failed to encode route.bind request: {err}"),
4640 )
4641 })?;
4642 let relay_frame = Frame::build_with_version(
4643 negotiated_ver,
4644 FrameType::Request,
4645 control_flags(),
4646 0,
4647 0,
4648 relay_corr,
4649 relay_body,
4650 )
4651 .map_err(RouterError::FrameBuild)?;
4652
4653 if let Err(err) = module_sink.send(relay_frame).await {
4654 reservation.release_and_disarm();
4655 return Ok(vec![self.route_open_refusal_frame(
4656 ctx,
4657 &frame,
4658 &target_module_id,
4659 "relay_send_failed",
4660 "target_unavailable",
4661 err.to_string(),
4662 )?]);
4663 }
4664
4665 #[cfg(test)]
4666 if let Some(hold) = &self.route_bind_relay_sent_hold {
4667 hold.wait().await;
4671 hold.wait().await;
4672 }
4673
4674 if self
4679 .forwarding
4680 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4681 .map_err(RouterError::Forwarding)?
4682 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4683 {
4684 self.send_abandoned_route_bind_goodbye(
4685 &module_sink,
4686 negotiated_ver,
4687 module_channel,
4688 module_epoch,
4689 );
4690 }
4691
4692 match timeout_at(relay_deadline, receiver).await {
4693 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4694 reservation.disarm();
4695 if breaker.record_accepted() {
4696 info!(
4697 module_id = %target_module_id,
4698 "route.bind breaker closed: the probe was accepted"
4699 );
4700 }
4701 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4702 Ok(Vec::new())
4703 }
4704 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4705 reservation.release_and_disarm();
4706 breaker.record_inconclusive();
4710 let scope_code = match body.code.as_str() {
4714 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4715 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4716 _ => None,
4717 };
4718 if let Some(code) = scope_code {
4719 self.observe_route_open_refusal(
4720 ctx,
4721 &target_module_id,
4722 "scope_changed_before_commit",
4723 code,
4724 );
4725 return Ok(vec![control_error_body_frame(&frame, body)?]);
4726 }
4727 self.counters
4728 .increment_route_open_refused("module_rejected");
4729 info!(
4730 target: "control",
4731 code = "module_rejected",
4732 module_code = ?body.code,
4733 module_id = ?target_module_id,
4734 connection_id = ctx.connection_id.get(),
4735 "route.open refused"
4736 );
4737 Ok(vec![control_error_body_frame(&frame, body)?])
4738 }
4739 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4740 reservation.release_and_disarm();
4741 breaker.record_inconclusive();
4742 tracing::warn!(
4749 module_id = %target_module_id,
4750 "route.bind relay abandoned: {message}"
4751 );
4752 Ok(vec![self.route_open_refusal_frame(
4753 ctx,
4754 &frame,
4755 &target_module_id,
4756 "relay_abandoned",
4757 "target_unavailable",
4758 message,
4759 )?])
4760 }
4761 Ok(Err(_)) => {
4762 reservation.release_and_disarm();
4763 breaker.record_inconclusive();
4764 Ok(vec![self.route_open_refusal_frame(
4765 ctx,
4766 &frame,
4767 &target_module_id,
4768 "relay_waiter_canceled",
4769 "target_unavailable",
4770 "route.bind relay waiter was canceled before the module responded",
4771 )?])
4772 }
4773 Err(_) => {
4774 reservation.release_and_disarm();
4775 if let Some(opened) = breaker.record_timeout(
4779 self.route_bind_breaker_threshold,
4780 self.route_bind_breaker_cooldown,
4781 ) {
4782 warn!(
4783 module_id = %target_module_id,
4784 consecutive_timeouts = opened.consecutive_timeouts,
4785 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4786 reopened_after_probe = opened.reopened_after_probe,
4787 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4788 );
4789 }
4790 tracing::warn!(
4798 module_id = %target_module_id,
4799 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4800 "route.bind relay timed out: module did not ack within budget"
4801 );
4802 Ok(vec![self.route_open_refusal_frame(
4803 ctx,
4804 &frame,
4805 &target_module_id,
4806 "relay_timed_out",
4807 "module_timeout",
4808 format!(
4809 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4810 route_bind_relay_timeout
4811 ),
4812 )?])
4813 }
4814 }
4815 }
4816
4817 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4818 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4819 snapshot: self.supervisor.spawn_snapshot(),
4820 };
4821 Ok(vec![control_response_body_frame(
4822 &frame,
4823 &response,
4824 "ClientControlResponse::SupervisorSpawnSnapshot",
4825 )?])
4826 }
4827
4828 fn handle_supervisor_spawn_subscribe(
4829 &self,
4830 ctx: &RouteCtx,
4831 frame: Frame,
4832 since: Option<SpawnCursor>,
4833 ) -> Result<Vec<Frame>, RouterError> {
4834 match self.supervisor.subscribe_spawns(
4835 ctx.connection_id,
4836 frame.header.corr,
4837 response_version(&frame),
4838 since,
4839 ctx.egress.clone(),
4840 ) {
4841 Ok(()) => Ok(Vec::new()),
4842 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4843 Ok(vec![control_error_body_frame(
4844 &frame,
4845 ErrorBody {
4846 code: "spawn_cursor_incarnation_mismatch".to_string(),
4847 message: "spawn cursor belongs to a different daemon incarnation"
4848 .to_string(),
4849 detail: Some(serde_json::json!({
4850 "current_daemon_incarnation": current
4851 })),
4852 },
4853 )?])
4854 }
4855 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4856 &frame,
4857 ErrorBody {
4858 code: "spawn_cursor_too_old".to_string(),
4859 message: "spawn cursor predates the retained event ring".to_string(),
4860 detail: Some(serde_json::json!({
4861 "oldest_retained_cursor": oldest
4862 })),
4863 },
4864 )?]),
4865 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4866 0,
4867 frame.header.corr,
4868 format!("failed to open supervisor spawn subscription: {error}"),
4869 )),
4870 }
4871 }
4872
4873 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4874 let generation = self
4875 .registry
4876 .generation()
4877 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4878 let mut modules = Vec::new();
4879 for module in self.supervisor.list() {
4880 let status = module.status_for_control("list").map_err(|err| {
4881 RouterError::backend(
4882 0,
4883 frame.header.corr,
4884 format!("failed to read supervisor status: {err}"),
4885 )
4886 })?;
4887 let (configured, _) = module.configuration().map_err(|err| {
4888 RouterError::backend(
4889 0,
4890 frame.header.corr,
4891 format!("failed to read module configuration: {err}"),
4892 )
4893 })?;
4894 let image = module.running_image_agreement().await;
4896 let resources = Some(module.child_resource_usage());
4899 let pending_reload = Some(reload_verdict(
4900 &configured.program,
4901 status.spawned_from.as_deref(),
4902 image,
4903 ));
4904 modules.push(
4907 SupervisorEntry::new(
4908 status.module_id,
4909 status.state.to_string(),
4910 status.enabled,
4911 status.live,
4912 status.health.status,
4913 )
4914 .with_launch_nonce_env(Some(!cfg!(unix)))
4915 .with_launch_nonce_source(status.launch_nonce_source)
4916 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
4917 .with_protocol(status.protocol)
4918 .with_pending_reload(pending_reload)
4919 .with_last_probe_ms(status.health.last_probe_ms)
4920 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4921 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4922 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4923 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4924 .with_restart_count(Some(status.restart_count))
4925 .with_max_restarts(Some(status.max_restarts))
4926 .with_lifetime_restarts(Some(status.lifetime_restarts))
4927 .with_spawn_generation(Some(status.spawn_generation))
4928 .with_restart_window_secs(Some(status.restart_window.as_secs()))
4929 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4930 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4931 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4932 .with_resources(resources),
4933 );
4934 }
4935 let response = ClientControlResponse::SupervisorList {
4936 generation,
4937 modules,
4938 };
4939 Ok(vec![control_response_body_frame(
4940 &frame,
4941 &response,
4942 "ClientControlResponse::SupervisorList",
4943 )?])
4944 }
4945
4946 fn handle_supervisor_stderr_tail(
4947 &self,
4948 frame: Frame,
4949 module_id: String,
4950 max_lines: Option<u32>,
4951 max_bytes: Option<u32>,
4952 ) -> Result<Vec<Frame>, RouterError> {
4953 let Some(module) = self.supervisor.get(&module_id) else {
4954 return Ok(vec![control_error_frame(
4955 &frame,
4956 "unknown_module",
4957 format!("module_id '{module_id}' is not supervised"),
4958 )?]);
4959 };
4960
4961 let snapshot = module.stderr_tail(
4962 max_lines.map(|value| value as usize),
4963 max_bytes.map(|value| value as usize),
4964 );
4965
4966 let response = ClientControlResponse::SupervisorStderrTail {
4967 module_id,
4968 tail: StderrTail {
4969 capture: match snapshot.capture {
4970 CaptureState::Captured => StderrCaptureState::Captured,
4971 CaptureState::Incomplete { reason } => {
4972 StderrCaptureState::Incomplete { reason }
4973 }
4974 CaptureState::NotCaptured { reason } => {
4975 StderrCaptureState::NotCaptured { reason }
4976 }
4977 },
4978 entries: snapshot
4979 .entries
4980 .into_iter()
4981 .map(|entry| match entry {
4982 TailEntry::Line {
4983 text,
4984 truncated,
4985 at_ms,
4986 } => StderrTailEntry::Line {
4987 text,
4988 truncated,
4989 at_ms,
4990 },
4991 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4992 })
4993 .collect(),
4994 dropped_lines: snapshot.dropped_lines,
4995 },
4996 };
4997 Ok(vec![control_response_body_frame(
4998 &frame,
4999 &response,
5000 "ClientControlResponse::SupervisorStderrTail",
5001 )?])
5002 }
5003
5004 async fn handle_supervisor_terminals(
5005 &self,
5006 frame: Frame,
5007 module_id: String,
5008 ) -> Result<Vec<Frame>, RouterError> {
5009 let Some(module) = self.supervisor.get(&module_id) else {
5010 return Ok(vec![control_error_frame(
5011 &frame,
5012 "unknown_module",
5013 format!("module_id '{module_id}' is not supervised"),
5014 )?]);
5015 };
5016
5017 let terminals = module
5020 .read_durable_terminal_history()
5021 .await
5022 .map_err(|error| {
5023 RouterError::backend(
5024 0,
5025 frame.header.corr,
5026 format!("failed to read terminal history: {error}"),
5027 )
5028 })?;
5029 let response = ClientControlResponse::SupervisorTerminals {
5030 module_id,
5031 terminals,
5032 };
5033 Ok(vec![control_response_body_frame(
5034 &frame,
5035 &response,
5036 "ClientControlResponse::SupervisorTerminals",
5037 )?])
5038 }
5039
5040 fn handle_supervisor_routes(
5041 &self,
5042 frame: Frame,
5043 module_id: Option<String>,
5044 ) -> Result<Vec<Frame>, RouterError> {
5045 let modules = self
5046 .forwarding
5047 .route_census(module_id.as_deref())
5048 .map_err(RouterError::Forwarding)?
5049 .into_iter()
5050 .map(|(module_id, routes)| SupervisorRouteModule {
5051 module_id,
5052 routes: routes
5053 .into_iter()
5054 .map(|route| SupervisorRoute {
5055 consumer: match route.principal {
5056 Principal::Reserved { module_id } => {
5057 SupervisorRouteConsumer::Reserved { module_id }
5058 }
5059 Principal::Direct | Principal::Unverified => {
5060 SupervisorRouteConsumer::Direct {
5061 connection_id: route.goodbye_target.connection_id.get(),
5062 }
5063 }
5064 },
5065 age_ms: Instant::now()
5066 .saturating_duration_since(route.bound_at)
5067 .as_millis()
5068 .try_into()
5069 .unwrap_or(u64::MAX),
5070 draining: route.draining,
5071 drain_reason: route.drain_reason,
5072 })
5073 .collect(),
5074 })
5075 .collect();
5076 let response = ClientControlResponse::SupervisorRoutes { modules };
5077 Ok(vec![control_response_body_frame(
5078 &frame,
5079 &response,
5080 "ClientControlResponse::SupervisorRoutes",
5081 )?])
5082 }
5083
5084 async fn handle_supervisor_provenance(
5085 &self,
5086 frame: Frame,
5087 module_id: Option<String>,
5088 ) -> Result<Vec<Frame>, RouterError> {
5089 let mut selected = if let Some(module_id) = module_id {
5090 let Some(module) = self.supervisor.get(&module_id) else {
5091 return Ok(vec![control_error_frame(
5092 &frame,
5093 "unknown_module",
5094 format!("module_id '{module_id}' is not supervised"),
5095 )?]);
5096 };
5097 vec![module]
5098 } else {
5099 self.supervisor.list()
5100 };
5101
5102 let mut modules = Vec::with_capacity(selected.len());
5103 for module in selected.drain(..) {
5104 let (status, observed_image) = module
5105 .status_and_running_image_agreement()
5106 .await
5107 .map_err(|err| {
5108 RouterError::backend(
5109 0,
5110 frame.header.corr,
5111 format!("failed to read supervisor status: {err}"),
5112 )
5113 })?;
5114 let module_declared = self
5115 .registry
5116 .get_module(&status.module_id)
5117 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5118 .and_then(|registration| registration.manifest.provenance)
5119 .map(|build| ModuleDeclaredProvenance::Reported { build })
5120 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5121 #[cfg(test)]
5122 let running_image = match &self.provenance_probe_override {
5123 Some(result) => result.clone(),
5124 None => observed_image,
5125 };
5126 #[cfg(not(test))]
5127 let running_image = observed_image;
5128 modules.push(SupervisorModuleProvenance {
5129 module_id: status.module_id,
5130 module_declared,
5131 daemon_observed: SupervisorObservedProcess {
5132 launch_nonce_source: status.launch_nonce_source,
5133 pid: status.pid,
5134 spawned_at_ms: status.spawned_at_ms,
5135 spawned_from: status.spawned_from,
5136 running_image,
5137 },
5138 });
5139 }
5140 let daemon = SupervisorDaemonProvenance {
5141 daemon_build: self.daemon_provenance.build.clone(),
5142 daemon_observed: DaemonObservedProcess {
5143 pid: self.daemon_provenance.pid,
5144 started_at_ms: self
5145 .daemon_provenance
5146 .start_clock
5147 .map(|clock| clock.started_at_ms())
5148 .or(self.daemon_provenance.started_at_ms),
5149 running_image: self
5150 .daemon_provenance
5151 .probe
5152 .observe(
5153 self.daemon_provenance.pid,
5154 self.daemon_provenance.executable_path.as_deref(),
5155 self.daemon_provenance.executable_identity.clone(),
5156 self.daemon_provenance.process_start_time,
5157 )
5158 .await,
5159 },
5160 };
5161 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5162 Ok(vec![control_response_body_frame(
5163 &frame,
5164 &response,
5165 "ClientControlResponse::SupervisorProvenance",
5166 )?])
5167 }
5168
5169 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5170 self.refresh_capability_requirements();
5171 let generation = self
5172 .registry
5173 .generation()
5174 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5175 let modules = self
5176 .supervisor
5177 .list()
5178 .into_iter()
5179 .map(|module| {
5180 let status = module.status_for_control("health").map_err(|err| {
5181 RouterError::backend(
5182 0,
5183 frame.header.corr,
5184 format!("failed to read supervisor health: {err}"),
5185 )
5186 })?;
5187 let module_id = status.module_id;
5188 let capability_detail = self
5189 .capability_evaluator
5190 .required_problem_detail(&module_id);
5191 Ok(SupervisorHealthEntry {
5192 module_id,
5193 status: status.health.status,
5194 detail: append_capability_problem_detail(
5195 status.health.detail,
5196 capability_detail,
5197 ),
5198 metrics: status.health.metrics,
5199 consecutive_failures: status.health.consecutive_failures,
5200 late_answer_count: status.health.late_answer_count,
5201 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5202 last_action: status.health.last_action,
5203 last_action_ms: status.health.last_action_ms,
5204 last_probe_ms: status.health.last_probe_ms,
5205 })
5206 })
5207 .collect::<Result<Vec<_>, RouterError>>()?;
5208 let response = ClientControlResponse::SupervisorHealth {
5209 generation,
5210 modules,
5211 };
5212 Ok(vec![control_response_body_frame(
5213 &frame,
5214 &response,
5215 "ClientControlResponse::SupervisorHealth",
5216 )?])
5217 }
5218
5219 async fn handle_supervisor_restart(
5220 &self,
5221 frame: Frame,
5222 module_id: String,
5223 drain_timeout_ms: Option<u64>,
5224 requester: Principal,
5225 ) -> Result<Vec<Frame>, RouterError> {
5226 let operation_lock = self.supervisor.operation_lock();
5227 let _operation_guard = operation_lock.lock().await;
5228 let Some(module) = self.supervisor.get(&module_id) else {
5229 return Ok(vec![control_error_frame(
5230 &frame,
5231 "unknown_module",
5232 format!("module_id '{module_id}' is not supervised"),
5233 )?]);
5234 };
5235
5236 self.route_outages.mark_operator_action(&module_id);
5237 if let Err(err) = module
5238 .restart_requested(drain_timeout_ms, Some(requester))
5239 .await
5240 {
5241 self.route_outages
5242 .operator_action_ended_unrefused(&module_id);
5243 let (code, message) = match err {
5244 crate::supervise::SuperviseError::Disabled { .. } => {
5245 ("module_disabled", err.to_string())
5246 }
5247 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5248 ("swap_in_progress", err.to_string())
5249 }
5250 _ => (
5251 "target_unavailable",
5252 format!("failed to restart module_id '{module_id}': {err}"),
5253 ),
5254 };
5255 return Ok(vec![control_error_frame(&frame, code, message)?]);
5256 }
5257
5258 let response = ClientControlResponse::SupervisorAck {
5259 module_id,
5260 applied: true,
5261 };
5262 Ok(vec![control_response_body_frame(
5263 &frame,
5264 &response,
5265 "ClientControlResponse::SupervisorAck",
5266 )?])
5267 }
5268
5269 async fn handle_supervisor_swap(
5273 &self,
5274 frame: Frame,
5275 module_id: String,
5276 ready_timeout_ms: Option<u64>,
5277 requester: Principal,
5278 ) -> Result<Vec<Frame>, RouterError> {
5279 let module = {
5286 let operation_lock = self.supervisor.operation_lock();
5287 let _operation_guard = operation_lock.lock().await;
5288 self.supervisor.get(&module_id)
5289 };
5290 let Some(module) = module else {
5291 return Ok(vec![control_error_frame(
5292 &frame,
5293 "unknown_module",
5294 format!("module_id '{module_id}' is not supervised"),
5295 )?]);
5296 };
5297
5298 self.route_outages.mark_operator_action(&module_id);
5299 if let Err(err) = module
5300 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5301 .await
5302 {
5303 self.route_outages
5304 .operator_action_ended_unrefused(&module_id);
5305 use crate::supervise::SuperviseError;
5306 let message = err.to_string();
5307 let error = match err {
5308 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5309 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5310 code: "swap_refused".to_string(),
5311 message,
5312 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5313 },
5314 SuperviseError::SwapFailed {
5315 arm,
5316 candidate_exit,
5317 ..
5318 } => ErrorBody {
5319 code: "swap_failed".to_string(),
5320 message,
5321 detail: Some(serde_json::json!({
5322 "arm": arm.as_str(),
5323 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5324 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5325 })),
5326 },
5327 _ => ErrorBody::new(
5328 "target_unavailable",
5329 format!("failed to swap module_id '{module_id}': {message}"),
5330 ),
5331 };
5332 return Ok(vec![control_error_body_frame(&frame, error)?]);
5333 }
5334 self.route_outages
5338 .operator_action_ended_unrefused(&module_id);
5339
5340 let response = ClientControlResponse::SupervisorAck {
5341 module_id,
5342 applied: true,
5343 };
5344 Ok(vec![control_response_body_frame(
5345 &frame,
5346 &response,
5347 "ClientControlResponse::SupervisorAck",
5348 )?])
5349 }
5350
5351 async fn handle_supervisor_reload(
5352 &self,
5353 frame: Frame,
5354 module_id: String,
5355 requester: Principal,
5356 ) -> Result<Vec<Frame>, RouterError> {
5357 let operation_lock = self.supervisor.operation_lock();
5358 let _operation_guard = operation_lock.lock().await;
5359 let Some(module) = self.supervisor.get(&module_id) else {
5360 return Ok(vec![control_error_frame(
5361 &frame,
5362 "unknown_module",
5363 format!("module_id '{module_id}' is not supervised"),
5364 )?]);
5365 };
5366
5367 self.route_outages.mark_operator_action(&module_id);
5368 if let Err(err) = module.reload_requested(Some(requester)).await {
5369 self.route_outages
5370 .operator_action_ended_unrefused(&module_id);
5371 let (code, message) = match err {
5372 crate::supervise::SuperviseError::Disabled { .. } => {
5373 ("module_disabled", err.to_string())
5374 }
5375 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5376 ("swap_in_progress", err.to_string())
5377 }
5378 _ => (
5379 "reload_failed",
5380 format!("failed to reload module_id '{module_id}': {err}"),
5381 ),
5382 };
5383 return Ok(vec![control_error_frame(&frame, code, message)?]);
5384 }
5385
5386 let response = ClientControlResponse::SupervisorAck {
5387 module_id,
5388 applied: true,
5389 };
5390 Ok(vec![control_response_body_frame(
5391 &frame,
5392 &response,
5393 "ClientControlResponse::SupervisorAck",
5394 )?])
5395 }
5396
5397 async fn handle_supervisor_rescan(
5398 &self,
5399 frame: Frame,
5400 preview: bool,
5401 requester: Principal,
5402 ) -> Result<Vec<Frame>, RouterError> {
5403 let Some(context) = self.rescan.clone() else {
5404 return Ok(vec![control_error_frame(
5405 &frame,
5406 "rescan_unavailable",
5407 "the daemon was not started with a reloadable config path".to_string(),
5408 )?]);
5409 };
5410
5411 let operation_lock = self.supervisor.operation_lock();
5412 let _operation_guard = operation_lock.lock().await;
5413 let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5414 Ok(config) => config,
5415 Err(err) => {
5416 return Ok(vec![control_error_frame(
5417 &frame,
5418 "invalid_daemon_config",
5419 format!("supervisor rescan rejected daemon config: {err}"),
5420 )?])
5421 }
5422 };
5423 let Some((config, activity_config)) = loaded else {
5431 return Ok(vec![control_error_frame(
5432 &frame,
5433 "invalid_daemon_config",
5434 format!(
5435 "daemon config not found at {}; refusing to rescan (an absent config would \
5436 retire every supervised module)",
5437 context.config_path.display()
5438 ),
5439 )?]);
5440 };
5441 let (
5442 configured_port,
5443 storage_config,
5444 admission_facts_carrier_module_id,
5445 admission_facts_targets,
5446 scope_authority_owners,
5447 privileged_operator_prompt,
5448 modules,
5449 reserved_capabilities,
5450 ) = (
5451 config.port,
5452 config.storage,
5453 config.admission_facts_carrier_module_id,
5454 config.admission_facts_targets,
5455 config.scope_authority_owners,
5456 config.privileged_operator_prompt,
5457 config.modules,
5458 config.reserved_capabilities,
5459 );
5460
5461 let mut restart_required = Vec::new();
5470 for section in RestartRequiredSection::ALL {
5471 let changed = match section {
5472 RestartRequiredSection::Port => configured_port != context.configured_port,
5473 RestartRequiredSection::Storage => storage_config != context.storage_config,
5474 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5475 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5476 }
5477 RestartRequiredSection::AdmissionFactsTargets => {
5478 admission_facts_targets != context.admission_facts_targets
5479 }
5480 RestartRequiredSection::ScopeAuthorityOwners => {
5481 scope_authority_owners != context.scope_authority_owners
5482 }
5483 RestartRequiredSection::PrivilegedOperatorPrompt => {
5484 privileged_operator_prompt != context.privileged_operator_prompt
5485 }
5486 };
5487 if changed {
5488 restart_required.push(section.label().to_string());
5489 }
5490 }
5491 if !restart_required.is_empty() {
5492 warn!(
5493 config_path = %context.config_path.display(),
5494 sections = %restart_required.join(", "),
5495 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5496 );
5497 }
5498
5499 for configured in &modules {
5500 if let Err(err) = validate_spec(&configured.module_spec()) {
5501 return Ok(vec![control_error_frame(
5502 &frame,
5503 "invalid_daemon_config",
5504 format!("supervisor rescan rejected daemon config: {err}"),
5505 )?]);
5506 }
5507 }
5508
5509 let configured_capabilities = modules
5510 .iter()
5511 .map(|module| (module.module_id.clone(), module.enabled))
5512 .collect::<Vec<_>>();
5513 let preview_capability_warnings = if preview {
5514 let (_, registrations) = self.runtime_capability_snapshot()?;
5515 let current_modules = self
5516 .supervisor
5517 .list()
5518 .into_iter()
5519 .map(|module| module.module_id().to_string())
5520 .collect::<BTreeSet<_>>();
5521 let resulting_modules = configured_capabilities.clone();
5522 let removed = current_modules
5523 .into_iter()
5524 .filter(|module_id| {
5525 !resulting_modules
5526 .iter()
5527 .any(|(configured_id, _)| configured_id == module_id)
5528 })
5529 .collect::<Vec<_>>();
5530 self.capability_evaluator.preview_removal_warnings(
5531 resulting_modules,
5532 &removed,
5533 ®istrations,
5534 )
5535 } else {
5536 Vec::new()
5537 };
5538 let result = match self
5539 .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5540 .await
5541 {
5542 Ok(result) => result,
5543 Err(message) => {
5544 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5545 }
5546 };
5547 if !preview {
5548 if let Some(ledger) = &self.activity {
5549 ledger.module_rescanned(&result.added, &result.removed, requester);
5550 }
5551 if let Err(code) = self.configure_activity(activity_config).await {
5552 warn!(
5553 code,
5554 "activity retention could not be applied during rescan"
5555 );
5556 }
5557 self.capability_evaluator
5558 .configure(configured_capabilities, reserved_capabilities);
5559 self.capability_evaluator.wake_deadline_loop();
5560 self.refresh_capability_requirements();
5561 }
5562 let mut result = result;
5563 result.restart_required = restart_required;
5564 result.capability_warnings = preview_capability_warnings;
5565 let response = ClientControlResponse::SupervisorRescan { result };
5566 Ok(vec![control_response_body_frame(
5567 &frame,
5568 &response,
5569 "ClientControlResponse::SupervisorRescan",
5570 )?])
5571 }
5572
5573 async fn handle_supervisor_release_reserved(
5574 &self,
5575 frame: Frame,
5576 module_id: String,
5577 ) -> Result<Vec<Frame>, RouterError> {
5578 let Some(context) = self.rescan.clone() else {
5579 return Ok(vec![control_error_frame(
5580 &frame,
5581 "release_unavailable",
5582 "reserved-id release requires a daemon started with a reloadable config path",
5583 )?]);
5584 };
5585 let operation_lock = self.supervisor.operation_lock();
5586 let _operation_guard = operation_lock.lock().await;
5587 let loaded = match crate::daemon_config::load(&context.config_path) {
5588 Ok(Some(config)) => config,
5589 Ok(None) => {
5590 return Ok(vec![control_error_frame(
5591 &frame,
5592 "invalid_daemon_config",
5593 format!(
5594 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5595 context.config_path.display()
5596 ),
5597 )?])
5598 }
5599 Err(err) => {
5600 return Ok(vec![control_error_frame(
5601 &frame,
5602 "invalid_daemon_config",
5603 format!("unable to verify reserved-id release against daemon config: {err}"),
5604 )?])
5605 }
5606 };
5607 if loaded
5608 .modules
5609 .iter()
5610 .any(|configured| configured.module_id == module_id)
5611 {
5612 return Ok(vec![control_error_frame(
5613 &frame,
5614 "reserved_module_configured",
5615 format!(
5616 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5617 ),
5618 )?]);
5619 }
5620 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5621 return Ok(vec![control_error_frame(
5622 &frame,
5623 "reserved_gate_not_retained",
5624 format!(
5625 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5626 ),
5627 )?]);
5628 }
5629
5630 let response = ClientControlResponse::SupervisorAck {
5631 module_id,
5632 applied: true,
5633 };
5634 Ok(vec![control_response_body_frame(
5635 &frame,
5636 &response,
5637 "ClientControlResponse::SupervisorAck",
5638 )?])
5639 }
5640
5641 async fn reconcile_supervised_modules(
5650 &self,
5651 supervisor: &Supervisor,
5652 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5653 preview: bool,
5654 requester: Principal,
5655 ) -> Result<SupervisorRescanResult, String> {
5656 let mut current = BTreeMap::new();
5657 for module in self.supervisor.list() {
5658 let (spec, health) = module.configuration().map_err(|err| {
5659 format!(
5660 "failed to read configuration for module_id '{}': {err}",
5661 module.module_id()
5662 )
5663 })?;
5664 let enabled = module
5665 .status()
5666 .map_err(|err| {
5667 format!(
5668 "failed to read status for module_id '{}': {err}",
5669 module.module_id()
5670 )
5671 })?
5672 .enabled;
5673 current.insert(
5674 module.module_id().to_string(),
5675 (module, spec, health, enabled),
5676 );
5677 }
5678 let configured = configured_modules
5679 .into_iter()
5680 .map(|module| (module.module_id.clone(), module))
5681 .collect::<BTreeMap<_, _>>();
5682
5683 let added = configured
5684 .keys()
5685 .filter(|module_id| !current.contains_key(*module_id))
5686 .cloned()
5687 .collect::<Vec<_>>();
5688 let removed = current
5689 .keys()
5690 .filter(|module_id| !configured.contains_key(*module_id))
5691 .cloned()
5692 .collect::<Vec<_>>();
5693 let mut changed_pending_reload = Vec::new();
5694 let mut configuration_changes = BTreeSet::new();
5695 let mut enabled_changes = BTreeSet::new();
5696 let mut unchanged = 0_u32;
5697
5698 for (module_id, configured_module) in &configured {
5699 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5700 else {
5701 continue;
5702 };
5703 let launch_changed = *current_spec != configured_module.module_spec();
5707 let configuration_changed =
5708 launch_changed || *current_health != configured_module.health;
5709 let enabled_changed = *current_enabled != configured_module.enabled;
5710 if configuration_changed {
5711 configuration_changes.insert(module_id.clone());
5712 }
5713 if launch_changed {
5714 changed_pending_reload.push(module_id.clone());
5715 }
5716 if enabled_changed {
5717 enabled_changes.insert(module_id.clone());
5718 }
5719 if !configuration_changed && !enabled_changed {
5720 unchanged = unchanged.saturating_add(1);
5721 }
5722 }
5723
5724 if preview {
5730 return Ok(SupervisorRescanResult {
5731 added,
5732 removed,
5733 changed_pending_reload,
5734 enabled_changes: enabled_changes.iter().cloned().collect(),
5735 unchanged,
5736 preview: true,
5737 restart_required: Vec::new(),
5741 capability_warnings: Vec::new(),
5742 });
5743 }
5744
5745 for module_id in &removed {
5746 let module = ¤t
5747 .get(module_id)
5748 .expect("removed module came from current supervisor state")
5749 .0;
5750 module
5751 .retire_requested(Some(requester.clone()))
5752 .await
5753 .map_err(|err| {
5754 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5755 })?;
5756 self.supervisor.record_rescan_removal(module_id);
5790 self.supervisor.retire(module_id);
5791 self.route_outages.forget(module_id);
5792 }
5793
5794 for module_id in configured.keys() {
5795 let Some((module, _, _, _)) = current.get(module_id) else {
5796 continue;
5797 };
5798 let configured_module = configured
5799 .get(module_id)
5800 .expect("configured module id came from configured map");
5801 if configuration_changes.contains(module_id) {
5802 module
5803 .update_configuration(
5804 configured_module.module_spec(),
5805 configured_module.health.clone(),
5806 configured_module.drain_timeout_ms,
5807 )
5808 .await
5809 .map_err(|err| {
5810 format!(
5811 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5812 )
5813 })?;
5814 }
5815 if enabled_changes.contains(module_id) {
5816 self.route_outages.mark_operator_action(module_id);
5819 module
5820 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
5821 .await
5822 .map_err(|err| {
5823 self.route_outages.operator_action_ended_unrefused(module_id);
5824 format!(
5825 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5826 configured_module.enabled
5827 )
5828 })?;
5829 }
5830 }
5831
5832 for module_id in &added {
5833 let configured_module = configured
5834 .get(module_id)
5835 .expect("added module id came from configured map");
5836 supervisor
5837 .supervise_configured_requested(
5838 configured_module.module_spec(),
5839 configured_module.enabled,
5840 configured_module.health.clone(),
5841 configured_module.drain_timeout_ms,
5842 configured_module.restart,
5843 Some(requester.clone()),
5844 )
5845 .map_err(|err| {
5846 format!("failed to add module_id '{module_id}' during rescan: {err}")
5847 })?;
5848 }
5849
5850 Ok(SupervisorRescanResult {
5851 added,
5852 removed,
5853 changed_pending_reload,
5854 enabled_changes: enabled_changes.iter().cloned().collect(),
5855 unchanged,
5856 preview: false,
5857 restart_required: Vec::new(),
5860 capability_warnings: Vec::new(),
5861 })
5862 }
5863
5864 async fn handle_supervisor_set_enabled(
5865 &self,
5866 frame: Frame,
5867 module_id: String,
5868 enabled: bool,
5869 requester: Principal,
5870 ) -> Result<Vec<Frame>, RouterError> {
5871 let operation_lock = self.supervisor.operation_lock();
5872 let _operation_guard = operation_lock.lock().await;
5873 let Some(module) = self.supervisor.get(&module_id) else {
5874 return Ok(vec![control_error_frame(
5875 &frame,
5876 "unknown_module",
5877 format!("module_id '{module_id}' is not supervised"),
5878 )?]);
5879 };
5880
5881 self.route_outages.mark_operator_action(&module_id);
5884 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
5885 Ok(applied) => applied,
5886 Err(err) => {
5887 self.route_outages
5888 .operator_action_ended_unrefused(&module_id);
5889 return Ok(vec![control_error_frame(
5890 &frame,
5891 "target_unavailable",
5892 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5893 )?]);
5894 }
5895 };
5896 if !applied {
5897 self.route_outages
5900 .operator_action_ended_unrefused(&module_id);
5901 }
5902
5903 self.capability_evaluator.wake_deadline_loop();
5904 self.refresh_capability_requirements();
5905 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5906 Ok(vec![control_response_body_frame(
5907 &frame,
5908 &response,
5909 "ClientControlResponse::SupervisorAck",
5910 )?])
5911 }
5912
5913 async fn handle_supervisor_health_probe(
5914 &self,
5915 frame: Frame,
5916 module_id: String,
5917 ) -> Result<Vec<Frame>, RouterError> {
5918 self.refresh_capability_requirements();
5919 let Some(registration) = self
5920 .registry
5921 .get_module(&module_id)
5922 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5923 else {
5924 return Ok(vec![control_error_frame(
5925 &frame,
5926 "unknown_module",
5927 format!("module_id '{module_id}' is not registered"),
5928 )?]);
5929 };
5930
5931 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5944 {
5945 return Ok(vec![control_error_frame(
5946 &frame,
5947 "health_not_advertised",
5948 format!("module_id '{module_id}' did not advertise health.check"),
5949 )?]);
5950 }
5951
5952 let deadline = Instant::now() + self.health_probe_timeout;
5953 let pending = match self.forwarding.begin_module_control_rpc_for(
5954 &module_id,
5955 MODULE_CONTROL_OP_HEALTH_CHECK,
5956 deadline,
5957 ) {
5958 Ok(pending) => pending,
5959 Err(err) => {
5960 return Ok(vec![control_error_frame(
5961 &frame,
5962 forwarding_error_code(&err),
5963 err.to_string(),
5964 )?])
5965 }
5966 };
5967
5968 let PendingModuleControlRpc {
5969 endpoint,
5970 module_sink,
5971 negotiated_ver,
5972 corr: probe_corr,
5973 receiver,
5974 } = pending;
5975 let mut guard =
5976 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5977 let probe_body =
5978 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5979 RouterError::backend(
5980 0,
5981 frame.header.corr,
5982 format!("failed to encode health.check request: {err}"),
5983 )
5984 })?;
5985 let probe_frame = Frame::build_with_version(
5986 negotiated_ver,
5987 FrameType::Request,
5988 control_flags(),
5989 0,
5990 0,
5991 probe_corr,
5992 probe_body,
5993 )
5994 .map_err(RouterError::FrameBuild)?;
5995
5996 if let Err(err) = module_sink.send(probe_frame).await {
5997 return Ok(vec![control_error_frame(
5998 &frame,
5999 "target_unavailable",
6000 err.to_string(),
6001 )?]);
6002 }
6003
6004 match timeout_at(deadline, receiver).await {
6005 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6006 guard.disarm();
6007 let Some(report) = response.health_report() else {
6008 return Ok(vec![control_error_frame(
6009 &frame,
6010 "invalid_control_body",
6011 "health.check RPC returned a non-health response",
6012 )?]);
6013 };
6014 let HealthReport {
6019 status,
6020 detail,
6021 metrics,
6022 } = report;
6023 let capability_detail = self
6024 .capability_evaluator
6025 .required_problem_detail(&module_id);
6026 let response = ClientControlResponse::SupervisorHealthProbe {
6027 module_id,
6028 status,
6029 detail: append_capability_problem_detail(detail, capability_detail),
6030 metrics,
6031 };
6032 Ok(vec![control_response_body_frame(
6033 &frame,
6034 &response,
6035 "ClientControlResponse::SupervisorHealthProbe",
6036 )?])
6037 }
6038 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6039 guard.disarm();
6040 Ok(vec![control_error_body_frame(&frame, body)?])
6041 }
6042 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6043 guard.disarm();
6044 Ok(vec![control_error_frame(
6045 &frame,
6046 "target_unavailable",
6047 message,
6048 )?])
6049 }
6050 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6051 guard.disarm();
6052 Ok(vec![control_error_frame(
6053 &frame,
6054 "invalid_control_body",
6055 message,
6056 )?])
6057 }
6058 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6059 guard.disarm();
6060 Ok(vec![control_error_frame(
6061 &frame,
6062 "invalid_control_body",
6063 format!("expected module-control op '{expected}', got '{actual}'"),
6064 )?])
6065 }
6066 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6067 guard.disarm();
6068 Ok(vec![control_error_frame(
6069 &frame,
6070 "module_timeout",
6071 format!(
6072 "module_id '{module_id}' answered health.check after {:?}",
6073 self.health_probe_timeout
6074 ),
6075 )?])
6076 }
6077 Ok(Err(_)) => Ok(vec![control_error_frame(
6078 &frame,
6079 "target_unavailable",
6080 "health.check waiter was canceled before the module responded",
6081 )?]),
6082 Err(_) => Ok(vec![control_error_frame(
6083 &frame,
6084 "module_timeout",
6085 format!(
6086 "module_id '{module_id}' did not answer health.check within {:?}",
6087 self.health_probe_timeout
6088 ),
6089 )?]),
6090 }
6091 }
6092
6093 fn supervisor_status(
6094 &self,
6095 module_id: &str,
6096 corr: u64,
6097 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6098 self.supervisor
6099 .get(module_id)
6100 .map(|module| {
6101 let warming = module.is_warming_for_control("status").map_err(|err| {
6102 RouterError::backend(
6103 0,
6104 corr,
6105 format!(
6106 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6107 ),
6108 )
6109 })?;
6110 module.status_for_control("status").map_err(|err| {
6111 RouterError::backend(
6112 0,
6113 corr,
6114 format!(
6115 "failed to read supervisor status for module_id '{module_id}': {err}"
6116 ),
6117 )
6118 }).map(|status| (status, warming))
6119 })
6120 .transpose()
6121 }
6122
6123 fn guard_module_control_op(
6124 &self,
6125 frame: &Frame,
6126 module_id: &str,
6127 op: &str,
6128 ) -> Result<Option<Frame>, RouterError> {
6129 if self.module_grants_op(module_id, op, frame.header.corr)? {
6130 return Ok(None);
6131 }
6132
6133 Ok(Some(control_error_frame(
6134 frame,
6135 "op_not_allowed",
6136 format!("module_id '{module_id}' did not grant control op '{op}'"),
6137 )?))
6138 }
6139
6140 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6141 let Some(registration) = self
6142 .registry
6143 .get_module(module_id)
6144 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6145 else {
6146 return Ok(false);
6147 };
6148 Ok(module_registration_grants_op(®istration.control_ops, op))
6149 }
6150
6151 fn handle_status_update(
6152 &self,
6153 endpoint: ModuleEndpointId,
6154 frame: Frame,
6155 ) -> Result<Vec<Frame>, RouterError> {
6156 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6157 Ok(update) => update,
6158 Err(err) => {
6159 if is_known_module_push_op(&frame.body) {
6164 return Ok(vec![control_error_frame(
6165 &frame,
6166 "invalid_control_body",
6167 format!("malformed module control push body: {err}"),
6168 )?]);
6169 }
6170 return Ok(Vec::new());
6171 }
6172 };
6173
6174 match update {
6175 ModuleControlPush::RouteStatus {
6176 route_channel,
6177 route_epoch,
6178 status,
6179 } => {
6180 self.forwarding
6181 .cache_status(endpoint, route_channel, route_epoch, status)
6182 .map_err(RouterError::Forwarding)?;
6183 }
6184 }
6185 Ok(Vec::new())
6186 }
6187
6188 fn handle_route_poll(
6189 &self,
6190 ctx: &RouteCtx,
6191 frame: Frame,
6192 route_channel: u16,
6193 route_epoch: u32,
6194 kind: PollKind,
6195 ) -> Result<Vec<Frame>, RouterError> {
6196 let snapshot = self
6197 .forwarding
6198 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6199 .map_err(RouterError::Forwarding)?;
6200 let response = match (kind, snapshot) {
6201 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6202 ClientControlResponse::RoutePoll {
6203 route_channel,
6204 route_epoch,
6205 status,
6206 live: None,
6207 }
6208 }
6209 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6210 route_channel,
6211 route_epoch,
6212 status: None,
6213 live: None,
6214 },
6215 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6216 let live = self
6242 .process_liveness
6243 .as_ref()
6244 .and_then(|source| source.process_live(&module_id))
6245 .unwrap_or(true);
6246 ClientControlResponse::RoutePoll {
6247 route_channel,
6248 route_epoch,
6249 status: None,
6250 live: Some(live),
6251 }
6252 }
6253 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6254 route_channel,
6255 route_epoch,
6256 status: None,
6257 live: Some(false),
6258 },
6259 };
6260
6261 Ok(vec![control_response_body_frame(
6262 &frame,
6263 &response,
6264 "ClientControlResponse::RoutePoll",
6265 )?])
6266 }
6267
6268 pub(crate) fn observe_module_control_completion(
6269 &self,
6270 completion: ModuleControlRpcCompletion,
6271 ) -> bool {
6272 match completion {
6273 ModuleControlRpcCompletion::Unknown => false,
6274 ModuleControlRpcCompletion::Settled => true,
6275 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6276 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6277 info!(
6278 module_id = %module_id,
6279 latency_ms,
6280 "late health.check answer proves the module is alive"
6281 );
6282 match self
6283 .supervisor
6284 .record_late_health_answer(&module_id, latency_ms)
6285 {
6286 Ok(true) => {}
6287 Ok(false) => debug!(
6288 module_id = %module_id,
6289 latency_ms,
6290 "late health.check answer has no active supervisor snapshot"
6291 ),
6292 Err(err) => warn!(
6293 module_id = %module_id,
6294 latency_ms,
6295 error = %err,
6296 "failed to record late health.check answer"
6297 ),
6298 }
6299 true
6300 }
6301 }
6302 }
6303
6304 fn refuse_to_end_module_connection_for_a_client(
6325 &self,
6326 module_connection_id: ConnectionId,
6327 corr: u64,
6328 err: ForwardingError,
6329 ) -> Result<(), RouterError> {
6330 if let ForwardingError::ConnectionClosing { connection_id } = err {
6331 if connection_id != module_connection_id {
6332 warn!(
6333 module_connection_id = module_connection_id.get(),
6334 client_connection_id = connection_id.get(),
6335 corr,
6336 "dropping a route.bind response for a closing client; the module connection keeps serving"
6337 );
6338 return Ok(());
6339 }
6340 }
6341 Err(RouterError::Forwarding(err))
6342 }
6343
6344 fn handle_module_relay_response(
6345 &self,
6346 connection_id: ConnectionId,
6347 frame: Frame,
6348 ) -> Result<Vec<Frame>, RouterError> {
6349 let mut secondary_error = None;
6350 let outcome = match frame.header.ty {
6351 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6352 Ok(probe) if probe.op == "route.bind" => {
6353 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6354 Ok(ModuleControlResponse::RouteBindAck {}) => {
6355 RouteBindRelayOutcome::Accepted
6356 }
6357 Ok(other) => {
6358 let message =
6359 format!("route.bind response carried unexpected body: {other:?}");
6360 secondary_error = Some(control_error_frame(
6361 &frame,
6362 "invalid_control_body",
6363 message.clone(),
6364 )?);
6365 RouteBindRelayOutcome::ModuleGone(message)
6366 }
6367 Err(err) => {
6368 let message = format!("malformed route.bind response body: {err}");
6369 secondary_error = Some(control_error_frame(
6370 &frame,
6371 "invalid_control_body",
6372 message.clone(),
6373 )?);
6374 RouteBindRelayOutcome::ModuleGone(message)
6375 }
6376 }
6377 }
6378 Ok(probe) => {
6379 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6380 {
6381 Ok(response) => ModuleControlRpcOutcome::Response(response),
6382 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6383 "malformed {} response body: {err}",
6384 probe.op
6385 )),
6386 };
6387 let completion = self
6388 .forwarding
6389 .complete_module_control_rpc(
6390 connection_id,
6391 frame.header.corr,
6392 Some(&probe.op),
6393 outcome,
6394 )
6395 .map_err(RouterError::Forwarding)?;
6396 if !self.observe_module_control_completion(completion) {
6397 debug!(
6398 connection_id = connection_id.get(),
6399 corr = frame.header.corr,
6400 op = %probe.op,
6401 "dropping late or unknown module-control RPC response"
6402 );
6403 }
6404 return Ok(Vec::new());
6405 }
6406 Err(err) => {
6407 if let Some(expected_op) = self
6408 .forwarding
6409 .pending_module_control_op(connection_id, frame.header.corr)
6410 .map_err(RouterError::Forwarding)?
6411 {
6412 let completion = self
6413 .forwarding
6414 .complete_module_control_rpc(
6415 connection_id,
6416 frame.header.corr,
6417 None,
6418 ModuleControlRpcOutcome::MalformedResponse(format!(
6419 "malformed {expected_op} response body: {err}"
6420 )),
6421 )
6422 .map_err(RouterError::Forwarding)?;
6423 if !self.observe_module_control_completion(completion) {
6424 debug!(
6425 connection_id = connection_id.get(),
6426 corr = frame.header.corr,
6427 "dropping late malformed module-control RPC response"
6428 );
6429 }
6430 return Ok(Vec::new());
6431 }
6432 let message = format!("malformed route.bind response body: {err}");
6433 secondary_error = Some(control_error_frame(
6434 &frame,
6435 "invalid_control_body",
6436 message.clone(),
6437 )?);
6438 RouteBindRelayOutcome::ModuleGone(message)
6439 }
6440 },
6441 FrameType::Error => {
6442 if self
6443 .forwarding
6444 .pending_module_control_op(connection_id, frame.header.corr)
6445 .map_err(RouterError::Forwarding)?
6446 .is_some()
6447 {
6448 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6449 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6450 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6451 "malformed module-control ERROR body: {err}"
6452 )),
6453 };
6454 let completion = self
6455 .forwarding
6456 .complete_module_control_rpc(
6457 connection_id,
6458 frame.header.corr,
6459 None,
6460 outcome,
6461 )
6462 .map_err(RouterError::Forwarding)?;
6463 if !self.observe_module_control_completion(completion) {
6464 debug!(
6465 connection_id = connection_id.get(),
6466 corr = frame.header.corr,
6467 "dropping late or unknown module-control RPC error"
6468 );
6469 }
6470 return Ok(Vec::new());
6471 }
6472 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6473 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6474 Err(err) => {
6475 let message = format!("malformed route.bind ERROR body: {err}");
6476 secondary_error = Some(control_error_frame(
6477 &frame,
6478 "invalid_control_body",
6479 message.clone(),
6480 )?);
6481 RouteBindRelayOutcome::ModuleGone(message)
6482 }
6483 }
6484 }
6485 ty => {
6486 return Ok(vec![control_error_frame(
6487 &frame,
6488 "unsupported_control_frame",
6489 format!("unsupported module channel-0 frame {ty:?}"),
6490 )?])
6491 }
6492 };
6493
6494 let settled =
6495 self.forwarding
6496 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6497 let completion = match settled {
6498 Ok(completion) => completion,
6499 Err(err) => {
6500 self.refuse_to_end_module_connection_for_a_client(
6501 connection_id,
6502 frame.header.corr,
6503 err,
6504 )?;
6505 return Ok(secondary_error.into_iter().collect());
6506 }
6507 };
6508 if let Some(target) = completion.abandoned.as_ref() {
6509 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6510 }
6511 if !completion.settled {
6512 debug!(
6513 connection_id = connection_id.get(),
6514 corr = frame.header.corr,
6515 frame_type = ?frame.header.ty,
6516 "dropping late or unknown route.bind relay response"
6517 );
6518 }
6519 Ok(secondary_error.into_iter().collect())
6520 }
6521
6522 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6523 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6524 self.cleanup_connection_with_end_reason(
6528 connection_id,
6529 RegistrationEndReason::ExplicitGoodbye,
6530 )
6531 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6532 Ok(Vec::new())
6533 }
6534}
6535
6536impl Default for ControlHandler {
6537 fn default() -> Self {
6538 Self::new(Arc::new(Registry::default()))
6539 }
6540}
6541
6542impl crate::supervise::SwapPromotionObserver for ControlHandler {
6543 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6544 self.apply_registration_capabilities(registration);
6545 }
6546}
6547
6548fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6549 CapabilityRequirementStatus {
6550 consumer: status.consumer,
6551 capability: status.capability,
6552 need: match status.need {
6553 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6554 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6555 },
6556 verdict: status.verdict.as_str().to_string(),
6557 episode_seq: status.episode_seq,
6558 config_satisfiable: status.config_satisfiable,
6559 runtime_available: status.runtime_available,
6560 detail: status.detail,
6561 }
6562}
6563
6564fn append_capability_problem_detail(
6565 detail: Option<String>,
6566 capability_detail: Option<String>,
6567) -> Option<String> {
6568 match (detail, capability_detail) {
6569 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6570 (Some(detail), None) => Some(detail),
6571 (None, Some(capability_detail)) => Some(capability_detail),
6572 (None, None) => None,
6573 }
6574}
6575
6576fn subc_ops() -> Vec<String> {
6577 SUBC_CONTROL_OPS
6578 .iter()
6579 .map(|op| (*op).to_string())
6580 .collect()
6581}
6582
6583fn module_subc_ops() -> Vec<String> {
6584 SUBC_CONTROL_OPS
6585 .iter()
6586 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6587 .map(|op| (*op).to_string())
6588 .collect()
6589}
6590
6591#[cfg(test)]
6592fn module_baseline_control_ops() -> Vec<String> {
6593 MODULE_BASELINE_CONTROL_OPS
6594 .iter()
6595 .map(|op| (*op).to_string())
6596 .collect()
6597}
6598
6599fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6600 let mut seen = HashSet::new();
6601 let mut effective = Vec::new();
6602 for op in MODULE_BASELINE_CONTROL_OPS {
6603 if seen.insert((*op).to_string()) {
6604 effective.push((*op).to_string());
6605 }
6606 }
6607 for op in declared.unwrap_or_default() {
6608 if seen.insert(op.clone()) {
6609 effective.push(op);
6610 }
6611 }
6612 effective
6613}
6614
6615fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6616 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6617}
6618
6619fn target_module_id(target: &RouteTarget) -> &str {
6620 match target {
6621 RouteTarget::ToolProvider { module_id }
6622 | RouteTarget::ManagementSurface { module_id }
6623 | RouteTarget::InternalService { module_id, .. } => module_id,
6624 }
6625}
6626
6627fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6628 roles.iter().any(|role| match (target, role) {
6629 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6630 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6631 (
6632 RouteTarget::InternalService { service_id, .. },
6633 ProviderRole::InternalService {
6634 service_id: provided,
6635 ..
6636 },
6637 ) => service_id == provided,
6638 _ => false,
6639 })
6640}
6641
6642fn is_routable_role(role: &ProviderRole) -> bool {
6643 matches!(
6644 role,
6645 ProviderRole::ToolProvider { .. }
6646 | ProviderRole::ManagementSurface { .. }
6647 | ProviderRole::InternalService { .. }
6648 )
6649}
6650
6651#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6652enum ControlRequestBodyError {
6653 UnknownOp,
6654 InvalidBody,
6655}
6656
6657#[derive(Debug, Deserialize)]
6658struct ControlOpProbe {
6659 op: String,
6660}
6661
6662const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6665
6666fn is_known_module_push_op(body: &[u8]) -> bool {
6667 serde_json::from_slice::<ControlOpProbe>(body)
6668 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6669 .unwrap_or(false)
6670}
6671
6672fn is_known_module_request_op(body: &[u8]) -> bool {
6673 serde_json::from_slice::<ControlOpProbe>(body)
6674 .map(|probe| is_module_to_subc_op(&probe.op))
6675 .unwrap_or(false)
6676}
6677
6678fn is_module_to_subc_op(op: &str) -> bool {
6679 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6680}
6681
6682fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6683 debug!(
6684 op = %op,
6685 connection_id = connection_id.get(),
6686 corr,
6687 "control dispatch"
6688 );
6689}
6690
6691fn log_slow_control_dispatch(
6692 dispatch_started_at: Option<StdInstant>,
6693 op: &'static str,
6694 connection_id: ConnectionId,
6695 corr: u64,
6696) {
6697 let Some(dispatch_started_at) = dispatch_started_at else {
6698 return;
6699 };
6700 let elapsed = dispatch_started_at.elapsed();
6701 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6702 warn!(
6703 op = %op,
6704 connection_id = connection_id.get(),
6705 corr,
6706 elapsed_ms = elapsed.as_millis() as u64,
6707 "slow control dispatch"
6708 );
6709 }
6710}
6711
6712#[cfg(test)]
6714type TestCallerLookup =
6715 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
6716
6717fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
6719 match request {
6720 ClientControlRequest::SupervisorRestart { module_id, .. } => {
6721 Some(PrivilegedOperation::Restart { module_id })
6722 }
6723 ClientControlRequest::SupervisorSwap { module_id, .. } => {
6724 Some(PrivilegedOperation::Swap { module_id })
6725 }
6726 ClientControlRequest::SupervisorReload { module_id } => {
6727 Some(PrivilegedOperation::Reload { module_id })
6728 }
6729 ClientControlRequest::SupervisorRescan { preview } => {
6730 if *preview {
6731 None
6732 } else {
6733 Some(PrivilegedOperation::Rescan)
6734 }
6735 }
6736 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
6737 Some(PrivilegedOperation::SetEnabled {
6738 module_id,
6739 enabled: *enabled,
6740 })
6741 }
6742 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
6743 Some(PrivilegedOperation::ReleaseReserved { module_id })
6744 }
6745 ClientControlRequest::ServerDescribe {}
6746 | ClientControlRequest::CatalogList { .. }
6747 | ClientControlRequest::RouteOpen { .. }
6748 | ClientControlRequest::RoutePoll { .. }
6749 | ClientControlRequest::SupervisorList {}
6750 | ClientControlRequest::SupervisorHealth {}
6751 | ClientControlRequest::SupervisorHealthProbe { .. }
6752 | ClientControlRequest::SupervisorRoutes { .. }
6753 | ClientControlRequest::SupervisorProvenance { .. }
6754 | ClientControlRequest::SupervisorStderrTail { .. }
6755 | ClientControlRequest::SupervisorTerminals { .. }
6756 | ClientControlRequest::SupervisorSpawnSnapshot {}
6757 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
6758 }
6759}
6760
6761fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6762 match request {
6763 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6764 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6765 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6766 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6767 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6768 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6769 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6770 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6771 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6772 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6773 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6774 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6775 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6776 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6777 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6778 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6779 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6780 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6781 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6782 }
6783}
6784
6785fn principal_label(principal: &Principal) -> String {
6786 match principal {
6787 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6788 Principal::Direct => "direct".to_string(),
6789 other => format!("{other:?}"),
6790 }
6791}
6792
6793fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6794 match request {
6795 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6796 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6797 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6798 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6799 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6800 }
6801}
6802
6803fn parse_client_control_request(
6804 body: &[u8],
6805) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6806 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6807 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6808 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6809 ControlRequestBodyError::InvalidBody
6810 }
6811 Ok(_) => ControlRequestBodyError::UnknownOp,
6812 Err(_) => ControlRequestBodyError::InvalidBody,
6813 };
6814 (err, classification)
6815 })
6816}
6817
6818fn parse_module_control_request_from_module(
6819 body: &[u8],
6820) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6821 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6822 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6823 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6824 Ok(_) => ControlRequestBodyError::UnknownOp,
6825 Err(_) => ControlRequestBodyError::InvalidBody,
6826 };
6827 (err, classification)
6828 })
6829}
6830
6831#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6832enum ProviderRoleKind {
6833 ToolProvider,
6834 PipelineStage,
6835 ManagementSurface,
6836 InternalService,
6837}
6838
6839fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6840 match role {
6841 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6842 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6843 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6844 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6845 }
6846}
6847
6848fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6849 roles.iter().map(provider_role_kind).collect()
6850}
6851
6852const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6855
6856#[derive(Debug, Default, PartialEq, Eq)]
6858struct ScopeOutcomeCounts {
6859 created: usize,
6860 replaced: usize,
6861 updated: usize,
6862 unchanged: usize,
6863 refused: usize,
6864}
6865
6866impl ScopeOutcomeCounts {
6867 fn of(results: &[ScopeRecordResult]) -> Self {
6868 let mut counts = Self::default();
6869 for result in results {
6870 let slot = match result.outcome {
6871 ScopeRecordOutcome::Created => &mut counts.created,
6872 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6873 ScopeRecordOutcome::Updated => &mut counts.updated,
6874 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6875 ScopeRecordOutcome::Refused => &mut counts.refused,
6876 };
6877 *slot += 1;
6878 }
6879 counts
6880 }
6881}
6882
6883#[cfg(test)]
6884mod scope_outcome_count_tests {
6885 use super::*;
6886
6887 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6888 ScopeRecordResult {
6889 scope_ref: "r".to_string(),
6890 scope_epoch: 1,
6891 outcome,
6892 code: None,
6893 message: None,
6894 version: None,
6895 parent_state: None,
6896 }
6897 }
6898
6899 #[test]
6902 fn every_outcome_is_counted_in_its_own_field() {
6903 let results = [
6904 result(ScopeRecordOutcome::Created),
6905 result(ScopeRecordOutcome::Created),
6906 result(ScopeRecordOutcome::Replaced),
6907 result(ScopeRecordOutcome::Updated),
6908 result(ScopeRecordOutcome::Unchanged),
6909 result(ScopeRecordOutcome::Refused),
6910 result(ScopeRecordOutcome::Refused),
6911 result(ScopeRecordOutcome::Refused),
6912 ];
6913 assert_eq!(
6914 ScopeOutcomeCounts::of(&results),
6915 ScopeOutcomeCounts {
6916 created: 2,
6917 replaced: 1,
6918 updated: 1,
6919 unchanged: 1,
6920 refused: 3,
6921 }
6922 );
6923 }
6924}
6925
6926fn capability_census_trigger(
6930 old: Option<&CapabilityDeclarations>,
6931 new: Option<&CapabilityDeclarations>,
6932) -> bool {
6933 let old_provides = old
6934 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6935 .unwrap_or_default();
6936 let old_denies = old
6937 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6938 .unwrap_or_default();
6939 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6940 provides: Vec::new(),
6941 requires: Vec::new(),
6942 must_never_reach: Vec::new(),
6943 });
6944
6945 new.provides
6946 .iter()
6947 .any(|capability| !old_provides.contains(capability))
6948 || new
6949 .must_never_reach
6950 .iter()
6951 .any(|capability| !old_denies.contains(capability))
6952}
6953
6954fn denied_capability<'a>(
6957 opening_manifest: &'a ModuleManifest,
6958 target_manifest: &ModuleManifest,
6959) -> Option<&'a str> {
6960 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6961 let target_capabilities = target_manifest.capabilities.as_ref()?;
6962 opening_capabilities
6963 .must_never_reach
6964 .iter()
6965 .find(|denied| {
6966 target_capabilities
6967 .provides
6968 .iter()
6969 .any(|provided| provided == *denied)
6970 })
6971 .map(String::as_str)
6972}
6973
6974fn catalog_update_frozen_field_message(
6975 registered: &ModuleManifest,
6976 provides: &[ProviderRole],
6977) -> Option<String> {
6978 let old_has_provides = !registered.provides.is_empty();
6979 let new_has_provides = !provides.is_empty();
6980 if old_has_provides != new_has_provides {
6981 return Some(format!(
6982 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6983 registered.module_id
6984 ));
6985 }
6986
6987 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6988 return Some(format!(
6989 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6990 registered.module_id
6991 ));
6992 }
6993
6994 let registered_concurrency = manifest_concurrency(registered);
6995 let mut candidate = registered.clone();
6996 candidate.provides = provides.to_vec();
6997 let candidate_concurrency = manifest_concurrency(&candidate);
6998 if candidate_concurrency != registered_concurrency {
6999 return Some(format!(
7000 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7001 registered.module_id, registered_concurrency, candidate_concurrency
7002 ));
7003 }
7004
7005 None
7008}
7009
7010fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7011 manifest.provides.iter().any(is_routable_role)
7012}
7013
7014fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7020 manifest
7021 .provides
7022 .iter()
7023 .find_map(|provider| match provider {
7024 ProviderRole::ToolProvider { concurrency, .. }
7025 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7026 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7027 })
7028 .unwrap_or(Concurrency::ModuleManaged)
7029}
7030
7031fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7038 let has_management_surface = manifest
7039 .provides
7040 .iter()
7041 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7042 if !has_management_surface {
7043 return false;
7044 }
7045 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7046 return false;
7047 };
7048 let Some(provides) = raw
7049 .get("manifest")
7050 .and_then(|manifest| manifest.get("provides"))
7051 .and_then(serde_json::Value::as_array)
7052 else {
7053 return false;
7054 };
7055 provides.iter().any(|role| {
7060 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7061 && role.get("concurrency").is_none()
7062 })
7063}
7064
7065fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7066 if peer_version != PROTOCOL_VERSION {
7067 return Err(format!(
7068 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7069 ));
7070 }
7071 Ok(PROTOCOL_VERSION)
7072}
7073
7074fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7075 Frame::build_with_version(
7076 response_version(frame),
7077 FrameType::Pong,
7078 frame.header.flags,
7079 0,
7080 0,
7081 frame.header.corr,
7082 Vec::new(),
7083 )
7084 .map_err(RouterError::FrameBuild)
7085}
7086
7087pub(crate) fn control_error_frame(
7088 frame: &Frame,
7089 code: &'static str,
7090 message: impl Into<String>,
7091) -> Result<Frame, RouterError> {
7092 control_error_body_frame(
7093 frame,
7094 ErrorBody {
7095 code: code.to_string(),
7096 message: message.into(),
7097 detail: None,
7098 },
7099 )
7100}
7101
7102fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7103 let body = serde_json::to_vec(&error).map_err(|err| {
7104 RouterError::backend(
7105 0,
7106 frame.header.corr,
7107 format!("failed to encode control ERROR: {err}"),
7108 )
7109 })?;
7110
7111 Frame::build_with_version(
7112 response_version(frame),
7113 FrameType::Error,
7114 control_flags(),
7115 0,
7116 0,
7117 frame.header.corr,
7118 body,
7119 )
7120 .map_err(RouterError::FrameBuild)
7121}
7122
7123fn control_response_body_frame<T: Serialize>(
7124 frame: &Frame,
7125 reply: &T,
7126 label: &'static str,
7127) -> Result<Frame, RouterError> {
7128 let body = serde_json::to_vec(reply).map_err(|err| {
7129 RouterError::backend(
7130 0,
7131 frame.header.corr,
7132 format!("failed to encode {label}: {err}"),
7133 )
7134 })?;
7135
7136 Frame::build_with_version(
7137 response_version(frame),
7138 FrameType::Response,
7139 control_flags(),
7140 0,
7141 0,
7142 frame.header.corr,
7143 body,
7144 )
7145 .map_err(RouterError::FrameBuild)
7146}
7147
7148fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7178 match err {
7179 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7180 ForwardingError::NoModuleConnection => "target_unavailable",
7181 ForwardingError::ModuleReloading { .. } => "module_reloading",
7182 ForwardingError::ClientRouteChannelExhausted { .. }
7183 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7184 ForwardingError::StaleModuleEndpoint
7185 | ForwardingError::UnknownReservation { .. }
7186 | ForwardingError::ConnectionClosing { .. }
7187 | ForwardingError::ClientEgressClosed { .. }
7188 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7189 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7192 ForwardingError::RelayCorrelationExhausted
7193 | ForwardingError::RouteOpenBuild(_)
7194 | ForwardingError::Poisoned => "forwarding_error",
7195 }
7196}
7197
7198fn response_version(frame: &Frame) -> u8 {
7199 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7200 frame.header.ver
7201 } else {
7202 PROTOCOL_VERSION
7203 }
7204}
7205
7206fn control_flags() -> Flags {
7207 Flags::new(false, Priority::Passive, false)
7208}
7209
7210fn send_goodbye_target_best_effort(
7215 counters: &DaemonCounters,
7216 target: &GoodbyeTarget,
7217 context: &'static str,
7218) {
7219 let Ok(frame) = Frame::build_with_version(
7220 target.negotiated_ver,
7221 FrameType::Goodbye,
7222 control_flags(),
7223 target.channel,
7224 target.epoch,
7225 0,
7226 Vec::new(),
7227 ) else {
7228 return;
7229 };
7230 crate::forwarding::send_module_route_goodbye(
7231 counters,
7232 &target.sink,
7233 frame,
7234 target.module_id.as_deref(),
7235 context,
7236 );
7237}
7238
7239pub(crate) fn send_route_control_pushes(
7240 forwarding: &ForwardingTable,
7241 routes: Vec<EndpointRoute>,
7242 push: ClientControlPush,
7243) {
7244 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7245 for route in routes {
7246 let target = route.goodbye_target;
7247 if let Some((existing, channels)) = targets
7248 .iter_mut()
7249 .find(|(existing, _)| existing.connection_id == target.connection_id)
7250 {
7251 debug_assert_eq!(
7252 existing.negotiated_ver, target.negotiated_ver,
7253 "one connection cannot negotiate multiple frame versions"
7254 );
7255 if !channels.contains(&target.channel) {
7256 channels.push(target.channel);
7257 }
7258 continue;
7259 }
7260 let channel = target.channel;
7261 targets.push((target, vec![channel]));
7262 }
7263 for (target, mut channels) in targets {
7264 channels.sort_unstable();
7265 let mut push = push.clone();
7266 match &mut push {
7267 ClientControlPush::RouteClosing {
7268 channels: covered, ..
7269 }
7270 | ClientControlPush::RouteClosed {
7271 channels: covered, ..
7272 } => *covered = channels,
7273 }
7274 let body = match serde_json::to_vec(&push) {
7275 Ok(body) => body,
7276 Err(err) => {
7277 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7278 continue;
7279 }
7280 };
7281 let frame = match Frame::build_with_version(
7282 target.negotiated_ver,
7283 FrameType::Push,
7284 control_flags(),
7285 0,
7286 0,
7287 0,
7288 body.clone(),
7289 ) {
7290 Ok(frame) => frame,
7291 Err(err) => {
7292 warn!(
7293 route_channel = target.channel,
7294 error = %err,
7295 "failed to build route lifecycle control PUSH frame"
7296 );
7297 continue;
7298 }
7299 };
7300 if let Err(err) = target.sink.try_send(frame) {
7301 if target.close_on_delivery_failure() {
7302 warn!(
7303 target_connection_id = target.connection_id.get(),
7304 route_channel = target.channel,
7305 error = %err,
7306 "route lifecycle control PUSH was not delivered to client; closing target connection"
7307 );
7308 let _ = forwarding.escalate_client_delivery_failure(
7309 target.connection_id,
7310 target.channel,
7311 target.epoch,
7312 CloseReason::new(
7313 "route_lifecycle_push_delivery_failed",
7314 format!(
7315 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7316 target.channel
7317 ),
7318 ),
7319 crate::forwarding::UndeliveredFrame {
7320 module_id: target.module_id.as_deref(),
7321 sink: &target.sink,
7322 },
7323 );
7324 }
7325 }
7326 }
7327}
7328
7329#[cfg(test)]
7330mod tests {
7331 mod activity_read {
7332 include!("activity_read_tests.rs");
7333 }
7334
7335 mod lifecycle_activity {
7336 include!("control_lifecycle_tests.rs");
7337 }
7338 #[cfg(unix)]
7339 #[tokio::test]
7340 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7341 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7342 let path = dir.join("subc.jsonc");
7343 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7344 "program":"/bin/sleep","args":["60"],"protocol":"none",
7345 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7346 }}}).to_string()).unwrap();
7347 let mut configured = crate::daemon_config::load(&path)
7348 .unwrap()
7349 .unwrap()
7350 .modules
7351 .pop()
7352 .unwrap();
7353 let registry = std::sync::Arc::new(crate::Registry::default());
7354 let handle = crate::SupervisorHandle::new();
7355 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7356 .with_handle(handle.clone());
7357 let module = supervisor
7358 .supervise_configured_with_health(
7359 configured.module_spec(),
7360 true,
7361 configured.health.clone(),
7362 None,
7363 configured.restart,
7364 )
7365 .unwrap();
7366 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7367 let before = module.status().unwrap().pid;
7368 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7369 configured.health.cadence = std::time::Duration::from_secs(3600);
7370 let health_only = handler
7371 .reconcile_supervised_modules(
7372 &supervisor,
7373 vec![configured.clone()],
7374 false,
7375 Principal::Direct,
7376 )
7377 .await
7378 .unwrap();
7379 assert!(
7380 health_only.changed_pending_reload.is_empty(),
7381 "health policy is already applied live"
7382 );
7383 assert_eq!(module.status().unwrap().pid, before);
7384 assert_eq!(
7385 module.configuration().unwrap().1.http,
7386 configured.health.http
7387 );
7388 configured.args = vec!["61".into()];
7389 let launch = handler
7390 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7391 .await
7392 .unwrap();
7393 assert_eq!(launch.changed_pending_reload, ["stock"]);
7394 assert_eq!(
7395 module.status().unwrap().pid,
7396 before,
7397 "a launch edit is stored until reload"
7398 );
7399 module.drain().await.unwrap();
7400 }
7401 use cortexkit_test_support::ScratchDir;
7402 use std::{
7403 collections::BTreeMap,
7404 fmt,
7405 path::PathBuf,
7406 sync::{Arc, Mutex},
7407 time::Duration,
7408 };
7409
7410 use serde_json::{json, Value};
7411 use subc_protocol::{
7412 manifest::{
7413 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7414 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7415 },
7416 session::HealthStatus,
7417 FrameType,
7418 };
7419
7420 use super::*;
7421 use crate::{
7422 forwarding::{DataRoute, DataRouteState},
7423 registry::ChannelState,
7424 router::FrameSink,
7425 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7426 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7427 RouteCtx, Router,
7428 };
7429 use tokio::{
7430 sync::mpsc,
7431 time::{sleep, Instant},
7432 };
7433 use tracing::{
7434 field::{Field, Visit},
7435 Event, Subscriber,
7436 };
7437 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7438
7439 fn fake_aft_stub_path() -> PathBuf {
7458 let mut path = std::env::current_exe().expect("current_exe available in tests");
7459 path.pop(); path.pop(); path.push(if cfg!(windows) {
7462 "fake-aft-stub.exe"
7463 } else {
7464 "fake-aft-stub"
7465 });
7466 assert!(
7467 path.exists(),
7468 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7469 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7470 path.display()
7471 );
7472 path
7473 }
7474
7475 fn client_retries(code: &str) -> bool {
7480 subc_protocol::error_codes::is_retryable_route_open(code)
7481 }
7482
7483 #[test]
7494 fn retryability_of_forwarding_codes_matches_the_failure() {
7495 let transient = [
7498 ForwardingError::NoModuleConnection,
7499 ForwardingError::ModuleReloading {
7500 module_id: "m".into(),
7501 },
7502 ForwardingError::StaleModuleEndpoint,
7503 ForwardingError::UnknownReservation {
7504 client_channel: 1,
7505 module_channel: 1,
7506 },
7507 ForwardingError::ConnectionClosing {
7508 connection_id: ConnectionId::new(1),
7509 },
7510 ForwardingError::ClientEgressClosed {
7511 connection_id: ConnectionId::new(1),
7512 },
7513 ForwardingError::ModuleEgressUnavailable {
7514 connection_id: ConnectionId::new(1),
7515 },
7516 ];
7517 for err in transient {
7518 let code = forwarding_error_code(&err);
7519 assert!(
7520 client_retries(code),
7521 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7522 );
7523 }
7524
7525 let permanent = [
7530 ForwardingError::ConnectionRoleConflict {
7531 connection_id: ConnectionId::new(1),
7532 },
7533 ForwardingError::ClientRouteChannelExhausted {
7534 connection_id: ConnectionId::new(1),
7535 },
7536 ForwardingError::ModuleRouteChannelExhausted {
7537 endpoint: ModuleEndpointId {
7538 connection_id: ConnectionId::new(1),
7539 generation: 1,
7540 },
7541 },
7542 ForwardingError::RelayCorrelationExhausted,
7543 ForwardingError::RouteOpenBuild("x".into()),
7544 ForwardingError::Poisoned,
7545 ];
7546 for err in permanent {
7547 let code = forwarding_error_code(&err);
7548 assert!(
7549 !client_retries(code),
7550 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7551 );
7552 }
7553 }
7554
7555 #[tokio::test]
7583 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7584 let handler = ControlHandler::default();
7585 let frame =
7586 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7587
7588 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7592 assert_eq!(
7593 stamped,
7594 Principal::Direct,
7595 "a caller that proved nothing must not be stamped as a supervised module"
7596 );
7597
7598 let forged = handler
7603 .route_open_principal(
7604 &frame,
7605 Some(ConsumerIdentity {
7606 module_id: "aft".to_string(),
7607 launch_nonce: "not-a-real-nonce".to_string(),
7608 }),
7609 )
7610 .unwrap();
7611 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7612 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7613 }
7614
7615 #[test]
7632 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7633 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7634 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7635 let ClientControlRequest::RouteOpen {
7636 consumer_identity, ..
7637 } = parsed
7638 else {
7639 panic!("route.open body must parse as RouteOpen");
7640 };
7641 assert_eq!(
7642 consumer_identity,
7643 Some(ConsumerIdentity {
7644 module_id: "aft".to_string(),
7645 launch_nonce: "n".to_string(),
7646 }),
7647 "the wire field name must reach the value route_open_principal reads"
7648 );
7649 }
7650
7651 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7652 ModuleManifest::builder(module_id, "0.1.0")
7653 .protocol_ver(protocol_ver)
7654 .provides(vec![ProviderRole::ToolProvider {
7655 tools: vec![Tool {
7656 name: "read".to_string(),
7657 description: None,
7658 execution_mode: ExecutionMode::Pure,
7659 schema: json!({"type": "object"}),
7660 }],
7661 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7662 concurrency: Concurrency::ModuleManaged,
7663 emits_push: true,
7664 sub_supervises: true,
7665 }])
7666 .build()
7667 }
7668
7669 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7670 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7671 }
7672
7673 fn hello_frame_with_control_ops(
7674 module_id: &str,
7675 protocol_ver: u8,
7676 corr: u64,
7677 control_ops: Option<Vec<String>>,
7678 ) -> Frame {
7679 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7680 }
7681
7682 fn hello_frame_with_nonce(
7683 module_id: &str,
7684 protocol_ver: u8,
7685 corr: u64,
7686 launch_nonce: Option<&str>,
7687 ) -> Frame {
7688 hello_frame_full(
7689 module_id,
7690 protocol_ver,
7691 corr,
7692 None,
7693 launch_nonce.map(ToOwned::to_owned),
7694 )
7695 }
7696
7697 fn hello_frame_full(
7698 module_id: &str,
7699 protocol_ver: u8,
7700 corr: u64,
7701 control_ops: Option<Vec<String>>,
7702 launch_nonce: Option<String>,
7703 ) -> Frame {
7704 let body = serde_json::to_vec(&ModuleHelloBody {
7705 manifest: manifest(module_id, protocol_ver),
7706 protocol_ver,
7707 control_ops,
7708 launch_nonce,
7709 })
7710 .unwrap();
7711 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7712 }
7713
7714 fn non_routable_hello_frame_with_control_ops(
7715 module_id: &str,
7716 corr: u64,
7717 control_ops: Option<Vec<String>>,
7718 ) -> Frame {
7719 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7720 manifest.provides.clear();
7721 let body = serde_json::to_vec(&ModuleHelloBody {
7722 manifest,
7723 protocol_ver: PROTOCOL_VERSION,
7724 control_ops,
7725 launch_nonce: None,
7726 })
7727 .unwrap();
7728 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7729 }
7730
7731 fn capability_grammar_hello_frame(
7732 capabilities: Value,
7733 runtime_computed: Option<Value>,
7734 corr: u64,
7735 ) -> Frame {
7736 let mut body = serde_json::to_value(ModuleHelloBody {
7737 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7738 protocol_ver: PROTOCOL_VERSION,
7739 control_ops: None,
7740 launch_nonce: None,
7741 })
7742 .expect("HELLO body serializes");
7743 body["manifest"]["capabilities"] = capabilities;
7744 if let Some(runtime_computed) = runtime_computed {
7745 body["runtime_computed"] = runtime_computed;
7746 }
7747 Frame::build(
7748 FrameType::Hello,
7749 control_flags(),
7750 0,
7751 0,
7752 corr,
7753 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7754 )
7755 .expect("HELLO frame builds")
7756 }
7757
7758 fn channel_request(channel: u16, corr: u64) -> Frame {
7759 Frame::build(
7760 FrameType::Request,
7761 Flags::new(true, Priority::Interactive, false),
7762 channel,
7763 0,
7764 corr,
7765 b"opaque".to_vec(),
7766 )
7767 .unwrap()
7768 }
7769
7770 fn route_ctx(
7771 connection_id: ConnectionId,
7772 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7773 let (tx, rx) = mpsc::channel(8);
7774 (
7775 RouteCtx {
7776 connection_id,
7777 egress: FrameSink::new(tx),
7778 },
7779 rx,
7780 )
7781 }
7782
7783 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7784 serde_json::from_slice(&frame.body).unwrap()
7785 }
7786
7787 async fn hello_via_sink(
7792 handler: &ControlHandler,
7793 ctx: &RouteCtx,
7794 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7795 hello: Frame,
7796 ) -> Frame {
7797 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7798 assert!(
7799 replies.is_empty(),
7800 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7801 );
7802 let ack = rx
7803 .try_recv()
7804 .expect("HELLO_ACK is queued on the module sink")
7805 .frame;
7806 assert_eq!(ack.header.ty, FrameType::HelloAck);
7807 ack
7808 }
7809
7810 fn parse_error(frame: &Frame) -> Value {
7811 serde_json::from_slice(&frame.body).unwrap()
7812 }
7813
7814 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7815 serde_json::from_slice(&frame.body).unwrap()
7816 }
7817
7818 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7819 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7820 route_channel,
7821 route_epoch: 0,
7822 kind,
7823 })
7824 .unwrap();
7825 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7826 }
7827
7828 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7829 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7830 module_id: module_id.to_string(),
7831 })
7832 .unwrap();
7833 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7834 }
7835
7836 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7837 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7838 }
7839
7840 fn route_open_frame_with_consumer_capabilities(
7841 corr: u64,
7842 module_id: &str,
7843 project_root: ScratchDir,
7844 consumer_capabilities: Option<Vec<String>>,
7845 ) -> Frame {
7846 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7847 target: RouteTarget::ToolProvider {
7848 module_id: module_id.to_string(),
7849 },
7850 identity: BindIdentity::new(
7851 project_root.path().to_path_buf(),
7852 "unit".to_string(),
7853 "session".to_string(),
7854 ),
7855 consumer_identity: None,
7856 consumer_capabilities,
7857 role_versions: None,
7858 admission_facts: None,
7859 scope: None,
7860 })
7861 .unwrap();
7862 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7863 }
7864
7865 fn route_open_frame_with_role_versions(
7866 corr: u64,
7867 module_id: &str,
7868 project_root: ScratchDir,
7869 role_versions: Option<BTreeMap<String, String>>,
7870 ) -> Frame {
7871 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7872 target: RouteTarget::ToolProvider {
7873 module_id: module_id.to_string(),
7874 },
7875 identity: BindIdentity::new(
7876 project_root.path().to_path_buf(),
7877 "unit".to_string(),
7878 format!("session-{corr}"),
7879 ),
7880 consumer_identity: None,
7881 consumer_capabilities: None,
7882 role_versions,
7883 admission_facts: None,
7884 scope: None,
7885 })
7886 .unwrap();
7887 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7888 }
7889
7890 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7891 entries
7892 .iter()
7893 .map(|(role, version)| (role.to_string(), version.to_string()))
7894 .collect()
7895 }
7896
7897 fn route_open_frame_with_admission_facts(
7898 corr: u64,
7899 module_id: &str,
7900 project_root: ScratchDir,
7901 consumer_identity: Option<subc_control::ConsumerIdentity>,
7902 facts: Option<Value>,
7903 ) -> Frame {
7904 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7905 target: RouteTarget::ToolProvider {
7906 module_id: module_id.to_string(),
7907 },
7908 identity: BindIdentity::new(
7909 project_root.path().to_path_buf(),
7910 "unit".to_string(),
7911 format!("session-{corr}"),
7912 ),
7913 consumer_identity,
7914 consumer_capabilities: None,
7915 role_versions: None,
7916 admission_facts: facts,
7917 scope: None,
7918 })
7919 .unwrap();
7920 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7921 }
7922
7923 #[derive(Clone, Default)]
7924 struct EventCapture {
7925 events: Arc<Mutex<Vec<CapturedEvent>>>,
7926 }
7927
7928 #[derive(Clone, Debug)]
7929 struct CapturedEvent {
7930 target: String,
7931 level: tracing::Level,
7932 fields: BTreeMap<String, String>,
7933 }
7934
7935 impl EventCapture {
7936 fn events(&self) -> Vec<CapturedEvent> {
7937 self.events.lock().unwrap().clone()
7938 }
7939 }
7940
7941 impl<S> Layer<S> for EventCapture
7942 where
7943 S: Subscriber,
7944 {
7945 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7946 let mut visitor = EventFieldVisitor::default();
7947 event.record(&mut visitor);
7948 self.events.lock().unwrap().push(CapturedEvent {
7949 target: event.metadata().target().to_string(),
7950 level: *event.metadata().level(),
7951 fields: visitor.fields,
7952 });
7953 }
7954 }
7955
7956 #[derive(Default)]
7957 struct EventFieldVisitor {
7958 fields: BTreeMap<String, String>,
7959 }
7960
7961 impl Visit for EventFieldVisitor {
7962 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7963 self.fields
7964 .insert(field.name().to_string(), format!("{value:?}"));
7965 }
7966 }
7967
7968 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7969 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7970 status,
7971 detail: Some("warming".to_string()),
7972 metrics: Some(json!({"queue_depth": 3})),
7973 })
7974 .unwrap();
7975 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7976 }
7977
7978 fn route_bind_ack(corr: u64) -> Frame {
7979 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7980 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7981 }
7982
7983 fn unique_project_root(label: &str) -> ScratchDir {
7984 ScratchDir::new(label)
7985 }
7986
7987 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7988 match parse_route_poll(frame) {
7989 ClientControlResponse::RoutePoll {
7990 status: None,
7991 live: Some(live),
7992 ..
7993 } => assert_eq!(live, expected_live),
7994 other => panic!("unexpected route.poll response: {other:?}"),
7995 }
7996 }
7997
7998 fn bind_liveness_route(
7999 registry: &Registry,
8000 forwarding: &ForwardingTable,
8001 module_id: &str,
8002 ) -> (RouteCtx, u16, u32) {
8003 let module_connection = ConnectionId::new(101);
8004 let client_connection = ConnectionId::new(202);
8005 let registration = registry
8006 .register_with_control_ops(
8007 manifest(module_id, PROTOCOL_VERSION),
8008 PROTOCOL_VERSION,
8009 module_connection,
8010 module_baseline_control_ops(),
8011 )
8012 .unwrap();
8013 let (module_tx, _module_rx) = mpsc::channel(8);
8014 let endpoint = forwarding
8015 .register_module_connection(
8016 module_connection,
8017 module_id.to_string(),
8018 PROTOCOL_VERSION,
8019 manifest_concurrency(®istration.manifest),
8020 FrameSink::new(module_tx),
8021 )
8022 .unwrap();
8023 let (client_ctx, _client_rx) = route_ctx(client_connection);
8024 let pending = forwarding
8025 .begin_route_bind_relay_for_test(
8026 client_connection,
8027 client_ctx.egress.clone(),
8028 1,
8029 module_id,
8030 )
8031 .unwrap();
8032 assert_eq!(pending.endpoint, endpoint);
8033 let route_channel = pending.client_channel;
8034 let route_epoch = pending.client_epoch;
8035 forwarding
8036 .complete_pending_relay(
8037 module_connection,
8038 pending.corr,
8039 RouteBindRelayOutcome::Accepted,
8040 )
8041 .unwrap();
8042 (client_ctx, route_channel, route_epoch)
8043 }
8044
8045 struct FakeProcessLiveness {
8046 live: Option<bool>,
8047 }
8048
8049 impl ModuleProcessLiveness for FakeProcessLiveness {
8050 fn process_live(&self, _module_id: &str) -> Option<bool> {
8051 self.live
8052 }
8053 }
8054
8055 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8056 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8057 {
8058 let registry = Arc::new(Registry::default());
8059 let supervisor_handle = SupervisorHandle::new();
8060 let supervisor = Supervisor::new_for_test(
8061 Arc::clone(®istry),
8062 RestartPolicy::new(1, Duration::from_millis(10)),
8063 )
8064 .with_handle(supervisor_handle.clone());
8065 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8066 let module = supervisor
8067 .spawn(ModuleSpec {
8068 module_id: "stderr-tail-wire".to_string(),
8069 program: fake_aft_stub_path(),
8070 args: Vec::new(),
8071 env: vec![
8072 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8073 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8074 ],
8075 reserved: false,
8076 reserved_prefixes: Vec::new(),
8077 protocol: ModuleProtocol::Subc,
8078 overlap: Default::default(),
8079 })
8080 .unwrap();
8081
8082 let deadline = Instant::now() + Duration::from_secs(5);
8083 loop {
8084 let tail = module.stderr_tail(None, None);
8085 if tail
8086 .entries
8087 .iter()
8088 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8089 && tail.entries.iter().any(|entry| {
8090 matches!(
8091 entry,
8092 TailEntry::Line {
8093 truncated: true,
8094 ..
8095 }
8096 )
8097 })
8098 {
8099 break;
8100 }
8101 assert!(
8102 Instant::now() < deadline,
8103 "module did not produce a truncated line and restart boundary: {tail:?}"
8104 );
8105 sleep(Duration::from_millis(10)).await;
8106 }
8107
8108 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8109 let request = ClientControlRequest::SupervisorStderrTail {
8110 module_id: "stderr-tail-wire".to_string(),
8111 max_lines: None,
8112 max_bytes: None,
8113 };
8114 let frame = Frame::build(
8115 FrameType::Request,
8116 control_flags(),
8117 0,
8118 0,
8119 1,
8120 serde_json::to_vec(&request).unwrap(),
8121 )
8122 .unwrap();
8123 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8124 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8125 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8126 serde_json::from_slice(&responses[0].body).unwrap()
8127 else {
8128 panic!("expected supervisor.stderr_tail response");
8129 };
8130
8131 assert!(
8132 tail.entries
8133 .iter()
8134 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8135 "the control response lost the restart boundary"
8136 );
8137 let Some(StderrTailEntry::Line {
8138 text,
8139 truncated,
8140 at_ms,
8141 }) = tail.entries.iter().find(|entry| {
8142 matches!(
8143 entry,
8144 StderrTailEntry::Line {
8145 truncated: true,
8146 ..
8147 }
8148 )
8149 })
8150 else {
8151 panic!("the control response lost the truncated line");
8152 };
8153 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8154 assert!(*truncated);
8155 assert!(
8156 at_ms.is_some(),
8157 "the control response lost the line's capture time"
8158 );
8159 }
8160
8161 #[tokio::test(flavor = "current_thread")]
8166 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8167 let dir = ScratchDir::new("terminals-off-worker");
8168 let journal_path = dir.join("terminals.jsonl");
8169 let registry = Arc::new(Registry::default());
8170 let supervisor_handle = SupervisorHandle::new();
8171 let supervisor =
8172 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8173 .with_handle(supervisor_handle.clone())
8174 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8175 let module = supervisor
8176 .spawn(ModuleSpec {
8177 module_id: "terminal-off-worker".to_string(),
8178 program: fake_aft_stub_path(),
8179 args: Vec::new(),
8180 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8181 reserved: false,
8182 reserved_prefixes: Vec::new(),
8183 protocol: ModuleProtocol::Subc,
8184 overlap: Default::default(),
8185 })
8186 .unwrap();
8187 let deadline = Instant::now() + Duration::from_secs(5);
8188 while module.terminal_history().entries.len() != 2 {
8189 assert!(Instant::now() < deadline, "module did not record two exits");
8190 sleep(Duration::from_millis(10)).await;
8191 }
8192
8193 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8194 let handler =
8195 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8196 let frame = Frame::build(
8197 FrameType::Request,
8198 control_flags(),
8199 0,
8200 0,
8201 1,
8202 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8203 module_id: "terminal-off-worker".to_string(),
8204 })
8205 .unwrap(),
8206 )
8207 .unwrap();
8208 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8209 let spawned_at = std::time::Instant::now();
8210 let read = tokio::spawn({
8211 let handler = Arc::clone(&handler);
8212 async move { handler.handle_control_frame(&ctx, frame).await }
8213 });
8214 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8217 .await
8218 .unwrap()
8219 .expect("the history read reached its pause");
8220 let elapsed = spawned_at.elapsed();
8221 assert!(
8222 elapsed < Duration::from_secs(2) && !read.is_finished(),
8223 "this task could not run while the history read was paused \
8224 (resumed after {elapsed:?}, read finished: {})",
8225 read.is_finished()
8226 );
8227
8228 drop(release);
8229 let responses = read.await.unwrap().unwrap();
8230 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8231 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8232 panic!("expected supervisor.terminals response");
8233 };
8234 assert_eq!(terminals.entries.len(), 2);
8235 assert_eq!(terminals.journal_skipped_lines, 0);
8236 assert_eq!(terminals.journal_read_errors, 0);
8237 }
8238
8239 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8240 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8241 let registry = Arc::new(Registry::default());
8242 let supervisor_handle = SupervisorHandle::new();
8243 let supervisor =
8244 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8245 .with_handle(supervisor_handle.clone());
8246 let module = supervisor
8247 .spawn(ModuleSpec {
8248 module_id: "terminal-golden".to_string(),
8249 program: fake_aft_stub_path(),
8250 args: Vec::new(),
8251 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8252 reserved: false,
8253 reserved_prefixes: Vec::new(),
8254 protocol: ModuleProtocol::Subc,
8255 overlap: Default::default(),
8256 })
8257 .unwrap();
8258
8259 let deadline = Instant::now() + Duration::from_secs(5);
8260 while module.terminal_history().entries.len() != 2 {
8261 assert!(
8262 Instant::now() < deadline,
8263 "module did not retain two terminal exits: {:?}",
8264 module.terminal_history()
8265 );
8266 sleep(Duration::from_millis(10)).await;
8267 }
8268
8269 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8270 let request = ClientControlRequest::SupervisorTerminals {
8271 module_id: "terminal-golden".to_string(),
8272 };
8273 let frame = Frame::build(
8274 FrameType::Request,
8275 control_flags(),
8276 0,
8277 0,
8278 1,
8279 serde_json::to_vec(&request).unwrap(),
8280 )
8281 .unwrap();
8282 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8283 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8284 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8285 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8286 panic!("expected supervisor.terminals response");
8287 };
8288 assert_eq!(terminals.entries.len(), 2);
8289 assert_eq!(terminals.dropped, 0);
8290
8291 let mut rendered = serde_json::to_value(response).unwrap();
8292 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8295 for (index, entry) in rendered["entries"]
8296 .as_array_mut()
8297 .expect("terminal response entries array")
8298 .iter_mut()
8299 .enumerate()
8300 {
8301 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8302 }
8303
8304 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8305 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8306 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8307 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8308 std::fs::write(&golden_path, &serialized).unwrap();
8309 }
8310 let expected: Value =
8311 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8312 assert_eq!(rendered, expected);
8313 }
8314
8315 #[test]
8316 fn hello_registers_manifest_and_returns_ack() {
8317 let registry = Arc::new(Registry::default());
8318 let handler = ControlHandler::new(Arc::clone(®istry));
8319 let conn = ConnectionId::new(1);
8320
8321 let responses = handler
8322 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8323 .unwrap();
8324
8325 assert_eq!(responses.len(), 1);
8326 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8327 assert_eq!(responses[0].header.channel, 0);
8328 assert_eq!(responses[0].header.corr, 7);
8329 let ack = parse_ack(&responses[0]);
8330 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8331 assert!(ack
8332 .subc_capabilities
8333 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8334 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8335 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8336 assert!(ack
8337 .subc_ops
8338 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8339 assert!(ack
8340 .subc_ops
8341 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8342
8343 let registration = registry.get_module("aft").unwrap().unwrap();
8344 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8345 assert_eq!(registration.state, ChannelState::Active);
8346 assert_eq!(registration.connection_id, conn);
8347 assert_eq!(registration.control_ops, module_baseline_control_ops());
8348 }
8349
8350 #[test]
8351 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8352 let invalid_identifiers = [
8353 ("case_change", "credentials-Provider/v1"),
8354 ("leading_zero", "credentials-provider/v01"),
8355 ("trailing_hyphen", "credentials-provider-/v1"),
8356 ("consecutive_hyphens", "credentials--provider/v1"),
8357 ("uppercase", "Credentials-provider/v1"),
8358 ("missing_v", "credentials-provider/1"),
8359 ("whitespace", "credentials provider/v1"),
8360 ("zero_version", "credentials-provider/v0"),
8361 ("out_of_range_version", "credentials-provider/v4294967296"),
8362 (
8363 "overlength_name",
8364 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8365 ),
8366 ];
8367 let mut cases = invalid_identifiers
8368 .into_iter()
8369 .map(|(name, identifier)| {
8370 (
8371 format!("identifier_{name}"),
8372 "capabilities.provides[0]".to_string(),
8373 identifier.to_string(),
8374 json!({ "provides": [identifier] }),
8375 None,
8376 )
8377 })
8378 .collect::<Vec<_>>();
8379 cases.extend([
8380 (
8381 "unknown_need".to_string(),
8382 "capabilities.requires[0].need".to_string(),
8383 "deferred".to_string(),
8384 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8385 None,
8386 ),
8387 (
8388 "duplicate_provides".to_string(),
8389 "capabilities.provides[1]".to_string(),
8390 "credentials-provider/v1".to_string(),
8391 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8392 None,
8393 ),
8394 (
8395 "duplicate_must_never_reach".to_string(),
8396 "capabilities.must_never_reach[1]".to_string(),
8397 "credentials-provider/v1".to_string(),
8398 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8399 None,
8400 ),
8401 (
8402 "duplicate_requires_same_need".to_string(),
8403 "capabilities.requires[1]".to_string(),
8404 "credentials-provider/v1".to_string(),
8405 json!({ "requires": [
8406 { "capability": "credentials-provider/v1", "need": "required" },
8407 { "capability": "credentials-provider/v1", "need": "required" }
8408 ] }),
8409 None,
8410 ),
8411 (
8412 "duplicate_requires_conflicting_need".to_string(),
8413 "capabilities.requires[1]".to_string(),
8414 "credentials-provider/v1".to_string(),
8415 json!({ "requires": [
8416 { "capability": "credentials-provider/v1", "need": "required" },
8417 { "capability": "credentials-provider/v1", "need": "optional" }
8418 ] }),
8419 None,
8420 ),
8421 (
8422 "capabilities_root_pointer".to_string(),
8423 "runtime_computed[0]".to_string(),
8424 "/capabilities".to_string(),
8425 json!({}),
8426 Some(json!(["/capabilities"])),
8427 ),
8428 (
8429 "capabilities_descendant_pointer".to_string(),
8430 "runtime_computed[0]".to_string(),
8431 "/capabilities/provides".to_string(),
8432 json!({}),
8433 Some(json!(["/capabilities/provides"])),
8434 ),
8435 (
8436 "malformed_pointer_without_leading_slash".to_string(),
8437 "runtime_computed[0]".to_string(),
8438 "capabilities".to_string(),
8439 json!({}),
8440 Some(json!(["capabilities"])),
8441 ),
8442 (
8443 "malformed_pointer_escape".to_string(),
8444 "runtime_computed[0]".to_string(),
8445 "/roles/~2/tools".to_string(),
8446 json!({}),
8447 Some(json!(["/roles/~2/tools"])),
8448 ),
8449 (
8450 "unknown_capabilities_field".to_string(),
8451 "capabilities.future".to_string(),
8452 "<array>".to_string(),
8453 json!({ "future": [] }),
8454 None,
8455 ),
8456 ]);
8457
8458 for (index, (name, field, value, capabilities, runtime_computed)) in
8459 cases.into_iter().enumerate()
8460 {
8461 let registry = Arc::new(Registry::default());
8462 let handler = ControlHandler::new(Arc::clone(®istry));
8463 let response = handler
8464 .handle_control(
8465 ConnectionId::new((index + 1) as u64),
8466 capability_grammar_hello_frame(
8467 capabilities,
8468 runtime_computed,
8469 index as u64 + 1,
8470 ),
8471 )
8472 .expect("invalid HELLO returns a refusal");
8473
8474 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8475 let error = parse_error(&response[0]);
8476 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8477 let message = error["message"]
8478 .as_str()
8479 .expect("error message is a string");
8480 assert!(
8481 message.contains(&field),
8482 "{name}: field missing from {message}"
8483 );
8484 assert!(
8485 message.contains(&value),
8486 "{name}: value missing from {message}"
8487 );
8488 assert_eq!(
8489 registry
8490 .active_registration_count()
8491 .expect("registry reads"),
8492 0,
8493 "{name}: refused HELLO must not create a catalog entry"
8494 );
8495 }
8496 }
8497
8498 #[test]
8499 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8500 let registry = Arc::new(Registry::default());
8501 let handler = ControlHandler::new(Arc::clone(®istry));
8502 let capabilities = json!({
8503 "provides": ["credentials-provider/v1"],
8504 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8505 "must_never_reach": ["federation-transport/v1"]
8506 });
8507 let response = handler
8508 .handle_control(
8509 ConnectionId::new(99),
8510 capability_grammar_hello_frame(
8511 capabilities.clone(),
8512 Some(json!(["/roles/0/tools"])),
8513 99,
8514 ),
8515 )
8516 .expect("valid HELLO registers");
8517 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8518
8519 let request = Frame::build(
8520 FrameType::Request,
8521 control_flags(),
8522 0,
8523 0,
8524 100,
8525 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8526 .expect("catalog request serializes"),
8527 )
8528 .expect("catalog request frame builds");
8529 let response = handler
8530 .handle_catalog_list(request, None)
8531 .expect("catalog list succeeds");
8532 let ClientControlResponse::CatalogList { modules, .. } =
8533 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8534 else {
8535 panic!("catalog request must return catalog.list");
8536 };
8537 assert_eq!(modules.len(), 1);
8538 assert_eq!(
8539 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8540 capabilities
8541 );
8542 }
8543
8544 #[test]
8545 fn catalog_list_mirrors_management_operation_description() {
8546 let registry = Arc::new(Registry::default());
8547 let handler = ControlHandler::new(Arc::clone(®istry));
8548 let description = "List managed records and return their identifiers and metadata.";
8549 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8550 manifest.provides = vec![ProviderRole::ManagementSurface {
8551 operations: vec![ManagementOperation {
8552 name: "records.list".to_string(),
8553 kind: ManagementOperationKind::Query,
8554 description: Some(description.to_string()),
8555 }],
8556 config_schema: json!({"type": "object"}),
8557 observability: vec![ObservabilitySurface {
8558 name: "records.stats".to_string(),
8559 kind: ObservabilityKind::Snapshot,
8560 }],
8561 identity_scope: vec![IdentityScope::Project],
8562 concurrency: Concurrency::ModuleManaged,
8563 }];
8564 registry
8565 .register_with_control_ops(
8566 manifest,
8567 PROTOCOL_VERSION,
8568 ConnectionId::new(99),
8569 Vec::new(),
8570 )
8571 .expect("described management manifest registers");
8572
8573 let request = Frame::build(
8574 FrameType::Request,
8575 control_flags(),
8576 0,
8577 0,
8578 100,
8579 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8580 .expect("catalog request serializes"),
8581 )
8582 .expect("catalog request frame builds");
8583 let response = handler
8584 .handle_catalog_list(request, None)
8585 .expect("catalog list succeeds");
8586 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8587 assert_eq!(
8588 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8589 "catalog.list must preserve the declared operation description verbatim"
8590 );
8591 }
8592
8593 #[test]
8594 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8595 let registry = Arc::new(Registry::default());
8596 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8597 [("vault".to_string(), true), ("squatter".to_string(), true)],
8598 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8599 );
8600 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8601 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8602 provides: vec!["credentials-provider/v1".to_string()],
8603 requires: Vec::new(),
8604 must_never_reach: Vec::new(),
8605 });
8606 let frame = Frame::build(
8607 FrameType::Hello,
8608 control_flags(),
8609 0,
8610 0,
8611 77,
8612 serde_json::to_vec(&ModuleHelloBody {
8613 manifest: squatter,
8614 protocol_ver: PROTOCOL_VERSION,
8615 control_ops: None,
8616 launch_nonce: None,
8617 })
8618 .expect("HELLO serializes"),
8619 )
8620 .expect("HELLO frame builds");
8621 let response = handler
8622 .handle_control(ConnectionId::new(77), frame)
8623 .expect("reserved claim receives a typed refusal");
8624 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8625 assert_eq!(
8626 registry
8627 .active_registration_count()
8628 .expect("registry reads"),
8629 0,
8630 "a reserved capability refusal must not leave a catalog entry"
8631 );
8632 }
8633
8634 #[test]
8635 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8636 for settlement in ["timeout", "inconclusive", "drop"] {
8637 let breakers = RouteBindBreakers::default();
8638 let RouteBindAdmission::Admitted {
8639 guard: mut old,
8640 probe: false,
8641 } = breakers.admit("prov")
8642 else {
8643 panic!("ordinary relay admitted")
8644 };
8645 let RouteBindAdmission::Admitted {
8646 guard: mut opener, ..
8647 } = breakers.admit("prov")
8648 else {
8649 panic!("second relay admitted")
8650 };
8651 assert!(
8652 !opener
8653 .record_timeout(1, Duration::ZERO)
8654 .unwrap()
8655 .reopened_after_probe
8656 );
8657 let RouteBindAdmission::Admitted {
8658 guard: mut probe,
8659 probe: true,
8660 } = breakers.admit("prov")
8661 else {
8662 panic!("one cooldown probe admitted")
8663 };
8664 match settlement {
8665 "timeout" => assert!(
8666 !old.record_timeout(1, Duration::ZERO)
8667 .unwrap()
8668 .reopened_after_probe
8669 ),
8670 "inconclusive" => old.record_inconclusive(),
8671 "drop" => drop(old),
8672 _ => unreachable!(),
8673 }
8674 assert!(
8675 matches!(
8676 breakers.admit("prov"),
8677 RouteBindAdmission::Refused {
8678 probe_in_flight: true,
8679 ..
8680 }
8681 ),
8682 "{settlement} of a pre-open relay cannot release the real probe"
8683 );
8684 assert!(
8685 probe
8686 .record_timeout(1, Duration::ZERO)
8687 .unwrap()
8688 .reopened_after_probe
8689 );
8690 assert!(matches!(
8691 breakers.admit("prov"),
8692 RouteBindAdmission::Admitted { probe: true, .. }
8693 ));
8694 }
8695 let breakers = RouteBindBreakers::default();
8696 let admit = || match breakers.admit("prov") {
8697 RouteBindAdmission::Admitted { guard, .. } => guard,
8698 _ => panic!("relay admitted"),
8699 };
8700 admit().record_timeout(1, Duration::ZERO);
8701 let mut old_probe = admit();
8702 breakers.reset_for_new_module_connection("prov");
8703 admit().record_timeout(1, Duration::ZERO);
8704 let _new_probe = admit();
8705 old_probe.record_inconclusive();
8706 assert!(matches!(
8707 breakers.admit("prov"),
8708 RouteBindAdmission::Refused {
8709 probe_in_flight: true,
8710 ..
8711 }
8712 ));
8713 }
8714
8715 #[tokio::test]
8716 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8717 for candidate in [false, true] {
8718 let registry = Arc::new(Registry::default());
8719 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8720 [("vault".to_string(), true), ("squatter".to_string(), true)],
8721 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8722 );
8723 let conn = ConnectionId::new(77);
8724 let (ctx, mut rx) = route_ctx(conn);
8725 let initial = capability_manifest("squatter", &[], &[]);
8726 if candidate {
8727 registry
8728 .register_candidate_with_control_ops(
8729 initial.clone(),
8730 PROTOCOL_VERSION,
8731 conn,
8732 module_baseline_control_ops(),
8733 )
8734 .unwrap();
8735 handler
8736 .forwarding
8737 .register_candidate_module_connection(
8738 conn,
8739 "squatter".to_string(),
8740 PROTOCOL_VERSION,
8741 manifest_concurrency(&initial),
8742 ctx.egress.clone(),
8743 )
8744 .unwrap();
8745 } else {
8746 hello_via_sink(
8747 &handler,
8748 &ctx,
8749 &mut rx,
8750 hello_frame_with_manifest(initial.clone(), 1),
8751 )
8752 .await;
8753 }
8754 let response = handler
8755 .handle_control_frame(
8756 &ctx,
8757 catalog_update_with_capabilities_frame(
8758 2,
8759 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8760 .capabilities
8761 .unwrap(),
8762 ),
8763 )
8764 .await
8765 .unwrap();
8766 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8767 assert_eq!(
8768 registry
8769 .get_module_by_connection(conn)
8770 .unwrap()
8771 .unwrap()
8772 .manifest,
8773 initial
8774 );
8775 }
8776 }
8777
8778 #[test]
8779 fn server_describe_surfaces_required_capability_verdict_fields() {
8780 let registry = Arc::new(Registry::default());
8781 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8782 [
8783 ("consumer".to_string(), true),
8784 ("provider".to_string(), false),
8785 ],
8786 BTreeMap::new(),
8787 );
8788 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8789 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8790 provides: Vec::new(),
8791 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8792 capability: "credentials-provider/v1".to_string(),
8793 need: subc_protocol::manifest::CapabilityNeed::Required,
8794 }],
8795 must_never_reach: Vec::new(),
8796 });
8797 let hello = Frame::build(
8798 FrameType::Hello,
8799 control_flags(),
8800 0,
8801 0,
8802 78,
8803 serde_json::to_vec(&ModuleHelloBody {
8804 manifest: consumer,
8805 protocol_ver: PROTOCOL_VERSION,
8806 control_ops: None,
8807 launch_nonce: None,
8808 })
8809 .expect("HELLO serializes"),
8810 )
8811 .expect("HELLO frame builds");
8812 handler
8813 .handle_control(ConnectionId::new(78), hello)
8814 .expect("consumer registers");
8815 let describe = Frame::build(
8816 FrameType::Request,
8817 control_flags(),
8818 0,
8819 0,
8820 79,
8821 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8822 .expect("request serializes"),
8823 )
8824 .expect("describe frame builds");
8825 let response = handler
8826 .handle_server_describe(describe)
8827 .expect("server.describe succeeds");
8828 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8829 let requirement = &rendered["capability_requirements"][0];
8830 assert_eq!(requirement["consumer"], "consumer");
8831 assert_eq!(requirement["verdict"], "never_provided");
8832 assert_eq!(requirement["episode_seq"], 1);
8833 assert_eq!(requirement["config_satisfiable"], false);
8834 assert_eq!(requirement["runtime_available"], false);
8835 assert!(requirement["detail"]
8836 .as_str()
8837 .expect("detail string")
8838 .contains("credentials-provider/v1"));
8839 }
8840
8841 #[test]
8842 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8843 let registry = Arc::new(Registry::default());
8844 let handler = ControlHandler::new(Arc::clone(®istry));
8845 let hello = handler
8846 .handle_control(
8847 ConnectionId::new(101),
8848 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8849 )
8850 .expect("legacy HELLO registers");
8851 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8852
8853 let request = Frame::build(
8854 FrameType::Request,
8855 control_flags(),
8856 0,
8857 0,
8858 102,
8859 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8860 .expect("catalog request serializes"),
8861 )
8862 .expect("catalog request frame builds");
8863 let response = handler
8864 .handle_catalog_list(request, None)
8865 .expect("catalog list succeeds");
8866 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8867 assert!(
8868 body["modules"][0].get("capabilities").is_none(),
8869 "legacy manifest must retain an absent capabilities field on catalog.list"
8870 );
8871 }
8872
8873 #[test]
8874 fn hello_ack_omits_storage_when_no_storage_config() {
8875 let registry = Arc::new(Registry::default());
8876 let handler = ControlHandler::new(Arc::clone(®istry));
8877 let responses = handler
8878 .handle_control(
8879 ConnectionId::new(1),
8880 hello_frame("aft", PROTOCOL_VERSION, 7),
8881 )
8882 .unwrap();
8883 let ack = parse_ack(&responses[0]);
8884 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8885 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8886 }
8887
8888 #[tokio::test]
8889 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8890 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8891 let registry = Arc::new(Registry::default());
8892 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
8893 let responses = handler
8894 .handle_control(
8895 ConnectionId::new(1),
8896 hello_frame("aft", PROTOCOL_VERSION, 7),
8897 )
8898 .unwrap();
8899 let ack = parse_ack(&responses[0]);
8900 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8901
8902 let described = handler
8903 .handle_control_frame(
8904 &route_ctx(ConnectionId::new(2)).0,
8905 Frame::build(
8906 FrameType::Request,
8907 control_flags(),
8908 0,
8909 0,
8910 9,
8911 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8912 )
8913 .unwrap(),
8914 )
8915 .await
8916 .unwrap();
8917 let ClientControlResponse::ServerDescribe { machine_id, .. } =
8918 serde_json::from_slice(&described[0].body).unwrap()
8919 else {
8920 panic!("server.describe answered with another shape");
8921 };
8922 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8923 }
8924
8925 #[test]
8926 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8927 let registry = Arc::new(Registry::default());
8930 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
8931 crate::daemon_config::StorageConfig::Sqlite {
8932 data_home: std::path::PathBuf::from("/data"),
8933 },
8934 ));
8935
8936 let responses = handler
8937 .handle_control(
8938 ConnectionId::new(1),
8939 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8940 )
8941 .unwrap();
8942 let ack = parse_ack(&responses[0]);
8943 assert_eq!(
8944 ack.storage,
8945 Some(serde_json::json!({
8946 "module_id": "alfonso-routing",
8947 "storage_namespace": "default",
8948 "isolation": { "kind": "module" },
8949 "backend": {
8950 "backend": "sqlite",
8951 "path": "/data/cortexkit/alfonso-routing/store.db"
8952 }
8953 })),
8954 "the delivered descriptor is the module's own sqlite store path"
8955 );
8956 }
8957
8958 #[test]
8959 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8960 let registry = Arc::new(Registry::default());
8961 let handler = ControlHandler::new(Arc::clone(®istry));
8962 let conn = ConnectionId::new(1);
8963 let responses = handler
8964 .handle_control(
8965 conn,
8966 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8967 )
8968 .unwrap();
8969 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8970 let registration = registry.get_module("aft").unwrap().unwrap();
8971 assert_eq!(registration.control_ops, module_baseline_control_ops());
8972
8973 let frame =
8974 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8975 assert!(handler
8976 .guard_module_control_op(&frame, "aft", "route.bind")
8977 .unwrap()
8978 .is_none());
8979 let error = handler
8980 .guard_module_control_op(&frame, "aft", "test.synthetic")
8981 .unwrap()
8982 .expect("synthetic ungranted op should be rejected");
8983 assert_eq!(error.header.ty, FrameType::Error);
8984 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8985 }
8986
8987 #[test]
8988 fn hello_control_ops_some_adds_optional_grants() {
8989 let registry = Arc::new(Registry::default());
8990 let handler = ControlHandler::new(Arc::clone(®istry));
8991 handler
8992 .handle_control(
8993 ConnectionId::new(1),
8994 hello_frame_with_control_ops(
8995 "aft",
8996 PROTOCOL_VERSION,
8997 7,
8998 Some(vec![
8999 "future.synthetic".to_string(),
9000 "route.bind".to_string(),
9001 ]),
9002 ),
9003 )
9004 .unwrap();
9005 let registration = registry.get_module("aft").unwrap().unwrap();
9006 assert_eq!(
9007 registration.control_ops,
9008 vec![
9009 "route.bind".to_string(),
9010 "route.status".to_string(),
9011 "future.synthetic".to_string(),
9012 ]
9013 );
9014 let frame =
9015 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9016 assert!(handler
9017 .guard_module_control_op(&frame, "aft", "future.synthetic")
9018 .unwrap()
9019 .is_none());
9020 }
9021
9022 #[tokio::test]
9023 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9024 let registry = Arc::new(Registry::default());
9025 let forwarding = Arc::new(ForwardingTable::default());
9026 let handler =
9027 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9028 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9029 hello_via_sink(
9030 &handler,
9031 &module_ctx,
9032 &mut module_rx,
9033 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9034 )
9035 .await;
9036
9037 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9038 let responses = handler
9039 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9040 .await
9041 .unwrap();
9042 assert_eq!(responses.len(), 1);
9043 assert_eq!(responses[0].header.ty, FrameType::Error);
9044 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9045 assert!(module_rx.try_recv().is_err());
9046 }
9047
9048 #[tokio::test]
9049 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9050 let registry = Arc::new(Registry::default());
9051 let forwarding = Arc::new(ForwardingTable::default());
9052 let handler =
9053 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9054 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9055 hello_via_sink(
9056 &handler,
9057 &module_ctx,
9058 &mut module_rx,
9059 hello_frame_with_control_ops(
9060 "aft",
9061 PROTOCOL_VERSION,
9062 7,
9063 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9064 ),
9065 )
9066 .await;
9067
9068 let project_root = unique_project_root("demux");
9069 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9070 let route_handler = handler.clone();
9071 let route_task = tokio::spawn(async move {
9072 route_handler
9073 .handle_control_frame(
9074 &route_client_ctx,
9075 route_open_frame(100, "aft", project_root),
9076 )
9077 .await
9078 .unwrap()
9079 });
9080 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9081 .await
9082 .unwrap()
9083 .unwrap();
9084 assert!(matches!(
9085 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9086 ModuleControlRequest::RouteBind { .. }
9087 ));
9088
9089 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9090 let health_handler = handler.clone();
9091 let health_task = tokio::spawn(async move {
9092 health_handler
9093 .handle_control_frame(
9094 &health_client_ctx,
9095 supervisor_health_probe_frame(101, "aft"),
9096 )
9097 .await
9098 .unwrap()
9099 });
9100 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9101 .await
9102 .unwrap()
9103 .unwrap();
9104 assert_eq!(
9105 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9106 ModuleControlRequest::HealthCheck {}
9107 );
9108
9109 handler
9110 .handle_control_frame(
9111 &module_ctx,
9112 health_response(health_frame.header.corr, HealthStatus::Degraded),
9113 )
9114 .await
9115 .unwrap();
9116 let health_response = health_task.await.unwrap();
9117 assert_eq!(health_response.len(), 1);
9118 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9119 ClientControlResponse::SupervisorHealthProbe {
9120 module_id,
9121 status,
9122 detail,
9123 metrics,
9124 } => {
9125 assert_eq!(module_id, "aft");
9126 assert_eq!(status, HealthStatus::Degraded);
9127 assert_eq!(detail.as_deref(), Some("warming"));
9128 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9129 }
9130 other => panic!("unexpected health response: {other:?}"),
9131 }
9132
9133 handler
9134 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9135 .await
9136 .unwrap();
9137 let route_response = route_task.await.unwrap();
9138 assert!(route_response.is_empty());
9139 let published = route_client_rx.recv().await.unwrap();
9140 assert!(matches!(
9141 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9142 ClientControlResponse::RouteOpen { .. }
9143 ));
9144 }
9145
9146 async fn relay_route_open(
9151 handler: &ControlHandler,
9152 client_connection: ConnectionId,
9153 client_egress: &FrameSink,
9154 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9155 corr: u64,
9156 module_id: &str,
9157 project_root_label: &str,
9158 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9159 let ctx = RouteCtx {
9160 connection_id: client_connection,
9161 egress: client_egress.clone(),
9162 };
9163 let handler = handler.clone();
9164 let project_root = unique_project_root(project_root_label);
9165 let module_id = module_id.to_string();
9166 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9167 let task = tokio::spawn(async move {
9168 let _guard = tracing::dispatcher::set_default(&dispatch);
9169 handler
9170 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9171 .await
9172 .unwrap()
9173 });
9174 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9175 .await
9176 .expect("module receives the relayed route.bind")
9177 .expect("module egress is open");
9178 (task, bind.frame)
9179 }
9180
9181 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9182 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9183 ModuleControlRequest::RouteBind {
9184 route_channel,
9185 epoch,
9186 ..
9187 } => (route_channel, epoch),
9188 other => panic!("expected a route.bind request, got {other:?}"),
9189 }
9190 }
9191
9192 fn published_route(frame: &Frame) -> (u16, u32) {
9193 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9194 ClientControlResponse::RouteOpen {
9195 route_channel,
9196 route_epoch,
9197 } => (route_channel, route_epoch),
9198 other => panic!("expected a route.open response, got {other:?}"),
9199 }
9200 }
9201
9202 #[tokio::test]
9222 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9223 let registry = Arc::new(Registry::default());
9224 let forwarding = Arc::new(ForwardingTable::default());
9225 let handler =
9226 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9227
9228 let module_connection = ConnectionId::new(30);
9229 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9230 hello_via_sink(
9231 &handler,
9232 &module_ctx,
9233 &mut module_rx,
9234 hello_frame("aft", PROTOCOL_VERSION, 7),
9235 )
9236 .await;
9237
9238 let dying_client = ConnectionId::new(31);
9239 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9240
9241 let (first_task, first_bind) = relay_route_open(
9244 &handler,
9245 dying_client,
9246 &dying_ctx.egress,
9247 &mut module_rx,
9248 100,
9249 "aft",
9250 "closing-first",
9251 )
9252 .await;
9253 handler
9254 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9255 .await
9256 .unwrap();
9257 assert!(first_task.await.unwrap().is_empty());
9258 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9259
9260 let (second_task, second_bind) = relay_route_open(
9262 &handler,
9263 dying_client,
9264 &dying_ctx.egress,
9265 &mut module_rx,
9266 101,
9267 "aft",
9268 "closing-second",
9269 )
9270 .await;
9271 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9272
9273 assert!(forwarding
9276 .escalate_client_delivery_failure(
9277 dying_client,
9278 first_channel,
9279 first_epoch,
9280 CloseReason::new(
9281 "module_to_client_delivery_failed",
9282 "client egress refused a module frame",
9283 ),
9284 crate::forwarding::UndeliveredFrame {
9285 module_id: None,
9286 sink: &dying_ctx.egress,
9287 },
9288 )
9289 .unwrap());
9290 assert!(!dying_ctx.egress.is_closed());
9291
9292 let ack = handler
9294 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9295 .await;
9296 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9297 if module_loop_error.is_some() {
9298 handler.cleanup_connection(module_connection).unwrap();
9302 }
9303 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9307 .await
9308 .ok()
9309 .flatten();
9310
9311 assert!(
9313 registry
9314 .get_module_by_connection(module_connection)
9315 .unwrap()
9316 .is_some(),
9317 "one client's closing connection ended the shared module connection: \
9318 {module_loop_error:?}"
9319 );
9320 let cotenant = ConnectionId::new(32);
9322 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9323 let (cotenant_task, cotenant_bind) = relay_route_open(
9324 &handler,
9325 cotenant,
9326 &cotenant_ctx.egress,
9327 &mut module_rx,
9328 102,
9329 "aft",
9330 "closing-cotenant",
9331 )
9332 .await;
9333 handler
9334 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9335 .await
9336 .unwrap();
9337 assert!(cotenant_task.await.unwrap().is_empty());
9338 let (cotenant_channel, cotenant_epoch) =
9339 published_route(&cotenant_rx.recv().await.unwrap());
9340 assert!(matches!(
9341 forwarding
9342 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9343 .unwrap(),
9344 DataRoute::Client(DataRouteState::Bound(_))
9345 ));
9346
9347 let goodbye = post_ack_module_frame
9350 .expect("module receives a GOODBYE for the abandoned route channel");
9351 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9352 assert_eq!(goodbye.header.channel, abandoned_channel);
9353 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9354
9355 assert!(dying_rx.try_recv().is_err());
9359 let second_response = second_task.await.unwrap();
9360 assert_eq!(second_response.len(), 1);
9361 assert_eq!(
9362 parse_error(&second_response[0])["code"],
9363 "target_unavailable"
9364 );
9365 }
9366
9367 async fn catalog_snapshot(
9370 handler: &ControlHandler,
9371 client_ctx: &RouteCtx,
9372 corr: u64,
9373 ) -> (u64, Vec<(String, bool)>) {
9374 let body =
9375 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9376 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9377 let reply = handler
9378 .handle_control_frame(client_ctx, request)
9379 .await
9380 .unwrap()
9381 .pop()
9382 .unwrap();
9383 let ClientControlResponse::CatalogList {
9384 generation,
9385 modules,
9386 ..
9387 } = serde_json::from_slice(&reply.body).unwrap()
9388 else {
9389 panic!("not a catalog.list reply");
9390 };
9391 (
9392 generation,
9393 modules
9394 .into_iter()
9395 .map(|entry| (entry.module_id, entry.ready))
9396 .collect(),
9397 )
9398 }
9399
9400 fn hello_held_before_forwarding() -> (
9406 ControlHandler,
9407 Arc<std::sync::Barrier>,
9408 std::thread::JoinHandle<Vec<Frame>>,
9409 RouteCtx,
9410 mpsc::Receiver<crate::router::OutboundFrame>,
9411 ) {
9412 let hold = Arc::new(std::sync::Barrier::new(2));
9413 let handler = ControlHandler::with_forwarding(
9414 Arc::new(Registry::default()),
9415 Arc::new(ForwardingTable::default()),
9416 )
9417 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9418 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9419 let hello = {
9420 let handler = handler.clone();
9421 let module_ctx = module_ctx.clone();
9422 std::thread::spawn(move || {
9423 handler
9424 .handle_hello(
9425 module_ctx.connection_id,
9426 Some(module_ctx.egress.clone()),
9427 hello_frame("aft", PROTOCOL_VERSION, 7),
9428 )
9429 .unwrap()
9430 })
9431 };
9432 (handler, hold, hello, module_ctx, module_rx)
9433 }
9434
9435 #[tokio::test]
9444 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9445 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9446 let client = ConnectionId::new(41);
9447 let (client_ctx, mut client_rx) = route_ctx(client);
9448
9449 hold.wait();
9450 assert!(handler.registry.get_module("aft").unwrap().is_some());
9452 assert!(!handler
9453 .forwarding
9454 .has_live_module_connection("aft")
9455 .unwrap());
9456 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9457 assert!(
9458 listed.is_empty(),
9459 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9460 );
9461 let refused = handler
9462 .handle_control_frame(
9463 &client_ctx,
9464 route_open_frame(301, "aft", unique_project_root("listed-window")),
9465 )
9466 .await
9467 .unwrap();
9468 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9469 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9470 let duplicate = handler
9471 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9472 .await
9473 .unwrap();
9474 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9475
9476 hold.wait();
9477 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9478 let ack = module_rx.try_recv().unwrap().frame;
9479 assert_eq!(ack.header.ty, FrameType::HelloAck);
9480 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9481 assert_eq!(listed, vec![("aft".to_string(), true)]);
9482
9483 let (open_task, bind) = relay_route_open(
9484 &handler,
9485 client,
9486 &client_ctx.egress,
9487 &mut module_rx,
9488 303,
9489 "aft",
9490 "listed-after",
9491 )
9492 .await;
9493 assert!(handler
9494 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9495 .await
9496 .unwrap()
9497 .is_empty());
9498 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9499 published_route(&client_rx.try_recv().unwrap().frame);
9500 }
9501
9502 #[tokio::test]
9509 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9510 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9511 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9512 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9513
9514 hold.wait();
9515 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9516 assert!(listed.is_empty());
9517 assert_eq!(
9518 during, before,
9519 "the generation moved while the module was not listed"
9520 );
9521
9522 hold.wait();
9523 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9524 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9525 assert_eq!(listed, vec![("aft".to_string(), true)]);
9526 assert_eq!(
9527 after,
9528 before + 1,
9529 "the module appeared without exactly one generation bump"
9530 );
9531 }
9532
9533 #[tokio::test]
9538 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
9539 let registry = Arc::new(Registry::default());
9540 let handler = ControlHandler::with_forwarding(
9541 Arc::clone(®istry),
9542 Arc::new(ForwardingTable::default()),
9543 );
9544 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9545 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9546 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9547 drop(module_rx);
9548
9549 let replies = handler
9550 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9551 .await
9552 .unwrap();
9553 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
9554 assert!(registry.get_module("aft").unwrap().is_none());
9555 assert!(registry
9556 .get_module_by_connection(module_ctx.connection_id)
9557 .unwrap()
9558 .is_none());
9559 assert_eq!(
9560 catalog_snapshot(&handler, &client_ctx, 301).await,
9561 (before, Vec::new())
9562 );
9563 }
9564
9565 async fn route_open_held_after_bind_send() -> (
9569 ControlHandler,
9570 Arc<tokio::sync::Barrier>,
9571 RouteCtx,
9572 mpsc::Receiver<crate::router::OutboundFrame>,
9573 ) {
9574 let hold = Arc::new(tokio::sync::Barrier::new(2));
9575 let handler = ControlHandler::with_forwarding(
9576 Arc::new(Registry::default()),
9577 Arc::new(ForwardingTable::default()),
9578 )
9579 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9580 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9581 hello_via_sink(
9582 &handler,
9583 &module_ctx,
9584 &mut module_rx,
9585 hello_frame("aft", PROTOCOL_VERSION, 7),
9586 )
9587 .await;
9588 (handler, hold, module_ctx, module_rx)
9589 }
9590
9591 #[tokio::test]
9598 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9599 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9600 let client = ConnectionId::new(41);
9601 let (client_ctx, mut client_rx) = route_ctx(client);
9602
9603 let (open_task, bind) = relay_route_open(
9604 &handler,
9605 client,
9606 &client_ctx.egress,
9607 &mut module_rx,
9608 200,
9609 "aft",
9610 "acked-before-mark",
9611 )
9612 .await;
9613 let (module_channel, module_epoch) = route_bind_channel(&bind);
9614 hold.wait().await;
9616
9617 assert!(handler
9619 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9620 .await
9621 .unwrap()
9622 .is_empty());
9623 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9624
9625 hold.wait().await;
9626 assert!(
9627 open_task.await.unwrap().is_empty(),
9628 "route.open answers accepted"
9629 );
9630
9631 match module_rx.try_recv() {
9634 Err(mpsc::error::TryRecvError::Empty) => {}
9635 Ok(outbound) => panic!(
9636 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9637 outbound.frame.header.ty,
9638 outbound.frame.header.channel,
9639 outbound.frame.header.epoch
9640 ),
9641 Err(err) => panic!("module egress closed: {err}"),
9642 }
9643
9644 let router = Router::with_control_handler(Arc::new(handler.clone()));
9647 let request = Frame::build(
9648 FrameType::Request,
9649 Flags::new(true, Priority::Interactive, false),
9650 client_channel,
9651 client_epoch,
9652 201,
9653 b"first request".to_vec(),
9654 )
9655 .unwrap();
9656 router
9657 .route_for_connection(&client_ctx, request)
9658 .await
9659 .unwrap();
9660 let delivered = module_rx.try_recv().unwrap().frame;
9661 assert_eq!(delivered.header.ty, FrameType::Request);
9662 assert_eq!(
9663 (delivered.header.channel, delivered.header.epoch),
9664 (module_channel, module_epoch)
9665 );
9666 assert_eq!(delivered.body, b"first request");
9667 }
9668
9669 #[tokio::test]
9675 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9676 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9677 let client = ConnectionId::new(42);
9678 let (client_ctx, _client_rx) = route_ctx(client);
9679
9680 let (open_task, bind) = relay_route_open(
9681 &handler,
9682 client,
9683 &client_ctx.egress,
9684 &mut module_rx,
9685 300,
9686 "aft",
9687 "released-before-mark",
9688 )
9689 .await;
9690 let (module_channel, module_epoch) = route_bind_channel(&bind);
9691 hold.wait().await;
9692
9693 handler.cleanup_connection(client).unwrap();
9694 assert!(
9695 module_rx.try_recv().is_err(),
9696 "client teardown must not GOODBYE a bind route.open is still queueing"
9697 );
9698
9699 hold.wait().await;
9700 let response = open_task.await.unwrap();
9701 assert_eq!(response.len(), 1);
9702 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9703
9704 let goodbye = module_rx
9705 .try_recv()
9706 .expect("module is told to drop the binding the daemon abandoned")
9707 .frame;
9708 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9709 assert_eq!(
9710 (goodbye.header.channel, goodbye.header.epoch),
9711 (module_channel, module_epoch)
9712 );
9713 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9714 }
9715
9716 #[tokio::test]
9721 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
9722 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9723 let client = ConnectionId::new(43);
9724 let (client_ctx, client_rx) = route_ctx(client);
9725
9726 let (open_task, bind) = relay_route_open(
9727 &handler,
9728 client,
9729 &client_ctx.egress,
9730 &mut module_rx,
9731 400,
9732 "aft",
9733 "gone-client-before-mark",
9734 )
9735 .await;
9736 let (module_channel, module_epoch) = route_bind_channel(&bind);
9737 hold.wait().await;
9738
9739 drop(client_rx);
9740 assert!(client_ctx.egress.is_closed());
9741 handler
9742 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9743 .await
9744 .unwrap();
9745
9746 hold.wait().await;
9747 let response = open_task.await.unwrap();
9748 assert_eq!(response.len(), 1);
9749 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9750
9751 let goodbye = module_rx
9752 .try_recv()
9753 .expect("module is told to drop the binding it acked")
9754 .frame;
9755 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9756 assert_eq!(
9757 (goodbye.header.channel, goodbye.header.epoch),
9758 (module_channel, module_epoch)
9759 );
9760 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9761 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
9762 }
9763
9764 #[test]
9771 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9772 let handler = ControlHandler::default();
9773 let module_connection = ConnectionId::new(30);
9774 let client_connection = ConnectionId::new(31);
9775
9776 handler
9777 .refuse_to_end_module_connection_for_a_client(
9778 module_connection,
9779 77,
9780 ForwardingError::ConnectionClosing {
9781 connection_id: client_connection,
9782 },
9783 )
9784 .expect("a closing client must never end the module connection");
9785
9786 assert!(matches!(
9787 handler.refuse_to_end_module_connection_for_a_client(
9788 module_connection,
9789 78,
9790 ForwardingError::ConnectionClosing {
9791 connection_id: module_connection,
9792 },
9793 ),
9794 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9795 connection_id
9796 })) if connection_id == module_connection
9797 ));
9798 assert!(matches!(
9799 handler.refuse_to_end_module_connection_for_a_client(
9800 module_connection,
9801 79,
9802 ForwardingError::Poisoned,
9803 ),
9804 Err(RouterError::Forwarding(ForwardingError::Poisoned))
9805 ));
9806 assert!(matches!(
9807 handler.refuse_to_end_module_connection_for_a_client(
9808 module_connection,
9809 80,
9810 ForwardingError::StaleModuleEndpoint,
9811 ),
9812 Err(RouterError::Forwarding(
9813 ForwardingError::StaleModuleEndpoint
9814 ))
9815 ));
9816 }
9817
9818 #[tokio::test]
9831 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9832 let registry = Arc::new(Registry::default());
9833 let forwarding = Arc::new(ForwardingTable::default());
9834 let supervisor = SupervisorHandle::new();
9835 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9836 let handler =
9837 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9838 .with_supervisor(supervisor);
9839
9840 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9841 hello_via_sink(
9842 &handler,
9843 &target_ctx,
9844 &mut target_rx,
9845 hello_frame("target", PROTOCOL_VERSION, 1),
9846 )
9847 .await;
9848
9849 let wrong_nonce = handler
9853 .handle_control_frame(
9854 &route_ctx(ConnectionId::new(91)).0,
9855 route_open_frame_with_admission_facts(
9856 20,
9857 "target",
9858 unique_project_root("admission-facts"),
9859 Some(subc_control::ConsumerIdentity {
9860 module_id: "fed".to_string(),
9861 launch_nonce: "not-the-real-nonce".to_string(),
9862 }),
9863 None,
9864 ),
9865 )
9866 .await
9867 .unwrap();
9868 assert_eq!(
9869 parse_error(&wrong_nonce[0])["code"],
9870 "bad_consumer_identity",
9871 "a mismatched launch nonce must be refused, not stamped Reserved"
9872 );
9873
9874 let never_spawned = handler
9878 .handle_control_frame(
9879 &route_ctx(ConnectionId::new(92)).0,
9880 route_open_frame_with_admission_facts(
9881 21,
9882 "target",
9883 unique_project_root("admission-facts"),
9884 Some(subc_control::ConsumerIdentity {
9885 module_id: "never-spawned".to_string(),
9886 launch_nonce: "any-nonce".to_string(),
9887 }),
9888 None,
9889 ),
9890 )
9891 .await
9892 .unwrap();
9893 assert_eq!(
9894 parse_error(&never_spawned[0])["code"],
9895 "bad_consumer_identity",
9896 "an unspawned module_id must be refused rather than accepted for lack of a record"
9897 );
9898 }
9899
9900 #[tokio::test]
9923 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
9924 let registry = Arc::new(Registry::default());
9925 let forwarding = Arc::new(ForwardingTable::default());
9926 let supervisor = SupervisorHandle::new();
9927 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9928 let handler =
9929 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9930 .with_supervisor(supervisor);
9931
9932 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
9933 hello_via_sink(
9934 &handler,
9935 &target_ctx,
9936 &mut target_rx,
9937 hello_frame("target", PROTOCOL_VERSION, 1),
9938 )
9939 .await;
9940
9941 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
9942 let route_handler = handler.clone();
9943 let route_task = tokio::spawn(async move {
9944 route_handler
9945 .handle_control_frame(
9946 &client_ctx,
9947 route_open_frame_with_admission_facts(
9948 30,
9949 "target",
9950 unique_project_root("admission-facts"),
9951 Some(subc_control::ConsumerIdentity {
9952 module_id: "fed".to_string(),
9953 launch_nonce: "fed-nonce".to_string(),
9954 }),
9955 None,
9956 ),
9957 )
9958 .await
9959 .unwrap()
9960 });
9961
9962 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9969 .await
9970 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9971 .expect("module control channel closed before route.bind");
9972 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9973 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9974 panic!("expected route.bind")
9975 };
9976 assert_eq!(
9977 principal,
9978 Some(Principal::Reserved {
9979 module_id: "fed".to_string()
9980 }),
9981 "a correctly attested consumer must be stamped Reserved for its own id"
9982 );
9983
9984 handler
9985 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9986 .await
9987 .unwrap();
9988 assert!(route_task.await.unwrap().is_empty());
9989 assert!(
9990 matches!(
9991 serde_json::from_slice::<ClientControlResponse>(
9992 &client_rx.recv().await.unwrap().body
9993 )
9994 .unwrap(),
9995 ClientControlResponse::RouteOpen { .. }
9996 ),
9997 "the route must actually open, not merely avoid an error"
9998 );
9999 }
10000
10001 #[tokio::test(start_paused = true)]
10002 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10003 let registry = Arc::new(Registry::default());
10004 let forwarding = Arc::new(ForwardingTable::default());
10005 let supervisor = SupervisorHandle::new();
10006 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10007 let handler =
10008 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10009 .with_supervisor(supervisor);
10010
10011 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10012 hello_via_sink(
10013 &handler,
10014 &target_ctx,
10015 &mut target_rx,
10016 hello_frame("target", PROTOCOL_VERSION, 1),
10017 )
10018 .await;
10019
10020 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10021 let direct_handler = handler.clone();
10022 let direct_open = tokio::spawn(async move {
10023 direct_handler
10024 .handle_control_frame(
10025 &direct_ctx,
10026 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10027 )
10028 .await
10029 .unwrap()
10030 });
10031 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10032 .await
10033 .expect("no direct route.bind within 5s")
10034 .expect("target control channel closed before direct route.bind");
10035 handler
10036 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10037 .await
10038 .unwrap();
10039 assert!(direct_open.await.unwrap().is_empty());
10040 let _ = direct_rx.recv().await.unwrap();
10041
10042 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10043 let reserved_handler = handler.clone();
10044 let reserved_open = tokio::spawn(async move {
10045 reserved_handler
10046 .handle_control_frame(
10047 &reserved_ctx,
10048 route_open_frame_with_admission_facts(
10049 3,
10050 "target",
10051 unique_project_root("admission-facts"),
10052 Some(ConsumerIdentity {
10053 module_id: "fed".to_string(),
10054 launch_nonce: "fed-nonce".to_string(),
10055 }),
10056 None,
10057 ),
10058 )
10059 .await
10060 .unwrap()
10061 });
10062 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10063 .await
10064 .expect("no reserved route.bind within 5s")
10065 .expect("target control channel closed before reserved route.bind");
10066 handler
10067 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10068 .await
10069 .unwrap();
10070 assert!(reserved_open.await.unwrap().is_empty());
10071 let _ = reserved_rx.recv().await.unwrap();
10072
10073 forwarding
10074 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10075 .unwrap();
10076 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10077 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10078 module_id: Some("target".to_string()),
10079 })
10080 .unwrap();
10081 let census_frame =
10082 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10083 let response = handler
10084 .handle_control_frame(&census_ctx, census_frame)
10085 .await
10086 .unwrap()
10087 .pop()
10088 .unwrap();
10089 let actual: Value = serde_json::from_slice(&response.body).unwrap();
10090 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10091 assert!(matches!(
10092 decoded,
10093 ClientControlResponse::SupervisorRoutes { .. }
10094 ));
10095 let routes = actual["modules"][0]["routes"].as_array().unwrap();
10096 assert_eq!(routes.len(), 2);
10097 assert!(routes.iter().all(|route| route["draining"] == true));
10098 assert!(
10101 routes.iter().all(|route| route["drain_reason"] == "reload"),
10102 "draining routes must name the drain's reason: {routes:?}"
10103 );
10104 assert!(routes.iter().any(|route| {
10105 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10106 }));
10107 assert!(routes.iter().any(|route| {
10108 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10109 }));
10110
10111 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10112 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10113 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10114 std::fs::write(
10115 &golden_path,
10116 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10117 )
10118 .unwrap();
10119 }
10120 let expected: Value =
10121 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10122 assert_eq!(actual, expected);
10123 }
10124
10125 async fn query_live_roots(
10126 handler: &ControlHandler,
10127 module_ctx: &RouteCtx,
10128 ) -> ModuleControlResponseToModule {
10129 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10130 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10131 let response = handler
10132 .handle_control_frame(module_ctx, frame)
10133 .await
10134 .unwrap()
10135 .pop()
10136 .unwrap();
10137 serde_json::from_slice(&response.body).unwrap()
10138 }
10139
10140 #[tokio::test(start_paused = true)]
10141 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10142 let registry = Arc::new(Registry::default());
10143 let forwarding = Arc::new(ForwardingTable::default());
10144 let handler = ControlHandler::with_forwarding(registry, forwarding);
10145 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10146 hello_via_sink(
10147 &handler,
10148 &target_ctx,
10149 &mut target_rx,
10150 hello_frame("target", PROTOCOL_VERSION, 1),
10151 )
10152 .await;
10153 let root = unique_project_root("live-roots-known");
10154 let path = ProjectRootId::from_path_allowing_missing(root.path())
10155 .unwrap()
10156 .as_path()
10157 .to_path_buf();
10158 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10159 let open_handler = handler.clone();
10160 let opened = tokio::spawn(async move {
10161 open_handler
10162 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10163 .await
10164 .unwrap()
10165 });
10166 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10167 .await
10168 .unwrap()
10169 .unwrap();
10170 handler
10171 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10172 .await
10173 .unwrap();
10174 assert!(opened.await.unwrap().is_empty());
10175 let _ = client_rx.recv().await.unwrap();
10176
10177 let root = unique_project_root("live-roots-pending");
10178 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10179 .unwrap()
10180 .as_path()
10181 .to_path_buf();
10182 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10183 let open_handler = handler.clone();
10184 let pending = tokio::spawn(async move {
10185 open_handler
10186 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10187 .await
10188 .unwrap()
10189 });
10190 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10191 .await
10192 .unwrap()
10193 .unwrap();
10194 let actual = query_live_roots(&handler, &target_ctx).await;
10195 let ModuleControlResponseToModule::LiveRoots {
10196 roots,
10197 unknown_root_bindings,
10198 total_bindings,
10199 } = actual
10200 else {
10201 panic!("expected live roots")
10202 };
10203 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10204 assert_eq!(unknown_root_bindings, 0);
10205 assert_eq!(
10206 roots.len(),
10207 2,
10208 "root-known arm must retain each canonical root"
10209 );
10210 assert_eq!(
10211 total_bindings,
10212 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10213 );
10214 let counts = roots
10215 .iter()
10216 .map(|root| (root.project_root.clone(), root.bound, root.pending))
10217 .collect::<Vec<_>>();
10218 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10219 expected.sort_by(|a, b| a.0.cmp(&b.0));
10220 assert_eq!(
10221 counts, expected,
10222 "roots must sort by path and count pending separately"
10223 );
10224 handler
10225 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10226 .await
10227 .unwrap();
10228 assert!(pending.await.unwrap().is_empty());
10229 }
10230
10231 #[tokio::test(start_paused = true)]
10232 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10233 let forwarding = Arc::new(ForwardingTable::default());
10234 let handler =
10235 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10236 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10237 hello_via_sink(
10238 &handler,
10239 &target_ctx,
10240 &mut target_rx,
10241 hello_frame("target", PROTOCOL_VERSION, 1),
10242 )
10243 .await;
10244 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10245 let pending = forwarding
10246 .begin_route_bind_relay_for_test(
10247 client_ctx.connection_id,
10248 client_ctx.egress.clone(),
10249 2,
10250 "target",
10251 )
10252 .unwrap();
10253 forwarding
10254 .complete_pending_relay(
10255 target_ctx.connection_id,
10256 pending.corr,
10257 RouteBindRelayOutcome::Accepted,
10258 )
10259 .unwrap();
10260 let actual = query_live_roots(&handler, &target_ctx).await;
10261 let ModuleControlResponseToModule::LiveRoots {
10262 roots,
10263 unknown_root_bindings,
10264 total_bindings,
10265 } = actual
10266 else {
10267 panic!("expected live roots")
10268 };
10269 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10270 assert_eq!(
10271 unknown_root_bindings, 1,
10272 "unknown-root arm must not read as no bindings"
10273 );
10274 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10275 assert_eq!(
10276 total_bindings,
10277 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10278 );
10279 }
10280
10281 #[tokio::test(start_paused = true)]
10289 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10290 let forwarding = Arc::new(ForwardingTable::default());
10291 let handler =
10292 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10293 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10294 let replies = handler
10295 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10296 .await
10297 .unwrap();
10298 let queued_by_hello = module_rx.len();
10299
10300 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10301 let open_handler = handler.clone();
10302 let open = tokio::spawn(async move {
10303 open_handler
10304 .handle_control_frame(
10305 &client_ctx,
10306 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10307 )
10308 .await
10309 .unwrap()
10310 });
10311 let mut spins = 0;
10313 while module_rx.len() == queued_by_hello {
10314 spins += 1;
10315 assert!(spins < 10_000, "route.open never queued a route.bind");
10316 tokio::task::yield_now().await;
10317 }
10318
10319 for reply in replies {
10321 module_ctx.egress.send(reply).await.unwrap();
10322 }
10323
10324 let first = module_rx.recv().await.unwrap().frame;
10325 assert_eq!(
10326 first.header.ty,
10327 FrameType::HelloAck,
10328 "the first frame a registering module reads must be its HELLO_ACK"
10329 );
10330 assert_eq!(first.header.corr, 7);
10331 let second = module_rx.recv().await.unwrap().frame;
10332 assert_eq!(second.header.ty, FrameType::Request);
10333 assert!(
10334 matches!(
10335 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10336 ModuleControlRequest::RouteBind { .. }
10337 ),
10338 "the route.bind follows the ack"
10339 );
10340 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10341
10342 handler
10343 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10344 .await
10345 .unwrap();
10346 assert!(open.await.unwrap().is_empty());
10347 let _ = client_rx.recv().await.unwrap();
10348 }
10349
10350 #[tokio::test(start_paused = true)]
10351 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10352 let handler = ControlHandler::with_forwarding(
10353 Arc::new(Registry::default()),
10354 Arc::new(ForwardingTable::default()),
10355 );
10356 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10357 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10358 hello_via_sink(
10359 &handler,
10360 &first_ctx,
10361 &mut first_rx,
10362 hello_frame("first", PROTOCOL_VERSION, 1),
10363 )
10364 .await;
10365 hello_via_sink(
10366 &handler,
10367 &second_ctx,
10368 &mut second_rx,
10369 hello_frame("second", PROTOCOL_VERSION, 2),
10370 )
10371 .await;
10372 let root = unique_project_root("second-only");
10373 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10374 let cloned = handler.clone();
10375 let open = tokio::spawn(async move {
10376 cloned
10377 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10378 .await
10379 .unwrap()
10380 });
10381 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10382 .await
10383 .unwrap()
10384 .unwrap();
10385 let first = query_live_roots(&handler, &first_ctx).await;
10386 let second = query_live_roots(&handler, &second_ctx).await;
10387 assert!(
10388 matches!(
10389 first,
10390 ModuleControlResponseToModule::LiveRoots {
10391 total_bindings: 0,
10392 ..
10393 }
10394 ),
10395 "cross-module scope must not expose another module's roots"
10396 );
10397 assert!(
10398 matches!(
10399 second,
10400 ModuleControlResponseToModule::LiveRoots {
10401 total_bindings: 1,
10402 ..
10403 }
10404 ),
10405 "second module must see its pending route"
10406 );
10407 handler
10408 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10409 .await
10410 .unwrap();
10411 assert!(open.await.unwrap().is_empty());
10412 }
10413
10414 #[tokio::test(start_paused = true)]
10415 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10416 let handler = ControlHandler::with_forwarding(
10417 Arc::new(Registry::default()),
10418 Arc::new(ForwardingTable::default()),
10419 );
10420 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10421 hello_via_sink(
10422 &handler,
10423 &target_ctx,
10424 &mut target_rx,
10425 hello_frame("target", PROTOCOL_VERSION, 1),
10426 )
10427 .await;
10428 let actual = query_live_roots(&handler, &target_ctx).await;
10429 let ModuleControlResponseToModule::LiveRoots {
10430 roots,
10431 unknown_root_bindings,
10432 total_bindings,
10433 } = actual
10434 else {
10435 panic!("expected live roots")
10436 };
10437 assert!(roots.is_empty());
10438 assert_eq!(unknown_root_bindings, 0);
10439 assert_eq!(total_bindings, 0);
10440 assert_eq!(
10441 total_bindings,
10442 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10443 );
10444 }
10445
10446 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10453 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10454 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10455 let text = std::fs::read_to_string(&path)
10456 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10457 let vectors: Vec<(String, Value)> = text
10458 .lines()
10459 .filter(|line| !line.trim().is_empty())
10460 .map(|line| {
10461 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10462 let id = entry["corpus_id"]
10463 .as_str()
10464 .expect("every vector carries a corpus_id")
10465 .to_string();
10466 (id, entry["package"].clone())
10467 })
10468 .collect();
10469 assert_eq!(
10473 vectors.len(),
10474 3,
10475 "vendored fed corpus changed size; re-sync from subc-federation"
10476 );
10477
10478 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10493 let richest = vectors
10494 .iter()
10495 .filter_map(|(_, package)| package.as_object())
10496 .map(|object| {
10497 object
10498 .keys()
10499 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10500 .count()
10501 })
10502 .max()
10503 .unwrap_or(0);
10504 assert!(
10505 richest >= 2,
10506 "vendored corpus no longer carries a package with unmodelled fields, \
10507 so the relay test can no longer distinguish a verbatim relay from a lossy one"
10508 );
10509
10510 vectors
10511 }
10512
10513 #[tokio::test]
10521 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10522 for (corpus_id, package) in fed_admission_facts_vectors() {
10523 let registry = Arc::new(Registry::default());
10524 let forwarding = Arc::new(ForwardingTable::default());
10525 let supervisor = SupervisorHandle::new();
10526 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10527 let handler =
10528 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10529 .with_supervisor(supervisor)
10530 .with_admission_facts_config(
10531 Some("fed".to_string()),
10532 Some(vec!["target".to_string()]),
10533 );
10534
10535 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10536 hello_via_sink(
10537 &handler,
10538 &target_ctx,
10539 &mut target_rx,
10540 hello_frame("target", PROTOCOL_VERSION, 1),
10541 )
10542 .await;
10543
10544 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10545 let route_handler = handler.clone();
10546 let expected = package.clone();
10547 let route_task = tokio::spawn(async move {
10548 route_handler
10549 .handle_control_frame(
10550 &client_ctx,
10551 route_open_frame_with_admission_facts(
10552 20,
10553 "target",
10554 unique_project_root("admission-facts"),
10555 Some(subc_control::ConsumerIdentity {
10556 module_id: "fed".to_string(),
10557 launch_nonce: "fed-nonce".to_string(),
10558 }),
10559 Some(package),
10560 ),
10561 )
10562 .await
10563 .unwrap()
10564 });
10565
10566 let bind_frame = target_rx.recv().await.unwrap();
10567 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10568 let ModuleControlRequest::RouteBind {
10569 admission_facts, ..
10570 } = bind
10571 else {
10572 panic!("{corpus_id}: expected route.bind")
10573 };
10574 assert_eq!(
10575 admission_facts,
10576 Some(expected),
10577 "{corpus_id}: relay must not add, drop or reshape any field"
10578 );
10579
10580 handler
10581 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10582 .await
10583 .unwrap();
10584 route_task.await.unwrap();
10585 }
10586 }
10587
10588 #[tokio::test]
10589 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10590 let registry = Arc::new(Registry::default());
10591 let forwarding = Arc::new(ForwardingTable::default());
10592 let supervisor = SupervisorHandle::new();
10593 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10594 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10595 let handler =
10596 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10597 .with_supervisor(supervisor)
10598 .with_admission_facts_config(
10599 Some("fed".to_string()),
10600 Some(vec!["target".to_string()]),
10601 );
10602
10603 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10604 hello_via_sink(
10605 &handler,
10606 &target_ctx,
10607 &mut target_rx,
10608 hello_frame("target", PROTOCOL_VERSION, 1),
10609 )
10610 .await;
10611 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10612 hello_via_sink(
10613 &handler,
10614 &other_ctx,
10615 &mut other_rx,
10616 hello_frame("other", PROTOCOL_VERSION, 2),
10617 )
10618 .await;
10619
10620 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10621 let expected_facts = facts.clone();
10622 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10623 let route_handler = handler.clone();
10624 let route_task = tokio::spawn(async move {
10625 route_handler
10626 .handle_control_frame(
10627 &client_ctx,
10628 route_open_frame_with_admission_facts(
10629 10,
10630 "target",
10631 unique_project_root("admission-facts"),
10632 Some(subc_control::ConsumerIdentity {
10633 module_id: "fed".to_string(),
10634 launch_nonce: "fed-nonce".to_string(),
10635 }),
10636 Some(facts.clone()),
10637 ),
10638 )
10639 .await
10640 .unwrap()
10641 });
10642 let bind_frame = target_rx.recv().await.unwrap();
10643 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10644 let ModuleControlRequest::RouteBind {
10645 admission_facts, ..
10646 } = bind
10647 else {
10648 panic!("expected route.bind")
10649 };
10650 assert_eq!(admission_facts, Some(expected_facts));
10651 handler
10652 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10653 .await
10654 .unwrap();
10655 assert!(route_task.await.unwrap().is_empty());
10656 assert!(matches!(
10657 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10658 .unwrap(),
10659 ClientControlResponse::RouteOpen { .. }
10660 ));
10661
10662 let direct = handler
10663 .handle_control_frame(
10664 &route_ctx(ConnectionId::new(73)).0,
10665 route_open_frame_with_admission_facts(
10666 11,
10667 "target",
10668 unique_project_root("admission-facts"),
10669 None,
10670 Some(json!({"x": 1})),
10671 ),
10672 )
10673 .await
10674 .unwrap();
10675 assert_eq!(
10676 parse_error(&direct[0])["code"],
10677 "admission_facts_not_permitted"
10678 );
10679
10680 let different_reserved = handler
10681 .handle_control_frame(
10682 &route_ctx(ConnectionId::new(77)).0,
10683 route_open_frame_with_admission_facts(
10684 15,
10685 "target",
10686 unique_project_root("admission-facts"),
10687 Some(subc_control::ConsumerIdentity {
10688 module_id: "other".to_string(),
10689 launch_nonce: "other-nonce".to_string(),
10690 }),
10691 Some(json!({"x": 1})),
10692 ),
10693 )
10694 .await
10695 .unwrap();
10696 assert_eq!(
10697 parse_error(&different_reserved[0])["code"],
10698 "admission_facts_not_permitted"
10699 );
10700
10701 let other_target = handler
10702 .handle_control_frame(
10703 &route_ctx(ConnectionId::new(74)).0,
10704 route_open_frame_with_admission_facts(
10705 12,
10706 "other",
10707 unique_project_root("admission-facts"),
10708 Some(subc_control::ConsumerIdentity {
10709 module_id: "fed".to_string(),
10710 launch_nonce: "fed-nonce".to_string(),
10711 }),
10712 Some(json!({"x": 1})),
10713 ),
10714 )
10715 .await
10716 .unwrap();
10717 assert_eq!(
10718 parse_error(&other_target[0])["code"],
10719 "admission_facts_target_not_allowed"
10720 );
10721
10722 let nonexistent = handler
10723 .handle_control_frame(
10724 &route_ctx(ConnectionId::new(75)).0,
10725 route_open_frame_with_admission_facts(
10726 13,
10727 "missing",
10728 unique_project_root("admission-facts"),
10729 None,
10730 Some(json!({"x": 1})),
10731 ),
10732 )
10733 .await
10734 .unwrap();
10735 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
10736
10737 let described = handler
10738 .handle_control_frame(
10739 &route_ctx(ConnectionId::new(76)).0,
10740 Frame::build(
10741 FrameType::Request,
10742 control_flags(),
10743 0,
10744 0,
10745 14,
10746 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10747 )
10748 .unwrap(),
10749 )
10750 .await
10751 .unwrap();
10752 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10753 serde_json::from_slice(&described[0].body).unwrap()
10754 else {
10755 panic!("expected server.describe response")
10756 };
10757 assert!(capabilities
10758 .iter()
10759 .any(|cap| cap == "admission_facts_relay_v1"));
10760 }
10761
10762 #[tokio::test]
10763 async fn admission_facts_without_configured_carrier_are_rejected() {
10764 let registry = Arc::new(Registry::default());
10765 let forwarding = Arc::new(ForwardingTable::default());
10766 let handler = ControlHandler::with_forwarding(registry, forwarding);
10767 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10768 hello_via_sink(
10769 &handler,
10770 &target_ctx,
10771 &mut target_rx,
10772 hello_frame("target", PROTOCOL_VERSION, 1),
10773 )
10774 .await;
10775
10776 let responses = handler
10777 .handle_control_frame(
10778 &route_ctx(ConnectionId::new(79)).0,
10779 route_open_frame_with_admission_facts(
10780 16,
10781 "target",
10782 unique_project_root("admission-facts"),
10783 None,
10784 Some(json!({"x": 1})),
10785 ),
10786 )
10787 .await
10788 .unwrap();
10789 assert_eq!(
10790 parse_error(&responses[0])["code"],
10791 "admission_facts_not_permitted"
10792 );
10793 }
10794
10795 #[tokio::test]
10796 async fn route_open_relays_consumer_capabilities_verbatim() {
10797 let registry = Arc::new(Registry::default());
10798 let forwarding = Arc::new(ForwardingTable::default());
10799 let handler =
10800 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10801 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10802 hello_via_sink(
10803 &handler,
10804 &module_ctx,
10805 &mut module_rx,
10806 hello_frame("aft", PROTOCOL_VERSION, 7),
10807 )
10808 .await;
10809
10810 let expected = vec!["elicitation".to_string(), "roots".to_string()];
10811 let expected_for_request = expected.clone();
10812 let project_root = unique_project_root("consumer-capabilities-present");
10813 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10814 let route_handler = handler.clone();
10815 let route_task = tokio::spawn(async move {
10816 route_handler
10817 .handle_control_frame(
10818 &client_ctx,
10819 route_open_frame_with_consumer_capabilities(
10820 401,
10821 "aft",
10822 project_root,
10823 Some(expected_for_request),
10824 ),
10825 )
10826 .await
10827 .unwrap()
10828 });
10829 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10830 .await
10831 .unwrap()
10832 .unwrap();
10833 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10834 let ModuleControlRequest::RouteBind {
10835 consumer_capabilities,
10836 ..
10837 } = bind
10838 else {
10839 panic!("expected route.bind request, got {bind:?}");
10840 };
10841 assert_eq!(consumer_capabilities, Some(expected.clone()));
10842
10843 handler
10844 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10845 .await
10846 .unwrap();
10847 let route_response = route_task.await.unwrap();
10848 assert!(route_response.is_empty());
10849 let published = client_rx.recv().await.unwrap();
10850 assert!(matches!(
10851 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10852 ClientControlResponse::RouteOpen { .. }
10853 ));
10854 }
10855
10856 #[tokio::test]
10857 async fn route_open_without_consumer_capabilities_relays_none() {
10858 let registry = Arc::new(Registry::default());
10859 let forwarding = Arc::new(ForwardingTable::default());
10860 let handler =
10861 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10862 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10863 hello_via_sink(
10864 &handler,
10865 &module_ctx,
10866 &mut module_rx,
10867 hello_frame("aft", PROTOCOL_VERSION, 7),
10868 )
10869 .await;
10870
10871 let project_root = unique_project_root("consumer-capabilities-absent");
10872 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10873 let route_handler = handler.clone();
10874 let route_task = tokio::spawn(async move {
10875 route_handler
10876 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
10877 .await
10878 .unwrap()
10879 });
10880 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10881 .await
10882 .unwrap()
10883 .unwrap();
10884 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10885 let ModuleControlRequest::RouteBind {
10886 consumer_capabilities,
10887 ..
10888 } = bind
10889 else {
10890 panic!("expected route.bind request, got {bind:?}");
10891 };
10892 assert_eq!(consumer_capabilities, None);
10893
10894 handler
10895 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10896 .await
10897 .unwrap();
10898 let route_response = route_task.await.unwrap();
10899 assert!(route_response.is_empty());
10900 let published = client_rx.recv().await.unwrap();
10901 assert!(matches!(
10902 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10903 ClientControlResponse::RouteOpen { .. }
10904 ));
10905 }
10906
10907 async fn bind_role_versions_for(
10911 sent: Option<BTreeMap<String, String>>,
10912 connection: u64,
10913 ) -> Option<BTreeMap<String, String>> {
10914 let registry = Arc::new(Registry::default());
10915 let forwarding = Arc::new(ForwardingTable::default());
10916 let handler =
10917 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10918 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
10919 hello_via_sink(
10920 &handler,
10921 &module_ctx,
10922 &mut module_rx,
10923 hello_frame("aft", PROTOCOL_VERSION, 7),
10924 )
10925 .await;
10926 let project_root = unique_project_root("role-versions");
10927 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
10928 let route_handler = handler.clone();
10929 let route_task = tokio::spawn(async move {
10930 route_handler
10931 .handle_control_frame(
10932 &client_ctx,
10933 route_open_frame_with_role_versions(403, "aft", project_root, sent),
10934 )
10935 .await
10936 .unwrap()
10937 });
10938 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10939 .await
10940 .expect("a well-formed route.open reaches the module as a bind")
10941 .unwrap();
10942 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10943 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
10944 panic!("expected route.bind request, got {bind:?}");
10945 };
10946 handler
10947 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10948 .await
10949 .unwrap();
10950 assert!(route_task.await.unwrap().is_empty());
10951 let published = client_rx.recv().await.unwrap();
10952 assert!(matches!(
10953 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10954 ClientControlResponse::RouteOpen { .. }
10955 ));
10956 role_versions
10957 }
10958
10959 #[tokio::test]
10960 async fn route_open_relays_role_versions_verbatim() {
10961 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10962 assert_eq!(
10963 bind_role_versions_for(Some(sent.clone()), 141).await,
10964 Some(sent)
10965 );
10966 }
10967
10968 #[tokio::test]
10971 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10972 assert_eq!(bind_role_versions_for(None, 143).await, None);
10973 assert_eq!(
10974 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10975 None
10976 );
10977 }
10978
10979 #[tokio::test]
10980 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10981 let registry = Arc::new(Registry::default());
10982 let forwarding = Arc::new(ForwardingTable::default());
10983 let handler =
10984 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10985 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10986 hello_via_sink(
10987 &handler,
10988 &module_ctx,
10989 &mut module_rx,
10990 hello_frame("aft", PROTOCOL_VERSION, 7),
10991 )
10992 .await;
10993
10994 let nine: BTreeMap<String, String> = (0..9)
10995 .map(|index| (format!("role-{index}"), "v1".to_string()))
10996 .collect();
10997 for (label, malformed) in [
10998 (
10999 "invalid role name",
11000 role_versions(&[("Tool_Provider", "v1")]),
11001 ),
11002 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11003 ("nine entries", nine),
11004 ] {
11005 let responses = tokio::time::timeout(
11008 Duration::from_secs(1),
11009 handler.handle_control_frame(
11010 &route_ctx(ConnectionId::new(148)).0,
11011 route_open_frame_with_role_versions(
11012 404,
11013 "aft",
11014 unique_project_root("role-versions-malformed"),
11015 Some(malformed),
11016 ),
11017 ),
11018 )
11019 .await
11020 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11021 .unwrap();
11022 assert_eq!(responses.len(), 1, "{label}");
11023 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11024 let error = parse_error(&responses[0]);
11025 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11026 assert_eq!(
11027 error["detail"]["field"], "role_versions",
11028 "{label}: {error}"
11029 );
11030 assert!(
11031 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11032 "{label}: a malformed declaration is terminal"
11033 );
11034 assert!(
11035 module_rx.try_recv().is_err(),
11036 "{label}: the module must never see a bind"
11037 );
11038 }
11039 }
11040
11041 #[tokio::test]
11045 async fn route_role_versions_capability_is_advertised() {
11046 let handler = ControlHandler::new(Arc::new(Registry::default()));
11047 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11048 let ack = hello_via_sink(
11049 &handler,
11050 &ctx,
11051 &mut rx,
11052 hello_frame("m", PROTOCOL_VERSION, 1),
11053 )
11054 .await;
11055 let ack = parse_ack(&ack);
11056 assert!(
11057 ack.subc_capabilities
11058 .iter()
11059 .any(|c| c == "route-role-versions/v1"),
11060 "{:?}",
11061 ack.subc_capabilities
11062 );
11063
11064 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11065 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11066 let reply = handler
11067 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11068 .await
11069 .unwrap()
11070 .pop()
11071 .unwrap();
11072 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11073 serde_json::from_slice(&reply.body).unwrap()
11074 else {
11075 panic!("not a server.describe reply");
11076 };
11077 assert!(
11078 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11079 "{capabilities:?}"
11080 );
11081 }
11082
11083 #[tokio::test]
11087 async fn catalog_events_capability_is_advertised() {
11088 let handler = ControlHandler::new(Arc::new(Registry::default()));
11089 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11090 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11091 let reply = handler
11092 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11093 .await
11094 .unwrap()
11095 .pop()
11096 .unwrap();
11097 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11098 serde_json::from_slice(&reply.body).unwrap()
11099 else {
11100 panic!("not a server.describe reply");
11101 };
11102 assert!(
11103 capabilities
11104 .iter()
11105 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11106 "{capabilities:?}"
11107 );
11108 }
11109
11110 #[tokio::test]
11111 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11112 let handler = ControlHandler::new(Arc::new(Registry::default()));
11113 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11114 let ack = hello_via_sink(
11115 &handler,
11116 &module,
11117 &mut module_rx,
11118 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11119 )
11120 .await;
11121 let ack = parse_ack(&ack);
11122 assert!(
11123 ack.subc_capabilities
11124 .iter()
11125 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11126 "{:?}",
11127 ack.subc_capabilities
11128 );
11129 assert!(ack
11130 .subc_capabilities
11131 .iter()
11132 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11133
11134 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11135 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11136 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11137 let reply = handler
11138 .handle_control_frame(&client, frame)
11139 .await
11140 .unwrap()
11141 .pop()
11142 .unwrap();
11143 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11144 serde_json::from_slice(&reply.body).unwrap()
11145 else {
11146 panic!("not a server.describe reply");
11147 };
11148 assert!(
11149 capabilities
11150 .iter()
11151 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11152 "{capabilities:?}"
11153 );
11154 assert!(capabilities
11155 .iter()
11156 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11157 }
11158
11159 #[tokio::test]
11160 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11161 let handler = ControlHandler::new(Arc::new(Registry::default()));
11162 let (module, mut rx) = route_ctx(ConnectionId::new(154));
11163 let ack = hello_via_sink(
11164 &handler,
11165 &module,
11166 &mut rx,
11167 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11168 )
11169 .await;
11170 assert!(parse_ack(&ack)
11171 .subc_capabilities
11172 .iter()
11173 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11174 let (client, _rx) = route_ctx(ConnectionId::new(155));
11175 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11176 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11177 let reply = handler
11178 .handle_control_frame(&client, frame)
11179 .await
11180 .unwrap()
11181 .pop()
11182 .unwrap();
11183 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11184 serde_json::from_slice(&reply.body).unwrap()
11185 else {
11186 panic!("not a server.describe reply")
11187 };
11188 assert!(capabilities
11189 .iter()
11190 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11191 }
11192
11193 #[tokio::test]
11194 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11195 let registry = Arc::new(Registry::default());
11196 let forwarding = Arc::new(ForwardingTable::default());
11197 let handler =
11198 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11199 .with_health_probe_timeout(Duration::from_secs(5));
11200 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11201 hello_via_sink(
11202 &handler,
11203 &module_ctx,
11204 &mut module_rx,
11205 non_routable_hello_frame_with_control_ops(
11206 "mcp",
11207 300,
11208 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11209 ),
11210 )
11211 .await;
11212 assert!(registry
11213 .get_module("mcp")
11214 .unwrap()
11215 .unwrap()
11216 .manifest
11217 .provides
11218 .is_empty());
11219
11220 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11221 let route_response = handler
11222 .handle_control_frame(
11223 &route_client_ctx,
11224 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11225 )
11226 .await
11227 .unwrap();
11228 assert_eq!(route_response[0].header.ty, FrameType::Error);
11229 assert_eq!(
11230 parse_error(&route_response[0])["code"],
11231 "target_unavailable"
11232 );
11233 assert!(parse_error(&route_response[0])["message"]
11234 .as_str()
11235 .unwrap()
11236 .contains("does not provide the requested target"));
11237 assert!(module_rx.try_recv().is_err());
11238
11239 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11240 let health_handler = handler.clone();
11241 let health_task = tokio::spawn(async move {
11242 health_handler
11243 .handle_control_frame(
11244 &health_client_ctx,
11245 supervisor_health_probe_frame(302, "mcp"),
11246 )
11247 .await
11248 .unwrap()
11249 });
11250 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11251 .await
11252 .unwrap()
11253 .unwrap();
11254 assert_eq!(
11255 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11256 ModuleControlRequest::HealthCheck {}
11257 );
11258 handler
11259 .handle_control_frame(
11260 &module_ctx,
11261 health_response(health_frame.header.corr, HealthStatus::Ok),
11262 )
11263 .await
11264 .unwrap();
11265 let health_response = health_task.await.unwrap();
11266 assert_eq!(health_response[0].header.ty, FrameType::Response);
11267 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11268 ClientControlResponse::SupervisorHealthProbe {
11269 module_id, status, ..
11270 } => {
11271 assert_eq!(module_id, "mcp");
11272 assert_eq!(status, HealthStatus::Ok);
11273 }
11274 other => panic!("unexpected health response: {other:?}"),
11275 }
11276
11277 forwarding
11282 .cleanup_connection(module_ctx.connection_id)
11283 .unwrap();
11284 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11285 let cleanup_response = tokio::time::timeout(
11286 Duration::from_millis(200),
11287 handler.handle_control_frame(
11288 &cleanup_probe_ctx,
11289 supervisor_health_probe_frame(303, "mcp"),
11290 ),
11291 )
11292 .await
11293 .expect("probe should fail immediately when the control lane is gone")
11294 .unwrap();
11295 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11296 assert_eq!(
11297 parse_error(&cleanup_response[0])["code"],
11298 "target_unavailable"
11299 );
11300 assert!(parse_error(&cleanup_response[0])["message"]
11301 .as_str()
11302 .unwrap()
11303 .contains("no module connection"));
11304
11305 handler
11306 .cleanup_connection(module_ctx.connection_id)
11307 .unwrap();
11308 }
11309
11310 #[tokio::test]
11311 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11312 let registry = Arc::new(Registry::default());
11313 let supervisor_handle = SupervisorHandle::new();
11314 let supervisor =
11315 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11316 .with_handle(supervisor_handle.clone())
11317 .with_connection_file_path(
11318 std::env::temp_dir()
11319 .join(format!("subc-route-open-warming-{}", std::process::id())),
11320 );
11321 let module = supervisor
11322 .supervise_configured(
11323 ModuleSpec {
11324 module_id: "warming".to_string(),
11325 program: fake_aft_stub_path(),
11326 args: Vec::new(),
11327 env: Vec::new(),
11328 reserved: false,
11329 reserved_prefixes: Vec::new(),
11330 protocol: ModuleProtocol::Subc,
11331 overlap: Default::default(),
11332 },
11333 true,
11334 )
11335 .unwrap();
11336 assert_eq!(module.state().unwrap(), ModuleState::Running);
11337
11338 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11339 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11340 let response = handler
11341 .handle_control_frame(
11342 &ctx,
11343 route_open_frame(304, "warming", unique_project_root("warming")),
11344 )
11345 .await
11346 .unwrap();
11347 module.stop().await.unwrap();
11348
11349 assert_eq!(response[0].header.ty, FrameType::Error);
11350 let error = parse_error(&response[0]);
11351 assert_eq!(error["code"], "module_warming");
11352 assert!(error["message"]
11353 .as_str()
11354 .unwrap()
11355 .contains("state=running, enabled=true, live=false"));
11356 }
11357
11358 #[test]
11359 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11360 let handler = ControlHandler::new(Arc::new(Registry::default()));
11361 let capture = EventCapture::default();
11362 let _subscriber =
11363 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11364 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11365 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11366 let pending = (0..limit).collect::<Vec<_>>();
11367 let response = handler
11368 .route_open_capacity_refusal(
11369 &ctx,
11370 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11371 "busy",
11372 pending.len(),
11373 limit,
11374 )
11375 .unwrap();
11376 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11377 let event = capture
11378 .events()
11379 .into_iter()
11380 .find(|event| {
11381 event.target == "control"
11382 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11383 })
11384 .expect("connection admission refusal event");
11385 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11386 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11387 }
11388
11389 #[test]
11390 fn route_open_target_cap_logs_admission_reason_and_capacity() {
11391 let handler = ControlHandler::new(Arc::new(Registry::default()));
11392 let capture = EventCapture::default();
11393 let _subscriber =
11394 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11395 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11396 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11397 let guards = (0..limit)
11398 .map(|_| {
11399 handler
11400 .route_bind_concurrency
11401 .try_admit("busy", limit)
11402 .unwrap()
11403 })
11404 .collect::<Vec<_>>();
11405 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11406 Err(in_flight) => in_flight,
11407 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11408 };
11409 let response = handler
11410 .route_open_target_capacity_refusal(
11411 &ctx,
11412 &route_open_frame(397, "busy", unique_project_root("target-cap")),
11413 "busy",
11414 in_flight,
11415 )
11416 .unwrap();
11417 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11418 let event = capture
11419 .events()
11420 .into_iter()
11421 .find(|event| {
11422 event.target == "control"
11423 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11424 })
11425 .expect("target admission refusal event");
11426 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11427 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11428 drop(guards);
11429 }
11430
11431 #[tokio::test(flavor = "current_thread")]
11432 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11433 let handler = ControlHandler::new(Arc::new(Registry::default()));
11434 let capture = EventCapture::default();
11435 let _subscriber =
11436 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11437 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11438 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11439
11440 let _ = handler
11441 .handle_client_control_request(
11442 &ctx,
11443 frame,
11444 ClientControlRequest::SupervisorSetEnabled {
11445 module_id: "broca".to_string(),
11446 enabled: false,
11447 },
11448 )
11449 .await
11450 .unwrap();
11451
11452 let events = capture
11453 .events()
11454 .into_iter()
11455 .filter(|event| {
11456 event.target == "control"
11457 && event.fields.get("message").map(String::as_str)
11458 == Some("supervisor request received")
11459 })
11460 .collect::<Vec<_>>();
11461 assert_eq!(events.len(), 1, "one supervisor request log line");
11462 let event = &events[0];
11463 assert_eq!(event.level, tracing::Level::INFO);
11464 assert_eq!(
11465 event.fields.get("op"),
11466 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11467 );
11468 assert_eq!(
11469 event.fields.get("module_id"),
11470 Some(&"\"broca\"".to_string())
11471 );
11472 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11473 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11474 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11475 }
11476
11477 #[tokio::test(flavor = "current_thread")]
11478 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11479 let registry = Arc::new(Registry::default());
11480 let module_connection = ConnectionId::new(303);
11481 registry
11482 .register_with_control_ops(
11483 manifest("aft", PROTOCOL_VERSION),
11484 PROTOCOL_VERSION,
11485 module_connection,
11486 module_baseline_control_ops(),
11487 )
11488 .unwrap();
11489 let handler = ControlHandler::new(registry);
11490 let capture = EventCapture::default();
11491 let _subscriber =
11492 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11493 let (ctx, _rx) = route_ctx(module_connection);
11494 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11495
11496 let _ = handler
11497 .handle_client_control_request(
11498 &ctx,
11499 frame,
11500 ClientControlRequest::SupervisorSetEnabled {
11501 module_id: "broca".to_string(),
11502 enabled: false,
11503 },
11504 )
11505 .await
11506 .unwrap();
11507
11508 let callers = capture
11509 .events()
11510 .into_iter()
11511 .filter(|event| {
11512 event.fields.get("message").map(String::as_str)
11513 == Some("supervisor request received")
11514 })
11515 .map(|event| event.fields.get("caller").cloned())
11516 .collect::<Vec<_>>();
11517 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
11518 }
11519
11520 #[tokio::test(flavor = "current_thread")]
11521 async fn supervisor_list_does_not_log_request_received() {
11522 let handler = ControlHandler::new(Arc::new(Registry::default()));
11523 let capture = EventCapture::default();
11524 let _subscriber =
11525 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11526 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
11527 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
11528
11529 handler
11530 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
11531 .await
11532 .unwrap();
11533
11534 assert!(capture.events().into_iter().all(|event| {
11535 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11536 }));
11537 }
11538
11539 #[tokio::test(flavor = "current_thread")]
11540 async fn supervisor_rescan_preview_does_not_log_request_received() {
11541 let handler = ControlHandler::new(Arc::new(Registry::default()));
11542 let capture = EventCapture::default();
11543 let _subscriber =
11544 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11545 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
11546 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
11547
11548 handler
11549 .handle_client_control_request(
11550 &ctx,
11551 frame,
11552 ClientControlRequest::SupervisorRescan { preview: true },
11553 )
11554 .await
11555 .unwrap();
11556
11557 assert!(capture.events().into_iter().all(|event| {
11558 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11559 }));
11560 }
11561
11562 struct ApproveOperator;
11563 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
11564 fn prompt(
11565 &self,
11566 _: &str,
11567 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
11568 ) -> crate::operator_confirm::ProviderResult {
11569 crate::operator_confirm::ProviderResult::Approved
11570 }
11571 }
11572
11573 #[tokio::test]
11574 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
11575 assert_eq!(
11576 ControlHandler::default().privileged_operator_prompt,
11577 PrivilegedOperatorPrompt::Log
11578 );
11579 for (initial, changed, running) in [
11580 ("log", "enforce", PrivilegedOperatorPrompt::Log),
11581 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
11582 ] {
11583 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
11584 let path = dir.join("subc.jsonc");
11585 std::fs::write(
11586 &path,
11587 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
11588 )
11589 .unwrap();
11590 let config = crate::daemon_config::load(&path).unwrap().unwrap();
11591 let registry = Arc::new(Registry::default());
11592 let handle = SupervisorHandle::new();
11593 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
11594 .with_handle(handle.clone());
11595 let handler = ControlHandler::new(registry)
11596 .with_supervisor(handle)
11597 .with_privileged_operator_prompt(config.privileged_operator_prompt)
11598 .with_supervisor_rescan(supervisor, &path, config.port);
11599 handler.forwarding.operator_confirms().configure(
11600 Arc::new(ApproveOperator),
11601 crate::operator_confirm::OperatorLimits::default(),
11602 );
11603 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
11604
11605 for (disk_mode, needs_restart) in [
11606 (Some(initial), false),
11607 (Some(changed), true),
11608 (Some(changed), true),
11609 (None, running == PrivilegedOperatorPrompt::Enforce),
11610 (Some(initial), false),
11611 ] {
11612 let mut doc = json!({"version":1});
11613 if let Some(mode) = disk_mode {
11614 doc["privileged_operator_prompt"] = json!(mode);
11615 }
11616 std::fs::write(&path, doc.to_string()).unwrap();
11617 for preview in [true, false] {
11618 let request = ClientControlRequest::SupervisorRescan { preview };
11619 let frame = Frame::build(
11620 FrameType::Request,
11621 control_flags(),
11622 0,
11623 0,
11624 1,
11625 serde_json::to_vec(&request).unwrap(),
11626 )
11627 .unwrap();
11628 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11629 assert_eq!(frames.len(), 1);
11630 let ClientControlResponse::SupervisorRescan { result } =
11631 serde_json::from_slice(&frames[0].body).unwrap()
11632 else {
11633 panic!("expected a rescan response");
11634 };
11635 let expected = if needs_restart {
11636 vec!["privileged_operator_prompt"]
11637 } else {
11638 Vec::new()
11639 };
11640 assert_eq!(result.restart_required, expected);
11641 assert_eq!(handler.privileged_operator_prompt, running);
11642 assert_eq!(
11643 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
11644 running
11645 );
11646 }
11647 }
11648 }
11649 }
11650
11651 #[tokio::test]
11652 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
11653 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
11654 let path = dir.join("subc.jsonc");
11655 let registry = Arc::new(Registry::default());
11656 let handle = SupervisorHandle::new();
11657 let supervisor =
11658 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
11659 let handler = ControlHandler::new(registry)
11660 .with_supervisor(handle)
11661 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
11662 .with_supervisor_rescan(supervisor, &path, None);
11663 handler.forwarding.operator_confirms().configure(
11664 Arc::new(ApproveOperator),
11665 crate::operator_confirm::OperatorLimits::default(),
11666 );
11667 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
11668 for preview in [true, false] {
11669 std::fs::write(
11670 &path,
11671 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
11672 )
11673 .unwrap();
11674 let request = ClientControlRequest::SupervisorRescan { preview };
11675 let frame = Frame::build(
11676 FrameType::Request,
11677 control_flags(),
11678 0,
11679 0,
11680 1,
11681 serde_json::to_vec(&request).unwrap(),
11682 )
11683 .unwrap();
11684 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11685 assert_eq!(frames.len(), 1);
11686 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
11687 assert_eq!(
11688 handler.privileged_operator_prompt,
11689 PrivilegedOperatorPrompt::Enforce
11690 );
11691 }
11692 }
11693
11694 #[tokio::test]
11699 async fn route_open_refusal_names_the_check_that_refused() {
11700 let handler = ControlHandler::new(Arc::new(Registry::default()));
11701 let capture = EventCapture::default();
11702 let _subscriber =
11703 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11704 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11705 let response = handler
11706 .handle_control_frame(
11707 &ctx,
11708 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
11709 )
11710 .await
11711 .unwrap();
11712
11713 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11714 let event = capture
11715 .events()
11716 .into_iter()
11717 .find(|event| {
11718 event.target == "control"
11719 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11720 })
11721 .expect("route.open refusal event");
11722 assert_eq!(
11723 event.fields.get("reason"),
11724 Some(&"\"not_registered\"".to_string())
11725 );
11726 }
11727
11728 #[tokio::test]
11729 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
11730 let registry = Arc::new(Registry::default());
11731 let supervisor_handle = SupervisorHandle::new();
11732 let supervisor =
11733 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11734 .with_handle(supervisor_handle.clone())
11735 .with_connection_file_path(std::env::temp_dir().join(format!(
11736 "subc-route-open-refusal-info-{}",
11737 std::process::id()
11738 )));
11739 let module = supervisor
11740 .supervise_configured(
11741 ModuleSpec {
11742 module_id: "warming".to_string(),
11743 program: fake_aft_stub_path(),
11744 args: Vec::new(),
11745 env: Vec::new(),
11746 reserved: false,
11747 reserved_prefixes: Vec::new(),
11748 protocol: ModuleProtocol::Subc,
11749 overlap: Default::default(),
11750 },
11751 true,
11752 )
11753 .unwrap();
11754 assert_eq!(module.state().unwrap(), ModuleState::Running);
11755
11756 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11757 assert!(handler
11758 .counters()
11759 .snapshot()
11760 .get("route_open_refused_by_code")
11761 .is_none());
11762 let capture = EventCapture::default();
11763 let _subscriber =
11764 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11765 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
11766 let response = handler
11767 .handle_control_frame(
11768 &ctx,
11769 route_open_frame(394, "warming", unique_project_root("refusal-info")),
11770 )
11771 .await
11772 .unwrap();
11773 module.stop().await.unwrap();
11774
11775 assert_eq!(parse_error(&response[0])["code"], "module_warming");
11776 let event = capture
11777 .events()
11778 .into_iter()
11779 .find(|event| {
11780 event.target == "control"
11781 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
11782 })
11783 .expect("route.open refusal event");
11784 assert_eq!(
11785 event.fields.get("module_id"),
11786 Some(&"\"warming\"".to_string())
11787 );
11788 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
11789 assert_eq!(
11790 event.fields.get("reason"),
11791 Some(&"\"supervised_not_registered\"".to_string())
11792 );
11793 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
11794 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
11795 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
11796 assert_eq!(
11797 handler.counters().snapshot()["route_open_refused_by_code"],
11798 json!({ "module_warming": 1 })
11799 );
11800 }
11801
11802 const OUTAGE_START: &str = "route.open refusing module: not serving";
11803 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
11804
11805 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
11806 capture
11807 .events()
11808 .into_iter()
11809 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
11810 .collect()
11811 }
11812
11813 fn supervise_stub(
11814 registry: &Arc<Registry>,
11815 module_id: &str,
11816 enabled: bool,
11817 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
11818 let supervisor_handle = SupervisorHandle::new();
11819 let supervisor =
11820 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
11821 .with_handle(supervisor_handle.clone())
11822 .with_connection_file_path(std::env::temp_dir().join(format!(
11823 "subc-route-outage-{module_id}-{}",
11824 std::process::id()
11825 )));
11826 let module = supervisor
11827 .supervise_configured(
11828 ModuleSpec {
11829 module_id: module_id.to_string(),
11830 program: fake_aft_stub_path(),
11831 args: Vec::new(),
11832 env: Vec::new(),
11833 reserved: false,
11834 reserved_prefixes: Vec::new(),
11835 protocol: ModuleProtocol::Subc,
11836 overlap: Default::default(),
11837 },
11838 enabled,
11839 )
11840 .unwrap();
11841 (supervisor_handle, module)
11842 }
11843
11844 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
11845 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
11846 module_id: module_id.to_string(),
11847 drain_timeout_ms: Some(50),
11848 })
11849 .unwrap();
11850 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11851 }
11852
11853 #[test]
11857 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
11858 let registry = Arc::new(Registry::default());
11859 let forwarding = Arc::new(ForwardingTable::default());
11860 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11861 let second = ControlHandler::with_forwarding(registry, forwarding);
11862 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
11863 }
11864
11865 #[tokio::test(flavor = "current_thread")]
11869 async fn route_open_unknown_module_refusals_add_no_outage_state() {
11870 let handler = ControlHandler::new(Arc::new(Registry::default()));
11871 let capture = EventCapture::default();
11872 let _subscriber =
11873 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11874 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
11875 for corr in 0..8 {
11876 let response = handler
11877 .handle_control_frame(
11878 &ctx,
11879 route_open_frame(
11880 380 + corr,
11881 &format!("nobody-{corr}"),
11882 unique_project_root("outage-unknown"),
11883 ),
11884 )
11885 .await
11886 .unwrap();
11887 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11888 }
11889
11890 assert_eq!(handler.route_outages.tracked_module_count(), 0);
11891 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
11892 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
11893 }
11894
11895 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11900 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
11901 let registry = Arc::new(Registry::default());
11902 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
11903 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11904 let capture = EventCapture::default();
11905 let _subscriber =
11906 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11907 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
11908 handler.route_outages.record_accepted("outage-restart");
11911
11912 let response = handler
11913 .handle_control_frame(
11914 &ctx,
11915 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
11916 )
11917 .await
11918 .unwrap();
11919 assert_eq!(response[0].header.ty, FrameType::Error);
11920 let starts = outage_lines(&capture, OUTAGE_START);
11921 assert_eq!(starts.len(), 1, "{starts:?}");
11922 assert_eq!(starts[0].level, tracing::Level::WARN);
11923 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
11924 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
11925 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
11926 handler.route_outages.record_accepted("outage-restart");
11927 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
11928
11929 let restart = handler
11930 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
11931 .await
11932 .unwrap();
11933 assert_eq!(
11934 restart[0].header.ty,
11935 FrameType::Response,
11936 "{:?}",
11937 parse_error(&restart[0])
11938 );
11939 handler
11940 .handle_control_frame(
11941 &ctx,
11942 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
11943 )
11944 .await
11945 .unwrap();
11946 module.stop().await.unwrap();
11947
11948 let starts = outage_lines(&capture, OUTAGE_START);
11949 assert_eq!(starts.len(), 2, "{starts:?}");
11950 assert_eq!(starts[1].level, tracing::Level::INFO);
11951 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
11952 }
11953
11954 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11958 async fn failed_operator_restart_leaves_no_operator_mark() {
11959 let registry = Arc::new(Registry::default());
11960 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
11961 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11962 let capture = EventCapture::default();
11963 let _subscriber =
11964 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11965 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
11966 handler.route_outages.record_accepted("outage-disabled");
11967
11968 let restart = handler
11969 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
11970 .await
11971 .unwrap();
11972 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
11973 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
11974
11975 handler
11976 .handle_control_frame(
11977 &ctx,
11978 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
11979 )
11980 .await
11981 .unwrap();
11982 let starts = outage_lines(&capture, OUTAGE_START);
11983 assert_eq!(starts.len(), 1, "{starts:?}");
11984 assert_eq!(starts[0].level, tracing::Level::WARN);
11985 }
11986
11987 #[tokio::test(flavor = "current_thread")]
11988 async fn route_open_unknown_module_escapes_target_module_id() {
11989 let handler = ControlHandler::new(Arc::new(Registry::default()));
11990 let capture = EventCapture::default();
11991 let _subscriber =
11992 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11993 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
11994 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11995 let response = handler
11996 .handle_control_frame(
11997 &ctx,
11998 route_open_frame(
11999 395,
12000 hostile_module_id,
12001 unique_project_root("hostile-target-module-id"),
12002 ),
12003 )
12004 .await
12005 .unwrap();
12006
12007 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12008 let event = capture
12009 .events()
12010 .into_iter()
12011 .find(|event| {
12012 event.target == "control"
12013 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12014 })
12015 .expect("route.open unknown-module refusal event");
12016 let logged = event.fields.get("module_id").expect("module_id field");
12017 assert!(!logged.bytes().any(|byte| byte < 0x20));
12018 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12019 }
12020
12021 #[tokio::test(flavor = "current_thread")]
12022 async fn route_open_module_rejection_uses_daemon_counter_key() {
12023 let registry = Arc::new(Registry::default());
12024 let forwarding = Arc::new(ForwardingTable::default());
12025 let handler =
12026 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12027 let module_connection = ConnectionId::new(95);
12028 let (module_ctx, mut module_rx) = route_ctx(module_connection);
12029 hello_via_sink(
12030 &handler,
12031 &module_ctx,
12032 &mut module_rx,
12033 hello_frame("aft", PROTOCOL_VERSION, 395),
12034 )
12035 .await;
12036
12037 let client_connection = ConnectionId::new(96);
12038 let (client_ctx, _client_rx) = route_ctx(client_connection);
12039 let capture = EventCapture::default();
12040 let _subscriber =
12041 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12042 let (route_task, bind) = relay_route_open(
12043 &handler,
12044 client_connection,
12045 &client_ctx.egress,
12046 &mut module_rx,
12047 396,
12048 "aft",
12049 "hostile-module-code",
12050 )
12051 .await;
12052 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12053 let rejection = Frame::build(
12054 FrameType::Error,
12055 control_flags(),
12056 0,
12057 0,
12058 bind.header.corr,
12059 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12060 )
12061 .unwrap();
12062 handler
12063 .handle_control_frame(&module_ctx, rejection)
12064 .await
12065 .unwrap();
12066
12067 let response = route_task.await.unwrap();
12068 assert_eq!(parse_error(&response[0])["code"], hostile_code);
12069 let counters = handler.counters().snapshot();
12070 assert_eq!(
12071 counters["route_open_refused_by_code"],
12072 json!({ "module_rejected": 1 })
12073 );
12074 assert!(counters["route_open_refused_by_code"]
12075 .get(hostile_code)
12076 .is_none());
12077
12078 let event = capture
12079 .events()
12080 .into_iter()
12081 .find(|event| {
12082 event.target == "control"
12083 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12084 })
12085 .expect("route.open module-rejection refusal event");
12086 let logged = event.fields.get("module_code").expect("module_code field");
12087 assert!(!logged.bytes().any(|byte| byte < 0x20));
12088 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12089 }
12090
12091 #[tokio::test]
12092 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12093 let registry = Arc::new(Registry::default());
12094 let supervisor_handle = SupervisorHandle::new();
12095 let missing_program = std::env::temp_dir().join(format!(
12096 "subc-route-open-missing-program-{}",
12097 std::process::id()
12098 ));
12099 let supervisor =
12100 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12101 .with_handle(supervisor_handle.clone());
12102 let module = supervisor
12103 .supervise_configured(
12104 ModuleSpec {
12105 module_id: "failed".to_string(),
12106 program: missing_program,
12107 args: Vec::new(),
12108 env: Vec::new(),
12109 reserved: false,
12110 reserved_prefixes: Vec::new(),
12111 protocol: ModuleProtocol::Subc,
12112 overlap: Default::default(),
12113 },
12114 true,
12115 )
12116 .unwrap();
12117 assert_eq!(module.state().unwrap(), ModuleState::Failed);
12118
12119 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12120 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12121 let response = handler
12122 .handle_control_frame(
12123 &ctx,
12124 route_open_frame(305, "failed", unique_project_root("failed")),
12125 )
12126 .await
12127 .unwrap();
12128
12129 assert_eq!(response[0].header.ty, FrameType::Error);
12130 let error = parse_error(&response[0]);
12131 assert_eq!(error["code"], "target_unavailable");
12132 assert!(error["message"]
12133 .as_str()
12134 .unwrap()
12135 .contains("state=failed, enabled=true, live=false"));
12136 }
12137
12138 #[tokio::test]
12139 async fn route_open_role_mismatch_remains_target_unavailable() {
12140 let registry = Arc::new(Registry::default());
12141 let handler = ControlHandler::new(Arc::clone(®istry));
12142 handler
12143 .handle_control(
12144 ConnectionId::new(41),
12145 non_routable_hello_frame_with_control_ops("health-only", 306, None),
12146 )
12147 .unwrap();
12148
12149 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12150 let response = handler
12151 .handle_control_frame(
12152 &ctx,
12153 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12154 )
12155 .await
12156 .unwrap();
12157
12158 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12159 assert!(parse_error(&response[0])["message"]
12160 .as_str()
12161 .unwrap()
12162 .contains("does not provide the requested target"));
12163 }
12164
12165 #[tokio::test]
12166 async fn route_open_inactive_registration_remains_target_unavailable() {
12167 let registry = Arc::new(Registry::default());
12168 let handler = ControlHandler::new(Arc::clone(®istry));
12169 handler
12170 .handle_control(
12171 ConnectionId::new(43),
12172 hello_frame("inactive", PROTOCOL_VERSION, 308),
12173 )
12174 .unwrap();
12175 assert!(registry
12176 .set_module_state_for_test("inactive", ChannelState::Closed)
12177 .unwrap());
12178
12179 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12180 let response = handler
12181 .handle_control_frame(
12182 &ctx,
12183 route_open_frame(309, "inactive", unique_project_root("inactive")),
12184 )
12185 .await
12186 .unwrap();
12187
12188 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12189 assert!(parse_error(&response[0])["message"]
12190 .as_str()
12191 .unwrap()
12192 .contains("is not active"));
12193 }
12194
12195 #[tokio::test]
12196 async fn late_health_reply_is_recorded_through_the_module_response_path() {
12197 let registry = Arc::new(Registry::default());
12198 let forwarding = Arc::new(ForwardingTable::default());
12199 let supervisor_handle = SupervisorHandle::new();
12200 let supervisor =
12201 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
12202 .with_forwarding(Arc::clone(&forwarding))
12203 .with_handle(supervisor_handle.clone());
12204 let module = supervisor
12205 .supervise_configured(
12206 crate::ModuleSpec {
12207 module_id: "late-health-response".to_string(),
12208 program: PathBuf::from("disabled-module"),
12209 args: Vec::new(),
12210 env: Vec::new(),
12211 reserved: false,
12212 reserved_prefixes: Vec::new(),
12213 protocol: ModuleProtocol::Subc,
12214 overlap: Default::default(),
12215 },
12216 false,
12217 )
12218 .unwrap();
12219 let handler =
12220 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12221 .with_supervisor(supervisor_handle);
12222 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12223 handler
12224 .handle_control_frame(
12225 &module_ctx,
12226 hello_frame_with_control_ops(
12227 "late-health-response",
12228 PROTOCOL_VERSION,
12229 7,
12230 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12231 ),
12232 )
12233 .await
12234 .unwrap();
12235 let probe_started_at = Instant::now() - Duration::from_millis(80);
12236 let pending = forwarding
12237 .begin_health_probe_rpc_for(
12238 "late-health-response",
12239 MODULE_CONTROL_OP_HEALTH_CHECK,
12240 probe_started_at,
12241 Instant::now() - Duration::from_millis(1),
12242 )
12243 .unwrap();
12244 assert!(forwarding
12245 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12246 .unwrap());
12247
12248 let responses = handler
12249 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12250 .await
12251 .unwrap();
12252
12253 assert!(responses.is_empty());
12254 let health = module.status().unwrap().health;
12255 assert_eq!(health.late_answer_count, 1);
12256 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12257 }
12258
12259 #[tokio::test]
12260 async fn health_probe_timeout_and_module_death_are_typed() {
12261 let registry = Arc::new(Registry::default());
12262 let forwarding = Arc::new(ForwardingTable::default());
12263 let handler =
12264 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12265 .with_health_probe_timeout(Duration::from_millis(50));
12266 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12267 hello_via_sink(
12268 &handler,
12269 &module_ctx,
12270 &mut module_rx,
12271 hello_frame_with_control_ops(
12272 "aft",
12273 PROTOCOL_VERSION,
12274 7,
12275 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12276 ),
12277 )
12278 .await;
12279
12280 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12281 let responses = handler
12282 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12283 .await
12284 .unwrap();
12285 assert_eq!(responses[0].header.ty, FrameType::Error);
12286 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12287 let _ = module_rx.try_recv();
12288
12289 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12290 let health_handler = handler.clone();
12291 let death_task = tokio::spawn(async move {
12292 health_handler
12293 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12294 .await
12295 .unwrap()
12296 });
12297 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12298 .await
12299 .unwrap()
12300 .unwrap();
12301 handler
12302 .cleanup_connection(module_ctx.connection_id)
12303 .unwrap();
12304 let responses = death_task.await.unwrap();
12305 assert_eq!(responses[0].header.ty, FrameType::Error);
12306 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12307 }
12308
12309 #[test]
12310 fn hello_requires_exact_protocol_version() {
12311 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12312 let registry = Arc::new(Registry::default());
12313 let handler = ControlHandler::new(Arc::clone(®istry));
12314 let responses = handler
12315 .handle_control(
12316 ConnectionId::new(connection),
12317 hello_frame("aft", offered, 9),
12318 )
12319 .unwrap();
12320
12321 assert_eq!(responses.len(), 1);
12322 assert_eq!(responses[0].header.ty, FrameType::Error);
12323 let error = parse_error(&responses[0]);
12324 assert_eq!(error["code"], "version_unsupported");
12325 assert!(registry.get_module("aft").unwrap().is_none());
12326 assert_eq!(registry.active_registration_count().unwrap(), 0);
12327 }
12328 }
12329
12330 #[test]
12331 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12332 let registry = Arc::new(Registry::default());
12333 let forwarding = Arc::new(ForwardingTable::default());
12334 let handler =
12335 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12336 let module_connection = ConnectionId::new(301);
12337 let registration = registry
12338 .register_with_control_ops(
12339 manifest("aft-push", PROTOCOL_VERSION),
12340 PROTOCOL_VERSION,
12341 module_connection,
12342 module_baseline_control_ops(),
12343 )
12344 .unwrap();
12345 let (module_tx, _module_rx) = mpsc::channel(8);
12346 let endpoint = forwarding
12347 .register_module_connection(
12348 module_connection,
12349 "aft-push".to_string(),
12350 PROTOCOL_VERSION,
12351 manifest_concurrency(®istration.manifest),
12352 FrameSink::new(module_tx),
12353 )
12354 .unwrap();
12355
12356 let unknown = Frame::build(
12358 FrameType::Push,
12359 control_flags(),
12360 0,
12361 0,
12362 5,
12363 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12364 )
12365 .unwrap();
12366 let out = handler.handle_status_update(endpoint, unknown).unwrap();
12367 assert!(
12368 out.is_empty(),
12369 "unknown push op must be ignored, got {out:?}"
12370 );
12371
12372 let malformed = Frame::build(
12374 FrameType::Push,
12375 control_flags(),
12376 0,
12377 0,
12378 6,
12379 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
12380 )
12381 .unwrap();
12382 let out = handler.handle_status_update(endpoint, malformed).unwrap();
12383 assert_eq!(out.len(), 1);
12384 assert_eq!(out[0].header.ty, FrameType::Error);
12385 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
12386 }
12387
12388 #[test]
12389 fn hello_rejected_when_connection_already_owns_client_routes() {
12390 let registry = Arc::new(Registry::default());
12391 let forwarding = Arc::new(ForwardingTable::default());
12392 let handler =
12393 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12394 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
12396 let client_connection = ConnectionId::new(202);
12397
12398 let responses = handler
12401 .handle_control(
12402 client_connection,
12403 hello_frame("aft-second", PROTOCOL_VERSION, 9),
12404 )
12405 .unwrap();
12406 assert_eq!(responses[0].header.ty, FrameType::Error);
12407 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12408 assert!(registry.get_module("aft-second").unwrap().is_none());
12409 }
12410
12411 #[tokio::test]
12412 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12413 let registry = Arc::new(Registry::default());
12414 let forwarding = Arc::new(ForwardingTable::default());
12415 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12416 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12417 hello_via_sink(
12418 &handler,
12419 &module_ctx,
12420 &mut module_rx,
12421 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12422 )
12423 .await;
12424 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12425 let pending = forwarding
12426 .begin_route_bind_relay_for_test(
12427 client_ctx.connection_id,
12428 client_ctx.egress.clone(),
12429 2,
12430 "alpha",
12431 )
12432 .unwrap();
12433 forwarding
12434 .complete_pending_relay(
12435 module_ctx.connection_id,
12436 pending.corr,
12437 RouteBindRelayOutcome::Accepted,
12438 )
12439 .unwrap();
12440 client_rx.try_recv().unwrap();
12441 for module_id in ["beta", "alpha"] {
12442 let replies = handler
12443 .handle_control_frame(
12444 &module_ctx,
12445 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12446 )
12447 .await
12448 .unwrap();
12449 assert_eq!(replies.len(), 1, "second HELLO must be refused");
12450 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12451 }
12452 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12453 assert!(registry.get_module("beta").unwrap().is_none());
12454 assert!(matches!(
12455 forwarding
12456 .lookup_data_route(
12457 client_ctx.connection_id,
12458 pending.client_channel,
12459 pending.client_epoch,
12460 )
12461 .unwrap(),
12462 DataRoute::Client(DataRouteState::Bound(_))
12463 ));
12464 assert!(handler
12465 .hello_launch_nonces
12466 .lock()
12467 .unwrap()
12468 .presented(module_ctx.connection_id, Some("alpha-nonce")));
12469 assert!(module_rx.try_recv().is_err());
12470 }
12471
12472 #[test]
12473 fn reserved_module_hello_requires_matching_launch_nonce() {
12474 let registry = Arc::new(Registry::default());
12475 let supervisor = SupervisorHandle::new();
12476 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12479 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12480
12481 let no_nonce = handler
12483 .handle_control(
12484 ConnectionId::new(1),
12485 hello_frame("vault", PROTOCOL_VERSION, 1),
12486 )
12487 .unwrap();
12488 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
12489 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
12490 assert!(registry.get_module("vault").unwrap().is_none());
12491
12492 let wrong = handler
12494 .handle_control(
12495 ConnectionId::new(2),
12496 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
12497 )
12498 .unwrap();
12499 assert_eq!(wrong[0].header.ty, FrameType::Error);
12500 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
12501 assert!(registry.get_module("vault").unwrap().is_none());
12502
12503 let ok = handler
12505 .handle_control(
12506 ConnectionId::new(3),
12507 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
12508 )
12509 .unwrap();
12510 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
12511 assert!(registry.get_module("vault").unwrap().is_some());
12512 }
12513
12514 #[test]
12515 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
12516 let registry = Arc::new(Registry::default());
12517 let supervisor = SupervisorHandle::new();
12518 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12519 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12520 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12521
12522 let squat = handler
12523 .handle_control(
12524 ConnectionId::new(1),
12525 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
12526 )
12527 .unwrap();
12528 assert_eq!(squat[0].header.ty, FrameType::Error);
12529 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
12530 assert!(parse_error(&squat[0])["message"]
12531 .as_str()
12532 .unwrap()
12533 .contains("fed:"));
12534
12535 let accepted_peer = handler
12536 .handle_control(
12537 ConnectionId::new(2),
12538 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
12539 )
12540 .unwrap();
12541 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
12542
12543 let accepted_short = handler
12544 .handle_control(
12545 ConnectionId::new(3),
12546 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
12547 )
12548 .unwrap();
12549 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
12550
12551 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
12552 let response = handler
12553 .handle_control(
12554 ConnectionId::new(conn),
12555 hello_frame(module_id, PROTOCOL_VERSION, conn),
12556 )
12557 .unwrap();
12558 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
12559 }
12560 }
12561
12562 #[test]
12563 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
12564 let registry = Arc::new(Registry::default());
12565 let supervisor = SupervisorHandle::new();
12566 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12567 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12568 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
12569 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12570
12571 let owner_nonce = handler
12572 .handle_control(
12573 ConnectionId::new(1),
12574 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
12575 )
12576 .unwrap();
12577 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
12578 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
12579 assert!(registry.get_module("fed:special").unwrap().is_none());
12580
12581 let exact_nonce = handler
12582 .handle_control(
12583 ConnectionId::new(2),
12584 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
12585 )
12586 .unwrap();
12587 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
12588 assert!(registry.get_module("fed:special").unwrap().is_some());
12589 }
12590
12591 #[test]
12592 fn non_reserved_module_ignores_launch_nonce() {
12593 let registry = Arc::new(Registry::default());
12594 let handler = ControlHandler::new(Arc::clone(®istry));
12597 let no_nonce = handler
12598 .handle_control(
12599 ConnectionId::new(1),
12600 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
12601 )
12602 .unwrap();
12603 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
12604 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
12605
12606 let echoed_nonce = handler
12607 .handle_control(
12608 ConnectionId::new(2),
12609 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
12610 )
12611 .unwrap();
12612 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
12613 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
12614 }
12615
12616 #[test]
12617 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
12618 let handler = ControlHandler::default();
12619 let conn = ConnectionId::new(1);
12620 let malformed = Frame::build(
12621 FrameType::Hello,
12622 control_flags(),
12623 0,
12624 0,
12625 3,
12626 b"{not json".to_vec(),
12627 )
12628 .unwrap();
12629
12630 let error = handler.handle_control(conn, malformed).unwrap();
12631 assert_eq!(error[0].header.ty, FrameType::Error);
12632 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
12633
12634 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12635 let pong = handler.handle_control(conn, ping).unwrap();
12636 assert_eq!(pong[0].header.ty, FrameType::Pong);
12637 assert_eq!(pong[0].header.corr, 4);
12638 }
12639
12640 #[test]
12641 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
12642 let registry = Arc::new(Registry::default());
12643 let handler = ControlHandler::new(Arc::clone(®istry));
12644
12645 handler
12646 .handle_control(
12647 ConnectionId::new(1),
12648 hello_frame("aft", PROTOCOL_VERSION, 1),
12649 )
12650 .unwrap();
12651 let duplicate = handler
12652 .handle_control(
12653 ConnectionId::new(2),
12654 hello_frame("aft", PROTOCOL_VERSION, 2),
12655 )
12656 .unwrap();
12657
12658 assert_eq!(duplicate[0].header.ty, FrameType::Error);
12659 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12660 let registration = registry.get_module("aft").unwrap().unwrap();
12661 assert_eq!(registration.connection_id, ConnectionId::new(1));
12662 }
12663
12664 #[test]
12665 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
12666 let registry = Arc::new(Registry::default());
12667 let forwarding = Arc::new(ForwardingTable::default());
12668 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
12669 let handler =
12670 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12671 .with_process_liveness(process_liveness);
12672 let (ctx, route_channel, route_epoch) =
12673 bind_liveness_route(®istry, &forwarding, "aft-dead");
12674 let responses = handler
12675 .handle_route_poll(
12676 &ctx,
12677 route_poll_frame(41, PollKind::Liveness, route_channel),
12678 route_channel,
12679 route_epoch,
12680 PollKind::Liveness,
12681 )
12682 .unwrap();
12683
12684 assert_eq!(responses.len(), 1);
12685 assert_eq!(responses[0].header.ty, FrameType::Response);
12686 assert_route_poll_liveness(&responses[0], false);
12687 }
12688
12689 #[test]
12690 fn liveness_poll_without_process_source_uses_bound_route() {
12691 let registry = Arc::new(Registry::default());
12692 let forwarding = Arc::new(ForwardingTable::default());
12693 let handler =
12694 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12695 let (ctx, route_channel, route_epoch) =
12696 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
12697 let responses = handler
12698 .handle_route_poll(
12699 &ctx,
12700 route_poll_frame(42, PollKind::Liveness, route_channel),
12701 route_channel,
12702 route_epoch,
12703 PollKind::Liveness,
12704 )
12705 .unwrap();
12706
12707 assert_route_poll_liveness(&responses[0], true);
12708 }
12709
12710 #[test]
12711 fn liveness_poll_untracked_process_source_uses_bound_route() {
12712 let registry = Arc::new(Registry::default());
12713 let forwarding = Arc::new(ForwardingTable::default());
12714 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
12715 let handler =
12716 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12717 .with_process_liveness(process_liveness);
12718 let (ctx, route_channel, route_epoch) =
12719 bind_liveness_route(®istry, &forwarding, "aft-untracked");
12720 let responses = handler
12721 .handle_route_poll(
12722 &ctx,
12723 route_poll_frame(43, PollKind::Liveness, route_channel),
12724 route_channel,
12725 route_epoch,
12726 PollKind::Liveness,
12727 )
12728 .unwrap();
12729
12730 assert_route_poll_liveness(&responses[0], true);
12731 }
12732
12733 #[tokio::test]
12734 async fn unknown_op_returns_unknown_control_op() {
12735 let handler = ControlHandler::default();
12736 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
12737 let request = Frame::build(
12738 FrameType::Request,
12739 control_flags(),
12740 0,
12741 0,
12742 55,
12743 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
12744 )
12745 .unwrap();
12746
12747 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12748
12749 assert_eq!(response.len(), 1);
12750 assert_eq!(response[0].header.ty, FrameType::Error);
12751 assert_eq!(response[0].header.corr, 55);
12752 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
12753 }
12754
12755 #[tokio::test]
12756 async fn supervisor_provenance_rejects_unknown_exact_module() {
12757 let handler = ControlHandler::default();
12758 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
12759 let request = Frame::build(
12760 FrameType::Request,
12761 control_flags(),
12762 0,
12763 0,
12764 57,
12765 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
12766 )
12767 .unwrap();
12768
12769 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12770
12771 assert_eq!(response.len(), 1);
12772 assert_eq!(response[0].header.ty, FrameType::Error);
12773 assert_eq!(response[0].header.corr, 57);
12774 let error = parse_error(&response[0]);
12775 assert_eq!(error["code"], "unknown_module");
12776 assert_eq!(error["message"], "module_id 'missing' is not supervised");
12777 }
12778
12779 #[test]
12780 fn provenance_probe_override_keeps_handler_tests_deterministic() {
12781 let expected = subc_control::RunningImageAgreement::Unavailable {
12782 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12783 };
12784 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
12785 assert_eq!(handler.provenance_probe_override, Some(expected));
12786 }
12787
12788 #[test]
12789 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
12790 let verdict = reload_verdict(
12791 std::path::Path::new("/bin/new"),
12792 Some(std::path::Path::new("/bin/old")),
12793 subc_control::RunningImageAgreement::Unavailable {
12794 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12795 },
12796 );
12797 assert!(matches!(
12798 verdict.path,
12799 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
12800 if configured == std::path::Path::new("/bin/new")
12801 && spawned_from == std::path::Path::new("/bin/old")
12802 ));
12803 }
12804
12805 #[cfg(windows)]
12806 #[test]
12807 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
12808 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
12809 let configured = dir.join("module.exe");
12810 std::fs::write(&configured, b"executable path fixture").unwrap();
12811 let spawned = std::fs::canonicalize(&configured).unwrap();
12812 assert_ne!(
12813 configured, spawned,
12814 "fixture must exercise different path spellings"
12815 );
12816 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
12817 let lower_case = configured.to_string_lossy().to_lowercase();
12818 for spelling in [
12819 &configured,
12820 Path::new(&forward_slashes),
12821 Path::new(&lower_case),
12822 ] {
12823 let verdict = reload_verdict(
12824 spelling,
12825 Some(&spawned),
12826 subc_control::RunningImageAgreement::Unavailable {
12827 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12828 },
12829 );
12830 assert_eq!(
12831 verdict.path,
12832 ReloadPathAgreement::Match,
12833 "configured {spelling:?}, spawned {spawned:?}"
12834 );
12835 }
12836 }
12837
12838 #[cfg(windows)]
12839 #[test]
12840 fn windows_reload_path_keeps_different_existing_program_pending() {
12841 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
12842 let old = dir.join("old.exe");
12843 let new = dir.join("new.exe");
12844 std::fs::write(&old, b"same executable content").unwrap();
12845 std::fs::copy(&old, &new).unwrap();
12846 let spawned = std::fs::canonicalize(&old).unwrap();
12847 let image = subc_control::RunningImageAgreement::Unavailable {
12848 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12849 };
12850 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
12851 assert!(
12852 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
12853 if configured == new && spawned_from == spawned)
12854 );
12855 let missing = dir.join("missing.exe");
12856 assert!(matches!(
12857 reload_verdict(&missing, Some(&spawned), image).path,
12858 ReloadPathAgreement::Mismatch { .. }
12859 ));
12860 }
12861
12862 #[test]
12863 fn reload_verdict_detects_replaced_image_at_same_path() {
12864 let image = subc_control::RunningImageAgreement::Mismatch {
12865 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
12866 digest: "old".into(),
12867 },
12868 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
12869 digest: "new".into(),
12870 },
12871 };
12872 let verdict = reload_verdict(
12873 std::path::Path::new("/bin/same"),
12874 Some(std::path::Path::new("/bin/same")),
12875 image.clone(),
12876 );
12877 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12878 assert_eq!(verdict.image, image);
12879 }
12880
12881 #[test]
12882 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
12883 let image = subc_control::RunningImageAgreement::Unavailable {
12884 reason: subc_control::RunningImageUnavailableReason::NotRunning,
12885 };
12886 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
12887 assert_eq!(
12888 verdict.path,
12889 subc_control::ReloadPathAgreement::Unavailable {
12890 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
12891 }
12892 );
12893 assert_eq!(verdict.image, image);
12894
12895 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
12896 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12897 };
12898 let verdict = reload_verdict(
12899 std::path::Path::new("/bin/same"),
12900 Some(std::path::Path::new("/bin/same")),
12901 unconfirmed.clone(),
12902 );
12903 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12904 assert_eq!(verdict.image, unconfirmed);
12905 }
12906
12907 #[test]
12908 fn reload_verdict_preserves_each_image_unavailability_reason() {
12909 use subc_control::RunningImageUnavailableReason as Reason;
12910
12911 for reason in [
12912 Reason::NotRunning,
12913 Reason::UnsupportedPlatform,
12914 Reason::RunningExecutableUnreadable,
12915 Reason::SpawnedPathUnreadable,
12916 Reason::HashFailed,
12917 Reason::ProcessIdentityUnconfirmed,
12918 Reason::Unknown("future_probe_reason".to_string()),
12919 ] {
12920 let image = subc_control::RunningImageAgreement::Unavailable {
12921 reason: reason.clone(),
12922 };
12923 let verdict = reload_verdict(
12924 std::path::Path::new("/bin/same"),
12925 Some(std::path::Path::new("/bin/same")),
12926 image.clone(),
12927 );
12928 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12929 assert_eq!(verdict.image, image, "{reason:?}");
12930 }
12931 }
12932
12933 #[tokio::test]
12934 async fn malformed_control_bodies_return_invalid_control_body() {
12935 let handler = ControlHandler::default();
12936 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
12937
12938 for (corr, body) in [
12939 (56, br#"{"route_channel":1}"#.as_slice()),
12940 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
12941 (
12942 58,
12943 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
12944 ),
12945 ] {
12946 let request = Frame::build(
12947 FrameType::Request,
12948 control_flags(),
12949 0,
12950 0,
12951 corr,
12952 body.to_vec(),
12953 )
12954 .unwrap();
12955 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12956
12957 assert_eq!(response.len(), 1);
12958 assert_eq!(response[0].header.ty, FrameType::Error);
12959 assert_eq!(response[0].header.corr, corr);
12960 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
12961 }
12962 }
12963
12964 #[tokio::test]
12965 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
12966 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12967 let registry = Arc::new(Registry::default());
12968 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
12969 let router = Router::with_control_handler(Arc::clone(&control));
12970 let connection = router.begin_connection();
12971 let (ctx, mut rx) = route_ctx(connection.id());
12972
12973 router
12974 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
12975 .await
12976 .unwrap();
12977 let response = rx.recv().await.unwrap();
12978 let ack = parse_ack(&response);
12979 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12980 let channel = 1;
12981
12982 let goodbye =
12983 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
12984 router.route_for_connection(&ctx, goodbye).await.unwrap();
12985 assert!(rx.try_recv().is_err());
12986 assert!(registry.get_module("aft").unwrap().is_none());
12987
12988 router
12989 .route_for_connection(&ctx, channel_request(channel, 13))
12990 .await
12991 .unwrap();
12992 let error_frame = rx.recv().await.unwrap();
12993 assert_eq!(error_frame.header.ty, FrameType::Error);
12994 assert_eq!(error_frame.header.channel, channel);
12995 let captured = crate::router::test_log::captured_logs(&logs);
12996 assert_eq!(
12997 captured
12998 .lines()
12999 .filter(|line| {
13000 line.contains("module_id=aft")
13001 && line.contains("reason=explicit_goodbye")
13002 && line.contains("module registration ended")
13003 })
13004 .count(),
13005 1,
13006 "unexpected GOODBYE registry log: {captured}"
13007 );
13008 }
13009
13010 #[tokio::test]
13011 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13012 let registry = Arc::new(Registry::default());
13013 let handler = ControlHandler::new(registry).with_capability_config(
13014 [("prov".to_string(), true), ("cons".to_string(), true)],
13015 BTreeMap::new(),
13016 );
13017 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13018 register_capability_manifest(
13019 &handler,
13020 &provider_ctx,
13021 &mut provider_rx,
13022 capability_manifest("prov", &["thing/v1"], &[]),
13023 1,
13024 )
13025 .await;
13026 let mut consumer = capability_manifest("cons", &[], &[]);
13027 consumer.capabilities.as_mut().unwrap().requires.push(
13028 subc_protocol::manifest::CapabilityRequirement {
13029 capability: "thing/v1".to_string(),
13030 need: subc_protocol::manifest::CapabilityNeed::Required,
13031 },
13032 );
13033 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13034 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13035 assert_eq!(
13036 handler.capability_evaluator.verdict("cons", "thing/v1"),
13037 Some(CapabilityVerdict::Provided)
13038 );
13039 let (mut client_rx, _) = open_route_for_capability_test(
13040 &handler,
13041 &provider_ctx,
13042 &mut provider_rx,
13043 703,
13044 3,
13045 "prov",
13046 None,
13047 )
13048 .await;
13049 let goodbye =
13050 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13051 handler
13052 .handle_control_frame(&provider_ctx, goodbye)
13053 .await
13054 .unwrap();
13055 assert_eq!(
13056 handler.capability_evaluator.verdict("cons", "thing/v1"),
13057 Some(CapabilityVerdict::NeverProvided)
13058 );
13059 let closed = client_rx
13060 .try_recv()
13061 .expect("GOODBYE pushes route.closed before route GOODBYE");
13062 assert!(
13063 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13064 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13065 );
13066 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13067 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13068 }
13069
13070 struct RouteClosedFixture {
13071 handler: ControlHandler,
13072 #[cfg(unix)]
13073 supervisor: Supervisor,
13074 module_ctx: RouteCtx,
13075 #[cfg(unix)]
13076 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13077 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13078 route: Arc<crate::forwarding::RouteBinding>,
13079 }
13080
13081 async fn route_closed_fixture() -> RouteClosedFixture {
13082 let registry = Arc::new(Registry::default());
13083 let handler = ControlHandler::new(Arc::clone(®istry));
13084 #[cfg(unix)]
13085 let supervisor = Supervisor::new(registry, RestartPolicy::default())
13086 .with_forwarding(Arc::clone(&handler.forwarding));
13087 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13088 register_capability_manifest(
13089 &handler,
13090 &module_ctx,
13091 &mut module_rx,
13092 capability_manifest("prov", &[], &[]),
13093 1,
13094 )
13095 .await;
13096 let (client_rx, bind) = open_route_for_capability_test(
13097 &handler,
13098 &module_ctx,
13099 &mut module_rx,
13100 702,
13101 2,
13102 "prov",
13103 None,
13104 )
13105 .await;
13106 let ModuleControlRequest::RouteBind {
13107 route_channel: channel,
13108 epoch: route_epoch,
13109 ..
13110 } = bind
13111 else {
13112 panic!("expected route.bind");
13113 };
13114 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13115 handler
13116 .forwarding
13117 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13118 .unwrap()
13119 else {
13120 panic!("the acknowledged route must be bound");
13121 };
13122 RouteClosedFixture {
13123 handler,
13124 #[cfg(unix)]
13125 supervisor,
13126 module_ctx,
13127 #[cfg(unix)]
13128 module_rx,
13129 client_rx,
13130 route,
13131 }
13132 }
13133
13134 impl RouteClosedFixture {
13135 #[cfg(unix)]
13136 async fn cut(&mut self) {
13137 let notices = async {
13138 let mut draining = self.module_rx.recv().await.unwrap();
13139 assert!(matches!(
13140 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13141 &draining.body
13142 )
13143 .unwrap(),
13144 subc_protocol::session::ModuleControlCommand::Draining {
13145 reason: RouteCloseReason::Restart,
13146 ..
13147 }
13148 ));
13149 draining.flushed.take().unwrap().send(()).unwrap();
13150 let mut closing = self.client_rx.recv().await.unwrap();
13151 assert!(matches!(
13152 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13153 ClientControlPush::RouteClosing {
13154 reason: RouteCloseReason::Restart,
13155 ..
13156 }
13157 ));
13158 closing.flushed.take().unwrap().send(()).unwrap();
13159 };
13160 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13161 result.unwrap();
13162 }
13163
13164 fn disconnect(&mut self) -> ClientControlPush {
13165 self.handler
13166 .cleanup_connection(self.module_ctx.connection_id)
13167 .unwrap();
13168 let closed = self.client_rx.try_recv().unwrap();
13169 assert_eq!(closed.header.ty, FrameType::Push);
13170 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13171 assert!(matches!(
13172 &push,
13173 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13174 if channels == &[self.route.client_channel]
13175 ));
13176 assert_eq!(
13177 self.client_rx.try_recv().unwrap().header.ty,
13178 FrameType::Goodbye
13179 );
13180 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13181 push
13182 }
13183 }
13184
13185 #[cfg(unix)]
13186 #[tokio::test(start_paused = true)]
13187 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13188 let mut fixture = route_closed_fixture().await;
13189 fixture.cut().await;
13190 assert!(matches!(
13191 fixture.disconnect(),
13192 ClientControlPush::RouteClosed {
13193 reason: RouteCloseReason::Restart,
13194 drained: true,
13195 ..
13196 }
13197 ));
13198 }
13199
13200 #[cfg(unix)]
13201 #[tokio::test(start_paused = true)]
13202 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13203 let mut fixture = route_closed_fixture().await;
13204 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13205 let started = tokio::time::Instant::now();
13206 fixture.cut().await;
13207 assert!(started.elapsed() >= Duration::from_secs(2));
13208 assert_eq!(fixture.route.flow.in_flight(), 1);
13209 assert!(matches!(
13210 fixture.disconnect(),
13211 ClientControlPush::RouteClosed {
13212 reason: RouteCloseReason::Restart,
13213 drained: false,
13214 ..
13215 }
13216 ));
13217 }
13218
13219 #[tokio::test]
13220 async fn module_connection_crash_reports_crash_and_not_drained() {
13221 let mut fixture = route_closed_fixture().await;
13222 assert!(matches!(
13226 fixture.disconnect(),
13227 ClientControlPush::RouteClosed {
13228 reason: RouteCloseReason::Crash,
13229 drained: false,
13230 ..
13231 }
13232 ));
13233 }
13234
13235 #[tokio::test]
13236 async fn dropping_router_connection_releases_registration() {
13237 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13238 let registry = Arc::new(Registry::default());
13239 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13240 let router = Router::with_control_handler(Arc::clone(&control));
13241 let connection = router.begin_connection();
13242 let connection_id = connection.id();
13243 let (ctx, mut rx) = route_ctx(connection_id);
13244
13245 router
13246 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13247 .await
13248 .unwrap();
13249 let response = rx.recv().await.unwrap();
13250 let ack = parse_ack(&response);
13251 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13252 assert!(registry.get_module("aft").unwrap().is_some());
13253
13254 drop(connection);
13255
13256 assert!(registry.get_module("aft").unwrap().is_none());
13257 assert_eq!(registry.active_registration_count().unwrap(), 0);
13258
13259 control
13260 .cleanup_connection(ConnectionId::new(u64::MAX))
13261 .unwrap();
13262 let captured = crate::router::test_log::captured_logs(&logs);
13263 println!("captured registry lifecycle logs:\n{captured}");
13264 let events: Vec<_> = captured
13265 .lines()
13266 .filter(|line| {
13267 line.contains("module registered module_id=aft ")
13268 || line.contains("module registration ended")
13269 })
13270 .collect();
13271 assert_eq!(
13272 events.len(),
13273 2,
13274 "unexpected registry lifecycle logs: {captured}"
13275 );
13276 assert!(events[0].contains("module registered module_id=aft "));
13277 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13278 assert!(events[1].contains(&format!(
13279 "module_id=aft connection_id={}",
13280 connection_id.get()
13281 )));
13282 assert!(events[1].contains("reason=connection_closed"));
13283 assert!(events[1].contains("module registration ended"));
13284 assert_eq!(
13285 captured
13286 .lines()
13287 .filter(|line| line.contains("module registered module_id=aft "))
13288 .count(),
13289 1,
13290 "legacy registration admission line must appear once: {captured}"
13291 );
13292 }
13293
13294 #[tokio::test]
13295 async fn hello_registration_keeps_legacy_module_registered_line_once() {
13296 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13297 let registry = Arc::new(Registry::default());
13298 let control = ControlHandler::new(Arc::clone(®istry));
13299 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13300 hello_via_sink(
13301 &control,
13302 &ctx,
13303 &mut rx,
13304 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13305 )
13306 .await;
13307
13308 let captured = crate::router::test_log::captured_logs(&logs);
13309 let admissions: Vec<_> = captured
13310 .lines()
13311 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13312 .collect();
13313 assert_eq!(
13314 admissions.len(),
13315 1,
13316 "legacy admission line must remain exactly once: {captured}"
13317 );
13318 assert!(admissions[0].contains("routable_provider=true"));
13319 assert!(admissions[0].contains("connection_id=777"));
13320 }
13321
13322 fn capability_manifest(
13323 module_id: &str,
13324 provides: &[&str],
13325 must_never_reach: &[&str],
13326 ) -> ModuleManifest {
13327 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13328 manifest.capabilities = Some(CapabilityDeclarations {
13329 provides: provides
13330 .iter()
13331 .map(|capability| (*capability).to_string())
13332 .collect(),
13333 requires: Vec::new(),
13334 must_never_reach: must_never_reach
13335 .iter()
13336 .map(|capability| (*capability).to_string())
13337 .collect(),
13338 });
13339 manifest
13340 }
13341
13342 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13343 Frame::build(
13344 FrameType::Hello,
13345 control_flags(),
13346 0,
13347 0,
13348 corr,
13349 serde_json::to_vec(&ModuleHelloBody {
13350 protocol_ver: manifest.protocol_ver,
13351 manifest,
13352 control_ops: None,
13353 launch_nonce: None,
13354 })
13355 .expect("capability test HELLO serializes"),
13356 )
13357 .expect("capability test HELLO frame builds")
13358 }
13359
13360 fn catalog_update_with_capabilities_frame(
13361 corr: u64,
13362 capabilities: CapabilityDeclarations,
13363 ) -> Frame {
13364 Frame::build(
13365 FrameType::Request,
13366 control_flags(),
13367 0,
13368 0,
13369 corr,
13370 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13371 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
13372 capabilities: Some(capabilities),
13373 ready: None,
13374 })
13375 .expect("capability catalog.update serializes"),
13376 )
13377 .expect("capability catalog.update frame builds")
13378 }
13379
13380 async fn register_capability_manifest(
13381 handler: &ControlHandler,
13382 ctx: &RouteCtx,
13383 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13384 manifest: ModuleManifest,
13385 corr: u64,
13386 ) {
13387 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
13388 }
13389
13390 async fn open_route_for_capability_test(
13391 handler: &ControlHandler,
13392 target_ctx: &RouteCtx,
13393 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13394 client_connection_id: u64,
13395 corr: u64,
13396 target_module_id: &str,
13397 consumer_identity: Option<ConsumerIdentity>,
13398 ) -> (
13399 mpsc::Receiver<crate::router::OutboundFrame>,
13400 ModuleControlRequest,
13401 ) {
13402 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13403 let route_handler = handler.clone();
13404 let target_module_id = target_module_id.to_string();
13405 let route_task = tokio::spawn(async move {
13406 route_handler
13407 .handle_control_frame(
13408 &client_ctx,
13409 route_open_frame_with_admission_facts(
13410 corr,
13411 &target_module_id,
13412 unique_project_root("admission-facts"),
13413 consumer_identity,
13414 None,
13415 ),
13416 )
13417 .await
13418 .expect("capability test route.open succeeds")
13419 });
13420 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13421 .await
13422 .expect("capability test route.open must reach route.bind")
13423 .expect("target control receiver stays open");
13424 let bind_request: ModuleControlRequest =
13425 serde_json::from_slice(&bind.body).expect("route.bind decodes");
13426 handler
13427 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13428 .await
13429 .expect("capability test route.bind ACK succeeds");
13430 assert!(route_task.await.expect("route.open task joins").is_empty());
13431 let opened = client_rx
13432 .recv()
13433 .await
13434 .expect("successful route.open publishes a response");
13435 assert!(matches!(
13436 serde_json::from_slice::<ClientControlResponse>(&opened.body),
13437 Ok(ClientControlResponse::RouteOpen { .. })
13438 ));
13439 (client_rx, bind_request)
13440 }
13441
13442 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13443 assert_eq!(frame.header.ty, FrameType::Push);
13444 assert_eq!(frame.header.channel, 0);
13445 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13446 .expect("route.closed control push decodes");
13447 let ClientControlPush::RouteClosed { channels, .. } = &push else {
13448 panic!("expected route.closed");
13449 };
13450 assert_eq!(channels.len(), 1, "exactly one violating route closed");
13451 let channels = channels.clone();
13452 assert_eq!(
13453 push,
13454 ClientControlPush::RouteClosed {
13455 module_id: target_module_id.to_string(),
13456 channels,
13457 reason: RouteCloseReason::CapabilityDenied,
13458 drained: false,
13459 abandoned: 0,
13460 excluded_subscriptions: 0,
13461 terminal: Some(false),
13462 }
13463 );
13464 }
13465
13466 #[tokio::test]
13467 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
13468 let registry = Arc::new(Registry::default());
13469 let forwarding = Arc::new(ForwardingTable::default());
13470 let supervisor = SupervisorHandle::new();
13471 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13472 let handler =
13473 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13474 .with_supervisor(supervisor);
13475 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
13476 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
13477 register_capability_manifest(
13478 &handler,
13479 &target_ctx,
13480 &mut target_rx,
13481 capability_manifest("target", &["credentials-provider/v1"], &[]),
13482 1,
13483 )
13484 .await;
13485 register_capability_manifest(
13486 &handler,
13487 &opener_ctx,
13488 &mut opener_rx,
13489 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13490 2,
13491 )
13492 .await;
13493
13494 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
13495 let replies = handler
13496 .handle_control_frame(
13497 &client_ctx,
13498 route_open_frame_with_admission_facts(
13499 3,
13500 "target",
13501 unique_project_root("admission-facts"),
13502 Some(ConsumerIdentity {
13503 module_id: "opener".to_string(),
13504 launch_nonce: "opener-nonce".to_string(),
13505 }),
13506 None,
13507 ),
13508 )
13509 .await
13510 .expect("denied route.open returns a typed frame");
13511 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13512 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13513 assert!(
13514 target_rx.try_recv().is_err(),
13515 "forbidden route.open must not relay route.bind"
13516 );
13517 }
13518
13519 #[tokio::test]
13520 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
13521 let registry = Arc::new(Registry::default());
13522 let forwarding = Arc::new(ForwardingTable::default());
13523 let supervisor = SupervisorHandle::new();
13524 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13525 let handler =
13526 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13527 .with_supervisor(supervisor);
13528 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
13529 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
13530 register_capability_manifest(
13531 &handler,
13532 &target_ctx,
13533 &mut target_rx,
13534 capability_manifest("target", &["credentials-provider/v1"], &[]),
13535 1,
13536 )
13537 .await;
13538 register_capability_manifest(
13539 &handler,
13540 &old_opener_ctx,
13541 &mut old_opener_rx,
13542 capability_manifest("opener", &[], &[]),
13543 2,
13544 )
13545 .await;
13546 let (mut client_rx, _) = open_route_for_capability_test(
13547 &handler,
13548 &target_ctx,
13549 &mut target_rx,
13550 712,
13551 3,
13552 "target",
13553 Some(ConsumerIdentity {
13554 module_id: "opener".to_string(),
13555 launch_nonce: "opener-nonce".to_string(),
13556 }),
13557 )
13558 .await;
13559 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13560
13561 handler
13562 .cleanup_connection(old_opener_ctx.connection_id)
13563 .expect("old opener registration cleans up");
13564 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
13565 register_capability_manifest(
13566 &handler,
13567 &new_opener_ctx,
13568 &mut new_opener_rx,
13569 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13570 4,
13571 )
13572 .await;
13573
13574 assert_capability_denied_push(
13575 client_rx
13576 .try_recv()
13577 .expect("HELLO deny addition must emit route.closed")
13578 .frame,
13579 "target",
13580 );
13581 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13582 assert!(matches!(
13583 target_rx.try_recv(),
13584 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13585 ));
13586 }
13587
13588 #[tokio::test]
13589 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
13590 let registry = Arc::new(Registry::default());
13591 let forwarding = Arc::new(ForwardingTable::default());
13592 let supervisor = SupervisorHandle::new();
13593 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13594 let handler =
13595 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13596 .with_supervisor(supervisor);
13597 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
13598 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
13599 register_capability_manifest(
13600 &handler,
13601 &target_ctx,
13602 &mut target_rx,
13603 capability_manifest("target", &[], &[]),
13604 1,
13605 )
13606 .await;
13607 register_capability_manifest(
13608 &handler,
13609 &opener_ctx,
13610 &mut opener_rx,
13611 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13612 2,
13613 )
13614 .await;
13615 let (mut client_rx, _) = open_route_for_capability_test(
13616 &handler,
13617 &target_ctx,
13618 &mut target_rx,
13619 722,
13620 3,
13621 "target",
13622 Some(ConsumerIdentity {
13623 module_id: "opener".to_string(),
13624 launch_nonce: "opener-nonce".to_string(),
13625 }),
13626 )
13627 .await;
13628 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13629
13630 let replies = handler
13631 .handle_control_frame(
13632 &target_ctx,
13633 catalog_update_with_capabilities_frame(
13634 4,
13635 CapabilityDeclarations {
13636 provides: vec!["credentials-provider/v1".to_string()],
13637 requires: Vec::new(),
13638 must_never_reach: Vec::new(),
13639 },
13640 ),
13641 )
13642 .await
13643 .expect("claim catalog.update succeeds");
13644 assert!(matches!(
13645 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
13646 Ok(ModuleControlResponseToModule::CatalogUpdate {})
13647 ));
13648 assert_capability_denied_push(
13649 client_rx
13650 .try_recv()
13651 .expect("claim addition must emit route.closed")
13652 .frame,
13653 "target",
13654 );
13655 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13656 assert!(matches!(
13657 target_rx.try_recv(),
13658 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13659 ));
13660 }
13661
13662 #[tokio::test]
13663 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
13664 let registry = Arc::new(Registry::default());
13665 let forwarding = Arc::new(ForwardingTable::default());
13666 let supervisor = SupervisorHandle::new();
13667 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13668 let handler =
13669 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13670 .with_supervisor(supervisor);
13671 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
13672 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
13673 register_capability_manifest(
13674 &handler,
13675 &target_ctx,
13676 &mut target_rx,
13677 capability_manifest("target", &["credentials-provider/v1"], &[]),
13678 1,
13679 )
13680 .await;
13681 register_capability_manifest(
13682 &handler,
13683 &opener_ctx,
13684 &mut opener_rx,
13685 capability_manifest("opener", &[], &[]),
13686 2,
13687 )
13688 .await;
13689 let (mut client_rx, _) = open_route_for_capability_test(
13690 &handler,
13691 &target_ctx,
13692 &mut target_rx,
13693 732,
13694 3,
13695 "target",
13696 Some(ConsumerIdentity {
13697 module_id: "opener".to_string(),
13698 launch_nonce: "opener-nonce".to_string(),
13699 }),
13700 )
13701 .await;
13702 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13703
13704 handler
13705 .handle_control_frame(
13706 &target_ctx,
13707 catalog_update_with_capabilities_frame(
13708 4,
13709 CapabilityDeclarations {
13710 provides: Vec::new(),
13711 requires: Vec::new(),
13712 must_never_reach: Vec::new(),
13713 },
13714 ),
13715 )
13716 .await
13717 .expect("claim removal catalog.update succeeds");
13718 assert_eq!(
13719 forwarding.active_binding_count().unwrap(),
13720 1,
13721 "removing an attested target claim must leave the route census unchanged"
13722 );
13723 assert!(
13724 client_rx.try_recv().is_err(),
13725 "claim removal must not emit route.closed capability_denied"
13726 );
13727 assert!(
13728 target_rx.try_recv().is_err(),
13729 "claim removal must not send the target a route GOODBYE"
13730 );
13731 }
13732
13733 #[tokio::test]
13736 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
13737 let registry = Arc::new(Registry::default());
13738 let forwarding = Arc::new(ForwardingTable::default());
13739 let supervisor = SupervisorHandle::new();
13740 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13741 let handler =
13742 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13743 .with_supervisor(supervisor);
13744 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
13745 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
13746 register_capability_manifest(
13747 &handler,
13748 &target_ctx,
13749 &mut target_rx,
13750 capability_manifest("target", &["credentials-provider/v1"], &[]),
13751 1,
13752 )
13753 .await;
13754 register_capability_manifest(
13755 &handler,
13756 &opener_ctx,
13757 &mut opener_rx,
13758 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13759 2,
13760 )
13761 .await;
13762
13763 let (_client_rx, bind) = open_route_for_capability_test(
13764 &handler,
13765 &target_ctx,
13766 &mut target_rx,
13767 742,
13768 3,
13769 "target",
13770 None,
13771 )
13772 .await;
13773 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
13774 panic!("direct scope-honesty route must bind");
13775 };
13776 assert_eq!(principal, Some(Principal::Direct));
13777 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13778 }
13779
13780 #[tokio::test]
13783 async fn must_never_reach_self_route_is_capability_forbidden() {
13784 let registry = Arc::new(Registry::default());
13785 let forwarding = Arc::new(ForwardingTable::default());
13786 let supervisor = SupervisorHandle::new();
13787 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
13788 let handler =
13789 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13790 .with_supervisor(supervisor);
13791 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
13792 register_capability_manifest(
13793 &handler,
13794 &self_ctx,
13795 &mut self_rx,
13796 capability_manifest(
13797 "self-provider",
13798 &["credentials-provider/v1"],
13799 &["credentials-provider/v1"],
13800 ),
13801 1,
13802 )
13803 .await;
13804
13805 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
13806 let replies = handler
13807 .handle_control_frame(
13808 &client_ctx,
13809 route_open_frame_with_admission_facts(
13810 2,
13811 "self-provider",
13812 unique_project_root("admission-facts"),
13813 Some(ConsumerIdentity {
13814 module_id: "self-provider".to_string(),
13815 launch_nonce: "self-nonce".to_string(),
13816 }),
13817 None,
13818 ),
13819 )
13820 .await
13821 .expect("self-route refusal returns a typed frame");
13822 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13823 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13824 assert!(
13825 self_rx.try_recv().is_err(),
13826 "self denial must not relay route.bind"
13827 );
13828 }
13829
13830 #[test]
13831 fn unsupported_channel_zero_frame_returns_error() {
13832 let handler = ControlHandler::default();
13833 let request = Frame::build(
13834 FrameType::Request,
13835 control_flags(),
13836 0,
13837 0,
13838 21,
13839 b"opaque".to_vec(),
13840 )
13841 .unwrap();
13842
13843 let response = handler
13844 .handle_control(ConnectionId::new(1), request)
13845 .unwrap();
13846
13847 assert_eq!(response[0].header.ty, FrameType::Error);
13848 assert_eq!(
13849 parse_error(&response[0])["code"],
13850 "unsupported_control_frame"
13851 );
13852 }
13853
13854 mod swap {
13859 use super::*;
13860
13861 const INCUMBENT: ConnectionId = ConnectionId::new(30);
13862 const CANDIDATE: ConnectionId = ConnectionId::new(40);
13863
13864 struct Swap {
13865 registry: Arc<Registry>,
13866 forwarding: Arc<ForwardingTable>,
13867 handler: ControlHandler,
13868 incumbent_ctx: RouteCtx,
13869 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13870 candidate_ctx: RouteCtx,
13871 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13872 }
13873
13874 async fn swap_with_incumbent() -> Swap {
13875 let registry = Arc::new(Registry::default());
13876 let forwarding = Arc::new(ForwardingTable::default());
13877 let handler =
13878 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13879 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
13880 hello_via_sink(
13881 &handler,
13882 &incumbent_ctx,
13883 &mut incumbent_rx,
13884 hello_frame("aft", PROTOCOL_VERSION, 7),
13885 )
13886 .await;
13887 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
13888 Swap {
13889 registry,
13890 forwarding,
13891 handler,
13892 incumbent_ctx,
13893 incumbent_rx,
13894 candidate_ctx,
13895 candidate_rx,
13896 }
13897 }
13898
13899 fn register_candidate(swap: &Swap, ready: Option<bool>) {
13900 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
13901 candidate_manifest.ready = ready;
13902 let registration = swap
13903 .registry
13904 .register_candidate_with_control_ops(
13905 candidate_manifest,
13906 PROTOCOL_VERSION,
13907 CANDIDATE,
13908 module_baseline_control_ops(),
13909 )
13910 .unwrap();
13911 swap.forwarding
13912 .register_candidate_module_connection(
13913 CANDIDATE,
13914 "aft".to_string(),
13915 PROTOCOL_VERSION,
13916 manifest_concurrency(®istration.manifest),
13917 swap.candidate_ctx.egress.clone(),
13918 )
13919 .unwrap();
13920 }
13921
13922 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
13923 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
13924 swap.registry.promote_candidate("aft").unwrap().unwrap();
13925 cutover.incumbent.unwrap()
13926 }
13927
13928 #[tokio::test]
13937 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
13938 let mut swap = swap_with_incumbent().await;
13939 let hold = Arc::new(std::sync::Barrier::new(2));
13940 let handler = swap
13941 .handler
13942 .clone()
13943 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
13944 handler
13945 .supervisor
13946 .open_swap("aft", "candidate-nonce".to_string());
13947 let hello = {
13948 let handler = handler.clone();
13949 let candidate_ctx = swap.candidate_ctx.clone();
13950 std::thread::spawn(move || {
13951 handler
13952 .handle_hello(
13953 candidate_ctx.connection_id,
13954 Some(candidate_ctx.egress.clone()),
13955 hello_frame_with_nonce(
13956 "aft",
13957 PROTOCOL_VERSION,
13958 8,
13959 Some("candidate-nonce"),
13960 ),
13961 )
13962 .unwrap()
13963 })
13964 };
13965
13966 hold.wait();
13967 assert!(swap
13970 .registry
13971 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
13972 .unwrap()
13973 .is_some());
13974 assert!(swap
13975 .registry
13976 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
13977 .unwrap()
13978 .is_none());
13979 assert!(swap
13980 .forwarding
13981 .module_endpoint_for_connection(CANDIDATE)
13982 .unwrap()
13983 .is_none());
13984
13985 hold.wait();
13986 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
13987 assert_eq!(
13988 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
13989 FrameType::HelloAck
13990 );
13991 let published = swap
13992 .registry
13993 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
13994 .unwrap()
13995 .expect("candidate is published once its HELLO completes");
13996 assert_eq!(published.connection_id, CANDIDATE);
13997 assert!(swap
13998 .forwarding
13999 .module_endpoint_for_connection(CANDIDATE)
14000 .unwrap()
14001 .is_some());
14002 }
14003
14004 fn keyed_total(counters: &Value, key: &str) -> u64 {
14005 counters[key]
14006 .as_object()
14007 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
14008 .unwrap_or(0)
14009 }
14010
14011 #[tokio::test]
14012 async fn replacement_logs_old_end_and_new_admission_once() {
14013 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14014 let mut swap = swap_with_incumbent().await;
14015 swap.handler
14016 .supervisor
14017 .open_swap("aft", "candidate-nonce".to_string());
14018 hello_via_sink(
14019 &swap.handler,
14020 &swap.candidate_ctx,
14021 &mut swap.candidate_rx,
14022 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14023 )
14024 .await;
14025 cutover(&swap);
14026 swap.handler.cleanup_connection(INCUMBENT).unwrap();
14027
14028 let captured = crate::router::test_log::captured_logs(&logs);
14029 println!("captured replacement registry logs:\n{captured}");
14030 let new_admissions: Vec<_> = captured
14031 .lines()
14032 .filter(|line| {
14033 line.contains("connection_id=40")
14034 && line.contains("swap candidate registered; not routable until cutover")
14035 })
14036 .collect();
14037 assert_eq!(
14038 new_admissions.len(),
14039 1,
14040 "unexpected admission logs: {captured}"
14041 );
14042 assert!(new_admissions[0].contains("module_id=aft"));
14043 assert!(new_admissions[0].contains("connection_id=40"));
14044 assert!(
14045 new_admissions[0].contains("swap candidate registered; not routable until cutover")
14046 );
14047 assert!(new_admissions[0].contains("ready=true"));
14048
14049 let old_admissions: Vec<_> = captured
14050 .lines()
14051 .filter(|line| {
14052 line.contains("module registered module_id=aft ")
14053 && line.contains("connection_id=30")
14054 })
14055 .collect();
14056 assert_eq!(
14057 old_admissions.len(),
14058 1,
14059 "unexpected incumbent admission: {captured}"
14060 );
14061
14062 let old_ends: Vec<_> = captured
14063 .lines()
14064 .filter(|line| {
14065 line.contains("connection_id=30") && line.contains("module registration ended")
14066 })
14067 .collect();
14068 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14069 assert!(old_ends[0].contains("module_id=aft"));
14070 assert!(old_ends[0].contains("connection_id=30"));
14071 assert!(old_ends[0].contains("reason=replaced"));
14072 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14073 assert_eq!(
14074 captured
14075 .lines()
14076 .filter(|line| line.contains("module registration promoted"))
14077 .count(),
14078 1,
14079 "unexpected promotion logs: {captured}"
14080 );
14081 }
14082
14083 #[tokio::test]
14089 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14090 let mut swap = swap_with_incumbent().await;
14091 let handler = swap.handler.clone();
14092
14093 let cotenant = ConnectionId::new(31);
14095 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14096 let (cotenant_task, cotenant_bind) = relay_route_open(
14097 &handler,
14098 cotenant,
14099 &cotenant_ctx.egress,
14100 &mut swap.incumbent_rx,
14101 100,
14102 "aft",
14103 "swap-cotenant",
14104 )
14105 .await;
14106 handler
14107 .handle_control_frame(
14108 &swap.incumbent_ctx,
14109 route_bind_ack(cotenant_bind.header.corr),
14110 )
14111 .await
14112 .unwrap();
14113 assert!(cotenant_task.await.unwrap().is_empty());
14114 let (cotenant_channel, cotenant_epoch) =
14115 published_route(&cotenant_rx.recv().await.unwrap());
14116
14117 let caller = ConnectionId::new(32);
14119 let (caller_ctx, mut caller_rx) = route_ctx(caller);
14120 let (caller_task, caller_bind) = relay_route_open(
14121 &handler,
14122 caller,
14123 &caller_ctx.egress,
14124 &mut swap.incumbent_rx,
14125 101,
14126 "aft",
14127 "swap-caller",
14128 )
14129 .await;
14130 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14131
14132 register_candidate(&swap, None);
14133 cutover(&swap);
14134
14135 let ack = handler
14137 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14138 .await;
14139 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14140 if module_loop_error.is_some() {
14141 handler.cleanup_connection(INCUMBENT).unwrap();
14144 }
14145
14146 assert!(
14148 cotenant_rx.try_recv().is_err(),
14149 "the co-tenant route on the incumbent was torn down by one late ack: \
14150 {module_loop_error:?}"
14151 );
14152 assert!(matches!(
14153 swap.forwarding
14154 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14155 .unwrap(),
14156 DataRoute::Client(DataRouteState::Bound(_))
14157 ));
14158 assert_eq!(module_loop_error, None);
14159 assert!(swap
14160 .registry
14161 .get_module_by_connection(INCUMBENT)
14162 .unwrap()
14163 .is_some());
14164
14165 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14167 .await
14168 .expect("the incumbent is told to drop the abandoned binding")
14169 .unwrap()
14170 .frame;
14171 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14172 assert_eq!(goodbye.header.channel, abandoned_channel);
14173 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14174 assert!(swap.incumbent_rx.try_recv().is_err());
14175
14176 let response = caller_task.await.unwrap();
14178 assert_eq!(response.len(), 1);
14179 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14180 assert!(caller_rx.try_recv().is_err());
14181
14182 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14186 let counters = handler.counters().snapshot();
14187 assert_eq!(
14188 keyed_total(&counters, "route_open_accepted_by_principal"),
14189 1
14190 );
14191 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14192 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14193 }
14194
14195 #[tokio::test]
14199 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14200 let mut swap = swap_with_incumbent().await;
14201 register_candidate(&swap, None);
14202 let incumbent = cutover(&swap);
14203 swap.forwarding
14204 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14205 .unwrap()
14206 .expect("the incumbent is still registered");
14207
14208 let client = ConnectionId::new(33);
14209 let (client_ctx, mut client_rx) = route_ctx(client);
14210 let route_handler = swap.handler.clone();
14211 let open_ctx = RouteCtx {
14212 connection_id: client,
14213 egress: client_ctx.egress.clone(),
14214 };
14215 let mut route_task = tokio::spawn(async move {
14216 route_handler
14217 .handle_control_frame(
14218 &open_ctx,
14219 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14220 )
14221 .await
14222 .unwrap()
14223 });
14224 let bind = tokio::select! {
14225 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14226 response = &mut route_task => {
14227 let response = response.unwrap();
14228 panic!(
14229 "post-cutover route.open was refused instead of relayed to the candidate: {}",
14230 parse_error(&response[0])["code"]
14231 );
14232 }
14233 };
14234 swap.handler
14235 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14236 .await
14237 .unwrap();
14238 assert!(route_task.await.unwrap().is_empty());
14239 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14240 match swap
14241 .forwarding
14242 .lookup_data_route(client, channel, epoch)
14243 .unwrap()
14244 {
14245 DataRoute::Client(DataRouteState::Bound(route)) => {
14246 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14247 }
14248 other => panic!("expected a bound route on the candidate, got {other:?}"),
14249 }
14250 assert!(swap.incumbent_rx.try_recv().is_err());
14251 }
14252
14253 #[tokio::test]
14258 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14259 let swap = swap_with_incumbent().await;
14260 register_candidate(&swap, Some(false));
14261 let update = Frame::build(
14262 FrameType::Request,
14263 control_flags(),
14264 0,
14265 0,
14266 55,
14267 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14268 provides: manifest("aft", PROTOCOL_VERSION).provides,
14269 capabilities: None,
14270 ready: Some(true),
14271 })
14272 .unwrap(),
14273 )
14274 .unwrap();
14275
14276 let replies = swap
14277 .handler
14278 .handle_control_frame(&swap.candidate_ctx, update)
14279 .await
14280 .unwrap();
14281
14282 assert_eq!(replies.len(), 1);
14283 assert_eq!(
14284 replies[0].header.ty,
14285 FrameType::Response,
14286 "candidate catalog.update was refused: {:?}",
14287 serde_json::from_slice::<Value>(&replies[0].body).ok()
14288 );
14289 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14290 assert_eq!(
14291 swap.registry
14292 .get_module("aft")
14293 .unwrap()
14294 .unwrap()
14295 .connection_id,
14296 INCUMBENT
14297 );
14298 }
14299 }
14300
14301 mod swap_admission {
14305 use super::*;
14306
14307 const INCUMBENT_NONCE: &str = "incumbent-nonce";
14308 const CANDIDATE_NONCE: &str = "candidate-nonce";
14309
14310 fn handler_with_incumbent(
14311 module_id: &str,
14312 reserved: bool,
14313 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14314 let registry = Arc::new(Registry::default());
14315 let supervisor = SupervisorHandle::new();
14316 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14317 if reserved {
14318 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14319 }
14320 let handler =
14321 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
14322 let incumbent = handler
14323 .handle_control(
14324 ConnectionId::new(1),
14325 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14326 )
14327 .unwrap();
14328 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14329 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14330 (registry, supervisor, handler)
14331 }
14332
14333 #[test]
14339 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14340 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14341
14342 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14343 let replies = handler
14344 .handle_control(
14345 ConnectionId::new(connection),
14346 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
14347 )
14348 .unwrap();
14349 assert_eq!(replies[0].header.ty, FrameType::Error);
14350 assert_eq!(
14351 parse_error(&replies[0])["code"],
14352 "swap_token_invalid",
14353 "nonce {nonce:?}"
14354 );
14355 }
14356 assert!(registry.get_candidate("aft").unwrap().is_none());
14357 assert_eq!(
14358 registry.get_module("aft").unwrap().unwrap().connection_id,
14359 ConnectionId::new(1)
14360 );
14361
14362 let admitted = handler
14365 .handle_control(
14366 ConnectionId::new(4),
14367 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
14368 )
14369 .unwrap();
14370 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
14371 assert_eq!(
14372 registry
14373 .get_candidate("aft")
14374 .unwrap()
14375 .unwrap()
14376 .connection_id,
14377 ConnectionId::new(4)
14378 );
14379 assert_eq!(
14380 registry.get_module("aft").unwrap().unwrap().connection_id,
14381 ConnectionId::new(1),
14382 "the candidate must not take the active slot"
14383 );
14384 let replayed = handler
14385 .handle_control(
14386 ConnectionId::new(5),
14387 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
14388 )
14389 .unwrap();
14390 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
14391
14392 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
14396 let squatter = handler
14397 .handle_control(
14398 ConnectionId::new(6),
14399 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
14400 )
14401 .unwrap();
14402 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14403 assert!(
14404 registry.get_module("aft").unwrap().is_none(),
14405 "a squatter took the active slot of an id being swapped"
14406 );
14407 }
14408
14409 #[test]
14414 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14415 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14416
14417 let replies = handler
14418 .handle_control(
14419 ConnectionId::new(2),
14420 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14421 )
14422 .unwrap();
14423
14424 assert_eq!(
14425 replies[0].header.ty,
14426 FrameType::HelloAck,
14427 "reserved candidate refused: {:?}",
14428 serde_json::from_slice::<Value>(&replies[0].body).ok()
14429 );
14430 assert_eq!(
14431 registry
14432 .get_candidate("vault")
14433 .unwrap()
14434 .unwrap()
14435 .connection_id,
14436 ConnectionId::new(2)
14437 );
14438 }
14439
14440 #[test]
14443 fn without_an_open_swap_the_ordinary_gates_decide() {
14444 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14445 supervisor.close_swap("vault");
14446
14447 let candidate = handler
14448 .handle_control(
14449 ConnectionId::new(2),
14450 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14451 )
14452 .unwrap();
14453 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14454 let duplicate = handler
14455 .handle_control(
14456 ConnectionId::new(3),
14457 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
14458 )
14459 .unwrap();
14460 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14461 assert!(registry.get_candidate("vault").unwrap().is_none());
14462 }
14463 }
14464
14465 mod scopes {
14469 mod idle_tests {
14470 use super::*;
14471 include!("scope_idle_tests.rs");
14472 }
14473 mod activity {
14474 include!("activity_scope_tests.rs");
14475 }
14476 mod delta {
14477 include!("scope_handler_delta_tests.rs");
14478 }
14479 mod call_modes {
14480 include!("scope_call_mode_tests.rs");
14481 }
14482
14483 use subc_protocol::scope::{
14484 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
14485 ScopeStatus,
14486 };
14487
14488 use super::*;
14489
14490 const OWNER: &str = "prefrontal-core";
14491
14492 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
14493 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
14494 }
14495
14496 async fn call(
14497 handler: &ControlHandler,
14498 ctx: &RouteCtx,
14499 request: &ModuleControlRequestFromModule,
14500 ) -> Frame {
14501 let body = serde_json::to_vec(request).unwrap();
14502 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
14503 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
14504 assert_eq!(replies.len(), 1, "{replies:?}");
14505 replies.pop().unwrap()
14506 }
14507
14508 async fn sync(
14509 handler: &ControlHandler,
14510 ctx: &RouteCtx,
14511 generation: u64,
14512 scopes: Vec<ScopeRecord>,
14513 ) -> Result<ModuleControlResponseToModule, String> {
14514 let reply = call(
14515 handler,
14516 ctx,
14517 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
14518 )
14519 .await;
14520 match reply.header.ty {
14521 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
14522 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
14523 }
14524 }
14525
14526 async fn describe(
14527 handler: &ControlHandler,
14528 ctx: &RouteCtx,
14529 owner: &str,
14530 scope_ref: &str,
14531 ) -> ModuleControlResponseToModule {
14532 let reply = call(
14533 handler,
14534 ctx,
14535 &ModuleControlRequestFromModule::ScopeDescribe {
14536 owner: Principal::Reserved {
14537 module_id: owner.to_string(),
14538 },
14539 scope_ref: scope_ref.to_string(),
14540 },
14541 )
14542 .await;
14543 assert_eq!(
14544 reply.header.ty,
14545 FrameType::Response,
14546 "{:?}",
14547 parse_error(&reply)
14548 );
14549 serde_json::from_slice(&reply.body).unwrap()
14550 }
14551
14552 #[tokio::test]
14553 async fn scope_describe_never_echoes_bind_proof_or_hash() {
14554 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
14555 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
14556 let mut rig = rig().await;
14557 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
14558 .await;
14559 let _bound = rig
14560 .bound(
14561 Some(AFT),
14562 PLEXUS,
14563 Some(
14564 rig.selector("s", Some(1))
14565 .with_bind_proof(Some(PROOF.into())),
14566 ),
14567 )
14568 .await;
14569 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
14570 let output = serde_json::to_string(&reply).unwrap();
14571 assert!(output.contains("\"status\":\"live\""));
14572 assert!(!output.contains(PROOF));
14573 assert!(!output.contains(HASH));
14574 }
14575
14576 async fn module(
14578 handler: &ControlHandler,
14579 connection: u64,
14580 module_id: &str,
14581 nonce: Option<&str>,
14582 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14583 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
14584 hello_via_sink(
14585 handler,
14586 &ctx,
14587 &mut rx,
14588 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
14589 )
14590 .await;
14591 (ctx, rx)
14592 }
14593
14594 #[tokio::test]
14597 async fn a_client_connection_cannot_sync_or_describe() {
14598 let handler = ControlHandler::new(Arc::new(Registry::default()));
14599 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
14600 for request in [
14601 ModuleControlRequestFromModule::ScopeSync {
14602 generation: 1,
14603 scopes: vec![head("s", 1)],
14604 },
14605 ModuleControlRequestFromModule::ScopeDescribe {
14606 owner: Principal::Direct,
14607 scope_ref: "s".to_string(),
14608 },
14609 ] {
14610 let reply = call(&handler, &ctx, &request).await;
14611 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
14612 }
14613 assert!(
14614 !handler
14615 .scopes
14616 .read()
14617 .unwrap()
14618 .describe(
14619 &Principal::Reserved {
14620 module_id: OWNER.to_string()
14621 },
14622 "s"
14623 )
14624 .owner_synced
14625 );
14626 }
14627
14628 #[tokio::test]
14631 async fn a_module_without_a_supervised_launch_cannot_sync() {
14632 let handler = ControlHandler::new(Arc::new(Registry::default()));
14633 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
14634 assert_eq!(
14635 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
14636 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14637 );
14638 }
14639
14640 #[tokio::test]
14641 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
14642 let supervisor = SupervisorHandle::new();
14643 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14644 let handler = ControlHandler::new(Arc::new(Registry::default()))
14645 .with_supervisor(supervisor.clone());
14646 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14647 sync(&handler, &incumbent, 1, vec![head("s", 1)])
14648 .await
14649 .expect("the current launch syncs");
14650
14651 supervisor.open_swap(OWNER, "n2".to_string());
14654 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
14655 assert_eq!(
14656 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14657 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14658 );
14659 sync(&handler, &incumbent, 2, vec![head("s", 1)])
14660 .await
14661 .expect("the serving owner syncs during the swap");
14662
14663 supervisor.close_swap(OWNER);
14666 assert_eq!(
14667 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14668 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14669 );
14670 sync(&handler, &incumbent, 3, vec![head("s", 1)])
14671 .await
14672 .expect("the serving owner syncs after the rollback");
14673 handler.cleanup_connection(candidate.connection_id).unwrap();
14674
14675 supervisor.open_swap(OWNER, "n3".to_string());
14680 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
14681 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
14682 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
14683 .await
14684 .expect("the promoted launch takes authority");
14685 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14686 panic!("unexpected reply {reply:?}");
14687 };
14688 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
14689 assert_eq!(
14690 sync(&handler, &incumbent, 4, Vec::new()).await,
14691 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14692 );
14693 }
14694
14695 #[tokio::test]
14698 async fn closing_the_authority_connection_frees_sync_authority() {
14699 let supervisor = SupervisorHandle::new();
14700 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14701 let handler = ControlHandler::new(Arc::new(Registry::default()))
14702 .with_supervisor(supervisor.clone());
14703 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14704 sync(&handler, &first, 10, vec![head("s", 1)])
14705 .await
14706 .unwrap();
14707 handler.cleanup_connection(first.connection_id).unwrap();
14708
14709 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
14710 sync(&handler, &second, 1, vec![head("s", 1)])
14711 .await
14712 .expect("the next connection takes the released authority");
14713 }
14714
14715 #[tokio::test]
14716 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
14717 let supervisor = SupervisorHandle::new();
14718 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14719 let handler =
14720 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
14721 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
14722 sync(&handler, &first, 10, vec![head("s", 1)])
14723 .await
14724 .unwrap();
14725 handler
14726 .handle_control_frame(
14727 &first,
14728 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
14729 )
14730 .await
14731 .unwrap();
14732 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
14733 sync(&handler, &second, 1, vec![head("s", 1)])
14734 .await
14735 .expect("GOODBYE releases authority even if the old socket remains open");
14736 }
14737
14738 #[tokio::test]
14739 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
14740 let registry = Arc::new(Registry::default());
14741 let supervisor_handle = SupervisorHandle::new();
14742 let supervisor =
14743 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14744 .with_handle(supervisor_handle.clone())
14745 .with_daemon_incarnation("incarnation-7".to_string());
14746 supervisor
14749 .supervise_configured(
14750 ModuleSpec {
14751 module_id: OWNER.to_string(),
14752 program: PathBuf::from("/nonexistent/prefrontal-core"),
14753 args: Vec::new(),
14754 env: Vec::new(),
14755 reserved: false,
14756 reserved_prefixes: Vec::new(),
14757 protocol: ModuleProtocol::Subc,
14758 overlap: Default::default(),
14759 },
14760 false,
14761 )
14762 .unwrap();
14763 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
14764 let handler =
14765 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14766 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
14767
14768 let ModuleControlResponseToModule::ScopeDescribe {
14770 status,
14771 daemon_incarnation,
14772 owner_synced,
14773 owner_configured,
14774 scope,
14775 ..
14776 } = describe(&handler, &reader, OWNER, "s").await
14777 else {
14778 panic!("not a describe reply");
14779 };
14780 assert_eq!(status, ScopeStatus::NotLive);
14781 assert_eq!(daemon_incarnation, "incarnation-7");
14782 assert!(!owner_synced);
14783 assert!(owner_configured);
14784 assert!(scope.is_none());
14785
14786 let ModuleControlResponseToModule::ScopeDescribe {
14789 status,
14790 owner_configured,
14791 ..
14792 } = describe(&handler, &reader, "ghost", "s").await
14793 else {
14794 panic!("not a describe reply");
14795 };
14796 assert_eq!(status, ScopeStatus::NotLive);
14797 assert!(!owner_configured);
14798
14799 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
14801 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
14802 let ModuleControlResponseToModule::ScopeDescribe {
14803 status,
14804 scope_epoch,
14805 owner_synced,
14806 scope,
14807 ..
14808 } = describe(&handler, &reader, OWNER, "s").await
14809 else {
14810 panic!("not a describe reply");
14811 };
14812 assert_eq!(status, ScopeStatus::Live);
14813 assert_eq!(scope_epoch, Some(4));
14814 assert!(owner_synced);
14815 let stamp = scope.expect("a live scope carries its stamp");
14816 assert!(
14817 stamp.owner_authorized,
14818 "prefrontal-core is the default authority"
14819 );
14820 assert_eq!(stamp.kind, ScopeKind::Head);
14821 }
14822
14823 #[tokio::test]
14824 async fn scope_authority_owners_decides_owner_authorized() {
14825 let supervisor = SupervisorHandle::new();
14826 supervisor.set_spawn_nonce("broca", "b1".to_string());
14827 let handler = ControlHandler::new(Arc::new(Registry::default()))
14828 .with_supervisor(supervisor)
14829 .with_scope_authority_owners(vec!["broca".to_string()]);
14830 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
14831 let mut gated = head("s", 1);
14832 gated.attributes.agent_id = Some("agent".to_string());
14833 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
14834 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
14835 describe(&handler, &broca, "broca", "s").await
14836 else {
14837 panic!("not a describe reply");
14838 };
14839 assert!(scope.unwrap().owner_authorized);
14840 }
14841
14842 #[tokio::test]
14846 async fn scope_ops_and_the_scopes_capability_are_advertised() {
14847 let handler = ControlHandler::new(Arc::new(Registry::default()));
14848 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
14849 let ack = hello_via_sink(
14850 &handler,
14851 &ctx,
14852 &mut rx,
14853 hello_frame("m", PROTOCOL_VERSION, 1),
14854 )
14855 .await;
14856 let ack = parse_ack(&ack);
14857 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
14858 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
14859 }
14860 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
14861
14862 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
14863 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14864 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14865 let reply = handler
14866 .handle_control_frame(&client, frame)
14867 .await
14868 .unwrap()
14869 .pop()
14870 .unwrap();
14871 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14872 serde_json::from_slice(&reply.body).unwrap()
14873 else {
14874 panic!("not a server.describe reply");
14875 };
14876 assert!(
14877 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
14878 "{capabilities:?}"
14879 );
14880 }
14881
14882 const PLEXUS: &str = "plexus";
14885 const OTHER: &str = "other";
14886 const AFT: &str = "aft";
14887 const BROCA: &str = "broca";
14888 const MAGIC: &str = "magic-context";
14889
14890 fn nonce(module_id: &str) -> String {
14891 format!("nonce-{module_id}")
14892 }
14893
14894 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14895 let (tx, rx) = mpsc::channel(64);
14896 (
14897 RouteCtx {
14898 connection_id: ConnectionId::new(connection),
14899 egress: FrameSink::new(tx),
14900 },
14901 rx,
14902 )
14903 }
14904
14905 struct Rig {
14909 handler: ControlHandler,
14910 forwarding: Arc<ForwardingTable>,
14911 owner: RouteCtx,
14912 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14913 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
14914 generation: u64,
14915 next_connection: u64,
14916 _supervisor: Supervisor,
14917 }
14918
14919 async fn rig() -> Rig {
14920 rig_with_flow_support(true).await
14921 }
14922
14923 async fn rig_with_flow_support(flow_support: bool) -> Rig {
14924 rig_with_scope_capabilities(flow_support, false).await
14925 }
14926
14927 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
14928 let mut provides = Vec::new();
14929 if flow_support {
14930 provides.push("flow-scopes/v1");
14931 }
14932 if run_support {
14933 provides.push("agent-run-scopes/v1");
14934 }
14935 rig_with_target_capabilities(&provides).await
14936 }
14937
14938 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
14939 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
14940 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14941 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14944 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
14945 Frame::build(
14946 FrameType::Hello,
14947 control_flags(),
14948 0,
14949 0,
14950 connection,
14951 serde_json::to_vec(&body).unwrap(),
14952 )
14953 .unwrap()
14954 }
14955
14956 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
14957 let registry = Arc::new(Registry::default());
14958 let forwarding = Arc::new(ForwardingTable::default());
14959 let supervisor_handle = SupervisorHandle::new();
14960 let supervisor =
14961 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14962 .with_handle(supervisor_handle.clone());
14963 supervisor
14964 .supervise_configured(
14965 ModuleSpec {
14966 module_id: OWNER.to_string(),
14967 program: PathBuf::from("/nonexistent/prefrontal-core"),
14968 args: Vec::new(),
14969 env: Vec::new(),
14970 reserved: false,
14971 reserved_prefixes: Vec::new(),
14972 protocol: ModuleProtocol::Subc,
14973 overlap: Default::default(),
14974 },
14975 false,
14976 )
14977 .unwrap();
14978 for module_id in [OWNER, AFT, BROCA, MAGIC] {
14979 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
14980 }
14981 let handler =
14982 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14983 .with_supervisor(supervisor_handle);
14984 let (owner, mut owner_rx) = wide_ctx(1);
14985 hello_via_sink(
14986 &handler,
14987 &owner,
14988 &mut owner_rx,
14989 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
14990 )
14991 .await;
14992 let mut modules = BTreeMap::new();
14993 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
14994 let (ctx, mut rx) = wide_ctx(connection);
14995 let hello = hello_with_capabilities(module_id, connection, provides);
14996 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
14997 modules.insert(module_id.to_string(), (ctx, rx));
14998 }
14999 Rig {
15000 handler,
15001 forwarding,
15002 owner,
15003 _owner_rx: owner_rx,
15004 modules,
15005 generation: 0,
15006 next_connection: 100,
15007 _supervisor: supervisor,
15008 }
15009 }
15010
15011 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15012 ScopeCarrier::new(Principal::Reserved {
15013 module_id: module_id.to_string(),
15014 })
15015 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15016 }
15017
15018 fn session(scope_epoch: u64) -> ScopeRecord {
15021 let mut record = head("s", scope_epoch);
15022 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15023 record.attributes.agent_id = Some("agent-1".to_string());
15024 record.attributes.delegates = true;
15025 record
15026 }
15027
15028 impl Rig {
15029 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15030 self.generation += 1;
15031 sync(&self.handler, &self.owner, self.generation, scopes)
15032 .await
15033 .expect("the owner's sync is accepted");
15034 }
15035
15036 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15037 ScopeSelector::new(
15038 Principal::Reserved {
15039 module_id: OWNER.to_string(),
15040 },
15041 scope_ref.to_string(),
15042 scope_epoch,
15043 )
15044 }
15045
15046 fn open_frame(
15047 &mut self,
15048 opener: Option<&str>,
15049 target: &str,
15050 scope: Option<ScopeSelector>,
15051 ) -> (
15052 RouteCtx,
15053 mpsc::Receiver<crate::router::OutboundFrame>,
15054 Frame,
15055 ) {
15056 self.next_connection += 1;
15057 let (ctx, rx) = wide_ctx(self.next_connection);
15058 let root = unique_project_root("scoped-open");
15059 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15060 target: RouteTarget::ToolProvider {
15061 module_id: target.to_string(),
15062 },
15063 identity: BindIdentity::new(
15064 root.path().to_path_buf(),
15065 "unit".to_string(),
15066 "session".to_string(),
15067 ),
15068 consumer_identity: opener.map(|module_id| ConsumerIdentity {
15069 module_id: module_id.to_string(),
15070 launch_nonce: nonce(module_id),
15071 }),
15072 consumer_capabilities: None,
15073 role_versions: None,
15074 admission_facts: None,
15075 scope,
15076 })
15077 .unwrap();
15078 let frame = Frame::build(
15079 FrameType::Request,
15080 control_flags(),
15081 0,
15082 0,
15083 self.next_connection,
15084 body,
15085 )
15086 .unwrap();
15087 (ctx, rx, frame)
15088 }
15089
15090 async fn refused(
15092 &mut self,
15093 opener: Option<&str>,
15094 target: &str,
15095 scope: Option<ScopeSelector>,
15096 ) -> String {
15097 self.refusal_body(opener, target, scope).await["code"]
15098 .as_str()
15099 .unwrap()
15100 .to_string()
15101 }
15102
15103 async fn refusal_body(
15104 &mut self,
15105 opener: Option<&str>,
15106 target: &str,
15107 scope: Option<ScopeSelector>,
15108 ) -> Value {
15109 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15110 let replies = tokio::time::timeout(
15111 Duration::from_secs(2),
15112 self.handler.handle_control_frame(&ctx, frame),
15113 )
15114 .await
15115 .expect("the open must be refused before waiting for a bind ack")
15116 .unwrap();
15117 assert_eq!(replies.len(), 1, "{replies:?}");
15118 assert_eq!(replies[0].header.ty, FrameType::Error);
15119 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15120 assert!(
15121 module_rx.try_recv().is_err(),
15122 "a refused open relays nothing"
15123 );
15124 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15125 parse_error(&replies[0])
15126 }
15127
15128 async fn relayed(
15130 &mut self,
15131 opener: Option<&str>,
15132 target: &str,
15133 scope: Option<ScopeSelector>,
15134 ) -> Relayed {
15135 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15136 let handler = self.handler.clone();
15137 let task_ctx = ctx.clone();
15138 let task = tokio::spawn(async move {
15139 handler
15140 .handle_control_frame(&task_ctx, frame)
15141 .await
15142 .unwrap()
15143 });
15144 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15145 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15146 .await
15147 .expect("the target receives the relayed route.bind")
15148 .unwrap()
15149 .frame;
15150 Relayed {
15151 target: target.to_string(),
15152 client: ctx,
15153 client_rx: rx,
15154 task,
15155 bind,
15156 }
15157 }
15158
15159 async fn ack(&self, relayed: &Relayed) {
15160 let (module, _) = &self.modules[&relayed.target];
15161 self.handler
15162 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15163 .await
15164 .unwrap();
15165 }
15166
15167 async fn bound(
15169 &mut self,
15170 opener: Option<&str>,
15171 target: &str,
15172 scope: Option<ScopeSelector>,
15173 ) -> Bound {
15174 let relayed = self.relayed(opener, target, scope).await;
15175 self.ack(&relayed).await;
15176 let Relayed {
15177 target,
15178 client,
15179 mut client_rx,
15180 task,
15181 bind,
15182 } = relayed;
15183 assert!(
15184 task.await.unwrap().is_empty(),
15185 "the open is answered by commit"
15186 );
15187 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15188 Bound {
15189 target,
15190 client,
15191 client_rx,
15192 channel,
15193 epoch,
15194 bind,
15195 }
15196 }
15197
15198 fn live(&self, route: &Bound) -> bool {
15199 matches!(
15200 self.forwarding
15201 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15202 .unwrap(),
15203 DataRoute::Client(DataRouteState::Bound(_))
15204 )
15205 }
15206 }
15207
15208 struct Relayed {
15209 target: String,
15210 client: RouteCtx,
15211 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15212 task: tokio::task::JoinHandle<Vec<Frame>>,
15213 bind: Frame,
15214 }
15215
15216 struct Bound {
15217 target: String,
15218 client: RouteCtx,
15219 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15220 channel: u16,
15221 epoch: u32,
15222 bind: Frame,
15223 }
15224
15225 impl Bound {
15226 fn closed_reason(&mut self) -> RouteCloseReason {
15229 let mut reason = None;
15230 let mut goodbye = false;
15231 while let Ok(outbound) = self.client_rx.try_recv() {
15232 let frame = outbound.frame;
15233 match frame.header.ty {
15234 FrameType::Goodbye => {
15235 assert_eq!(
15236 (frame.header.channel, frame.header.epoch),
15237 (self.channel, self.epoch)
15238 );
15239 goodbye = true;
15240 }
15241 FrameType::Push => {
15242 let ClientControlPush::RouteClosed {
15243 reason: r,
15244 module_id,
15245 ..
15246 } = serde_json::from_slice(&frame.body).unwrap()
15247 else {
15248 panic!("unexpected push");
15249 };
15250 assert_eq!(module_id, self.target);
15251 reason = Some(r);
15252 }
15253 other => panic!("unexpected frame {other:?}"),
15254 }
15255 }
15256 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15257 reason.expect("the client is told why the route closed")
15258 }
15259
15260 fn untouched(&mut self) -> bool {
15261 self.client_rx.try_recv().is_err()
15262 }
15263
15264 fn stamp(&self) -> Option<ScopeStamp> {
15265 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15266 ModuleControlRequest::RouteBind { scope, .. } => scope,
15267 other => panic!("expected a route.bind, got {other:?}"),
15268 }
15269 }
15270 }
15271
15272 #[tokio::test]
15273 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15274 ) {
15275 let mut rig = rig().await;
15276 let mut record = session(1);
15277 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15278 record.child_owners = vec![Principal::Reserved {
15279 module_id: MAGIC.to_string(),
15280 }];
15281 rig.sync(vec![record]).await;
15282 let scope = || Some(rig_selector("s", Some(1)));
15283
15284 rig.bound(Some(OWNER), PLEXUS, scope()).await;
15287 rig.bound(Some(AFT), OTHER, scope()).await;
15288 rig.bound(Some(BROCA), PLEXUS, scope()).await;
15289
15290 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15294 assert_eq!(
15295 rig.refused(opener, target, scope()).await,
15296 error_codes::SCOPE_NOT_CARRIER,
15297 "{opener:?} -> {target}"
15298 );
15299 }
15300 }
15301
15302 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15303 ScopeSelector::new(
15304 Principal::Reserved {
15305 module_id: OWNER.to_string(),
15306 },
15307 scope_ref.to_string(),
15308 scope_epoch,
15309 )
15310 }
15311
15312 #[tokio::test]
15313 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
15314 let mut rig = rig().await;
15315 rig.sync(vec![session(1)]).await;
15316 for opener in [OWNER, AFT] {
15317 assert_eq!(
15318 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
15319 .await,
15320 error_codes::SCOPE_EPOCH_REQUIRED,
15321 "{opener}"
15322 );
15323 }
15324 }
15325
15326 #[tokio::test]
15327 async fn admission_separates_not_synced_not_live_and_ended() {
15328 let mut rig = rig().await;
15329 let code = rig
15331 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15332 .await;
15333 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
15334 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
15335
15336 let ghost = ScopeSelector::new(
15338 Principal::Reserved {
15339 module_id: "ghost".to_string(),
15340 },
15341 "s".to_string(),
15342 Some(1),
15343 );
15344 assert_eq!(
15345 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
15346 error_codes::SCOPE_NOT_LIVE
15347 );
15348
15349 rig.sync(vec![session(2)]).await;
15350 assert_eq!(
15351 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
15352 .await,
15353 error_codes::SCOPE_NOT_LIVE
15354 );
15355 for epoch in [1, 3] {
15356 assert_eq!(
15357 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
15358 .await,
15359 error_codes::SCOPE_ENDED,
15360 "epoch {epoch}"
15361 );
15362 }
15363 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
15365 .await;
15366 }
15367
15368 #[tokio::test]
15369 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
15370 let mut rig = rig().await;
15371 rig.sync(vec![session(1)]).await;
15372 let route = rig
15373 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15374 .await;
15375 let stamp = route.stamp().expect("a scoped bind carries the stamp");
15376 assert_eq!(stamp.scope_ref, "s");
15377 assert_eq!(stamp.scope_epoch, 1);
15378 assert_eq!(stamp.kind, ScopeKind::Head);
15379 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
15380 assert!(stamp.attributes.delegates);
15381 assert!(stamp.owner_authorized);
15382 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15383 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
15384
15385 let (broca, mut broca_rx) = wide_ctx(50);
15388 hello_via_sink(
15389 &rig.handler,
15390 &broca,
15391 &mut broca_rx,
15392 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
15393 )
15394 .await;
15395 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
15396 .await
15397 .unwrap();
15398 let own = ScopeSelector::new(
15399 Principal::Reserved {
15400 module_id: BROCA.to_string(),
15401 },
15402 "b".to_string(),
15403 Some(1),
15404 );
15405 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
15406 assert!(!route.stamp().unwrap().owner_authorized);
15407 }
15408
15409 #[tokio::test]
15410 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
15411 let mut rig = rig().await;
15412 let mut record = head("s", 1);
15413 record.carriers = vec![carrier(AFT, None)];
15414 let flow_id = "Flow:run-7/step_2!~";
15415 record.attributes.flow_id = Some(flow_id.to_string());
15416 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
15417 .await
15418 .unwrap();
15419 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15420 panic!("not a sync reply");
15421 };
15422 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
15423 let route = rig
15424 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15425 .await;
15426 assert!(rig.live(&route), "the stamped bind committed");
15427 let stamp = route.stamp().expect("a flow scope carries a stamp");
15428 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
15429 assert_eq!(stamp.attributes.agent_id, None);
15430 assert!(!stamp.attributes.delegates);
15431 assert!(stamp.owner_authorized);
15432 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15433 assert_eq!(unscoped.stamp(), None);
15434 }
15435
15436 #[tokio::test]
15437 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
15438 let mut rig = rig_with_flow_support(false).await;
15439 let mut record = session(1);
15440 record.attributes.flow_id = Some("flow:7".to_string());
15441 rig.sync(vec![record]).await;
15442 for opener in [OWNER, AFT] {
15443 let body = rig
15444 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15445 .await;
15446 assert_eq!(body["code"], "target_flow_unsupported");
15447 let message = body["message"].as_str().unwrap();
15448 for required in [PLEXUS, "flow-scopes/v1"] {
15449 assert!(message.contains(required), "{message}");
15450 }
15451 }
15452 }
15453
15454 #[tokio::test]
15455 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
15456 let mut rig = rig_with_flow_support(true).await;
15457 let mut record = session(1);
15458 record.attributes.flow_id = Some("flow:7".to_string());
15459 rig.sync(vec![record]).await;
15460 for opener in [OWNER, AFT] {
15461 let route = rig
15462 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15463 .await;
15464 assert!(rig.live(&route));
15465 assert_eq!(
15466 route.stamp().unwrap().attributes.flow_id.as_deref(),
15467 Some("flow:7")
15468 );
15469 }
15470 }
15471
15472 #[tokio::test]
15473 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
15474 use std::future::Future;
15475
15476 let mut rig = rig_with_flow_support(true).await;
15477 let mut record = session(1);
15478 record.attributes.flow_id = Some("flow:7".to_string());
15479 rig.sync(vec![record]).await;
15480 let (client, mut client_rx, frame) =
15481 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
15482 for _ in 0..64 {
15485 client.egress.try_send(route_bind_ack(1)).unwrap();
15486 }
15487 let handler = rig.handler.clone();
15488 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15489 std::future::poll_fn(|cx| {
15490 assert!(open.as_mut().poll(cx).is_pending());
15491 std::task::Poll::Ready(())
15492 })
15493 .await;
15494 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15495
15496 let old_connection = rig.modules[PLEXUS].0.connection_id;
15497 rig.handler.cleanup_connection(old_connection).unwrap();
15498 let (replacement, mut replacement_rx) = wide_ctx(200);
15499 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
15500 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15501 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15502 let hello = Frame::build(
15503 FrameType::Hello,
15504 control_flags(),
15505 0,
15506 0,
15507 200,
15508 serde_json::to_vec(&body).unwrap(),
15509 )
15510 .unwrap();
15511 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
15512
15513 client_rx.try_recv().unwrap();
15514 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15515 .await
15516 .expect("the replacement is refused without waiting for a bind ack")
15517 .unwrap();
15518 assert_eq!(replies.len(), 1);
15519 let body = parse_error(&replies[0]);
15520 assert_eq!(body["code"], "target_flow_unsupported");
15521 for required in [PLEXUS, "flow-scopes/v1"] {
15522 assert!(body["message"].as_str().unwrap().contains(required));
15523 }
15524 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
15525 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15526 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15527 assert!(rig
15528 .handler
15529 .registry
15530 .get_module_by_connection(replacement.connection_id)
15531 .unwrap()
15532 .is_some());
15533 }
15534
15535 #[tokio::test]
15536 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
15537 use std::future::Future;
15538 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15539 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15540 const NEXT_HASH: &str =
15541 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15542 let mut rig = rig().await;
15543 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
15544 record.carriers.push(ScopeCarrier::new(Principal::Direct));
15545 rig.sync(vec![record.clone()]).await;
15546 let (client, mut rx, frame) = rig.open_frame(
15547 None,
15548 PLEXUS,
15549 Some(
15550 rig.selector("s", Some(1))
15551 .with_bind_proof(Some(PROOF.into())),
15552 ),
15553 );
15554 for _ in 0..64 {
15557 client.egress.try_send(route_bind_ack(1)).unwrap();
15558 }
15559 let handler = rig.handler.clone();
15560 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15561 std::future::poll_fn(|cx| {
15562 assert!(open.as_mut().poll(cx).is_pending());
15563 std::task::Poll::Ready(())
15564 })
15565 .await;
15566 record.bind_proof_sha256 = Some(NEXT_HASH.into());
15567 rig.sync(vec![record]).await;
15568 rx.try_recv().unwrap();
15569 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15570 .await
15571 .unwrap()
15572 .unwrap();
15573 assert_eq!(
15574 parse_error(&replies[0])["code"],
15575 error_codes::SCOPE_BIND_PROOF_MISMATCH
15576 );
15577 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
15578 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15579 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15580 }
15581
15582 #[tokio::test]
15583 async fn scope_bind_proof_reserved_openers_ignore_proof() {
15584 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15585 let mut rig = rig().await;
15586 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15587 .await;
15588 for opener in [OWNER, AFT] {
15589 for proof in [None, Some("malformed+proof=")] {
15590 let route = rig
15591 .bound(
15592 Some(opener),
15593 PLEXUS,
15594 Some(
15595 rig.selector("s", Some(1))
15596 .with_bind_proof(proof.map(str::to_string)),
15597 ),
15598 )
15599 .await;
15600 assert!(rig.live(&route));
15601 }
15602 }
15603 }
15604
15605 #[tokio::test]
15606 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
15607 use std::future::Future;
15608 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15609 const NEXT_HASH: &str =
15610 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15611 let mut rig = rig().await;
15612 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15613 .await;
15614 let (client, mut rx, frame) =
15615 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
15616 for _ in 0..64 {
15617 client.egress.try_send(route_bind_ack(1)).unwrap();
15618 }
15619 let handler = rig.handler.clone();
15620 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15621 std::future::poll_fn(|cx| {
15622 assert!(open.as_mut().poll(cx).is_pending());
15623 std::task::Poll::Ready(())
15624 })
15625 .await;
15626 rig.sync(vec![
15627 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
15628 ])
15629 .await;
15630 rx.try_recv().unwrap();
15631 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15632 let bind = std::future::poll_fn(|cx| {
15633 assert!(open.as_mut().poll(cx).is_pending());
15634 module_rx.poll_recv(cx)
15635 })
15636 .await
15637 .unwrap()
15638 .frame;
15639 handler
15640 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
15641 .await
15642 .unwrap();
15643 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15644 .await
15645 .unwrap()
15646 .unwrap();
15647 assert_eq!(replies.len(), 1);
15648 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
15651 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15652 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
15653 let route = rig
15654 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
15655 .await;
15656 assert!(rig.live(&route));
15657 }
15658
15659 #[tokio::test]
15660 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
15661 {
15662 let mut rig = rig_with_flow_support(false).await;
15663 rig.sync(vec![session(1)]).await;
15664 let route = rig
15665 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15666 .await;
15667 assert!(rig.live(&route));
15668 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
15669 let mut record = session(1);
15670 record.attributes.flow_id = Some("flow:7".to_string());
15671 rig.sync(vec![record]).await;
15672 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
15673 assert!(rig.live(&unscoped));
15674 assert_eq!(unscoped.stamp(), None);
15675 }
15676
15677 #[tokio::test]
15678 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
15679 let mut rig = rig().await;
15680 let mut record = session(1);
15681 record.attributes.flow_id = Some("flow:7".to_string());
15682 rig.sync(vec![record.clone()]).await;
15683 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15684 let mut owner_route = rig
15685 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15686 .await;
15687 let mut carrier_route = rig
15688 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
15689 .await;
15690 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15691 rig.sync(vec![record.clone()]).await;
15692 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15693 assert!(rig.live(&owner_route) && owner_route.untouched());
15694 assert!(rig.live(&carrier_route) && carrier_route.untouched());
15695
15696 record.attributes.flow_id = Some("flow:8".to_string());
15697 rig.sync(vec![record]).await;
15698 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
15699 let before = before.unwrap();
15700 assert_eq!(after.scope_epoch, before.scope_epoch);
15701 assert!(after.version > before.version);
15702 for route in [&mut owner_route, &mut carrier_route] {
15703 assert!(!rig.live(route));
15704 assert_eq!(
15705 route.closed_reason(),
15706 RouteCloseReason::ScopeDelegationChanged
15707 );
15708 }
15709 assert!(rig.live(&unscoped) && unscoped.untouched());
15710 for target in [PLEXUS, OTHER] {
15713 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
15714 let goodbye = module_rx
15715 .try_recv()
15716 .expect("the provider sees the drain")
15717 .frame;
15718 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15719 assert!(module_rx.try_recv().is_err());
15720 }
15721 let rebound = rig
15722 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15723 .await;
15724 assert_eq!(
15725 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
15726 Some("flow:8")
15727 );
15728 }
15729
15730 #[tokio::test]
15735 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
15736 let mut rig = rig().await;
15737 rig.sync(vec![session(1)]).await;
15738 let mut cotenant = rig
15739 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15740 .await;
15741
15742 let mut changed = session(1);
15743 changed.child_owners.push(Principal::Reserved {
15744 module_id: MAGIC.to_string(),
15745 });
15746 let mut ended = None;
15747 for (code, next) in [
15748 (error_codes::SCOPE_CHANGED, vec![changed]),
15749 (error_codes::SCOPE_ENDED, Vec::new()),
15750 ] {
15751 let relayed = rig
15752 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15753 .await;
15754 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
15755 ended = Some(next.is_empty());
15756 rig.sync(next).await;
15757 rig.ack(&relayed).await;
15758 let replies = relayed.task.await.unwrap();
15759 assert_eq!(replies.len(), 1, "{replies:?}");
15760 assert_eq!(parse_error(&replies[0])["code"], code);
15761 assert_eq!(
15762 subc_protocol::error_codes::is_retryable_route_open(code),
15763 code == error_codes::SCOPE_CHANGED
15764 );
15765 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15766 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15768 let mut goodbyes = Vec::new();
15771 while let Ok(outbound) = plexus_rx.try_recv() {
15772 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
15773 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
15774 }
15775 assert!(
15776 goodbyes.contains(&(bind_channel, bind_epoch)),
15777 "{goodbyes:?}"
15778 );
15779 assert!(rig
15780 .handler
15781 .registry
15782 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
15783 .unwrap()
15784 .is_some());
15785 }
15786 assert_eq!(ended, Some(true));
15787 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
15790 }
15791
15792 #[tokio::test]
15795 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
15796 struct Case {
15797 name: &'static str,
15798 change: fn(&mut ScopeRecord),
15799 closed: [Option<RouteCloseReason>; 4],
15802 }
15803 use RouteCloseReason::*;
15804 let cases = [
15805 Case {
15806 name: "a carrier entry removed",
15807 change: |r| {
15808 r.carriers.retain(|c| {
15809 c.principal
15810 != Principal::Reserved {
15811 module_id: AFT.to_string(),
15812 }
15813 })
15814 },
15815 closed: [None, Some(ScopeCarrierRemoved), None, None],
15816 },
15817 Case {
15818 name: "a target removed from a carrier",
15819 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
15820 closed: [None, None, None, Some(ScopeCarrierRemoved)],
15821 },
15822 Case {
15823 name: "a bare carrier narrowed to targets",
15824 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
15825 closed: [None, Some(ScopeCarrierRemoved), None, None],
15826 },
15827 Case {
15828 name: "delegates turned off",
15829 change: |r| r.attributes.delegates = false,
15830 closed: [Some(ScopeDelegationChanged); 4],
15831 },
15832 Case {
15833 name: "agent_id changed",
15834 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
15835 closed: [Some(ScopeDelegationChanged); 4],
15836 },
15837 Case {
15838 name: "a carrier added, child owners changed, the record re-sent",
15839 change: |r| {
15840 r.carriers.push(carrier(MAGIC, None));
15841 r.child_owners.push(Principal::Reserved {
15842 module_id: MAGIC.to_string(),
15843 });
15844 },
15845 closed: [None; 4],
15846 },
15847 Case {
15848 name: "a target added",
15849 change: |r| {
15850 r.carriers[1]
15851 .targets
15852 .as_mut()
15853 .unwrap()
15854 .push("third".to_string())
15855 },
15856 closed: [None; 4],
15857 },
15858 Case {
15859 name: "delegates turned on",
15860 change: |r| r.attributes.delegates = true,
15861 closed: [None; 4],
15862 },
15863 ];
15864 for case in cases {
15865 let mut rig = rig().await;
15866 rig.sync(vec![session(1)]).await;
15867 let scope = || Some(rig_selector("s", Some(1)));
15868 let mut routes = [
15869 rig.bound(Some(OWNER), PLEXUS, scope()).await,
15870 rig.bound(Some(AFT), PLEXUS, scope()).await,
15871 rig.bound(Some(BROCA), PLEXUS, scope()).await,
15872 rig.bound(Some(BROCA), OTHER, scope()).await,
15873 ];
15874 let mut record = session(1);
15875 (case.change)(&mut record);
15876 rig.sync(vec![record]).await;
15877 for (index, expected) in case.closed.iter().enumerate() {
15878 let route = &mut routes[index];
15879 match expected {
15880 Some(reason) => {
15881 assert!(!rig.live(route), "{}: route {index} still live", case.name);
15882 assert_eq!(
15883 route.closed_reason(),
15884 *reason,
15885 "{}: route {index}",
15886 case.name
15887 );
15888 }
15889 None => {
15890 assert!(rig.live(route), "{}: route {index} closed", case.name);
15891 assert!(
15892 route.untouched(),
15893 "{}: route {index} was told something",
15894 case.name
15895 );
15896 }
15897 }
15898 }
15899 }
15900 }
15901
15902 #[tokio::test]
15903 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
15904 for next in [Vec::new(), vec![session(2)]] {
15906 let mut rig = rig().await;
15907 rig.sync(vec![session(1)]).await;
15908 let mut route = rig
15909 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15910 .await;
15911 rig.sync(next).await;
15912 assert!(!rig.live(&route));
15913 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
15914 }
15915
15916 let mut rig = rig().await;
15919 let mut child = session(1);
15920 child.scope_ref = "child".to_string();
15921 child.kind = ScopeKind::Worker;
15922 child.parent = Some(ScopeParent::new(
15923 Principal::Reserved {
15924 module_id: OWNER.to_string(),
15925 },
15926 "s".to_string(),
15927 1,
15928 ));
15929 rig.sync(vec![session(1), child.clone()]).await;
15930 let mut child_route = rig
15931 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
15932 .await;
15933 assert_eq!(
15934 child_route.stamp().unwrap().parent_state,
15935 Some(ParentState::Linked)
15936 );
15937 rig.sync(vec![child]).await;
15938 assert!(!rig.live(&child_route));
15939 assert_eq!(
15940 child_route.closed_reason(),
15941 RouteCloseReason::ScopeParentEnded
15942 );
15943 }
15944
15945 #[tokio::test]
15946 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
15947 ) {
15948 let mut rig = rig().await;
15949 rig.sync(vec![session(1)]).await;
15950 let mut route = rig
15951 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15952 .await;
15953 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15954 rig.sync(vec![session(1)]).await;
15955 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15956 assert!(rig.live(&route) && route.untouched());
15957
15958 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
15962 .forwarding
15963 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15964 .unwrap()
15965 else {
15966 panic!("the route is bound");
15967 };
15968 binding.flow.acquire_tagged(9, false).await.unwrap();
15969 let mut widened = session(1);
15970 widened.carriers.push(carrier(MAGIC, None));
15971 rig.sync(vec![widened]).await;
15972 assert!(rig.live(&route) && route.untouched());
15973 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15974 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
15975 assert_eq!(binding.flow.in_flight(), 1);
15978 binding
15979 .flow
15980 .acquire_tagged(10, false)
15981 .await
15982 .expect("the flow is still open");
15983 }
15984
15985 #[tokio::test]
15988 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
15989 let mut rig = rig().await;
15990 rig.sync(vec![session(1)]).await;
15991 let mut on_incumbent = rig
15992 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15993 .await;
15994
15995 let (candidate, _candidate_rx) = wide_ctx(9);
15998 let registration = rig
15999 .handler
16000 .registry
16001 .register_candidate_with_control_ops(
16002 manifest(PLEXUS, PROTOCOL_VERSION),
16003 PROTOCOL_VERSION,
16004 candidate.connection_id,
16005 module_baseline_control_ops(),
16006 )
16007 .unwrap();
16008 rig.forwarding
16009 .register_candidate_module_connection(
16010 candidate.connection_id,
16011 PLEXUS.to_string(),
16012 PROTOCOL_VERSION,
16013 manifest_concurrency(®istration.manifest),
16014 candidate.egress.clone(),
16015 )
16016 .unwrap();
16017 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
16018 rig.handler
16019 .registry
16020 .promote_candidate(PLEXUS)
16021 .unwrap()
16022 .unwrap();
16023 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
16024
16025 rig.sync(Vec::new()).await;
16026 assert!(!rig.live(&on_incumbent));
16027 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
16028 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16029 let goodbye = incumbent_rx
16030 .try_recv()
16031 .expect("the superseded endpoint is told")
16032 .frame;
16033 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16034 }
16035 }
16036}
16037
16038#[cfg(test)]
16039mod concurrency_default_exposure_tests {
16040 use super::*;
16041
16042 fn hello_body(role_json: &str) -> Vec<u8> {
16043 format!(
16044 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
16045 )
16046 .into_bytes()
16047 }
16048
16049 fn manifest_from(body: &[u8]) -> ModuleManifest {
16050 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
16051 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
16052 .expect("manifest parses")
16053 }
16054
16055 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
16056
16057 #[test]
16058 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
16059 let body = hello_body(&format!(
16060 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
16061 ));
16062 let manifest = manifest_from(&body);
16063 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16066 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
16067 }
16068
16069 #[test]
16070 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
16071 let body = hello_body(&format!(
16072 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
16073 ));
16074 let manifest = manifest_from(&body);
16075 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16076 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16077 }
16078
16079 #[test]
16080 fn non_management_roles_are_never_reported() {
16081 let body = hello_body(
16082 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
16083 );
16084 let manifest = manifest_from(&body);
16085 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16086 }
16087}