1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27 },
28 scope::{
29 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31 SCOPE_SYNC_OP,
32 },
33 session::{
34 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37 ROLE_VERSIONS_FIELD,
38 },
39 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40 Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46 caller_identity::CallerIdentity,
47 capability_requirements::{
48 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50 RuntimeModule,
51 },
52 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
53 forwarding::{
54 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
55 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
56 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
57 },
58 observability::{
59 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
60 },
61 privileged_prompt::{prompt_text, PrivilegedOperation},
62 provenance::{
63 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
64 },
65 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
66 router::{RouteCtx, RouterError},
67 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
68 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
69 stderr_tail::{CaptureState, TailEntry},
70 supervise::{
71 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
72 SupervisorHandle, SwapHelloAdmission,
73 },
74 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
75};
76
77pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
83
84const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
85const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
86const CAP_PING_PONG: &str = "ping_pong_v1";
87const CAP_SESSION_ATTACH: &str = "session_attach_v1";
88const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
89
90const SUBC_CONTROL_OPS: &[&str] = &[
91 ops::ACTIVITY_RECORD,
92 ops::ACTIVITY_QUERY,
93 ops::ACTIVITY_SUBSCRIBE,
94 ops::SERVER_DESCRIBE,
95 ops::CATALOG_LIST,
96 ops::ROUTE_OPEN,
97 ops::ROUTE_POLL,
98 ops::ROUTE_CLOSING,
99 ops::ROUTE_CLOSED,
100 ops::SUPERVISOR_LIST,
101 ops::SUPERVISOR_RESTART,
102 ops::SUPERVISOR_SWAP,
103 ops::SUPERVISOR_RELOAD,
104 ops::SUPERVISOR_RESCAN,
105 ops::SUPERVISOR_RELEASE_RESERVED,
106 ops::SUPERVISOR_SET_ENABLED,
107 ops::SUPERVISOR_HEALTH_PROBE,
108 ops::SUPERVISOR_HEALTH,
109 ops::SUPERVISOR_STDERR_TAIL,
110 ops::SUPERVISOR_TERMINALS,
111 ops::SUPERVISOR_ROUTES,
112 ops::SUPERVISOR_PROVENANCE,
113 ops::SUPERVISOR_SPAWN_SNAPSHOT,
114 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
115];
116
117const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
118 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
119 "supervisor.live_roots",
120 SCOPE_SYNC_OP,
121 SCOPE_APPLY_OP,
122 SCOPE_DESCRIBE_OP,
123 "operator.confirm",
124];
125
126const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
130
131const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
132
133pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
141
142pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
157
158pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
176
177const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
178const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
179
180fn reload_verdict(
181 configured: &Path,
182 spawned_from: Option<&Path>,
183 image: subc_control::RunningImageAgreement,
184) -> PendingReloadVerdict {
185 let path = match spawned_from {
186 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
187 ReloadPathAgreement::Match
188 }
189 Some(spawned_from) => ReloadPathAgreement::Mismatch {
190 configured: configured.to_path_buf(),
191 spawned_from: spawned_from.to_path_buf(),
192 },
193 None => ReloadPathAgreement::Unavailable {
194 reason: if matches!(
195 image,
196 subc_control::RunningImageAgreement::Unavailable {
197 reason: subc_control::RunningImageUnavailableReason::NotRunning
198 }
199 ) {
200 ReloadPathUnavailableReason::NotRunning
201 } else {
202 ReloadPathUnavailableReason::SpawnedPathUnavailable
203 },
204 },
205 };
206 PendingReloadVerdict { path, image }
207}
208
209fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
210 if configured == spawned_from {
211 return true;
212 }
213 #[cfg(windows)]
214 {
215 match (
221 std::fs::canonicalize(configured),
222 std::fs::canonicalize(spawned_from),
223 ) {
224 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
225 _ => false,
226 }
227 }
228 #[cfg(not(windows))]
229 {
230 false
231 }
232}
233
234#[derive(Clone)]
235struct DaemonProvenanceFacts {
236 build: DaemonBuildProvenance,
237 pid: Option<u32>,
238 started_at_ms: Option<u64>,
239 start_clock: Option<crate::clock::StartClock>,
240 executable_path: Option<PathBuf>,
241 executable_identity: Option<SpawnedFileIdentity>,
242 process_start_time: Option<u64>,
243 probe: ExecutableIdentityProbe,
244}
245
246impl Default for DaemonProvenanceFacts {
247 fn default() -> Self {
248 Self {
249 build: DaemonBuildProvenance {
250 build_git_sha: None,
251 build_lock_digest: None,
252 },
253 pid: None,
254 started_at_ms: None,
255 start_clock: None,
256 executable_path: None,
257 executable_identity: None,
258 process_start_time: None,
259 probe: ExecutableIdentityProbe::default(),
260 }
261 }
262}
263
264#[derive(Debug, Clone)]
265struct SupervisorRescanContext {
266 supervisor: Supervisor,
267 config_path: PathBuf,
268 configured_port: Option<u16>,
269 storage_config: Option<crate::daemon_config::StorageConfig>,
270 admission_facts_carrier_module_id: Option<String>,
271 admission_facts_targets: Option<Vec<String>>,
272 scope_authority_owners: Vec<String>,
273 privileged_operator_prompt: PrivilegedOperatorPrompt,
274}
275
276const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
297 "reloading",
298 "supervisor_not_live",
299 "registration_not_active",
300 "no_forwarding_connection",
301 "relay_send_failed",
302];
303
304#[derive(Clone)]
306pub struct ControlHandler {
307 registry: Arc<Registry>,
308 activity: Option<crate::activity::ActivityLedger>,
309 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
310 forwarding: Arc<ForwardingTable>,
311 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
312 supervisor: SupervisorHandle,
313 subc_capabilities: Arc<[String]>,
314 route_bind_relay_timeout: Duration,
318 route_bind_relay_timeouts: BTreeMap<String, Duration>,
322 route_bind_breakers: RouteBindBreakers,
325 route_bind_concurrency: RouteBindConcurrency,
328 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
333 route_bind_breaker_threshold: u32,
335 route_bind_breaker_cooldown: Duration,
337 health_probe_timeout: Duration,
338 storage_config: Option<crate::daemon_config::StorageConfig>,
341 machine_id: Option<crate::machine_id::MachineId>,
345 admission_facts_carrier_module_id: Option<String>,
346 admission_facts_targets: Option<Vec<String>>,
347 scopes: Arc<RwLock<ScopeTable>>,
351 scope_deadline_added: Arc<tokio::sync::Notify>,
355 scope_authority_owners: Vec<String>,
358 privileged_operator_prompt: PrivilegedOperatorPrompt,
360 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
363 #[cfg(test)]
364 caller_lookup: Option<TestCallerLookup>,
365 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
368 rescan: Option<SupervisorRescanContext>,
369 connected_clients: ConnectedClients,
370 counters: DaemonCounters,
371 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
372 daemon_provenance: DaemonProvenanceFacts,
373 #[cfg(test)]
374 control_dispatch_delay: Option<Duration>,
375 #[cfg(test)]
376 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
377 #[cfg(test)]
381 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
382 #[cfg(test)]
388 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
389}
390
391impl fmt::Debug for ControlHandler {
392 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
393 f.debug_struct("ControlHandler")
394 .field("registry", &self.registry)
395 .field("forwarding", &self.forwarding)
396 .field("process_liveness", &self.process_liveness.is_some())
397 .field("supervisor", &self.supervisor)
398 .field("subc_capabilities", &self.subc_capabilities)
399 .finish()
400 }
401}
402
403struct RouteOpenRequest {
404 target: RouteTarget,
405 identity: BindIdentity,
406 consumer_identity: Option<ConsumerIdentity>,
407 consumer_capabilities: Option<Vec<String>>,
408 role_versions: Option<BTreeMap<String, String>>,
409 admission_facts: Option<serde_json::Value>,
410 scope: Option<ScopeSelector>,
411}
412
413struct RouteBindReservationGuard {
414 forwarding: Arc<ForwardingTable>,
415 endpoint: ModuleEndpointId,
416 relay_corr: u64,
417 armed: bool,
418}
419
420struct ModuleControlRpcGuard {
421 forwarding: Arc<ForwardingTable>,
422 endpoint: ModuleEndpointId,
423 corr: u64,
424 armed: bool,
425}
426
427impl ModuleControlRpcGuard {
428 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
429 Self {
430 forwarding,
431 endpoint,
432 corr,
433 armed: true,
434 }
435 }
436
437 fn disarm(&mut self) {
438 self.armed = false;
439 }
440}
441
442impl Drop for ModuleControlRpcGuard {
443 fn drop(&mut self) {
444 if self.armed {
445 let _ = self
446 .forwarding
447 .cancel_module_control_rpc(self.endpoint, self.corr);
448 }
449 }
450}
451
452impl RouteBindReservationGuard {
453 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
454 Self {
455 forwarding,
456 endpoint,
457 relay_corr,
458 armed: true,
459 }
460 }
461
462 fn release_and_disarm(&mut self) {
463 if !self.armed {
464 return;
465 }
466 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
467 self.endpoint,
468 self.relay_corr,
469 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
470 ) {
471 send_goodbye_target_best_effort(
472 &self.forwarding.counters(),
473 &target,
474 "canceled route.bind",
475 );
476 }
477 self.armed = false;
478 }
479
480 fn disarm(&mut self) {
481 self.armed = false;
482 }
483}
484
485impl Drop for RouteBindReservationGuard {
486 fn drop(&mut self) {
487 self.release_and_disarm();
488 }
489}
490
491#[derive(Debug, Clone, Default)]
521pub(crate) struct RouteBindBreakers {
522 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
523}
524
525#[derive(Debug, Clone, Default)]
526pub(crate) struct RouteBindConcurrency {
527 modules: Arc<Mutex<HashMap<String, usize>>>,
528}
529
530struct RouteBindConcurrencyGuard {
531 concurrency: RouteBindConcurrency,
532 module_id: String,
533}
534
535impl RouteBindConcurrency {
536 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
539 let mut modules = self
540 .modules
541 .lock()
542 .expect("route.bind concurrency mutex poisoned");
543 let in_flight = modules.entry(module_id.to_string()).or_default();
544 if *in_flight >= limit {
545 return Err(*in_flight);
546 }
547 *in_flight += 1;
548 Ok(RouteBindConcurrencyGuard {
549 concurrency: self.clone(),
550 module_id: module_id.to_string(),
551 })
552 }
553}
554
555impl Drop for RouteBindConcurrencyGuard {
556 fn drop(&mut self) {
557 let mut modules = self
558 .concurrency
559 .modules
560 .lock()
561 .expect("route.bind concurrency mutex poisoned");
562 let remove = {
563 let in_flight = modules
564 .get_mut(&self.module_id)
565 .expect("admitted route.bind has a concurrency entry");
566 *in_flight -= 1;
567 *in_flight == 0
568 };
569 if remove {
570 modules.remove(&self.module_id);
571 }
572 }
573}
574
575#[derive(Debug, Default)]
576struct ModuleBreakerState {
577 consecutive_timeouts: u32,
579 cooldown_until: Option<Instant>,
582 probe_in_flight: Option<Arc<()>>,
587}
588
589enum RouteBindAdmission<'a> {
591 Admitted {
592 guard: RouteBindBreakerGuard<'a>,
593 probe: bool,
596 },
597 Refused {
598 consecutive_timeouts: u32,
599 retry_in: Duration,
603 probe_in_flight: bool,
604 },
605}
606
607struct RouteBindBreakerGuard<'a> {
614 breakers: RouteBindBreakers,
615 module_id: &'a str,
616 probe_token: Option<Arc<()>>,
617 settled: bool,
618}
619
620impl RouteBindBreakerGuard<'_> {
621 fn record_accepted(&mut self) -> bool {
625 self.settled = true;
626 self.breakers.record_accepted(self.module_id)
627 }
628
629 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
632 self.settled = true;
633 self.breakers.record_timeout(
634 self.module_id,
635 self.probe_token.as_ref(),
636 threshold,
637 cooldown,
638 )
639 }
640
641 fn record_inconclusive(&mut self) {
650 self.settled = true;
651 self.breakers
652 .record_inconclusive(self.module_id, self.probe_token.as_ref());
653 }
654}
655
656impl Drop for RouteBindBreakerGuard<'_> {
657 fn drop(&mut self) {
658 if !self.settled {
659 self.breakers
660 .record_inconclusive(self.module_id, self.probe_token.as_ref());
661 }
662 }
663}
664
665struct BreakerOpened {
669 consecutive_timeouts: u32,
670 reopened_after_probe: bool,
673}
674
675impl RouteBindBreakers {
676 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
677 self.modules
678 .lock()
679 .expect("route.bind breaker mutex poisoned")
680 }
681
682 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
685 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
686 probe: probe_token.is_some(),
687 guard: RouteBindBreakerGuard {
688 breakers: self.clone(),
689 module_id,
690 probe_token,
691 settled: false,
692 },
693 };
694
695 let mut modules = self.lock();
696 let Some(state) = modules.get_mut(module_id) else {
697 return admitted(None);
698 };
699 let Some(cooldown_until) = state.cooldown_until else {
700 return admitted(None);
701 };
702 if state.probe_in_flight.is_some() {
703 return RouteBindAdmission::Refused {
704 consecutive_timeouts: state.consecutive_timeouts,
705 retry_in: Duration::ZERO,
706 probe_in_flight: true,
707 };
708 }
709 let now = Instant::now();
710 if now < cooldown_until {
711 return RouteBindAdmission::Refused {
712 consecutive_timeouts: state.consecutive_timeouts,
713 retry_in: cooldown_until - now,
714 probe_in_flight: false,
715 };
716 }
717 let token = Arc::new(());
718 state.probe_in_flight = Some(Arc::clone(&token));
719 admitted(Some(token))
720 }
721
722 fn record_accepted(&self, module_id: &str) -> bool {
723 self.lock()
724 .remove(module_id)
725 .is_some_and(|state| state.cooldown_until.is_some())
726 }
727
728 fn record_timeout(
729 &self,
730 module_id: &str,
731 probe_token: Option<&Arc<()>>,
732 threshold: u32,
733 cooldown: Duration,
734 ) -> Option<BreakerOpened> {
735 let mut modules = self.lock();
736 let state = modules.entry(module_id.to_string()).or_default();
737 let was_open = state.cooldown_until.is_some();
738 let was_probe = Self::owns_probe(state, probe_token);
739 if was_probe {
740 state.probe_in_flight = None;
741 }
742 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
743 if state.consecutive_timeouts < threshold {
744 return None;
745 }
746 state.cooldown_until = Some(Instant::now() + cooldown);
747 Some(BreakerOpened {
748 consecutive_timeouts: state.consecutive_timeouts,
749 reopened_after_probe: was_open && was_probe,
750 })
751 }
752
753 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
754 state
758 .probe_in_flight
759 .as_ref()
760 .zip(token)
761 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
762 }
763
764 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
765 if let Some(state) = self.lock().get_mut(module_id) {
766 if Self::owns_probe(state, probe_token) {
767 state.probe_in_flight = None;
768 }
769 }
770 }
771
772 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
787 self.lock()
788 .remove(module_id)
789 .map(|state| state.consecutive_timeouts)
790 .filter(|discarded| *discarded > 0)
791 }
792
793 fn open_snapshot(&self) -> Option<serde_json::Value> {
800 let now = Instant::now();
801 let modules = self.lock();
802 let open = modules
803 .iter()
804 .filter_map(|(module_id, state)| {
805 let cooldown_until = state.cooldown_until?;
806 Some((
807 module_id.clone(),
808 serde_json::json!({
809 "consecutive_timeouts": state.consecutive_timeouts,
810 "cooldown_remaining_ms":
811 cooldown_until.saturating_duration_since(now).as_millis() as u64,
812 "probe_in_flight": state.probe_in_flight.is_some(),
813 }),
814 ))
815 })
816 .collect::<serde_json::Map<String, serde_json::Value>>();
817 (!open.is_empty()).then_some(serde_json::Value::Object(open))
818 }
819}
820
821impl ControlHandler {
822 pub fn new(registry: Arc<Registry>) -> Self {
823 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
824 }
825
826 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
827 let counters = forwarding.counters();
828 let route_bind_breakers = forwarding.route_bind_breakers();
832 let route_bind_concurrency = forwarding.route_bind_concurrency();
833 let route_outages = forwarding.route_outages();
834 Self {
835 registry,
836 activity: None,
837 activity_streams: Arc::new(Mutex::new(HashMap::new())),
838 forwarding,
839 process_liveness: None,
840 supervisor: SupervisorHandle::new(),
841 subc_capabilities: Arc::from([
842 CAP_MANIFEST_REGISTRATION.to_string(),
843 CAP_CHANNEL_LIFECYCLE.to_string(),
844 CAP_PING_PONG.to_string(),
845 CAP_SESSION_ATTACH.to_string(),
846 CAP_ADMISSION_FACTS_RELAY.to_string(),
847 CAP_SCOPES_V1.to_string(),
848 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
849 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
850 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
851 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
852 ]),
853 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
854 route_bind_relay_timeouts: BTreeMap::new(),
855 route_bind_breakers,
856 route_bind_concurrency,
857 route_outages,
858 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
859 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
860 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
861 storage_config: None,
862 machine_id: None,
863 admission_facts_carrier_module_id: None,
864 admission_facts_targets: None,
865 scopes: Arc::new(RwLock::new(ScopeTable::new(
866 crate::daemon_config::default_scope_authority_owners(),
867 ))),
868 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
869 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
870 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
871 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
872 #[cfg(test)]
873 caller_lookup: None,
874 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
875 rescan: None,
876 connected_clients: ConnectedClients::new(),
877 counters,
878 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
879 daemon_provenance: DaemonProvenanceFacts::default(),
880 #[cfg(test)]
881 control_dispatch_delay: None,
882 #[cfg(test)]
883 provenance_probe_override: None,
884 #[cfg(test)]
885 route_bind_relay_sent_hold: None,
886 #[cfg(test)]
887 hello_registry_to_forwarding_hold: None,
888 }
889 }
890
891 pub fn with_storage_config(
894 mut self,
895 storage_config: Option<crate::daemon_config::StorageConfig>,
896 ) -> Self {
897 self.storage_config = storage_config;
898 self
899 }
900
901 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
904 self.activity = Some(crate::activity::ActivityLedger::open(
905 run_dir.join("activity.jsonl"),
906 incarnation,
907 ));
908 self.supervisor.set_activity(self.activity.clone());
909 self
910 }
911
912 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
915 self.machine_id = machine_id;
916 self
917 }
918
919 pub fn with_admission_facts_config(
923 mut self,
924 carrier_module_id: Option<String>,
925 targets: Option<Vec<String>>,
926 ) -> Self {
927 self.admission_facts_carrier_module_id = carrier_module_id;
928 self.admission_facts_targets = targets;
929 self
930 }
931
932 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
936 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
937 self.scope_authority_owners = owners;
938 self
939 }
940
941 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
943 self.privileged_operator_prompt = mode;
944 self
945 }
946
947 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
950 self.route_bind_relay_timeout = timeout;
951 self
952 }
953
954 pub fn with_route_bind_relay_timeouts(
960 mut self,
961 timeouts: impl IntoIterator<Item = (String, Duration)>,
962 ) -> Self {
963 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
964 self
965 }
966
967 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
973 self.route_bind_relay_timeouts
974 .get(module_id)
975 .copied()
976 .unwrap_or(self.route_bind_relay_timeout)
977 }
978
979 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
987 self.route_bind_breaker_threshold = threshold.max(1);
988 self.route_bind_breaker_cooldown = cooldown;
989 self
990 }
991
992 #[cfg(test)]
993 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
994 self.health_probe_timeout = timeout;
995 self
996 }
997
998 #[cfg(test)]
999 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1000 self.control_dispatch_delay = Some(delay);
1001 self
1002 }
1003
1004 #[cfg(test)]
1008 pub(crate) fn with_route_bind_relay_sent_hold(
1009 mut self,
1010 hold: Arc<tokio::sync::Barrier>,
1011 ) -> Self {
1012 self.route_bind_relay_sent_hold = Some(hold);
1013 self
1014 }
1015
1016 #[cfg(test)]
1021 pub(crate) fn with_hello_registry_to_forwarding_hold(
1022 mut self,
1023 hold: Arc<std::sync::Barrier>,
1024 ) -> Self {
1025 self.hello_registry_to_forwarding_hold = Some(hold);
1026 self
1027 }
1028
1029 #[cfg(test)]
1030 fn hold_between_registry_and_forwarding(&self) {
1031 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1032 hold.wait();
1033 hold.wait();
1034 }
1035 }
1036
1037 pub fn with_process_liveness(
1038 mut self,
1039 process_liveness: Arc<dyn ModuleProcessLiveness>,
1040 ) -> Self {
1041 self.process_liveness = Some(process_liveness);
1042 self
1043 }
1044
1045 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1046 self.supervisor = supervisor;
1047 self.supervisor.set_activity(self.activity.clone());
1048 self
1049 }
1050
1051 pub fn with_daemon_provenance(
1052 mut self,
1053 pid: u32,
1054 started_at_ms: u64,
1055 executable_path: Option<PathBuf>,
1056 build_git_sha: Option<String>,
1057 build_lock_digest: Option<String>,
1058 ) -> Self {
1059 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1060 let process_start_time = process_start_time(pid);
1061 self.daemon_provenance = DaemonProvenanceFacts {
1062 build: DaemonBuildProvenance {
1063 build_git_sha,
1064 build_lock_digest,
1065 },
1066 pid: Some(pid),
1067 started_at_ms: Some(started_at_ms),
1068 start_clock: None,
1069 executable_path,
1070 executable_identity,
1071 process_start_time,
1072 probe: ExecutableIdentityProbe::default(),
1073 };
1074 self
1075 }
1076
1077 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1078 self.daemon_provenance.start_clock = Some(clock);
1079 self
1080 }
1081
1082 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1084 self.scopes
1085 .write()
1086 .unwrap_or_else(|poisoned| poisoned.into_inner())
1087 .set_wall_clock(Arc::new(clock));
1088 self
1089 }
1090
1091 #[cfg(test)]
1092 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1093 self.provenance_probe_override = Some(result);
1094 self
1095 }
1096
1097 pub fn with_capability_config(
1101 self,
1102 modules: impl IntoIterator<Item = (String, bool)>,
1103 reserved_capabilities: BTreeMap<String, String>,
1104 ) -> Self {
1105 self.capability_evaluator
1106 .configure(modules, reserved_capabilities);
1107 self
1108 }
1109
1110 pub fn with_supervisor_rescan(
1111 mut self,
1112 supervisor: Supervisor,
1113 config_path: impl Into<PathBuf>,
1114 configured_port: Option<u16>,
1115 ) -> Self {
1116 self.rescan = Some(SupervisorRescanContext {
1117 supervisor,
1118 config_path: config_path.into(),
1119 configured_port,
1120 storage_config: self.storage_config.clone(),
1121 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1122 admission_facts_targets: self.admission_facts_targets.clone(),
1123 scope_authority_owners: self.scope_authority_owners.clone(),
1124 privileged_operator_prompt: self.privileged_operator_prompt,
1125 });
1126 self
1127 }
1128
1129 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1130 self.connected_clients = connected_clients;
1131 self
1132 }
1133
1134 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1135 Arc::clone(&self.forwarding)
1136 }
1137
1138 pub(crate) fn counters(&self) -> DaemonCounters {
1139 self.counters.clone()
1140 }
1141
1142 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1145 tokio::spawn(async move {
1146 loop {
1147 self.capability_evaluator
1148 .wait_for_change_or_deadline()
1149 .await;
1150 self.refresh_capability_requirements();
1151 }
1152 });
1153 }
1154
1155 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1161 tokio::spawn(async move {
1162 loop {
1163 let has_deadlines = self
1164 .scopes
1165 .read()
1166 .unwrap_or_else(|poisoned| poisoned.into_inner())
1167 .has_deadlines();
1168 if !has_deadlines {
1169 self.scope_deadline_added.notified().await;
1170 continue;
1171 }
1172 tokio::time::sleep(Duration::from_secs(1)).await;
1173 if let Err(error) = self.sweep_expired_scopes() {
1174 warn!(%error, "scope expiry sweep failed");
1175 }
1176 }
1177 });
1178 }
1179
1180 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1181 let mut table = self
1182 .scopes
1183 .write()
1184 .unwrap_or_else(|poisoned| poisoned.into_inner());
1185 let swept = table.sweep_expired();
1186 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1187 drop(table);
1188 self.close_scope_drained_routes(drained);
1189 Ok(())
1190 }
1191
1192 fn publish_scope_changes(
1193 &self,
1194 changes: &[crate::scopes::ScopeTagChange],
1195 expired: &[crate::scopes::ScopeExpired],
1196 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1197 if changes.is_empty() {
1198 return Ok(Vec::new());
1199 }
1200 let drained = self
1201 .forwarding
1202 .publish_scope_changes(changes)
1203 .map_err(RouterError::Forwarding)?;
1204 let mut counts = HashMap::new();
1205 for route in &drained {
1206 *counts
1207 .entry((
1208 route.scope.owner.as_str(),
1209 route.scope.scope_ref.as_str(),
1210 route.scope.tag.scope_epoch,
1211 ))
1212 .or_insert(0usize) += 1;
1213 }
1214 for scope in expired {
1215 let routes_closed = counts
1216 .get(&(
1217 scope.owner.as_str(),
1218 scope.scope_ref.as_str(),
1219 scope.scope_epoch,
1220 ))
1221 .copied()
1222 .unwrap_or(0);
1223 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1224 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1225 routes_closed, "scope expired");
1226 }
1227 Ok(drained)
1228 }
1229
1230 fn runtime_capability_snapshot(
1231 &self,
1232 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1233 let runtime = self
1234 .supervisor
1235 .list()
1236 .into_iter()
1237 .map(|module| {
1238 let status = module.status().map_err(|err| {
1239 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1240 })?;
1241 Ok(RuntimeModule {
1242 module_id: status.module_id,
1243 state: status.state,
1244 enabled: status.enabled,
1245 })
1246 })
1247 .collect::<Result<Vec<_>, RouterError>>()?;
1248 let registrations = self.registry.list_registered_modules().map_err(|err| {
1254 RouterError::backend(
1255 0,
1256 0,
1257 format!("failed to list capability registrations: {err}"),
1258 )
1259 })?;
1260 let registrations = registrations
1261 .into_iter()
1262 .map(|registration| RegisteredModule {
1263 module_id: registration.manifest.module_id,
1264 module_version: registration.manifest.module_version,
1265 capabilities: registration.manifest.capabilities,
1266 })
1267 .collect();
1268 Ok((runtime, registrations))
1269 }
1270
1271 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1278 let cached_registration = RegisteredModule {
1279 module_id: registration.manifest.module_id.clone(),
1280 module_version: registration.manifest.module_version.clone(),
1281 capabilities: registration.manifest.capabilities.clone(),
1282 };
1283 if self.capability_evaluator.record_hello(&cached_registration) {
1284 warn!(
1285 module_id = %cached_registration.module_id,
1286 "capability claims drifted from the cached manifest"
1287 );
1288 }
1289 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1290 self.enforce_capability_denies();
1291 }
1292 self.refresh_capability_requirements();
1293 }
1294
1295 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1299 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1300 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1301 self.supervisor.set_swap_promotion_observer(observer);
1302 }
1303
1304 pub fn refresh_capability_requirements(&self) {
1305 match self.runtime_capability_snapshot() {
1306 Ok((runtime, registrations)) => {
1307 log_requirement_events(
1308 self.capability_evaluator
1309 .evaluate_now(&runtime, ®istrations),
1310 );
1311 }
1312 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1313 }
1314 }
1315
1316 fn enforce_capability_denies(&self) {
1320 let registrations = match self.registry.list_registered_modules() {
1321 Ok(snapshot) => snapshot,
1322 Err(err) => {
1323 warn!(error = %err, "failed to read registrations for capability deny census");
1324 return;
1325 }
1326 };
1327 let manifests = registrations
1328 .into_iter()
1329 .map(|registration| {
1330 (
1331 registration.manifest.module_id.clone(),
1332 registration.manifest,
1333 )
1334 })
1335 .collect::<BTreeMap<_, _>>();
1336 let census = match self.forwarding.route_census(None) {
1337 Ok(census) => census,
1338 Err(err) => {
1339 warn!(error = %err, "failed to read route census for capability deny enforcement");
1340 return;
1341 }
1342 };
1343
1344 for (target_module_id, routes) in census {
1345 let Some(target_manifest) = manifests.get(&target_module_id) else {
1346 continue;
1347 };
1348 let mut closed_routes = Vec::new();
1349 let mut module_goodbyes = Vec::new();
1350 for route in routes {
1351 let Principal::Reserved {
1352 module_id: opening_module_id,
1353 } = &route.principal
1354 else {
1355 continue;
1356 };
1357 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1358 continue;
1359 };
1360 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1361 continue;
1362 };
1363
1364 match self.forwarding.release_client_route(
1365 route.goodbye_target.connection_id,
1366 route.goodbye_target.channel,
1367 route.goodbye_target.epoch,
1368 ) {
1369 Ok(RouteRelease::Removed(module_goodbye)) => {
1370 warn!(
1371 opening_module_id,
1372 target_module_id,
1373 capability,
1374 "force-closing route because an attested capability deny edge now matches"
1375 );
1376 closed_routes.push(route);
1377 module_goodbyes.push(module_goodbye);
1378 }
1379 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1380 Err(err) => warn!(
1381 opening_module_id,
1382 target_module_id,
1383 capability,
1384 error = %err,
1385 "failed to force-close capability-denied route"
1386 ),
1387 }
1388 }
1389
1390 if closed_routes.is_empty() {
1391 continue;
1392 }
1393 send_route_control_pushes(
1394 &self.forwarding,
1395 closed_routes,
1396 ClientControlPush::RouteClosed {
1397 module_id: target_module_id,
1398 channels: Vec::new(),
1399 reason: RouteCloseReason::CapabilityDenied,
1400 drained: false,
1401 abandoned: 0,
1402 excluded_subscriptions: 0,
1403 terminal: Some(false),
1404 },
1405 );
1406 self.emit_route_goodbyes(module_goodbyes);
1407 }
1408 }
1409
1410 fn not_ready_reason(
1416 &self,
1417 registration: &crate::registry::ModuleRegistration,
1418 ) -> Option<NotReadyReason> {
1419 if !registration.ready {
1420 return Some(NotReadyReason {
1421 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1422 capability: None,
1423 });
1424 }
1425 self.first_unprovided_required_capability(registration)
1426 .map(|capability| NotReadyReason {
1427 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1428 capability: Some(capability),
1429 })
1430 }
1431
1432 fn first_unprovided_required_capability(
1450 &self,
1451 registration: &crate::registry::ModuleRegistration,
1452 ) -> Option<String> {
1453 let required = registration
1454 .manifest
1455 .capabilities
1456 .iter()
1457 .flat_map(|declarations| declarations.requires.iter())
1458 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1459 .map(|requirement| requirement.capability.as_str())
1460 .collect::<BTreeSet<_>>();
1461 if required.is_empty() {
1462 return None;
1463 }
1464 let module_id = registration.manifest.module_id.as_str();
1465 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1466 if required
1467 .iter()
1468 .any(|capability| verdict(capability).is_none())
1469 {
1470 self.refresh_capability_requirements();
1471 }
1472 required
1473 .into_iter()
1474 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1475 .map(str::to_string)
1476 }
1477
1478 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1479 self.capability_evaluator
1480 .statuses()
1481 .into_iter()
1482 .map(capability_requirement_status)
1483 .collect()
1484 }
1485
1486 fn deregister_connection(
1493 &self,
1494 connection_id: ConnectionId,
1495 reason: RegistrationEndReason,
1496 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1497 self.registry
1498 .deregister_connection_with_reason(connection_id, reason)
1499 }
1500
1501 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1502 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1503 return None;
1504 }
1505 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1506 parse_client_control_request(&frame.body)
1507 else {
1508 return None;
1509 };
1510 Some(target_module_id(&target).to_string())
1511 }
1512
1513 pub(crate) fn route_open_capacity_refusal(
1514 &self,
1515 ctx: &RouteCtx,
1516 frame: &Frame,
1517 target_module_id: &str,
1518 in_flight: usize,
1519 limit: usize,
1520 ) -> Result<Frame, RouterError> {
1521 self.route_open_admission_refusal_frame(
1522 ctx,
1523 frame,
1524 target_module_id,
1525 "open_admission_full",
1526 (in_flight, limit),
1527 format!(
1528 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1529 ),
1530 )
1531 }
1532
1533 fn route_open_target_capacity_refusal(
1534 &self,
1535 ctx: &RouteCtx,
1536 frame: &Frame,
1537 target_module_id: &str,
1538 in_flight: usize,
1539 ) -> Result<Frame, RouterError> {
1540 self.route_open_admission_refusal_frame(
1541 ctx,
1542 frame,
1543 target_module_id,
1544 "target_binds_full",
1545 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1546 format!(
1547 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1548 ),
1549 )
1550 }
1551
1552 fn route_open_admission_refusal_frame(
1560 &self,
1561 ctx: &RouteCtx,
1562 frame: &Frame,
1563 target_module_id: &str,
1564 reason: &'static str,
1565 (in_flight, limit): (usize, usize),
1566 message: impl Into<String>,
1567 ) -> Result<Frame, RouterError> {
1568 let code = error_codes::TARGET_UNAVAILABLE;
1569 self.counters.increment_route_open_refused(code);
1570 info!(
1571 target: "control",
1572 code,
1573 reason,
1574 module_id = ?target_module_id,
1575 connection_id = ctx.connection_id.get(),
1576 in_flight,
1577 limit,
1578 "route.open refused"
1579 );
1580 control_error_frame(frame, code, message.into())
1581 }
1582
1583 #[cfg(test)]
1589 pub fn handle_control(
1590 &self,
1591 connection_id: ConnectionId,
1592 frame: Frame,
1593 ) -> Result<Vec<Frame>, RouterError> {
1594 match frame.header.ty {
1595 FrameType::Ping => Ok(vec![pong(&frame)?]),
1596 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1597 FrameType::Goodbye => self.handle_goodbye(connection_id),
1598 ty => Ok(vec![control_error_frame(
1599 &frame,
1600 "unsupported_control_frame",
1601 format!("unsupported channel-0 frame {ty:?}"),
1602 )?]),
1603 }
1604 }
1605
1606 pub async fn handle_control_frame(
1607 &self,
1608 ctx: &RouteCtx,
1609 frame: Frame,
1610 ) -> Result<Vec<Frame>, RouterError> {
1611 self.handle_control_frame_timed(ctx, frame, None).await
1612 }
1613
1614 pub(crate) fn record_caller_endpoints(
1615 &self,
1616 connection: ConnectionId,
1617 endpoints: Option<(SocketAddr, SocketAddr)>,
1618 ) {
1619 let mut callers = self
1620 .caller_endpoints
1621 .lock()
1622 .unwrap_or_else(|p| p.into_inner());
1623 callers.remove(&connection);
1624 if let Some(endpoints) = endpoints {
1625 callers.insert(connection, endpoints);
1626 }
1627 }
1628
1629 #[cfg(test)]
1630 pub(crate) fn with_caller_lookup_for_test(
1631 mut self,
1632 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1633 ) -> Self {
1634 self.caller_lookup = Some(Arc::new(lookup));
1635 self
1636 }
1637
1638 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1639 let endpoints = self
1640 .caller_endpoints
1641 .lock()
1642 .unwrap_or_else(|p| p.into_inner())
1643 .get(&connection)
1644 .copied();
1645 #[cfg(test)]
1646 let lookup = self.caller_lookup.clone();
1647 tokio::task::spawn_blocking(move || {
1648 #[cfg(test)]
1649 if let Some(lookup) = lookup {
1650 return lookup(endpoints);
1651 }
1652 endpoints
1653 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1654 .unwrap_or_default()
1655 })
1656 .await
1657 .unwrap_or_default()
1658 }
1659
1660 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1661 frame.header.channel == 0
1662 && frame.header.ty == FrameType::Request
1663 && self
1664 .forwarding
1665 .module_endpoint_for_connection(ctx.connection_id)
1666 .is_ok_and(|endpoint| endpoint.is_none())
1667 && parse_client_control_request(&frame.body)
1668 .ok()
1669 .is_some_and(|request| privileged_operation(&request).is_some())
1670 }
1671
1672 pub(crate) async fn handle_control_frame_timed(
1673 &self,
1674 ctx: &RouteCtx,
1675 frame: Frame,
1676 dispatch_started_at: Option<StdInstant>,
1677 ) -> Result<Vec<Frame>, RouterError> {
1678 match frame.header.ty {
1679 FrameType::Ping => Ok(vec![pong(&frame)?]),
1680 FrameType::Hello => {
1681 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1682 }
1683 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1684 FrameType::Cancel => {
1685 if self
1688 .forwarding
1689 .operator_confirms()
1690 .cancel(ctx.connection_id, frame.header.corr)
1691 || self
1692 .supervisor
1693 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1694 || self.cancel_activity_subscription(ctx, &frame)?
1695 {
1696 Ok(Vec::new())
1697 } else {
1698 Ok(vec![control_error_frame(
1699 &frame,
1700 "unknown_subscription",
1701 "no subscription has this correlation id",
1702 )?])
1703 }
1704 }
1705 FrameType::Request => {
1706 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1709 if op
1710 .as_ref()
1711 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1712 {
1713 return self.handle_activity_record(ctx, frame).await;
1714 }
1715 if op.as_ref().is_some_and(|probe| {
1716 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1717 }) {
1718 return self.handle_activity_read(ctx, frame).await;
1719 }
1720 if op
1721 .as_ref()
1722 .is_some_and(|probe| probe.op == "operator.confirm")
1723 {
1724 return self.handle_operator_confirm(ctx, frame);
1725 }
1726 if self
1727 .forwarding
1728 .module_endpoint_for_connection(ctx.connection_id)
1729 .map_err(RouterError::Forwarding)?
1730 .is_some()
1731 {
1732 if !is_known_module_request_op(&frame.body) {
1733 return Ok(vec![control_error_frame(
1734 &frame,
1735 "unsupported_control_frame",
1736 "module-originated channel-0 REQUEST is not supported",
1737 )?]);
1738 }
1739 let request = match parse_module_control_request_from_module(&frame.body) {
1740 Ok(request) => request,
1741 Err((err, ControlRequestBodyError::UnknownOp)) => {
1742 return Ok(vec![control_error_frame(
1743 &frame,
1744 "unsupported_control_frame",
1745 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1746 )?])
1747 }
1748 Err((err, ControlRequestBodyError::InvalidBody)) => {
1749 return Ok(vec![control_error_frame(
1750 &frame,
1751 "invalid_control_body",
1752 format!("malformed module control body: {err}"),
1753 )?])
1754 }
1755 };
1756 let op = module_control_request_op(&request);
1757 let corr = frame.header.corr;
1758 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1759 let result =
1760 self.handle_module_control_request(ctx.connection_id, frame, request);
1761 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1762 return result;
1763 }
1764
1765 if is_known_module_request_op(&frame.body) {
1766 return Ok(vec![control_error_frame(
1767 &frame,
1768 "not_registered",
1769 "catalog.update requires an active module registration owned by this connection",
1770 )?]);
1771 }
1772
1773 let request = match parse_client_control_request(&frame.body) {
1774 Ok(request) => request,
1775 Err((err, ControlRequestBodyError::UnknownOp)) => {
1776 return Ok(vec![control_error_frame(
1777 &frame,
1778 "unknown_control_op",
1779 format!("unknown client control op: {err}"),
1780 )?])
1781 }
1782 Err((err, ControlRequestBodyError::InvalidBody)) => {
1783 return Ok(vec![control_error_frame(
1784 &frame,
1785 "invalid_control_body",
1786 format!("malformed client control body: {err}"),
1787 )?])
1788 }
1789 };
1790 let op = client_control_request_op(&request);
1791 let corr = frame.header.corr;
1792 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1793 #[cfg(test)]
1794 if let Some(delay) = self.control_dispatch_delay {
1795 tokio::time::sleep(delay).await;
1796 }
1797 let result = self
1798 .handle_client_control_request(ctx, frame, request)
1799 .await;
1800 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1801 result
1802 }
1803 FrameType::Push => {
1804 let Some(endpoint) = self
1805 .forwarding
1806 .module_endpoint_for_connection(ctx.connection_id)
1807 .map_err(RouterError::Forwarding)?
1808 else {
1809 return Ok(vec![control_error_frame(
1810 &frame,
1811 "unsupported_control_frame",
1812 "client-originated channel-0 PUSH is not supported",
1813 )?]);
1814 };
1815 self.handle_status_update(endpoint, frame)
1816 }
1817 FrameType::Response | FrameType::Error
1818 if self
1819 .forwarding
1820 .module_endpoint_for_connection(ctx.connection_id)
1821 .map_err(RouterError::Forwarding)?
1822 .is_some() =>
1823 {
1824 self.handle_module_relay_response(ctx.connection_id, frame)
1825 }
1826 ty => Ok(vec![control_error_frame(
1827 &frame,
1828 "unsupported_control_frame",
1829 format!("unsupported channel-0 frame {ty:?}"),
1830 )?]),
1831 }
1832 }
1833
1834 pub fn cleanup_connection(
1835 &self,
1836 connection_id: ConnectionId,
1837 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1838 self.caller_endpoints
1839 .lock()
1840 .unwrap_or_else(|p| p.into_inner())
1841 .remove(&connection_id);
1842 self.forwarding
1843 .operator_confirms()
1844 .connection_closed(connection_id);
1845 self.cleanup_connection_with_end_reason(
1846 connection_id,
1847 RegistrationEndReason::ConnectionClosed,
1848 )
1849 }
1850
1851 fn cleanup_connection_with_end_reason(
1852 &self,
1853 connection_id: ConnectionId,
1854 requested_reason: RegistrationEndReason,
1855 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1856 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1857 self.registry
1858 .get_module_by_connection(connection_id)?
1859 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1860 .and_then(|module| module.registration_end_reason().ok().flatten())
1861 .unwrap_or(requested_reason)
1862 } else {
1863 requested_reason
1864 };
1865 let crash_closed = self
1866 .registry
1867 .get_module_by_connection(connection_id)?
1868 .and_then(|registration| {
1869 self.forwarding
1870 .module_endpoint_for_connection(connection_id)
1871 .ok()
1872 .flatten()
1873 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1874 .map(|routes| (registration.manifest.module_id, routes))
1875 });
1876 let crash_closed = crash_closed.map(|(module_id, routes)| {
1877 let terminal = match self.supervisor.get(&module_id) {
1878 None => false,
1879 Some(module) => match module.will_recover_after_connection_loss() {
1880 Ok(will_recover) => !will_recover,
1881 Err(err) => {
1882 warn!(
1883 %module_id,
1884 error = %err,
1885 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1886 );
1887 false
1888 }
1889 },
1890 };
1891 let (reason, drained) = match self
1895 .forwarding
1896 .daemon_drain_result_for_connection(connection_id)
1897 {
1898 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
1899 Ok(None) => (RouteCloseReason::Crash, false),
1900 Err(err) => {
1901 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1902 (RouteCloseReason::Crash, false)
1903 }
1904 };
1905 (module_id, routes, reason, drained, terminal)
1906 });
1907 let registrations = self.deregister_connection(connection_id, end_reason);
1908 let cleanup = if crash_closed.is_some() {
1909 self.forwarding.cleanup_connection_counted(connection_id)
1910 } else {
1911 self.forwarding
1917 .cleanup_connection(connection_id)
1918 .map(|released| crate::forwarding::ConnectionCleanup {
1919 released,
1920 abandoned_relays: 0,
1921 })
1922 };
1923 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
1928 let abandoned = cleanup
1929 .as_ref()
1930 .map_or(0, |cleanup| cleanup.abandoned_relays);
1931 send_route_control_pushes(
1932 &self.forwarding,
1933 routes,
1934 ClientControlPush::RouteClosed {
1935 module_id,
1936 channels: Vec::new(),
1937 reason,
1938 drained,
1939 abandoned,
1940 excluded_subscriptions: 0,
1941 terminal: Some(terminal),
1942 },
1943 );
1944 }
1945 if let Ok(cleanup) = cleanup {
1946 self.emit_route_goodbyes(cleanup.released);
1947 }
1948 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1952 crate::supervise::notify_registration_release();
1953 self.capability_evaluator.wake_deadline_loop();
1954 self.refresh_capability_requirements();
1955 }
1956 self.supervisor.remove_spawn_subscribers(connection_id);
1957 self.activity_streams
1958 .lock()
1959 .unwrap_or_else(|p| p.into_inner())
1960 .retain(|(id, _), task| {
1961 if *id == connection_id {
1962 task.abort();
1963 false
1964 } else {
1965 true
1966 }
1967 });
1968 self.hello_launch_nonces
1971 .lock()
1972 .unwrap_or_else(|poisoned| poisoned.into_inner())
1973 .forget(connection_id);
1974 self.scopes
1975 .write()
1976 .unwrap_or_else(|poisoned| poisoned.into_inner())
1977 .release_connection(connection_id);
1978 registrations
1979 }
1980
1981 pub(crate) fn handle_route_goodbye(
1982 &self,
1983 connection_id: ConnectionId,
1984 route_channel: u16,
1985 route_epoch: u32,
1986 ) -> Result<bool, RouterError> {
1987 debug!(
1988 connection_id = connection_id.get(),
1989 route_channel, route_epoch, "handling route GOODBYE"
1990 );
1991 let RouteRelease::Removed(released_route) = self
1992 .forwarding
1993 .release_client_route(connection_id, route_channel, route_epoch)
1994 .map_err(RouterError::Forwarding)?
1995 else {
1996 return Ok(false);
1997 };
1998 self.emit_route_goodbyes(vec![released_route]);
1999 Ok(true)
2000 }
2001
2002 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2003 for released in released_routes {
2004 let frame = match Frame::build_with_version(
2005 released.negotiated_ver,
2006 FrameType::Goodbye,
2007 control_flags(),
2008 released.channel,
2009 released.epoch,
2010 0,
2011 Vec::new(),
2012 ) {
2013 Ok(frame) => frame,
2014 Err(err) => {
2015 warn!(
2016 route_channel = released.channel,
2017 error = %err,
2018 "failed to build route GOODBYE frame"
2019 );
2020 continue;
2021 }
2022 };
2023 if !released.close_on_delivery_failure() {
2024 crate::forwarding::send_module_route_goodbye(
2025 &self.counters,
2026 &released.sink,
2027 frame,
2028 released.module_id.as_deref(),
2029 "client route released",
2030 );
2031 continue;
2032 }
2033 if let Err(err) = released.sink.try_send(frame) {
2034 warn!(
2035 target_connection_id = released.connection_id.get(),
2036 route_channel = released.channel,
2037 error = %err,
2038 "route GOODBYE was not delivered to client; closing target connection"
2039 );
2040 if self
2041 .forwarding
2042 .escalate_client_delivery_failure(
2043 released.connection_id,
2044 released.channel,
2045 released.epoch,
2046 CloseReason::new(
2047 "route_goodbye_delivery_failed",
2048 format!(
2049 "failed to enqueue route GOODBYE for channel {}: {err}",
2050 released.channel
2051 ),
2052 ),
2053 crate::forwarding::UndeliveredFrame {
2054 module_id: released.module_id.as_deref(),
2055 sink: &released.sink,
2056 },
2057 )
2058 .unwrap_or(false)
2059 {
2060 self.counters.increment_goodbye_relay_client_failed();
2061 }
2062 }
2063 }
2064 }
2065
2066 fn send_abandoned_route_bind_goodbye(
2078 &self,
2079 module_sink: &crate::FrameSink,
2080 negotiated_ver: u8,
2081 module_channel: u16,
2082 module_epoch: u32,
2083 ) {
2084 let frame = match Frame::build_with_version(
2085 negotiated_ver,
2086 FrameType::Goodbye,
2087 control_flags(),
2088 module_channel,
2089 module_epoch,
2090 0,
2091 Vec::new(),
2092 ) {
2093 Ok(frame) => frame,
2094 Err(err) => {
2095 warn!(
2096 route_channel = module_channel,
2097 error = %err,
2098 "failed to build GOODBYE for abandoned route.bind"
2099 );
2100 return;
2101 }
2102 };
2103 crate::forwarding::send_module_route_goodbye(
2104 &self.counters,
2105 module_sink,
2106 frame,
2107 None,
2108 "abandoned route.bind",
2109 );
2110 }
2111
2112 fn handle_hello(
2113 &self,
2114 connection_id: ConnectionId,
2115 sink: Option<crate::FrameSink>,
2116 frame: Frame,
2117 ) -> Result<Vec<Frame>, RouterError> {
2118 debug!(
2119 connection_id = connection_id.get(),
2120 corr = frame.header.corr,
2121 "handling HELLO"
2122 );
2123 if self
2127 .registry
2128 .get_module_by_connection(connection_id)
2129 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2130 .is_some()
2131 {
2132 return Ok(vec![control_error_frame(
2133 &frame,
2134 "invalid_hello",
2135 "connection is already registered as a module",
2136 )?]);
2137 }
2138 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2139 Ok(value) => value,
2140 Err(err) => {
2141 return Ok(vec![control_error_frame(
2142 &frame,
2143 "invalid_hello",
2144 format!("malformed HELLO body: {err}"),
2145 )?])
2146 }
2147 };
2148 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2149 return Ok(vec![control_error_frame(
2150 &frame,
2151 "invalid_capability_grammar",
2152 err.to_string(),
2153 )?]);
2154 }
2155 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2156 return Ok(vec![control_error_frame(
2157 &frame,
2158 "invalid_manifest",
2159 err.to_string(),
2160 )?]);
2161 }
2162 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2163 return Ok(vec![control_error_frame(
2164 &frame,
2165 "invalid_event_declaration",
2166 err.to_string(),
2167 )?]);
2168 }
2169 if let Some(provenance) = hello_value
2170 .get("manifest")
2171 .and_then(|manifest| manifest.get("provenance"))
2172 {
2173 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2174 return Ok(vec![control_error_frame(
2175 &frame,
2176 "invalid_manifest",
2177 format!("malformed manifest provenance: {err}"),
2178 )?]);
2179 }
2180 }
2181 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2182 Ok(hello) => hello,
2183 Err(err) => {
2184 return Ok(vec![control_error_frame(
2185 &frame,
2186 "invalid_hello",
2187 format!("malformed HELLO body: {err}"),
2188 )?])
2189 }
2190 };
2191
2192 if hello.protocol_ver != hello.manifest.protocol_ver {
2193 return Ok(vec![control_error_frame(
2194 &frame,
2195 "invalid_manifest",
2196 format!(
2197 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2198 hello.protocol_ver, hello.manifest.protocol_ver
2199 ),
2200 )?]);
2201 }
2202
2203 if hello.manifest.module_id.trim().is_empty() {
2204 return Ok(vec![control_error_frame(
2205 &frame,
2206 "invalid_manifest",
2207 "manifest module_id must not be empty",
2208 )?]);
2209 }
2210
2211 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2212 Ok(negotiated_ver) => negotiated_ver,
2213 Err(message) => {
2214 return Ok(vec![control_error_frame(
2215 &frame,
2216 "version_unsupported",
2217 message,
2218 )?])
2219 }
2220 };
2221
2222 let swap_admission = self
2230 .supervisor
2231 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2232 if swap_admission == SwapHelloAdmission::Refused {
2233 warn!(
2234 module_id = %hello.manifest.module_id,
2235 connection_id = connection_id.get(),
2236 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2237 );
2238 return Ok(vec![control_error_frame(
2239 &frame,
2240 "swap_token_invalid",
2241 format!(
2242 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2243 hello.manifest.module_id
2244 ),
2245 )?]);
2246 }
2247 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2248
2249 if let Some(rejection) = (!swap_candidate)
2257 .then(|| {
2258 self.supervisor.reserved_hello_rejection(
2259 &hello.manifest.module_id,
2260 hello.launch_nonce.as_deref(),
2261 )
2262 })
2263 .flatten()
2264 {
2265 let message = match rejection {
2266 ReservedHelloRejection::Exact { module_id } => format!(
2267 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2268 ),
2269 ReservedHelloRejection::Prefix {
2270 prefix,
2271 owner_module_id,
2272 } => format!(
2273 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2274 hello.manifest.module_id
2275 ),
2276 };
2277 return Ok(vec![control_error_frame(
2278 &frame,
2279 "reserved_module",
2280 message,
2281 )?]);
2282 }
2283
2284 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2285 &hello.manifest.module_id,
2286 hello.manifest.capabilities.as_ref(),
2287 );
2288 if let Some(refusal) = reserved_capability_refusals.first() {
2289 let capability = refusal.capability.clone();
2290 let bound_module = refusal.claimants[0].clone();
2291 log_duplicate_claim_events(reserved_capability_refusals);
2292 return Ok(vec![control_error_frame(
2293 &frame,
2294 "reserved_capability",
2295 format!(
2296 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2297 capability, bound_module, hello.manifest.module_id
2298 ),
2299 )?]);
2300 }
2301
2302 if self
2305 .forwarding
2306 .connection_has_client_routes(connection_id)
2307 .map_err(RouterError::Forwarding)?
2308 {
2309 return Ok(vec![control_error_frame(
2310 &frame,
2311 "invalid_hello",
2312 "connection has open client routes and cannot also register as a module",
2313 )?]);
2314 }
2315
2316 self.hello_launch_nonces
2321 .lock()
2322 .unwrap_or_else(|poisoned| poisoned.into_inner())
2323 .record(connection_id, hello.launch_nonce.as_deref());
2324 let control_ops = effective_module_control_ops(hello.control_ops);
2325 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2328 if swap_candidate {
2329 return self.register_swap_candidate(
2330 connection_id,
2331 sink,
2332 &frame,
2333 hello.manifest,
2334 negotiated_ver,
2335 control_ops,
2336 hello_ack,
2337 );
2338 }
2339 let registration = match self.registry.register_unpublished_with_control_ops(
2346 hello.manifest,
2347 negotiated_ver,
2348 connection_id,
2349 control_ops,
2350 ) {
2351 Ok(registration) => registration,
2352 Err(RegistryError::DuplicateModuleId { module_id }) => {
2353 return Ok(vec![control_error_frame(
2354 &frame,
2355 "duplicate_module_id",
2356 format!(
2357 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2358 ),
2359 )?])
2360 }
2361 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2362 return Ok(vec![control_error_frame(
2363 &frame,
2364 "invalid_module_id",
2365 err.to_string(),
2366 )?])
2367 }
2368 Err(err) => {
2369 return Ok(vec![control_error_frame(
2370 &frame,
2371 "registry_error",
2372 err.to_string(),
2373 )?])
2374 }
2375 };
2376 #[cfg(test)]
2377 self.hold_between_registry_and_forwarding();
2378
2379 let reply = if let Some(sink) = sink {
2380 let concurrency = manifest_concurrency(®istration.manifest);
2396 if let Err(err) = self.forwarding.register_module_connection_acked(
2397 connection_id,
2398 registration.manifest.module_id.clone(),
2399 negotiated_ver,
2400 concurrency,
2401 sink,
2402 hello_ack,
2403 ) {
2404 if matches!(
2409 self.deregister_connection(
2410 connection_id,
2411 RegistrationEndReason::RegistrationFailed,
2412 ),
2413 Ok(r) if !r.is_empty()
2414 ) {
2415 crate::supervise::notify_registration_release();
2416 }
2417 return Ok(vec![control_error_frame(
2418 &frame,
2419 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2420 "invalid_hello"
2421 } else {
2422 forwarding_error_code(&err)
2423 },
2424 err.to_string(),
2425 )?]);
2426 }
2427 Vec::new()
2428 } else {
2429 vec![hello_ack]
2432 };
2433
2434 if !self
2439 .registry
2440 .publish_registration(connection_id)
2441 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2442 {
2443 debug!(
2444 module_id = %registration.manifest.module_id,
2445 connection_id = connection_id.get(),
2446 "registration was removed before it could be published"
2447 );
2448 }
2449
2450 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2459 info!(
2460 module_id = %registration.manifest.module_id,
2461 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2462 );
2463 }
2464
2465 self.apply_registration_capabilities(®istration);
2466
2467 self.supervisor
2468 .record_activity_registration(®istration.manifest.module_id);
2469
2470 info!(
2471 module_id = %registration.manifest.module_id,
2472 module_version = %registration.manifest.module_version,
2473 negotiated_ver,
2474 routable_provider = manifest_provides_routable_role(®istration.manifest),
2475 connection_id = connection_id.get(),
2476 "module registered"
2477 );
2478
2479 Ok(reply)
2480 }
2481
2482 #[allow(clippy::too_many_arguments)]
2499 fn register_swap_candidate(
2500 &self,
2501 connection_id: ConnectionId,
2502 sink: Option<crate::FrameSink>,
2503 frame: &Frame,
2504 manifest: ModuleManifest,
2505 negotiated_ver: u8,
2506 control_ops: Vec<String>,
2507 hello_ack: Frame,
2508 ) -> Result<Vec<Frame>, RouterError> {
2509 let module_id = manifest.module_id.clone();
2510 let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2511 manifest,
2512 negotiated_ver,
2513 connection_id,
2514 control_ops,
2515 ) {
2516 Ok(registration) => registration,
2517 Err(RegistryError::DuplicateModuleId { module_id }) => {
2518 return Ok(vec![control_error_frame(
2519 frame,
2520 "duplicate_module_id",
2521 format!(
2522 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2523 ),
2524 )?])
2525 }
2526 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2527 return Ok(vec![control_error_frame(
2528 frame,
2529 "invalid_module_id",
2530 err.to_string(),
2531 )?])
2532 }
2533 Err(err) => {
2534 return Ok(vec![control_error_frame(
2535 frame,
2536 "registry_error",
2537 err.to_string(),
2538 )?])
2539 }
2540 };
2541 #[cfg(test)]
2542 self.hold_between_registry_and_forwarding();
2543 let reply = if let Some(sink) = sink {
2544 let concurrency = manifest_concurrency(®istration.manifest);
2548 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2549 connection_id,
2550 module_id.clone(),
2551 negotiated_ver,
2552 concurrency,
2553 sink,
2554 hello_ack,
2555 ) {
2556 if matches!(
2557 self.deregister_connection(
2558 connection_id,
2559 RegistrationEndReason::RegistrationFailed,
2560 ),
2561 Ok(r) if !r.is_empty()
2562 ) {
2563 crate::supervise::notify_registration_release();
2564 }
2565 return Ok(vec![control_error_frame(
2566 frame,
2567 forwarding_error_code(&err),
2568 err.to_string(),
2569 )?]);
2570 }
2571 Vec::new()
2572 } else {
2573 vec![hello_ack]
2574 };
2575 if !self
2576 .registry
2577 .publish_registration(connection_id)
2578 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2579 {
2580 debug!(
2581 module_id = %module_id,
2582 connection_id = connection_id.get(),
2583 "swap candidate registration was removed before it could be published"
2584 );
2585 }
2586 self.supervisor.mark_swap_candidate_admitted(&module_id);
2587 info!(
2588 module_id = %module_id,
2589 module_version = %registration.manifest.module_version,
2590 negotiated_ver,
2591 ready = registration.ready,
2592 connection_id = connection_id.get(),
2593 "swap candidate registered; not routable until cutover"
2594 );
2595 Ok(reply)
2596 }
2597
2598 fn build_hello_ack(
2599 &self,
2600 frame: &Frame,
2601 negotiated_ver: u8,
2602 module_id: &str,
2603 ) -> Result<Frame, RouterError> {
2604 let ack = ModuleHelloAckBody {
2605 negotiated_ver,
2606 subc_ops: module_subc_ops(),
2607 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2608 storage: self
2609 .storage_config
2610 .as_ref()
2611 .map(|cfg| cfg.descriptor_for(module_id)),
2612 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2613 };
2614 let body = serde_json::to_vec(&ack).map_err(|err| {
2615 RouterError::backend(
2616 0,
2617 frame.header.corr,
2618 format!("failed to encode HELLO_ACK: {err}"),
2619 )
2620 })?;
2621
2622 Frame::build_with_version(
2623 negotiated_ver,
2624 FrameType::HelloAck,
2625 control_flags(),
2626 0,
2627 0,
2628 frame.header.corr,
2629 body,
2630 )
2631 .map_err(RouterError::FrameBuild)
2632 }
2633
2634 async fn handle_client_control_request(
2635 &self,
2636 ctx: &RouteCtx,
2637 frame: Frame,
2638 request: ClientControlRequest,
2639 ) -> Result<Vec<Frame>, RouterError> {
2640 if let Some(operation) = privileged_operation(&request) {
2641 let caller = self.resolve_caller(ctx.connection_id).await;
2642 let op = client_control_request_op(&request);
2643 let text = prompt_text(operation, &caller);
2644 match self.privileged_operator_prompt {
2645 PrivilegedOperatorPrompt::Log => {
2646 info!(target: "subc_daemon::privileged_prompt", operation = op,
2647 target = ?operation, caller = %caller.description(), summary = %text,
2648 "privileged request would prompt the operator");
2649 }
2650 PrivilegedOperatorPrompt::Enforce => {
2651 let key =
2652 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2653 let outcome = self
2654 .forwarding
2655 .operator_confirms()
2656 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2657 .await;
2658 if outcome.code != "confirmed" {
2659 return Ok(vec![outcome.frame(&frame)]);
2660 }
2661 }
2662 }
2663 }
2664 match request {
2665 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2666 ClientControlRequest::CatalogList { module_id } => {
2667 self.handle_catalog_list(frame, module_id)
2668 }
2669 ClientControlRequest::RouteOpen {
2670 target,
2671 identity,
2672 consumer_identity,
2673 consumer_capabilities,
2674 role_versions,
2675 admission_facts,
2676 scope,
2677 } => {
2678 self.handle_route_open(
2679 ctx,
2680 frame,
2681 RouteOpenRequest {
2682 target,
2683 identity,
2684 consumer_identity,
2685 consumer_capabilities,
2686 role_versions,
2687 admission_facts,
2688 scope,
2689 },
2690 )
2691 .await
2692 }
2693 ClientControlRequest::RoutePoll {
2694 route_channel,
2695 route_epoch,
2696 kind,
2697 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2698 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2699 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2700 self.handle_supervisor_spawn_snapshot(frame)
2701 }
2702 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2703 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2704 }
2705 ClientControlRequest::SupervisorRestart {
2706 module_id,
2707 drain_timeout_ms,
2708 } => {
2709 self.log_supervisor_request_received(
2710 ctx,
2711 frame.header.corr,
2712 ops::SUPERVISOR_RESTART,
2713 Some(&module_id),
2714 None,
2715 )?;
2716 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2717 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2718 .await
2719 }
2720 ClientControlRequest::SupervisorSwap {
2721 module_id,
2722 ready_timeout_ms,
2723 } => {
2724 self.log_supervisor_request_received(
2725 ctx,
2726 frame.header.corr,
2727 ops::SUPERVISOR_SWAP,
2728 Some(&module_id),
2729 None,
2730 )?;
2731 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2732 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2733 .await
2734 }
2735 ClientControlRequest::SupervisorReload { module_id } => {
2736 self.log_supervisor_request_received(
2737 ctx,
2738 frame.header.corr,
2739 ops::SUPERVISOR_RELOAD,
2740 Some(&module_id),
2741 None,
2742 )?;
2743 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2744 self.handle_supervisor_reload(frame, module_id, requester)
2745 .await
2746 }
2747 ClientControlRequest::SupervisorRescan { preview } => {
2748 if !preview {
2749 self.log_supervisor_request_received(
2750 ctx,
2751 frame.header.corr,
2752 ops::SUPERVISOR_RESCAN,
2753 None,
2754 None,
2755 )?;
2756 }
2757 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2758 self.handle_supervisor_rescan(frame, preview, requester)
2759 .await
2760 }
2761 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2762 self.log_supervisor_request_received(
2763 ctx,
2764 frame.header.corr,
2765 ops::SUPERVISOR_RELEASE_RESERVED,
2766 Some(&module_id),
2767 None,
2768 )?;
2769 self.handle_supervisor_release_reserved(frame, module_id)
2770 .await
2771 }
2772 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2773 self.log_supervisor_request_received(
2774 ctx,
2775 frame.header.corr,
2776 ops::SUPERVISOR_SET_ENABLED,
2777 Some(&module_id),
2778 Some(enabled),
2779 )?;
2780 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2781 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2782 .await
2783 }
2784 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2785 self.handle_supervisor_health_probe(frame, module_id).await
2786 }
2787 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2788 ClientControlRequest::SupervisorRoutes { module_id } => {
2789 self.handle_supervisor_routes(frame, module_id)
2790 }
2791 ClientControlRequest::SupervisorProvenance { module_id } => {
2792 self.handle_supervisor_provenance(frame, module_id).await
2793 }
2794 ClientControlRequest::SupervisorStderrTail {
2795 module_id,
2796 max_lines,
2797 max_bytes,
2798 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2799 ClientControlRequest::SupervisorTerminals { module_id } => {
2800 self.handle_supervisor_terminals(frame, module_id).await
2801 }
2802 }
2803 }
2804
2805 fn log_supervisor_request_received(
2806 &self,
2807 ctx: &RouteCtx,
2808 corr: u64,
2809 op: &'static str,
2810 module_id: Option<&str>,
2811 enabled: Option<bool>,
2812 ) -> Result<(), RouterError> {
2813 let caller = self.supervisor_requester(ctx, corr)?;
2814 let caller = principal_label(&caller);
2815 info!(
2818 target: "control",
2819 op,
2820 module_id,
2821 enabled,
2822 connection_id = ctx.connection_id.get(),
2823 caller = %caller,
2824 "supervisor request received"
2825 );
2826 Ok(())
2827 }
2828
2829 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2830 Ok(self
2831 .registry
2832 .get_module_by_connection(ctx.connection_id)
2833 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2834 .map(|registration| Principal::Reserved {
2835 module_id: registration.manifest.module_id,
2836 })
2837 .unwrap_or(Principal::Direct))
2838 }
2839
2840 fn handle_module_control_request(
2841 &self,
2842 connection_id: ConnectionId,
2843 frame: Frame,
2844 request: ModuleControlRequestFromModule,
2845 ) -> Result<Vec<Frame>, RouterError> {
2846 match request {
2847 ModuleControlRequestFromModule::CatalogUpdate {
2848 provides,
2849 capabilities,
2850 ready,
2851 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2852 ModuleControlRequestFromModule::LiveRoots {} => {
2853 let registered = self
2854 .registry
2855 .get_module_by_connection(connection_id)
2856 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2857 let Some(registration) = registered else {
2858 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2859 };
2860 let response = self
2861 .forwarding
2862 .live_roots(®istration.manifest.module_id)
2863 .map_err(RouterError::Forwarding)?;
2864 Ok(vec![control_response_body_frame(
2865 &frame,
2866 &response,
2867 "ModuleControlResponseToModule::LiveRoots",
2868 )?])
2869 }
2870 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2871 self.handle_scope_sync(connection_id, frame, generation, scopes)
2872 }
2873 ModuleControlRequestFromModule::ScopeApply {
2874 generation,
2875 upsert,
2876 end,
2877 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2878 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2879 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2880 }
2881 }
2882 }
2883
2884 fn handle_operator_confirm(
2885 &self,
2886 ctx: &RouteCtx,
2887 frame: Frame,
2888 ) -> Result<Vec<Frame>, RouterError> {
2889 use crate::operator_confirm::{audit, Outcome};
2890 let registration = self
2891 .registry
2892 .get_module_by_connection(ctx.connection_id)
2893 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2894 let Some(registration) = registration else {
2895 let outcome = Outcome::refusal("not_registered");
2896 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2897 return Ok(vec![outcome.frame(&frame)]);
2898 };
2899 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2900 Ok(request) => request,
2901 Err(_) => {
2902 let outcome = Outcome::refusal("invalid_control_body");
2903 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2904 return Ok(vec![outcome.frame(&frame)]);
2905 }
2906 };
2907 let module_id = registration.manifest.module_id;
2908 let nonce = self
2912 .hello_launch_nonces
2913 .lock()
2914 .unwrap_or_else(|p| p.into_inner())
2915 .nonce(ctx.connection_id)
2916 .map(str::to_owned);
2917 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2918 self.supervisor
2919 .spawned_consumer_authorized(&module_id, nonce)
2920 });
2921 let confirms = self.forwarding.operator_confirms();
2922 self.forwarding
2923 .with_operator_route(
2924 ctx.connection_id,
2925 request.route_channel,
2926 request.route_epoch,
2927 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2928 )
2929 .map_err(RouterError::Forwarding)
2930 }
2931
2932 fn handle_scope_sync(
2936 &self,
2937 connection_id: ConnectionId,
2938 frame: Frame,
2939 generation: u64,
2940 scopes: Vec<ScopeRecord>,
2941 ) -> Result<Vec<Frame>, RouterError> {
2942 self.handle_scope_change(connection_id, frame, generation, scopes, None)
2943 }
2944
2945 fn handle_scope_change(
2946 &self,
2947 connection_id: ConnectionId,
2948 frame: Frame,
2949 generation: u64,
2950 scopes: Vec<ScopeRecord>,
2951 end: Option<Vec<ScopeEnd>>,
2952 ) -> Result<Vec<Frame>, RouterError> {
2953 let is_apply = end.is_some();
2954 let Some(registration) = self
2955 .registry
2956 .get_module_by_connection(connection_id)
2957 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2958 else {
2959 return Ok(vec![control_error_frame(
2960 &frame,
2961 "not_registered",
2962 "scope.sync and scope.apply require an active module registration owned by this connection",
2963 )?]);
2964 };
2965 let owner = registration.manifest.module_id;
2966 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2967 let is_current_launch = |connection: ConnectionId| {
2968 self.hello_launch_nonces
2969 .lock()
2970 .unwrap_or_else(|poisoned| poisoned.into_inner())
2971 .presented(connection, current_nonce.as_deref())
2972 };
2973 let mut table = self
2978 .scopes
2979 .write()
2980 .unwrap_or_else(|poisoned| poisoned.into_inner());
2981 let outcome = match end {
2982 Some(end) => table.apply(
2983 &owner,
2984 connection_id,
2985 is_current_launch,
2986 generation,
2987 scopes,
2988 end,
2989 ),
2990 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2991 };
2992 let drained = match &outcome {
2993 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2994 Err(_) => Vec::new(),
2995 };
2996 let has_deadlines = outcome.is_ok() && table.has_deadlines();
2997 drop(table);
2998 if has_deadlines {
2999 self.scope_deadline_added.notify_one();
3000 }
3001 match outcome {
3002 Ok(applied) => {
3003 let counts = ScopeOutcomeCounts::of(&applied.results);
3004 info!(
3005 owner = %owner,
3006 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3007 generation,
3008 records = applied.results.len(),
3009 created = counts.created,
3010 replaced = counts.replaced,
3011 updated = counts.updated,
3012 unchanged = counts.unchanged,
3013 refused = counts.refused,
3014 ended = applied.ended.len(),
3015 tag_changes = applied.tag_changes.len(),
3016 routes_closed = drained.len(),
3017 "scope change accepted"
3018 );
3019 for refused in applied
3025 .results
3026 .iter()
3027 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3028 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3029 {
3030 warn!(
3031 owner = %owner,
3032 generation,
3033 scope_ref = %refused.scope_ref,
3034 scope_epoch = refused.scope_epoch,
3035 code = refused.code.as_deref().unwrap_or(""),
3036 "scope record refused"
3037 );
3038 }
3039 self.close_scope_drained_routes(drained);
3040 let response = if is_apply {
3041 ModuleControlResponseToModule::ScopeApply {
3042 generation,
3043 results: applied.results,
3044 end_results: applied.end_results,
3045 ended: applied.ended,
3046 }
3047 } else {
3048 ModuleControlResponseToModule::ScopeSync {
3049 generation,
3050 results: applied.results,
3051 ended: applied.ended,
3052 }
3053 };
3054 Ok(vec![control_response_body_frame(
3055 &frame,
3056 &response,
3057 "scope change response",
3058 )?])
3059 }
3060 Err(refusal) => {
3061 info!(
3062 owner = %owner,
3063 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3064 generation,
3065 code = refusal.code,
3066 "scope change refused"
3067 );
3068 Ok(vec![control_error_frame(
3069 &frame,
3070 refusal.code,
3071 refusal.message,
3072 )?])
3073 }
3074 }
3075 }
3076
3077 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3083 if drained.is_empty() {
3084 return;
3085 }
3086 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3087 BTreeMap::new();
3088 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3089 for route in drained {
3090 warn!(
3091 module_id = %route.module_id,
3092 reason = ?route.reason,
3093 client_connection_id = route.client.connection_id.get(),
3094 route_channel = route.client.channel,
3095 "closing route because its scope changed"
3096 );
3097 pushes
3098 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3099 .or_insert_with(|| (route.reason, Vec::new()))
3100 .1
3101 .push(EndpointRoute {
3102 goodbye_target: route.client.clone(),
3103 principal: Principal::Unverified,
3104 bound_at: Instant::now(),
3105 draining: false,
3106 drain_reason: None,
3107 });
3108 goodbyes.push(route.module);
3109 goodbyes.push(route.client);
3110 }
3111 for ((module_id, _), (reason, routes)) in pushes {
3112 send_route_control_pushes(
3113 &self.forwarding,
3114 routes,
3115 ClientControlPush::RouteClosed {
3116 module_id,
3117 channels: Vec::new(),
3118 reason,
3119 drained: false,
3120 abandoned: 0,
3121 excluded_subscriptions: 0,
3122 terminal: Some(false),
3123 },
3124 );
3125 }
3126 self.emit_route_goodbyes(goodbyes);
3127 }
3128
3129 fn handle_scope_describe(
3132 &self,
3133 connection_id: ConnectionId,
3134 frame: Frame,
3135 owner: Principal,
3136 scope_ref: String,
3137 ) -> Result<Vec<Frame>, RouterError> {
3138 let registered = self
3139 .registry
3140 .get_module_by_connection(connection_id)
3141 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3142 if registered.is_none() {
3143 return Ok(vec![control_error_frame(
3144 &frame,
3145 "not_registered",
3146 "scope.describe requires an active module registration owned by this connection",
3147 )?]);
3148 }
3149 let description = self
3150 .scopes
3151 .read()
3152 .unwrap_or_else(|poisoned| poisoned.into_inner())
3153 .describe(&owner, &scope_ref);
3154 let owner_configured = match &owner {
3155 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3160 _ => false,
3161 };
3162 let response = ModuleControlResponseToModule::ScopeDescribe {
3163 status: description.status,
3164 scope_epoch: description.scope_epoch,
3165 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3166 owner_synced: description.owner_synced,
3167 owner_configured,
3168 scope: description.stamp,
3169 };
3170 Ok(vec![control_response_body_frame(
3171 &frame,
3172 &response,
3173 "ModuleControlResponseToModule::ScopeDescribe",
3174 )?])
3175 }
3176
3177 fn handle_catalog_update(
3178 &self,
3179 connection_id: ConnectionId,
3180 frame: Frame,
3181 provides: Vec<ProviderRole>,
3182 capabilities: Option<CapabilityDeclarations>,
3183 ready: Option<bool>,
3184 ) -> Result<Vec<Frame>, RouterError> {
3185 self.refresh_capability_requirements();
3186 let Some(registration) = self
3187 .registry
3188 .get_module_by_connection(connection_id)
3189 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3190 else {
3191 return Ok(vec![control_error_frame(
3192 &frame,
3193 "not_registered",
3194 "catalog.update requires an active module registration owned by this connection",
3195 )?]);
3196 };
3197
3198 if let Some(message) =
3199 catalog_update_frozen_field_message(®istration.manifest, &provides)
3200 {
3201 return Ok(vec![control_error_frame(
3202 &frame,
3203 "catalog_update_frozen_field",
3204 message,
3205 )?]);
3206 }
3207
3208 let mut candidate = registration.manifest.clone();
3209 candidate.provides = provides.clone();
3210 candidate.capabilities = capabilities
3211 .clone()
3212 .or_else(|| registration.manifest.capabilities.clone());
3213 if let Err(err) = candidate.validate_capability_grammar() {
3214 return Ok(vec![control_error_frame(
3215 &frame,
3216 "invalid_capability_grammar",
3217 err.to_string(),
3218 )?]);
3219 }
3220
3221 let mut conflicts = self
3224 .capability_evaluator
3225 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3226 if let Some(conflict) = conflicts.first() {
3227 let message = format!(
3228 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3229 conflict.capability, conflict.claimants[0], candidate.module_id
3230 );
3231 for conflict in &mut conflicts {
3232 conflict.source = DuplicateClaimSource::CatalogUpdate;
3233 }
3234 log_duplicate_claim_events(conflicts);
3235 return Ok(vec![control_error_frame(
3236 &frame,
3237 "reserved_capability",
3238 message,
3239 )?]);
3240 }
3241
3242 let updated = self
3243 .registry
3244 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3245 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3246 if updated.is_none() {
3247 return Ok(vec![control_error_frame(
3248 &frame,
3249 "not_registered",
3250 "catalog.update requires an active module registration owned by this connection",
3251 )?]);
3252 }
3253 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3254 log_duplicate_claim_events(
3255 self.capability_evaluator
3256 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3257 );
3258 }
3259 if capability_census_trigger(
3260 registration.manifest.capabilities.as_ref(),
3261 updated
3262 .as_ref()
3263 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3264 ) {
3265 self.enforce_capability_denies();
3266 }
3267 self.refresh_capability_requirements();
3268
3269 let response = ModuleControlResponseToModule::CatalogUpdate {};
3270 control_response_body_frame(
3271 &frame,
3272 &response,
3273 "ModuleControlResponseToModule::CatalogUpdate",
3274 )
3275 .map(|frame| vec![frame])
3276 }
3277
3278 pub(crate) fn with_activity_ledger_config(
3279 mut self,
3280 run_dir: &Path,
3281 incarnation: String,
3282 config: crate::activity::ActivityConfig,
3283 ) -> Self {
3284 self.activity = Some(crate::activity::ActivityLedger::open_config(
3285 run_dir.join("activity.jsonl"),
3286 incarnation,
3287 config,
3288 ));
3289 self.supervisor.set_activity(self.activity.clone());
3290 self
3291 }
3292
3293 pub(crate) async fn configure_activity(
3294 &self,
3295 config: crate::activity::ActivityConfig,
3296 ) -> Result<(), &'static str> {
3297 match &self.activity {
3298 Some(ledger) => ledger.configure(config).await,
3299 None => Ok(()),
3300 }
3301 }
3302
3303 fn cancel_activity_subscription(
3304 &self,
3305 ctx: &RouteCtx,
3306 frame: &Frame,
3307 ) -> Result<bool, RouterError> {
3308 let task = self
3309 .activity_streams
3310 .lock()
3311 .unwrap_or_else(|p| p.into_inner())
3312 .remove(&(ctx.connection_id, frame.header.corr));
3313 let Some(task) = task else {
3314 return Ok(false);
3315 };
3316 task.abort();
3317 let end = Frame::build_with_version(
3318 response_version(frame),
3319 FrameType::StreamEnd,
3320 control_flags(),
3321 0,
3322 0,
3323 frame.header.corr,
3324 Vec::new(),
3325 )
3326 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3327 let sink = ctx.egress.clone();
3328 tokio::spawn(async move {
3329 let _ = sink.send(end).await;
3330 });
3331 Ok(true)
3332 }
3333
3334 async fn handle_activity_read(
3335 &self,
3336 ctx: &RouteCtx,
3337 frame: Frame,
3338 ) -> Result<Vec<Frame>, RouterError> {
3339 use subc_control::{
3340 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3341 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3342 };
3343 #[derive(Deserialize)]
3344 struct Identity {
3345 #[serde(default)]
3346 consumer_identity: Option<ConsumerIdentity>,
3347 }
3348 let principal = match self
3352 .forwarding
3353 .module_id_for_connection(ctx.connection_id)
3354 .map_err(RouterError::Forwarding)?
3355 {
3356 Some(module_id) => Ok(Principal::Reserved { module_id }),
3357 None => {
3358 let identity: Identity = match serde_json::from_slice(&frame.body) {
3359 Ok(identity) => identity,
3360 Err(_) => {
3361 return Ok(vec![control_error_frame(
3362 &frame,
3363 ACTIVITY_MALFORMED,
3364 "malformed activity read",
3365 )?])
3366 }
3367 };
3368 self.route_open_principal(&frame, identity.consumer_identity)?
3369 }
3370 };
3371 let admitted = matches!(principal, Ok(Principal::Direct))
3372 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3373 if !admitted {
3374 return Ok(vec![control_error_frame(
3375 &frame,
3376 ACTIVITY_READ_FORBIDDEN,
3377 "activity reads require an operator or trusted mirror principal",
3378 )?]);
3379 }
3380 let Some(ledger) = &self.activity else {
3381 return Ok(vec![control_error_frame(
3382 &frame,
3383 ACTIVITY_UNAVAILABLE,
3384 "activity ledger unavailable",
3385 )?]);
3386 };
3387 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3388 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3389 {
3390 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3391 Ok(request) => ledger.query(request).await,
3392 Err(_) => Err(ACTIVITY_MALFORMED),
3393 };
3394 return match result {
3395 Ok(reply) => Ok(vec![control_response_body_frame(
3396 &frame,
3397 &reply,
3398 ops::ACTIVITY_QUERY,
3399 )?]),
3400 Err(code) => Ok(vec![control_error_frame(
3401 &frame,
3402 code,
3403 "activity query refused",
3404 )?]),
3405 };
3406 }
3407 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3408 Ok(request) => request,
3409 Err(_) => {
3410 return Ok(vec![control_error_frame(
3411 &frame,
3412 ACTIVITY_MALFORMED,
3413 "malformed activity subscription",
3414 )?])
3415 }
3416 };
3417 let mut subscription = match ledger.subscribe(request.since).await {
3418 Ok(subscription) => subscription,
3419 Err(code) => {
3420 return Ok(vec![control_error_frame(
3421 &frame,
3422 code,
3423 "activity subscription refused",
3424 )?])
3425 }
3426 };
3427 let key = (ctx.connection_id, frame.header.corr);
3428 let sink = ctx.egress.clone();
3429 let mut streams = self
3430 .activity_streams
3431 .lock()
3432 .unwrap_or_else(|p| p.into_inner());
3433 streams.retain(|_, task| !task.is_finished());
3434 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3435 return Ok(vec![control_error_frame(
3436 &frame,
3437 "duplicate_subscription",
3438 "activity subscription correlation id already in use",
3439 )?]);
3440 }
3441 let task = tokio::spawn(async move {
3442 loop {
3443 let entry = if let Some(entry) = subscription.replay.pop_front() {
3444 Some(entry)
3445 } else {
3446 subscription.receiver.recv().await
3447 };
3448 let Some(entry) = entry else {
3449 break;
3450 };
3451 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3452 return;
3453 };
3454 let Ok(next) = Frame::build_with_version(
3455 response_version(&frame),
3456 FrameType::StreamData,
3457 control_flags(),
3458 0,
3459 0,
3460 frame.header.corr,
3461 body,
3462 ) else {
3463 return;
3464 };
3465 if sink.send(next).await.is_err() {
3466 return;
3467 }
3468 }
3469 if let Some(code) = subscription.terminal_code() {
3472 if let Ok(end) =
3473 control_error_frame(&frame, code, "activity subscriber fell behind")
3474 {
3475 let _ = sink.send(end).await;
3476 }
3477 }
3478 });
3479 streams.insert(key, task.abort_handle());
3480 Ok(Vec::new())
3481 }
3482
3483 async fn handle_activity_record(
3484 &self,
3485 ctx: &RouteCtx,
3486 frame: Frame,
3487 ) -> Result<Vec<Frame>, RouterError> {
3488 use subc_control::{
3489 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3490 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3491 };
3492 let result = async {
3493 let module = self
3494 .forwarding
3495 .module_id_for_connection(ctx.connection_id)
3496 .ok()
3497 .flatten()
3498 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3499 let request: ActivityRecordRequest =
3500 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3501 crate::activity::validate(&module, &request)?;
3502 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3503 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3504 let at_ms = std::time::SystemTime::now()
3505 .duration_since(std::time::UNIX_EPOCH)
3506 .unwrap_or_default()
3507 .as_millis() as u64;
3508 ledger.record(module, request, actor, at_ms).await
3509 }
3510 .await;
3511 match result {
3512 Ok(entry) => Ok(vec![control_response_body_frame(
3513 &frame,
3514 &ActivityRecordResponse { seq: entry.seq },
3515 "activity.record",
3516 )?]),
3517 Err(code) => Ok(vec![control_error_frame(
3518 &frame,
3519 code,
3520 "activity recording refused",
3521 )?]),
3522 }
3523 }
3524
3525 fn activity_actor(
3526 &self,
3527 connection: ConnectionId,
3528 module: &str,
3529 on: Option<&subc_control::ActivityOn>,
3530 ) -> Result<subc_control::ActivityActor, &'static str> {
3531 use crate::forwarding::{DataRoute, DataRouteState};
3532 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3533 let principal = |id: &str| Principal::Reserved {
3534 module_id: id.to_owned(),
3535 };
3536 let mut actor = ActivityActor {
3537 principal: principal(module),
3538 scope_owner: None,
3539 agent_id: None,
3540 run_id: None,
3541 flow_id: None,
3542 project_root: None,
3543 };
3544 let scope = match on {
3545 None => return Ok(actor),
3546 Some(ActivityOn::Scope(scope)) => {
3547 let selector = subc_protocol::scope::ScopeSelector::new(
3548 principal(module),
3549 scope.scope_ref.clone(),
3550 Some(scope.epoch),
3551 );
3552 self.scopes
3553 .read()
3554 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3555 .attribute(&principal(module), module, &selector, true)
3556 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3557 }
3558 Some(ActivityOn::Route(route)) => {
3559 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3560 .forwarding
3561 .lookup_data_route(connection, route.channel, route.epoch)
3562 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3563 else {
3564 return Err(ACTIVITY_ACTOR_UNKNOWN);
3565 };
3566 if binding.module_id != module {
3567 return Err(ACTIVITY_ACTOR_UNKNOWN);
3568 }
3569 actor.principal = binding.principal.clone();
3570 actor.project_root = binding
3571 .project_root
3572 .as_ref()
3573 .map(|root| root.as_path().to_path_buf());
3574 let Some(bound) = &binding.scope else {
3575 return Ok(actor);
3576 };
3577 let owner = principal(&bound.owner);
3578 let selector = subc_protocol::scope::ScopeSelector::new(
3579 owner.clone(),
3580 bound.scope_ref.clone(),
3581 Some(bound.tag.scope_epoch),
3582 );
3583 let admitted = self
3584 .scopes
3585 .read()
3586 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3587 .attribute(&owner, module, &selector, true)
3588 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3589 if admitted.tag != bound.tag {
3590 return Err(ACTIVITY_ACTOR_UNKNOWN);
3591 }
3592 admitted
3593 }
3594 };
3595 actor.scope_owner = Some(scope.stamp.owner);
3596 actor.agent_id = scope.stamp.attributes.agent_id;
3597 actor.run_id = scope.stamp.attributes.run_id;
3598 actor.flow_id = scope.stamp.attributes.flow_id;
3599 Ok(actor)
3600 }
3601
3602 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3603 self.refresh_capability_requirements();
3604 let mut counters = self.counters.snapshot();
3610 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3611 obj.insert(
3612 "activity_unparseable_lines".into(),
3613 ledger.read_errors().into(),
3614 );
3615 obj.insert(
3616 "activity_lifecycle_append_failures".into(),
3617 ledger.lifecycle_failures().into(),
3618 );
3619 obj.insert(
3620 "activity_started_digest_skips".into(),
3621 ledger.lifecycle_digest_skips().into(),
3622 );
3623 }
3624 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3625 self.forwarding.client_route_concentration(),
3626 counters.as_object_mut(),
3627 ) {
3628 obj.insert(
3629 "client_connections_with_routes".into(),
3630 connections_with_routes.into(),
3631 );
3632 obj.insert("max_routes_on_one_connection".into(), max.into());
3633 }
3634 if let (Some(open_breakers), Some(obj)) = (
3640 self.route_bind_breakers.open_snapshot(),
3641 counters.as_object_mut(),
3642 ) {
3643 obj.insert("route_bind_breakers_open".into(), open_breakers);
3644 }
3645 let response = ClientControlResponse::ServerDescribe {
3646 protocol_ver: PROTOCOL_VERSION,
3647 subc_ops: subc_ops(),
3648 capabilities: self.subc_capabilities.as_ref().to_vec(),
3649 connected_clients: self.connected_clients.count(),
3650 counters: Some(counters),
3651 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3652 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3653 capability_requirements: self.capability_requirement_statuses(),
3654 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3655 };
3656 Ok(vec![control_response_body_frame(
3657 &frame,
3658 &response,
3659 "ClientControlResponse::ServerDescribe",
3660 )?])
3661 }
3662
3663 fn handle_catalog_list(
3664 &self,
3665 frame: Frame,
3666 module_id: Option<String>,
3667 ) -> Result<Vec<Frame>, RouterError> {
3668 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3669 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3670 })?;
3671 let entries = modules
3672 .into_iter()
3673 .filter(|registration| {
3674 module_id
3675 .as_deref()
3676 .map(|wanted| registration.manifest.module_id == wanted)
3677 .unwrap_or(true)
3678 })
3679 .map(|registration| {
3680 let not_ready = self.not_ready_reason(®istration);
3681 let roles = registration.manifest.provides;
3682 CatalogEntry::new(
3683 registration.manifest.module_id,
3684 roles,
3685 registration.control_ops,
3686 )
3687 .with_ready(not_ready.is_none())
3688 .with_not_ready(not_ready)
3689 .with_module_version(Some(registration.manifest.module_version))
3690 .with_capabilities(registration.manifest.capabilities)
3691 .with_self_signals(registration.manifest.self_signals)
3692 .with_events(registration.manifest.events)
3693 })
3694 .collect();
3695 let response = ClientControlResponse::CatalogList {
3696 generation,
3697 modules: entries,
3698 subc_ops: subc_ops(),
3699 };
3700 Ok(vec![control_response_body_frame(
3701 &frame,
3702 &response,
3703 "ClientControlResponse::CatalogList",
3704 )?])
3705 }
3706
3707 fn route_open_principal(
3708 &self,
3709 frame: &Frame,
3710 consumer_identity: Option<ConsumerIdentity>,
3711 ) -> Result<Result<Principal, Frame>, RouterError> {
3712 let Some(consumer_identity) = consumer_identity else {
3713 return Ok(Ok(Principal::Direct));
3714 };
3715
3716 if self.supervisor.spawned_consumer_authorized(
3717 &consumer_identity.module_id,
3718 &consumer_identity.launch_nonce,
3719 ) {
3720 return Ok(Ok(Principal::Reserved {
3721 module_id: consumer_identity.module_id,
3722 }));
3723 }
3724
3725 Ok(Err(control_error_frame(
3726 frame,
3727 "bad_consumer_identity",
3728 format!(
3729 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3730 consumer_identity.module_id
3731 ),
3732 )?))
3733 }
3734
3735 fn route_open_refusal_frame(
3742 &self,
3743 ctx: &RouteCtx,
3744 frame: &Frame,
3745 module_id: &str,
3746 reason: &'static str,
3747 code: &'static str,
3748 message: impl Into<String>,
3749 ) -> Result<Frame, RouterError> {
3750 self.observe_route_open_refusal(ctx, module_id, reason, code);
3751 control_error_frame(frame, code, message.into())
3752 }
3753
3754 fn route_open_breaker_refusal_frame(
3772 &self,
3773 ctx: &RouteCtx,
3774 frame: &Frame,
3775 module_id: &str,
3776 consecutive_timeouts: u32,
3777 retry_in: Duration,
3778 probe_in_flight: bool,
3779 ) -> Result<Frame, RouterError> {
3780 self.counters
3781 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3782 debug!(
3783 target: "control",
3784 code = "module_timeout",
3785 module_id = ?module_id,
3786 connection_id = ctx.connection_id.get(),
3787 consecutive_timeouts,
3788 retry_in_ms = retry_in.as_millis() as u64,
3789 probe_in_flight,
3790 "route.open refused by open bind-relay breaker"
3791 );
3792 let detail = if probe_in_flight {
3799 "one test route is already being tried; retry once it settles".to_string()
3800 } else {
3801 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3802 };
3803 control_error_frame(
3804 frame,
3805 "module_timeout",
3806 format!(
3807 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3808 timed out in a row); {detail}; its established routes are unaffected"
3809 ),
3810 )
3811 }
3812
3813 fn observe_route_open_refusal(
3826 &self,
3827 ctx: &RouteCtx,
3828 module_id: &str,
3829 reason: &'static str,
3830 code: &'static str,
3831 ) {
3832 self.counters.increment_route_open_refused(code);
3833 info!(
3834 target: "control",
3835 code,
3836 reason,
3837 module_id = ?module_id,
3838 connection_id = ctx.connection_id.get(),
3839 "route.open refused"
3840 );
3841 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3842 self.route_outages.record_not_serving(module_id, reason);
3843 }
3844 }
3845
3846 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3891 self.route_outages.record_accepted(module_id);
3892 self.counters.increment_route_open_accepted(principal);
3893 info!(
3894 target: "control",
3895 principal,
3896 module_id,
3897 connection_id = ctx.connection_id.get(),
3898 "route.open accepted"
3899 );
3900 }
3901
3902 fn supervised_absent_route_open_refusal_frame(
3903 &self,
3904 ctx: &RouteCtx,
3905 frame: &Frame,
3906 module_id: &str,
3907 code: &'static str,
3908 status: &crate::supervise::ModuleStatus,
3909 ) -> Result<Frame, RouterError> {
3910 self.counters.increment_route_open_refused(code);
3911 info!(
3912 target: "control",
3913 code,
3914 reason = "supervised_not_registered",
3915 module_id = ?module_id,
3916 connection_id = ctx.connection_id.get(),
3917 state = %status.state,
3918 enabled = status.enabled,
3919 live = status.live,
3920 "route.open refused"
3921 );
3922 self.route_outages
3926 .record_not_serving(module_id, "supervised_not_registered");
3927 let message = if let Some(pid) = status.waiting_for_previous_pid {
3928 format!("waiting: previous instance (pid {pid}) is still running but hidden; it cannot be identified")
3929 } else {
3930 format!(
3931 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3932 status.state, status.enabled, status.live
3933 )
3934 };
3935 control_error_frame(frame, code, message)
3936 }
3937
3938 async fn handle_route_open(
3939 &self,
3940 ctx: &RouteCtx,
3941 frame: Frame,
3942 request: RouteOpenRequest,
3943 ) -> Result<Vec<Frame>, RouterError> {
3944 let RouteOpenRequest {
3945 target,
3946 mut identity,
3947 consumer_identity,
3948 consumer_capabilities,
3949 role_versions,
3950 admission_facts,
3951 scope,
3952 } = request;
3953 let target_module_id = target_module_id(&target).to_string();
3954 if self
3955 .registry
3956 .get_module_by_connection(ctx.connection_id)
3957 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3958 .is_some()
3959 {
3960 return Ok(vec![control_error_frame(
3961 &frame,
3962 "invalid_request",
3963 "module connections cannot open client routes",
3964 )?]);
3965 }
3966 debug!(
3967 connection_id = ctx.connection_id.get(),
3968 corr = frame.header.corr,
3969 module_id = %target_module_id,
3970 "handling route.open"
3971 );
3972
3973 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3979 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3980 self.observe_route_open_refusal(
3981 ctx,
3982 &target_module_id,
3983 "invalid_role_versions",
3984 error_codes::INVALID_REQUEST,
3985 );
3986 return Ok(vec![control_error_body_frame(
3987 &frame,
3988 ErrorBody {
3989 code: error_codes::INVALID_REQUEST.to_string(),
3990 message: error.to_string(),
3991 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3992 },
3993 )?]);
3994 }
3995
3996 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
3997 if status.waiting_for_previous_pid.is_some() {
3998 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3999 ctx,
4000 &frame,
4001 &target_module_id,
4002 "module_warming",
4003 &status,
4004 )?]);
4005 }
4006 }
4007
4008 let Some(registration) = self
4027 .registry
4028 .get_module(&target_module_id)
4029 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4030 else {
4031 if let Some((status, warming)) =
4032 self.supervisor_status(&target_module_id, frame.header.corr)?
4033 {
4034 if status.protocol == ModuleProtocol::None {
4041 return Ok(vec![self.route_open_refusal_frame(
4042 ctx,
4043 &frame,
4044 &target_module_id,
4045 "protocol_none",
4046 error_codes::MODULE_NO_PROTOCOL,
4047 format!(
4048 "module_id '{target_module_id}' is declared protocol: none; \
4049 it speaks no subc wire and serves no routes"
4050 ),
4051 )?]);
4052 }
4053 let code = if warming {
4054 "module_warming"
4055 } else {
4056 "target_unavailable"
4057 };
4058 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4059 ctx,
4060 &frame,
4061 &target_module_id,
4062 code,
4063 &status,
4064 )?]);
4065 }
4066 if let Some(removed_ago_ms) =
4067 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4068 {
4069 return Ok(vec![self.route_open_refusal_frame(
4070 ctx,
4071 &frame,
4072 &target_module_id,
4073 "removed",
4074 error_codes::MODULE_REMOVED,
4075 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4076 )?]);
4077 }
4078 return Ok(vec![self.route_open_refusal_frame(
4079 ctx,
4080 &frame,
4081 &target_module_id,
4082 "not_registered",
4083 error_codes::UNKNOWN_MODULE,
4084 format!("module_id '{target_module_id}' is not registered"),
4085 )?]);
4086 };
4087
4088 if !registration.ready {
4092 self.counters
4093 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4094 info!(
4095 target: "control",
4096 code = error_codes::MODULE_WARMING,
4097 module_id = ?target_module_id,
4098 connection_id = ctx.connection_id.get(),
4099 reason = "declared_not_ready",
4100 "route.open refused"
4101 );
4102 self.route_outages
4105 .record_not_serving(&target_module_id, "declared_not_ready");
4106 return Ok(vec![control_error_body_frame(
4107 &frame,
4108 ErrorBody {
4109 code: error_codes::MODULE_WARMING.to_string(),
4110 message: format!(
4111 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4112 ),
4113 detail: Some(serde_json::json!({
4114 "reason": "declared_not_ready"
4115 })),
4116 },
4117 )?]);
4118 }
4119
4120 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4139 self.counters
4140 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4141 info!(
4142 target: "control",
4143 code = error_codes::MODULE_WARMING,
4144 module_id = ?target_module_id,
4145 connection_id = ctx.connection_id.get(),
4146 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4147 capability = %capability,
4148 "route.open refused"
4149 );
4150 return Ok(vec![control_error_body_frame(
4151 &frame,
4152 ErrorBody {
4153 code: error_codes::MODULE_WARMING.to_string(),
4154 message: format!(
4155 "module_id '{target_module_id}' requires capability '{capability}', \
4156 which no registered module provides; retry"
4157 ),
4158 detail: Some(serde_json::json!({
4159 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4160 "capability": capability,
4161 })),
4162 },
4163 )?]);
4164 }
4165
4166 if !target_has_required_role(&target, ®istration.manifest.provides) {
4167 return Ok(vec![self.route_open_refusal_frame(
4168 ctx,
4169 &frame,
4170 &target_module_id,
4171 "role_not_provided",
4172 "target_unavailable",
4173 format!("module_id '{target_module_id}' does not provide the requested target"),
4174 )?]);
4175 }
4176
4177 if registration.state != ChannelState::Active {
4178 return Ok(vec![self.route_open_refusal_frame(
4179 ctx,
4180 &frame,
4181 &target_module_id,
4182 "registration_not_active",
4183 "target_unavailable",
4184 format!("module_id '{target_module_id}' is not active"),
4185 )?]);
4186 }
4187
4188 if self
4189 .forwarding
4190 .module_is_draining(&target_module_id)
4191 .map_err(RouterError::Forwarding)?
4192 {
4193 return Ok(vec![self.route_open_refusal_frame(
4194 ctx,
4195 &frame,
4196 &target_module_id,
4197 "reloading",
4198 "module_reloading",
4199 format!("module_id '{target_module_id}' is reloading"),
4200 )?]);
4201 }
4202
4203 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4204 process_liveness.process_live(&target_module_id) == Some(false)
4205 }) {
4206 if process_liveness.process_replacing(&target_module_id) {
4213 return Ok(vec![self.route_open_refusal_frame(
4214 ctx,
4215 &frame,
4216 &target_module_id,
4217 "reloading",
4218 "module_reloading",
4219 format!("module_id '{target_module_id}' is reloading"),
4220 )?]);
4221 }
4222 return Ok(vec![self.route_open_refusal_frame(
4223 ctx,
4224 &frame,
4225 &target_module_id,
4226 "supervisor_not_live",
4227 "target_unavailable",
4228 format!("module_id '{target_module_id}' is not live"),
4229 )?]);
4230 }
4231
4232 if !self
4233 .forwarding
4234 .has_live_module_connection(&target_module_id)
4235 .map_err(RouterError::Forwarding)?
4236 {
4237 return Ok(vec![self.route_open_refusal_frame(
4238 ctx,
4239 &frame,
4240 &target_module_id,
4241 "no_forwarding_connection",
4242 "target_unavailable",
4243 format!("module_id '{target_module_id}' has no live forwarding connection"),
4244 )?]);
4245 }
4246
4247 if let Some(error) =
4248 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4249 {
4250 self.observe_route_open_refusal(
4251 ctx,
4252 &target_module_id,
4253 "op_not_allowed",
4254 "op_not_allowed",
4255 );
4256 return Ok(vec![error]);
4257 }
4258
4259 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4260 Ok(principal) => principal,
4261 Err(error) => {
4262 self.observe_route_open_refusal(
4263 ctx,
4264 &target_module_id,
4265 "bad_consumer_identity",
4266 "bad_consumer_identity",
4267 );
4268 return Ok(vec![error]);
4269 }
4270 };
4271
4272 if let Principal::Reserved {
4276 module_id: opening_module_id,
4277 } = &principal
4278 {
4279 if let Some(opening_registration) = self
4280 .registry
4281 .get_module(opening_module_id)
4282 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4283 {
4284 if let Some(capability) =
4285 denied_capability(&opening_registration.manifest, ®istration.manifest)
4286 {
4287 warn!(
4288 opening_module_id,
4289 target_module_id,
4290 capability,
4291 "refusing route.open because an attested capability deny edge matches"
4292 );
4293 return Ok(vec![self.route_open_refusal_frame(
4294 ctx,
4295 &frame,
4296 &target_module_id,
4297 "capability_deny_edge",
4298 "capability_forbidden",
4299 format!(
4300 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4301 ),
4302 )?]);
4303 }
4304 }
4305 }
4306
4307 if admission_facts.is_some() {
4308 let carrier_matches = matches!(
4309 &principal,
4310 Principal::Reserved { module_id }
4311 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4312 );
4313 if !carrier_matches {
4314 return Ok(vec![self.route_open_refusal_frame(
4315 ctx,
4316 &frame,
4317 &target_module_id,
4318 "admission_facts_carrier_not_permitted",
4319 "admission_facts_not_permitted",
4320 "admission facts may only be carried by the configured reserved module",
4321 )?]);
4322 }
4323
4324 let target_allowed = self
4325 .admission_facts_targets
4326 .as_ref()
4327 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4328 if !target_allowed {
4329 return Ok(vec![self.route_open_refusal_frame(
4330 ctx,
4331 &frame,
4332 &target_module_id,
4333 "admission_facts_target_not_listed",
4334 "admission_facts_target_not_allowed",
4335 format!(
4336 "admission facts are not permitted for target module_id '{target_module_id}'"
4337 ),
4338 )?]);
4339 }
4340
4341 }
4345
4346 let (bound_scope, mut scope_stamp) = match &scope {
4352 None => (None, None),
4353 Some(selector) => {
4354 let owner_configured = match &selector.owner {
4355 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4361 _ => false,
4362 };
4363 let admitted = self
4364 .scopes
4365 .read()
4366 .unwrap_or_else(|poisoned| poisoned.into_inner())
4367 .admit(&principal, &target_module_id, selector, owner_configured)
4368 .and_then(|admission| {
4369 crate::scopes::check_target_flow_support(
4370 &admission.stamp,
4371 &target_module_id,
4372 registration.manifest.capabilities.as_ref(),
4373 )?;
4374 crate::scopes::check_target_agent_run_support(
4375 &admission.stamp,
4376 &target_module_id,
4377 registration.manifest.capabilities.as_ref(),
4378 )?;
4379 Ok(admission)
4380 });
4381 match admitted {
4382 Ok(admission) => (
4383 Some(BoundScope {
4384 owner: admission.owner,
4385 scope_ref: admission.stamp.scope_ref.clone(),
4386 tag: admission.tag,
4387 }),
4388 Some(admission.stamp),
4389 ),
4390 Err(refusal) => {
4391 return Ok(vec![self.route_open_refusal_frame(
4392 ctx,
4393 &frame,
4394 &target_module_id,
4395 refusal.code,
4396 refusal.code,
4397 refusal.message,
4398 )?]);
4399 }
4400 }
4401 }
4402 };
4403
4404 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4426 Ok(project_root) => project_root,
4427 Err(err) => {
4428 return Ok(vec![control_error_frame(
4429 &frame,
4430 "invalid_project_root",
4431 err.to_string(),
4432 )?])
4433 }
4434 };
4435 identity.project_root = project_root.as_path().to_path_buf();
4436
4437 let _concurrency_guard = match self
4448 .route_bind_concurrency
4449 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4450 {
4451 Ok(guard) => guard,
4452 Err(in_flight) => {
4453 return Ok(vec![self.route_open_target_capacity_refusal(
4454 ctx,
4455 &frame,
4456 &target_module_id,
4457 in_flight,
4458 )?]);
4459 }
4460 };
4461
4462 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4465 RouteBindAdmission::Admitted { guard, probe } => {
4466 if probe {
4467 info!(
4468 module_id = %target_module_id,
4469 connection_id = ctx.connection_id.get(),
4470 "route.bind breaker half-open: admitting one probe"
4471 );
4472 }
4473 guard
4474 }
4475 RouteBindAdmission::Refused {
4476 consecutive_timeouts,
4477 retry_in,
4478 probe_in_flight,
4479 } => {
4480 return Ok(vec![self.route_open_breaker_refusal_frame(
4481 ctx,
4482 &frame,
4483 &target_module_id,
4484 consecutive_timeouts,
4485 retry_in,
4486 probe_in_flight,
4487 )?]);
4488 }
4489 };
4490
4491 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4496 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4497 let pending = match self
4498 .forwarding
4499 .begin_route_bind_relay_for(
4500 ctx.connection_id,
4501 ctx.egress.clone(),
4502 response_version(&frame),
4503 frame.header.corr,
4504 &target_module_id,
4505 principal.clone(),
4506 bound_scope,
4507 Some(project_root),
4508 relay_deadline,
4509 )
4510 .await
4511 {
4512 Ok(pending) => pending,
4513 Err(err) => {
4514 return Ok(vec![self.route_open_refusal_frame(
4515 ctx,
4516 &frame,
4517 &target_module_id,
4518 "relay_reservation_failed",
4519 forwarding_error_code(&err),
4520 err.to_string(),
4521 )?])
4522 }
4523 };
4524 let crate::forwarding::PendingRouteBindRelay {
4525 endpoint,
4526 module_sink,
4527 negotiated_ver,
4528 client_channel,
4529 client_epoch,
4530 module_channel,
4531 module_epoch,
4532 corr: relay_corr,
4533 receiver,
4534 } = pending;
4535 let mut reservation =
4536 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4537
4538 let relay_registration = self
4543 .registry
4544 .get_module_by_connection(endpoint.connection_id)
4545 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4546 let relay_capabilities = relay_registration
4547 .as_ref()
4548 .and_then(|registration| registration.manifest.capabilities.as_ref());
4549 if let Some(selector) = &scope {
4550 if let Err(refusal) = self
4553 .scopes
4554 .read()
4555 .unwrap_or_else(|poisoned| poisoned.into_inner())
4556 .check_bind_proof(selector)
4557 {
4558 reservation.release_and_disarm();
4559 return Ok(vec![self.route_open_refusal_frame(
4560 ctx,
4561 &frame,
4562 &target_module_id,
4563 refusal.code,
4564 refusal.code,
4565 refusal.message,
4566 )?]);
4567 }
4568 }
4569 if let Some(stamp) = scope_stamp
4570 .as_ref()
4571 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4572 {
4573 if let Err(refusal) = crate::scopes::check_target_flow_support(
4574 stamp,
4575 &target_module_id,
4576 relay_capabilities,
4577 )
4578 .and_then(|()| {
4579 crate::scopes::check_target_agent_run_support(
4580 stamp,
4581 &target_module_id,
4582 relay_capabilities,
4583 )
4584 }) {
4585 reservation.release_and_disarm();
4586 return Ok(vec![self.route_open_refusal_frame(
4587 ctx,
4588 &frame,
4589 &target_module_id,
4590 refusal.code,
4591 refusal.code,
4592 refusal.message,
4593 )?]);
4594 }
4595 }
4596
4597 let call_mode = if relay_capabilities
4600 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4601 {
4602 Some(derive_call_mode(scope_stamp.as_ref()))
4603 } else {
4604 if let Some(stamp) = scope_stamp.as_mut() {
4605 stamp.expires_at_ms = None;
4606 }
4607 None
4608 };
4609
4610 debug!(
4611 connection_id = ctx.connection_id.get(),
4612 client_channel,
4613 client_epoch,
4614 module_channel,
4615 module_epoch,
4616 "reserved route handle pair"
4617 );
4618 let principal_label = principal_label(&principal);
4621 let relay = ModuleControlRequest::RouteBind {
4622 route_channel: module_channel,
4623 epoch: module_epoch,
4624 target,
4625 identity,
4626 principal: Some(principal),
4627 consumer_capabilities,
4628 role_versions,
4629 admission_facts,
4630 scope: scope_stamp,
4631 call_mode,
4632 };
4633 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4634 RouterError::backend(
4635 0,
4636 frame.header.corr,
4637 format!("failed to encode route.bind request: {err}"),
4638 )
4639 })?;
4640 let relay_frame = Frame::build_with_version(
4641 negotiated_ver,
4642 FrameType::Request,
4643 control_flags(),
4644 0,
4645 0,
4646 relay_corr,
4647 relay_body,
4648 )
4649 .map_err(RouterError::FrameBuild)?;
4650
4651 if let Err(err) = module_sink.send(relay_frame).await {
4652 reservation.release_and_disarm();
4653 return Ok(vec![self.route_open_refusal_frame(
4654 ctx,
4655 &frame,
4656 &target_module_id,
4657 "relay_send_failed",
4658 "target_unavailable",
4659 err.to_string(),
4660 )?]);
4661 }
4662
4663 #[cfg(test)]
4664 if let Some(hold) = &self.route_bind_relay_sent_hold {
4665 hold.wait().await;
4669 hold.wait().await;
4670 }
4671
4672 if self
4677 .forwarding
4678 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4679 .map_err(RouterError::Forwarding)?
4680 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4681 {
4682 self.send_abandoned_route_bind_goodbye(
4683 &module_sink,
4684 negotiated_ver,
4685 module_channel,
4686 module_epoch,
4687 );
4688 }
4689
4690 match timeout_at(relay_deadline, receiver).await {
4691 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4692 reservation.disarm();
4693 if breaker.record_accepted() {
4694 info!(
4695 module_id = %target_module_id,
4696 "route.bind breaker closed: the probe was accepted"
4697 );
4698 }
4699 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4700 Ok(Vec::new())
4701 }
4702 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4703 reservation.release_and_disarm();
4704 breaker.record_inconclusive();
4708 let scope_code = match body.code.as_str() {
4712 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4713 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4714 _ => None,
4715 };
4716 if let Some(code) = scope_code {
4717 self.observe_route_open_refusal(
4718 ctx,
4719 &target_module_id,
4720 "scope_changed_before_commit",
4721 code,
4722 );
4723 return Ok(vec![control_error_body_frame(&frame, body)?]);
4724 }
4725 self.counters
4726 .increment_route_open_refused("module_rejected");
4727 info!(
4728 target: "control",
4729 code = "module_rejected",
4730 module_code = ?body.code,
4731 module_id = ?target_module_id,
4732 connection_id = ctx.connection_id.get(),
4733 "route.open refused"
4734 );
4735 Ok(vec![control_error_body_frame(&frame, body)?])
4736 }
4737 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4738 reservation.release_and_disarm();
4739 breaker.record_inconclusive();
4740 tracing::warn!(
4747 module_id = %target_module_id,
4748 "route.bind relay abandoned: {message}"
4749 );
4750 Ok(vec![self.route_open_refusal_frame(
4751 ctx,
4752 &frame,
4753 &target_module_id,
4754 "relay_abandoned",
4755 "target_unavailable",
4756 message,
4757 )?])
4758 }
4759 Ok(Err(_)) => {
4760 reservation.release_and_disarm();
4761 breaker.record_inconclusive();
4762 Ok(vec![self.route_open_refusal_frame(
4763 ctx,
4764 &frame,
4765 &target_module_id,
4766 "relay_waiter_canceled",
4767 "target_unavailable",
4768 "route.bind relay waiter was canceled before the module responded",
4769 )?])
4770 }
4771 Err(_) => {
4772 reservation.release_and_disarm();
4773 if let Some(opened) = breaker.record_timeout(
4777 self.route_bind_breaker_threshold,
4778 self.route_bind_breaker_cooldown,
4779 ) {
4780 warn!(
4781 module_id = %target_module_id,
4782 consecutive_timeouts = opened.consecutive_timeouts,
4783 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4784 reopened_after_probe = opened.reopened_after_probe,
4785 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4786 );
4787 }
4788 tracing::warn!(
4796 module_id = %target_module_id,
4797 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4798 "route.bind relay timed out: module did not ack within budget"
4799 );
4800 Ok(vec![self.route_open_refusal_frame(
4801 ctx,
4802 &frame,
4803 &target_module_id,
4804 "relay_timed_out",
4805 "module_timeout",
4806 format!(
4807 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4808 route_bind_relay_timeout
4809 ),
4810 )?])
4811 }
4812 }
4813 }
4814
4815 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4816 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4817 snapshot: self.supervisor.spawn_snapshot(),
4818 };
4819 Ok(vec![control_response_body_frame(
4820 &frame,
4821 &response,
4822 "ClientControlResponse::SupervisorSpawnSnapshot",
4823 )?])
4824 }
4825
4826 fn handle_supervisor_spawn_subscribe(
4827 &self,
4828 ctx: &RouteCtx,
4829 frame: Frame,
4830 since: Option<SpawnCursor>,
4831 ) -> Result<Vec<Frame>, RouterError> {
4832 match self.supervisor.subscribe_spawns(
4833 ctx.connection_id,
4834 frame.header.corr,
4835 response_version(&frame),
4836 since,
4837 ctx.egress.clone(),
4838 ) {
4839 Ok(()) => Ok(Vec::new()),
4840 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4841 Ok(vec![control_error_body_frame(
4842 &frame,
4843 ErrorBody {
4844 code: "spawn_cursor_incarnation_mismatch".to_string(),
4845 message: "spawn cursor belongs to a different daemon incarnation"
4846 .to_string(),
4847 detail: Some(serde_json::json!({
4848 "current_daemon_incarnation": current
4849 })),
4850 },
4851 )?])
4852 }
4853 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4854 &frame,
4855 ErrorBody {
4856 code: "spawn_cursor_too_old".to_string(),
4857 message: "spawn cursor predates the retained event ring".to_string(),
4858 detail: Some(serde_json::json!({
4859 "oldest_retained_cursor": oldest
4860 })),
4861 },
4862 )?]),
4863 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4864 0,
4865 frame.header.corr,
4866 format!("failed to open supervisor spawn subscription: {error}"),
4867 )),
4868 }
4869 }
4870
4871 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4872 let generation = self
4873 .registry
4874 .generation()
4875 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4876 let mut modules = Vec::new();
4877 for module in self.supervisor.list() {
4878 let status = module.status_for_control("list").map_err(|err| {
4879 RouterError::backend(
4880 0,
4881 frame.header.corr,
4882 format!("failed to read supervisor status: {err}"),
4883 )
4884 })?;
4885 let (configured, _) = module.configuration().map_err(|err| {
4886 RouterError::backend(
4887 0,
4888 frame.header.corr,
4889 format!("failed to read module configuration: {err}"),
4890 )
4891 })?;
4892 let image = module.running_image_agreement().await;
4894 let resources = Some(module.child_resource_usage());
4897 let pending_reload = Some(reload_verdict(
4898 &configured.program,
4899 status.spawned_from.as_deref(),
4900 image,
4901 ));
4902 modules.push(
4905 SupervisorEntry::new(
4906 status.module_id,
4907 status.state.to_string(),
4908 status.enabled,
4909 status.live,
4910 status.health.status,
4911 )
4912 .with_launch_nonce_env(Some(!cfg!(unix)))
4913 .with_launch_nonce_source(status.launch_nonce_source)
4914 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
4915 .with_protocol(status.protocol)
4916 .with_pending_reload(pending_reload)
4917 .with_last_probe_ms(status.health.last_probe_ms)
4918 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4919 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4920 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4921 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4922 .with_restart_count(Some(status.restart_count))
4923 .with_max_restarts(Some(status.max_restarts))
4924 .with_lifetime_restarts(Some(status.lifetime_restarts))
4925 .with_spawn_generation(Some(status.spawn_generation))
4926 .with_restart_window_secs(Some(status.restart_window.as_secs()))
4927 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4928 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4929 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4930 .with_resources(resources),
4931 );
4932 }
4933 let response = ClientControlResponse::SupervisorList {
4934 generation,
4935 modules,
4936 };
4937 Ok(vec![control_response_body_frame(
4938 &frame,
4939 &response,
4940 "ClientControlResponse::SupervisorList",
4941 )?])
4942 }
4943
4944 fn handle_supervisor_stderr_tail(
4945 &self,
4946 frame: Frame,
4947 module_id: String,
4948 max_lines: Option<u32>,
4949 max_bytes: Option<u32>,
4950 ) -> Result<Vec<Frame>, RouterError> {
4951 let Some(module) = self.supervisor.get(&module_id) else {
4952 return Ok(vec![control_error_frame(
4953 &frame,
4954 "unknown_module",
4955 format!("module_id '{module_id}' is not supervised"),
4956 )?]);
4957 };
4958
4959 let snapshot = module.stderr_tail(
4960 max_lines.map(|value| value as usize),
4961 max_bytes.map(|value| value as usize),
4962 );
4963
4964 let response = ClientControlResponse::SupervisorStderrTail {
4965 module_id,
4966 tail: StderrTail {
4967 capture: match snapshot.capture {
4968 CaptureState::Captured => StderrCaptureState::Captured,
4969 CaptureState::Incomplete { reason } => {
4970 StderrCaptureState::Incomplete { reason }
4971 }
4972 CaptureState::NotCaptured { reason } => {
4973 StderrCaptureState::NotCaptured { reason }
4974 }
4975 },
4976 entries: snapshot
4977 .entries
4978 .into_iter()
4979 .map(|entry| match entry {
4980 TailEntry::Line {
4981 text,
4982 truncated,
4983 at_ms,
4984 } => StderrTailEntry::Line {
4985 text,
4986 truncated,
4987 at_ms,
4988 },
4989 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4990 })
4991 .collect(),
4992 dropped_lines: snapshot.dropped_lines,
4993 },
4994 };
4995 Ok(vec![control_response_body_frame(
4996 &frame,
4997 &response,
4998 "ClientControlResponse::SupervisorStderrTail",
4999 )?])
5000 }
5001
5002 async fn handle_supervisor_terminals(
5003 &self,
5004 frame: Frame,
5005 module_id: String,
5006 ) -> Result<Vec<Frame>, RouterError> {
5007 let Some(module) = self.supervisor.get(&module_id) else {
5008 return Ok(vec![control_error_frame(
5009 &frame,
5010 "unknown_module",
5011 format!("module_id '{module_id}' is not supervised"),
5012 )?]);
5013 };
5014
5015 let terminals = module
5018 .read_durable_terminal_history()
5019 .await
5020 .map_err(|error| {
5021 RouterError::backend(
5022 0,
5023 frame.header.corr,
5024 format!("failed to read terminal history: {error}"),
5025 )
5026 })?;
5027 let response = ClientControlResponse::SupervisorTerminals {
5028 module_id,
5029 terminals,
5030 };
5031 Ok(vec![control_response_body_frame(
5032 &frame,
5033 &response,
5034 "ClientControlResponse::SupervisorTerminals",
5035 )?])
5036 }
5037
5038 fn handle_supervisor_routes(
5039 &self,
5040 frame: Frame,
5041 module_id: Option<String>,
5042 ) -> Result<Vec<Frame>, RouterError> {
5043 let modules = self
5044 .forwarding
5045 .route_census(module_id.as_deref())
5046 .map_err(RouterError::Forwarding)?
5047 .into_iter()
5048 .map(|(module_id, routes)| SupervisorRouteModule {
5049 module_id,
5050 routes: routes
5051 .into_iter()
5052 .map(|route| SupervisorRoute {
5053 consumer: match route.principal {
5054 Principal::Reserved { module_id } => {
5055 SupervisorRouteConsumer::Reserved { module_id }
5056 }
5057 Principal::Direct | Principal::Unverified => {
5058 SupervisorRouteConsumer::Direct {
5059 connection_id: route.goodbye_target.connection_id.get(),
5060 }
5061 }
5062 },
5063 age_ms: Instant::now()
5064 .saturating_duration_since(route.bound_at)
5065 .as_millis()
5066 .try_into()
5067 .unwrap_or(u64::MAX),
5068 draining: route.draining,
5069 drain_reason: route.drain_reason,
5070 })
5071 .collect(),
5072 })
5073 .collect();
5074 let response = ClientControlResponse::SupervisorRoutes { modules };
5075 Ok(vec![control_response_body_frame(
5076 &frame,
5077 &response,
5078 "ClientControlResponse::SupervisorRoutes",
5079 )?])
5080 }
5081
5082 async fn handle_supervisor_provenance(
5083 &self,
5084 frame: Frame,
5085 module_id: Option<String>,
5086 ) -> Result<Vec<Frame>, RouterError> {
5087 let mut selected = if let Some(module_id) = module_id {
5088 let Some(module) = self.supervisor.get(&module_id) else {
5089 return Ok(vec![control_error_frame(
5090 &frame,
5091 "unknown_module",
5092 format!("module_id '{module_id}' is not supervised"),
5093 )?]);
5094 };
5095 vec![module]
5096 } else {
5097 self.supervisor.list()
5098 };
5099
5100 let mut modules = Vec::with_capacity(selected.len());
5101 for module in selected.drain(..) {
5102 let (status, observed_image) = module
5103 .status_and_running_image_agreement()
5104 .await
5105 .map_err(|err| {
5106 RouterError::backend(
5107 0,
5108 frame.header.corr,
5109 format!("failed to read supervisor status: {err}"),
5110 )
5111 })?;
5112 let module_declared = self
5113 .registry
5114 .get_module(&status.module_id)
5115 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5116 .and_then(|registration| registration.manifest.provenance)
5117 .map(|build| ModuleDeclaredProvenance::Reported { build })
5118 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5119 #[cfg(test)]
5120 let running_image = match &self.provenance_probe_override {
5121 Some(result) => result.clone(),
5122 None => observed_image,
5123 };
5124 #[cfg(not(test))]
5125 let running_image = observed_image;
5126 modules.push(SupervisorModuleProvenance {
5127 module_id: status.module_id,
5128 module_declared,
5129 daemon_observed: SupervisorObservedProcess {
5130 launch_nonce_source: status.launch_nonce_source,
5131 pid: status.pid,
5132 spawned_at_ms: status.spawned_at_ms,
5133 spawned_from: status.spawned_from,
5134 running_image,
5135 },
5136 });
5137 }
5138 let daemon = SupervisorDaemonProvenance {
5139 daemon_build: self.daemon_provenance.build.clone(),
5140 daemon_observed: DaemonObservedProcess {
5141 pid: self.daemon_provenance.pid,
5142 started_at_ms: self
5143 .daemon_provenance
5144 .start_clock
5145 .map(|clock| clock.started_at_ms())
5146 .or(self.daemon_provenance.started_at_ms),
5147 running_image: self
5148 .daemon_provenance
5149 .probe
5150 .observe(
5151 self.daemon_provenance.pid,
5152 self.daemon_provenance.executable_path.as_deref(),
5153 self.daemon_provenance.executable_identity.clone(),
5154 self.daemon_provenance.process_start_time,
5155 )
5156 .await,
5157 },
5158 };
5159 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5160 Ok(vec![control_response_body_frame(
5161 &frame,
5162 &response,
5163 "ClientControlResponse::SupervisorProvenance",
5164 )?])
5165 }
5166
5167 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5168 self.refresh_capability_requirements();
5169 let generation = self
5170 .registry
5171 .generation()
5172 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5173 let modules = self
5174 .supervisor
5175 .list()
5176 .into_iter()
5177 .map(|module| {
5178 let status = module.status_for_control("health").map_err(|err| {
5179 RouterError::backend(
5180 0,
5181 frame.header.corr,
5182 format!("failed to read supervisor health: {err}"),
5183 )
5184 })?;
5185 let module_id = status.module_id;
5186 let capability_detail = self
5187 .capability_evaluator
5188 .required_problem_detail(&module_id);
5189 Ok(SupervisorHealthEntry {
5190 module_id,
5191 status: status.health.status,
5192 detail: append_capability_problem_detail(
5193 status.health.detail,
5194 capability_detail,
5195 ),
5196 metrics: status.health.metrics,
5197 consecutive_failures: status.health.consecutive_failures,
5198 late_answer_count: status.health.late_answer_count,
5199 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5200 last_action: status.health.last_action,
5201 last_action_ms: status.health.last_action_ms,
5202 last_probe_ms: status.health.last_probe_ms,
5203 })
5204 })
5205 .collect::<Result<Vec<_>, RouterError>>()?;
5206 let response = ClientControlResponse::SupervisorHealth {
5207 generation,
5208 modules,
5209 };
5210 Ok(vec![control_response_body_frame(
5211 &frame,
5212 &response,
5213 "ClientControlResponse::SupervisorHealth",
5214 )?])
5215 }
5216
5217 async fn handle_supervisor_restart(
5218 &self,
5219 frame: Frame,
5220 module_id: String,
5221 drain_timeout_ms: Option<u64>,
5222 requester: Principal,
5223 ) -> Result<Vec<Frame>, RouterError> {
5224 let operation_lock = self.supervisor.operation_lock();
5225 let _operation_guard = operation_lock.lock().await;
5226 let Some(module) = self.supervisor.get(&module_id) else {
5227 return Ok(vec![control_error_frame(
5228 &frame,
5229 "unknown_module",
5230 format!("module_id '{module_id}' is not supervised"),
5231 )?]);
5232 };
5233
5234 self.route_outages.mark_operator_action(&module_id);
5235 if let Err(err) = module
5236 .restart_requested(drain_timeout_ms, Some(requester))
5237 .await
5238 {
5239 self.route_outages
5240 .operator_action_ended_unrefused(&module_id);
5241 let (code, message) = match err {
5242 crate::supervise::SuperviseError::Disabled { .. } => {
5243 ("module_disabled", err.to_string())
5244 }
5245 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5246 ("swap_in_progress", err.to_string())
5247 }
5248 _ => (
5249 "target_unavailable",
5250 format!("failed to restart module_id '{module_id}': {err}"),
5251 ),
5252 };
5253 return Ok(vec![control_error_frame(&frame, code, message)?]);
5254 }
5255
5256 let response = ClientControlResponse::SupervisorAck {
5257 module_id,
5258 applied: true,
5259 };
5260 Ok(vec![control_response_body_frame(
5261 &frame,
5262 &response,
5263 "ClientControlResponse::SupervisorAck",
5264 )?])
5265 }
5266
5267 async fn handle_supervisor_swap(
5271 &self,
5272 frame: Frame,
5273 module_id: String,
5274 ready_timeout_ms: Option<u64>,
5275 requester: Principal,
5276 ) -> Result<Vec<Frame>, RouterError> {
5277 let module = {
5284 let operation_lock = self.supervisor.operation_lock();
5285 let _operation_guard = operation_lock.lock().await;
5286 self.supervisor.get(&module_id)
5287 };
5288 let Some(module) = module else {
5289 return Ok(vec![control_error_frame(
5290 &frame,
5291 "unknown_module",
5292 format!("module_id '{module_id}' is not supervised"),
5293 )?]);
5294 };
5295
5296 self.route_outages.mark_operator_action(&module_id);
5297 if let Err(err) = module
5298 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5299 .await
5300 {
5301 self.route_outages
5302 .operator_action_ended_unrefused(&module_id);
5303 use crate::supervise::SuperviseError;
5304 let message = err.to_string();
5305 let error = match err {
5306 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5307 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5308 code: "swap_refused".to_string(),
5309 message,
5310 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5311 },
5312 SuperviseError::SwapFailed {
5313 arm,
5314 candidate_exit,
5315 ..
5316 } => ErrorBody {
5317 code: "swap_failed".to_string(),
5318 message,
5319 detail: Some(serde_json::json!({
5320 "arm": arm.as_str(),
5321 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5322 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5323 })),
5324 },
5325 _ => ErrorBody::new(
5326 "target_unavailable",
5327 format!("failed to swap module_id '{module_id}': {message}"),
5328 ),
5329 };
5330 return Ok(vec![control_error_body_frame(&frame, error)?]);
5331 }
5332 self.route_outages
5336 .operator_action_ended_unrefused(&module_id);
5337
5338 let response = ClientControlResponse::SupervisorAck {
5339 module_id,
5340 applied: true,
5341 };
5342 Ok(vec![control_response_body_frame(
5343 &frame,
5344 &response,
5345 "ClientControlResponse::SupervisorAck",
5346 )?])
5347 }
5348
5349 async fn handle_supervisor_reload(
5350 &self,
5351 frame: Frame,
5352 module_id: String,
5353 requester: Principal,
5354 ) -> Result<Vec<Frame>, RouterError> {
5355 let operation_lock = self.supervisor.operation_lock();
5356 let _operation_guard = operation_lock.lock().await;
5357 let Some(module) = self.supervisor.get(&module_id) else {
5358 return Ok(vec![control_error_frame(
5359 &frame,
5360 "unknown_module",
5361 format!("module_id '{module_id}' is not supervised"),
5362 )?]);
5363 };
5364
5365 self.route_outages.mark_operator_action(&module_id);
5366 if let Err(err) = module.reload_requested(Some(requester)).await {
5367 self.route_outages
5368 .operator_action_ended_unrefused(&module_id);
5369 let (code, message) = match err {
5370 crate::supervise::SuperviseError::Disabled { .. } => {
5371 ("module_disabled", err.to_string())
5372 }
5373 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5374 ("swap_in_progress", err.to_string())
5375 }
5376 _ => (
5377 "reload_failed",
5378 format!("failed to reload module_id '{module_id}': {err}"),
5379 ),
5380 };
5381 return Ok(vec![control_error_frame(&frame, code, message)?]);
5382 }
5383
5384 let response = ClientControlResponse::SupervisorAck {
5385 module_id,
5386 applied: true,
5387 };
5388 Ok(vec![control_response_body_frame(
5389 &frame,
5390 &response,
5391 "ClientControlResponse::SupervisorAck",
5392 )?])
5393 }
5394
5395 async fn handle_supervisor_rescan(
5396 &self,
5397 frame: Frame,
5398 preview: bool,
5399 requester: Principal,
5400 ) -> Result<Vec<Frame>, RouterError> {
5401 let Some(context) = self.rescan.clone() else {
5402 return Ok(vec![control_error_frame(
5403 &frame,
5404 "rescan_unavailable",
5405 "the daemon was not started with a reloadable config path".to_string(),
5406 )?]);
5407 };
5408
5409 let operation_lock = self.supervisor.operation_lock();
5410 let _operation_guard = operation_lock.lock().await;
5411 let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5412 Ok(config) => config,
5413 Err(err) => {
5414 return Ok(vec![control_error_frame(
5415 &frame,
5416 "invalid_daemon_config",
5417 format!("supervisor rescan rejected daemon config: {err}"),
5418 )?])
5419 }
5420 };
5421 let Some((config, activity_config)) = loaded else {
5429 return Ok(vec![control_error_frame(
5430 &frame,
5431 "invalid_daemon_config",
5432 format!(
5433 "daemon config not found at {}; refusing to rescan (an absent config would \
5434 retire every supervised module)",
5435 context.config_path.display()
5436 ),
5437 )?]);
5438 };
5439 let (
5440 configured_port,
5441 storage_config,
5442 admission_facts_carrier_module_id,
5443 admission_facts_targets,
5444 scope_authority_owners,
5445 privileged_operator_prompt,
5446 modules,
5447 reserved_capabilities,
5448 ) = (
5449 config.port,
5450 config.storage,
5451 config.admission_facts_carrier_module_id,
5452 config.admission_facts_targets,
5453 config.scope_authority_owners,
5454 config.privileged_operator_prompt,
5455 config.modules,
5456 config.reserved_capabilities,
5457 );
5458
5459 let mut restart_required = Vec::new();
5468 for section in RestartRequiredSection::ALL {
5469 let changed = match section {
5470 RestartRequiredSection::Port => configured_port != context.configured_port,
5471 RestartRequiredSection::Storage => storage_config != context.storage_config,
5472 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5473 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5474 }
5475 RestartRequiredSection::AdmissionFactsTargets => {
5476 admission_facts_targets != context.admission_facts_targets
5477 }
5478 RestartRequiredSection::ScopeAuthorityOwners => {
5479 scope_authority_owners != context.scope_authority_owners
5480 }
5481 RestartRequiredSection::PrivilegedOperatorPrompt => {
5482 privileged_operator_prompt != context.privileged_operator_prompt
5483 }
5484 };
5485 if changed {
5486 restart_required.push(section.label().to_string());
5487 }
5488 }
5489 if !restart_required.is_empty() {
5490 warn!(
5491 config_path = %context.config_path.display(),
5492 sections = %restart_required.join(", "),
5493 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5494 );
5495 }
5496
5497 for configured in &modules {
5498 if let Err(err) = validate_spec(&configured.module_spec()) {
5499 return Ok(vec![control_error_frame(
5500 &frame,
5501 "invalid_daemon_config",
5502 format!("supervisor rescan rejected daemon config: {err}"),
5503 )?]);
5504 }
5505 }
5506
5507 let configured_capabilities = modules
5508 .iter()
5509 .map(|module| (module.module_id.clone(), module.enabled))
5510 .collect::<Vec<_>>();
5511 let preview_capability_warnings = if preview {
5512 let (_, registrations) = self.runtime_capability_snapshot()?;
5513 let current_modules = self
5514 .supervisor
5515 .list()
5516 .into_iter()
5517 .map(|module| module.module_id().to_string())
5518 .collect::<BTreeSet<_>>();
5519 let resulting_modules = configured_capabilities.clone();
5520 let removed = current_modules
5521 .into_iter()
5522 .filter(|module_id| {
5523 !resulting_modules
5524 .iter()
5525 .any(|(configured_id, _)| configured_id == module_id)
5526 })
5527 .collect::<Vec<_>>();
5528 self.capability_evaluator.preview_removal_warnings(
5529 resulting_modules,
5530 &removed,
5531 ®istrations,
5532 )
5533 } else {
5534 Vec::new()
5535 };
5536 let result = match self
5537 .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5538 .await
5539 {
5540 Ok(result) => result,
5541 Err(message) => {
5542 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5543 }
5544 };
5545 if !preview {
5546 if let Some(ledger) = &self.activity {
5547 ledger.module_rescanned(&result.added, &result.removed, requester);
5548 }
5549 if let Err(code) = self.configure_activity(activity_config).await {
5550 warn!(
5551 code,
5552 "activity retention could not be applied during rescan"
5553 );
5554 }
5555 self.capability_evaluator
5556 .configure(configured_capabilities, reserved_capabilities);
5557 self.capability_evaluator.wake_deadline_loop();
5558 self.refresh_capability_requirements();
5559 }
5560 let mut result = result;
5561 result.restart_required = restart_required;
5562 result.capability_warnings = preview_capability_warnings;
5563 let response = ClientControlResponse::SupervisorRescan { result };
5564 Ok(vec![control_response_body_frame(
5565 &frame,
5566 &response,
5567 "ClientControlResponse::SupervisorRescan",
5568 )?])
5569 }
5570
5571 async fn handle_supervisor_release_reserved(
5572 &self,
5573 frame: Frame,
5574 module_id: String,
5575 ) -> Result<Vec<Frame>, RouterError> {
5576 let Some(context) = self.rescan.clone() else {
5577 return Ok(vec![control_error_frame(
5578 &frame,
5579 "release_unavailable",
5580 "reserved-id release requires a daemon started with a reloadable config path",
5581 )?]);
5582 };
5583 let operation_lock = self.supervisor.operation_lock();
5584 let _operation_guard = operation_lock.lock().await;
5585 let loaded = match crate::daemon_config::load(&context.config_path) {
5586 Ok(Some(config)) => config,
5587 Ok(None) => {
5588 return Ok(vec![control_error_frame(
5589 &frame,
5590 "invalid_daemon_config",
5591 format!(
5592 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5593 context.config_path.display()
5594 ),
5595 )?])
5596 }
5597 Err(err) => {
5598 return Ok(vec![control_error_frame(
5599 &frame,
5600 "invalid_daemon_config",
5601 format!("unable to verify reserved-id release against daemon config: {err}"),
5602 )?])
5603 }
5604 };
5605 if loaded
5606 .modules
5607 .iter()
5608 .any(|configured| configured.module_id == module_id)
5609 {
5610 return Ok(vec![control_error_frame(
5611 &frame,
5612 "reserved_module_configured",
5613 format!(
5614 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5615 ),
5616 )?]);
5617 }
5618 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5619 return Ok(vec![control_error_frame(
5620 &frame,
5621 "reserved_gate_not_retained",
5622 format!(
5623 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5624 ),
5625 )?]);
5626 }
5627
5628 let response = ClientControlResponse::SupervisorAck {
5629 module_id,
5630 applied: true,
5631 };
5632 Ok(vec![control_response_body_frame(
5633 &frame,
5634 &response,
5635 "ClientControlResponse::SupervisorAck",
5636 )?])
5637 }
5638
5639 async fn reconcile_supervised_modules(
5648 &self,
5649 supervisor: &Supervisor,
5650 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5651 preview: bool,
5652 requester: Principal,
5653 ) -> Result<SupervisorRescanResult, String> {
5654 let mut current = BTreeMap::new();
5655 for module in self.supervisor.list() {
5656 let (spec, health) = module.configuration().map_err(|err| {
5657 format!(
5658 "failed to read configuration for module_id '{}': {err}",
5659 module.module_id()
5660 )
5661 })?;
5662 let enabled = module
5663 .status()
5664 .map_err(|err| {
5665 format!(
5666 "failed to read status for module_id '{}': {err}",
5667 module.module_id()
5668 )
5669 })?
5670 .enabled;
5671 current.insert(
5672 module.module_id().to_string(),
5673 (module, spec, health, enabled),
5674 );
5675 }
5676 let configured = configured_modules
5677 .into_iter()
5678 .map(|module| (module.module_id.clone(), module))
5679 .collect::<BTreeMap<_, _>>();
5680
5681 let added = configured
5682 .keys()
5683 .filter(|module_id| !current.contains_key(*module_id))
5684 .cloned()
5685 .collect::<Vec<_>>();
5686 let removed = current
5687 .keys()
5688 .filter(|module_id| !configured.contains_key(*module_id))
5689 .cloned()
5690 .collect::<Vec<_>>();
5691 let mut changed_pending_reload = Vec::new();
5692 let mut configuration_changes = BTreeSet::new();
5693 let mut enabled_changes = BTreeSet::new();
5694 let mut unchanged = 0_u32;
5695
5696 for (module_id, configured_module) in &configured {
5697 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5698 else {
5699 continue;
5700 };
5701 let launch_changed = *current_spec != configured_module.module_spec();
5705 let configuration_changed =
5706 launch_changed || *current_health != configured_module.health;
5707 let enabled_changed = *current_enabled != configured_module.enabled;
5708 if configuration_changed {
5709 configuration_changes.insert(module_id.clone());
5710 }
5711 if launch_changed {
5712 changed_pending_reload.push(module_id.clone());
5713 }
5714 if enabled_changed {
5715 enabled_changes.insert(module_id.clone());
5716 }
5717 if !configuration_changed && !enabled_changed {
5718 unchanged = unchanged.saturating_add(1);
5719 }
5720 }
5721
5722 if preview {
5728 return Ok(SupervisorRescanResult {
5729 added,
5730 removed,
5731 changed_pending_reload,
5732 enabled_changes: enabled_changes.iter().cloned().collect(),
5733 unchanged,
5734 preview: true,
5735 restart_required: Vec::new(),
5739 capability_warnings: Vec::new(),
5740 });
5741 }
5742
5743 for module_id in &removed {
5744 let module = ¤t
5745 .get(module_id)
5746 .expect("removed module came from current supervisor state")
5747 .0;
5748 module
5749 .retire_requested(Some(requester.clone()))
5750 .await
5751 .map_err(|err| {
5752 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5753 })?;
5754 self.supervisor.record_rescan_removal(module_id);
5788 self.supervisor.retire(module_id);
5789 self.route_outages.forget(module_id);
5790 }
5791
5792 for module_id in configured.keys() {
5793 let Some((module, _, _, _)) = current.get(module_id) else {
5794 continue;
5795 };
5796 let configured_module = configured
5797 .get(module_id)
5798 .expect("configured module id came from configured map");
5799 if configuration_changes.contains(module_id) {
5800 module
5801 .update_configuration(
5802 configured_module.module_spec(),
5803 configured_module.health.clone(),
5804 configured_module.drain_timeout_ms,
5805 )
5806 .await
5807 .map_err(|err| {
5808 format!(
5809 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5810 )
5811 })?;
5812 }
5813 if enabled_changes.contains(module_id) {
5814 self.route_outages.mark_operator_action(module_id);
5817 module
5818 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
5819 .await
5820 .map_err(|err| {
5821 self.route_outages.operator_action_ended_unrefused(module_id);
5822 format!(
5823 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5824 configured_module.enabled
5825 )
5826 })?;
5827 }
5828 }
5829
5830 for module_id in &added {
5831 let configured_module = configured
5832 .get(module_id)
5833 .expect("added module id came from configured map");
5834 supervisor
5835 .supervise_configured_requested(
5836 configured_module.module_spec(),
5837 configured_module.enabled,
5838 configured_module.health.clone(),
5839 configured_module.drain_timeout_ms,
5840 configured_module.restart,
5841 Some(requester.clone()),
5842 )
5843 .map_err(|err| {
5844 format!("failed to add module_id '{module_id}' during rescan: {err}")
5845 })?;
5846 }
5847
5848 Ok(SupervisorRescanResult {
5849 added,
5850 removed,
5851 changed_pending_reload,
5852 enabled_changes: enabled_changes.iter().cloned().collect(),
5853 unchanged,
5854 preview: false,
5855 restart_required: Vec::new(),
5858 capability_warnings: Vec::new(),
5859 })
5860 }
5861
5862 async fn handle_supervisor_set_enabled(
5863 &self,
5864 frame: Frame,
5865 module_id: String,
5866 enabled: bool,
5867 requester: Principal,
5868 ) -> Result<Vec<Frame>, RouterError> {
5869 let operation_lock = self.supervisor.operation_lock();
5870 let _operation_guard = operation_lock.lock().await;
5871 let Some(module) = self.supervisor.get(&module_id) else {
5872 return Ok(vec![control_error_frame(
5873 &frame,
5874 "unknown_module",
5875 format!("module_id '{module_id}' is not supervised"),
5876 )?]);
5877 };
5878
5879 self.route_outages.mark_operator_action(&module_id);
5882 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
5883 Ok(applied) => applied,
5884 Err(err) => {
5885 self.route_outages
5886 .operator_action_ended_unrefused(&module_id);
5887 return Ok(vec![control_error_frame(
5888 &frame,
5889 "target_unavailable",
5890 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5891 )?]);
5892 }
5893 };
5894 if !applied {
5895 self.route_outages
5898 .operator_action_ended_unrefused(&module_id);
5899 }
5900
5901 self.capability_evaluator.wake_deadline_loop();
5902 self.refresh_capability_requirements();
5903 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5904 Ok(vec![control_response_body_frame(
5905 &frame,
5906 &response,
5907 "ClientControlResponse::SupervisorAck",
5908 )?])
5909 }
5910
5911 async fn handle_supervisor_health_probe(
5912 &self,
5913 frame: Frame,
5914 module_id: String,
5915 ) -> Result<Vec<Frame>, RouterError> {
5916 self.refresh_capability_requirements();
5917 let Some(registration) = self
5918 .registry
5919 .get_module(&module_id)
5920 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5921 else {
5922 return Ok(vec![control_error_frame(
5923 &frame,
5924 "unknown_module",
5925 format!("module_id '{module_id}' is not registered"),
5926 )?]);
5927 };
5928
5929 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5942 {
5943 return Ok(vec![control_error_frame(
5944 &frame,
5945 "health_not_advertised",
5946 format!("module_id '{module_id}' did not advertise health.check"),
5947 )?]);
5948 }
5949
5950 let deadline = Instant::now() + self.health_probe_timeout;
5951 let pending = match self.forwarding.begin_module_control_rpc_for(
5952 &module_id,
5953 MODULE_CONTROL_OP_HEALTH_CHECK,
5954 deadline,
5955 ) {
5956 Ok(pending) => pending,
5957 Err(err) => {
5958 return Ok(vec![control_error_frame(
5959 &frame,
5960 forwarding_error_code(&err),
5961 err.to_string(),
5962 )?])
5963 }
5964 };
5965
5966 let PendingModuleControlRpc {
5967 endpoint,
5968 module_sink,
5969 negotiated_ver,
5970 corr: probe_corr,
5971 receiver,
5972 } = pending;
5973 let mut guard =
5974 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5975 let probe_body =
5976 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5977 RouterError::backend(
5978 0,
5979 frame.header.corr,
5980 format!("failed to encode health.check request: {err}"),
5981 )
5982 })?;
5983 let probe_frame = Frame::build_with_version(
5984 negotiated_ver,
5985 FrameType::Request,
5986 control_flags(),
5987 0,
5988 0,
5989 probe_corr,
5990 probe_body,
5991 )
5992 .map_err(RouterError::FrameBuild)?;
5993
5994 if let Err(err) = module_sink.send(probe_frame).await {
5995 return Ok(vec![control_error_frame(
5996 &frame,
5997 "target_unavailable",
5998 err.to_string(),
5999 )?]);
6000 }
6001
6002 match timeout_at(deadline, receiver).await {
6003 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6004 guard.disarm();
6005 let Some(report) = response.health_report() else {
6006 return Ok(vec![control_error_frame(
6007 &frame,
6008 "invalid_control_body",
6009 "health.check RPC returned a non-health response",
6010 )?]);
6011 };
6012 let HealthReport {
6017 status,
6018 detail,
6019 metrics,
6020 } = report;
6021 let capability_detail = self
6022 .capability_evaluator
6023 .required_problem_detail(&module_id);
6024 let response = ClientControlResponse::SupervisorHealthProbe {
6025 module_id,
6026 status,
6027 detail: append_capability_problem_detail(detail, capability_detail),
6028 metrics,
6029 };
6030 Ok(vec![control_response_body_frame(
6031 &frame,
6032 &response,
6033 "ClientControlResponse::SupervisorHealthProbe",
6034 )?])
6035 }
6036 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6037 guard.disarm();
6038 Ok(vec![control_error_body_frame(&frame, body)?])
6039 }
6040 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6041 guard.disarm();
6042 Ok(vec![control_error_frame(
6043 &frame,
6044 "target_unavailable",
6045 message,
6046 )?])
6047 }
6048 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6049 guard.disarm();
6050 Ok(vec![control_error_frame(
6051 &frame,
6052 "invalid_control_body",
6053 message,
6054 )?])
6055 }
6056 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6057 guard.disarm();
6058 Ok(vec![control_error_frame(
6059 &frame,
6060 "invalid_control_body",
6061 format!("expected module-control op '{expected}', got '{actual}'"),
6062 )?])
6063 }
6064 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6065 guard.disarm();
6066 Ok(vec![control_error_frame(
6067 &frame,
6068 "module_timeout",
6069 format!(
6070 "module_id '{module_id}' answered health.check after {:?}",
6071 self.health_probe_timeout
6072 ),
6073 )?])
6074 }
6075 Ok(Err(_)) => Ok(vec![control_error_frame(
6076 &frame,
6077 "target_unavailable",
6078 "health.check waiter was canceled before the module responded",
6079 )?]),
6080 Err(_) => Ok(vec![control_error_frame(
6081 &frame,
6082 "module_timeout",
6083 format!(
6084 "module_id '{module_id}' did not answer health.check within {:?}",
6085 self.health_probe_timeout
6086 ),
6087 )?]),
6088 }
6089 }
6090
6091 fn supervisor_status(
6092 &self,
6093 module_id: &str,
6094 corr: u64,
6095 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6096 self.supervisor
6097 .get(module_id)
6098 .map(|module| {
6099 let warming = module.is_warming_for_control("status").map_err(|err| {
6100 RouterError::backend(
6101 0,
6102 corr,
6103 format!(
6104 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6105 ),
6106 )
6107 })?;
6108 module.status_for_control("status").map_err(|err| {
6109 RouterError::backend(
6110 0,
6111 corr,
6112 format!(
6113 "failed to read supervisor status for module_id '{module_id}': {err}"
6114 ),
6115 )
6116 }).map(|status| (status, warming))
6117 })
6118 .transpose()
6119 }
6120
6121 fn guard_module_control_op(
6122 &self,
6123 frame: &Frame,
6124 module_id: &str,
6125 op: &str,
6126 ) -> Result<Option<Frame>, RouterError> {
6127 if self.module_grants_op(module_id, op, frame.header.corr)? {
6128 return Ok(None);
6129 }
6130
6131 Ok(Some(control_error_frame(
6132 frame,
6133 "op_not_allowed",
6134 format!("module_id '{module_id}' did not grant control op '{op}'"),
6135 )?))
6136 }
6137
6138 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6139 let Some(registration) = self
6140 .registry
6141 .get_module(module_id)
6142 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6143 else {
6144 return Ok(false);
6145 };
6146 Ok(module_registration_grants_op(®istration.control_ops, op))
6147 }
6148
6149 fn handle_status_update(
6150 &self,
6151 endpoint: ModuleEndpointId,
6152 frame: Frame,
6153 ) -> Result<Vec<Frame>, RouterError> {
6154 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6155 Ok(update) => update,
6156 Err(err) => {
6157 if is_known_module_push_op(&frame.body) {
6162 return Ok(vec![control_error_frame(
6163 &frame,
6164 "invalid_control_body",
6165 format!("malformed module control push body: {err}"),
6166 )?]);
6167 }
6168 return Ok(Vec::new());
6169 }
6170 };
6171
6172 match update {
6173 ModuleControlPush::RouteStatus {
6174 route_channel,
6175 route_epoch,
6176 status,
6177 } => {
6178 self.forwarding
6179 .cache_status(endpoint, route_channel, route_epoch, status)
6180 .map_err(RouterError::Forwarding)?;
6181 }
6182 }
6183 Ok(Vec::new())
6184 }
6185
6186 fn handle_route_poll(
6187 &self,
6188 ctx: &RouteCtx,
6189 frame: Frame,
6190 route_channel: u16,
6191 route_epoch: u32,
6192 kind: PollKind,
6193 ) -> Result<Vec<Frame>, RouterError> {
6194 let snapshot = self
6195 .forwarding
6196 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6197 .map_err(RouterError::Forwarding)?;
6198 let response = match (kind, snapshot) {
6199 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6200 ClientControlResponse::RoutePoll {
6201 route_channel,
6202 route_epoch,
6203 status,
6204 live: None,
6205 }
6206 }
6207 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6208 route_channel,
6209 route_epoch,
6210 status: None,
6211 live: None,
6212 },
6213 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6214 let live = self
6240 .process_liveness
6241 .as_ref()
6242 .and_then(|source| source.process_live(&module_id))
6243 .unwrap_or(true);
6244 ClientControlResponse::RoutePoll {
6245 route_channel,
6246 route_epoch,
6247 status: None,
6248 live: Some(live),
6249 }
6250 }
6251 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6252 route_channel,
6253 route_epoch,
6254 status: None,
6255 live: Some(false),
6256 },
6257 };
6258
6259 Ok(vec![control_response_body_frame(
6260 &frame,
6261 &response,
6262 "ClientControlResponse::RoutePoll",
6263 )?])
6264 }
6265
6266 pub(crate) fn observe_module_control_completion(
6267 &self,
6268 completion: ModuleControlRpcCompletion,
6269 ) -> bool {
6270 match completion {
6271 ModuleControlRpcCompletion::Unknown => false,
6272 ModuleControlRpcCompletion::Settled => true,
6273 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6274 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6275 info!(
6276 module_id = %module_id,
6277 latency_ms,
6278 "late health.check answer proves the module is alive"
6279 );
6280 match self
6281 .supervisor
6282 .record_late_health_answer(&module_id, latency_ms)
6283 {
6284 Ok(true) => {}
6285 Ok(false) => debug!(
6286 module_id = %module_id,
6287 latency_ms,
6288 "late health.check answer has no active supervisor snapshot"
6289 ),
6290 Err(err) => warn!(
6291 module_id = %module_id,
6292 latency_ms,
6293 error = %err,
6294 "failed to record late health.check answer"
6295 ),
6296 }
6297 true
6298 }
6299 }
6300 }
6301
6302 fn refuse_to_end_module_connection_for_a_client(
6323 &self,
6324 module_connection_id: ConnectionId,
6325 corr: u64,
6326 err: ForwardingError,
6327 ) -> Result<(), RouterError> {
6328 if let ForwardingError::ConnectionClosing { connection_id } = err {
6329 if connection_id != module_connection_id {
6330 warn!(
6331 module_connection_id = module_connection_id.get(),
6332 client_connection_id = connection_id.get(),
6333 corr,
6334 "dropping a route.bind response for a closing client; the module connection keeps serving"
6335 );
6336 return Ok(());
6337 }
6338 }
6339 Err(RouterError::Forwarding(err))
6340 }
6341
6342 fn handle_module_relay_response(
6343 &self,
6344 connection_id: ConnectionId,
6345 frame: Frame,
6346 ) -> Result<Vec<Frame>, RouterError> {
6347 let mut secondary_error = None;
6348 let outcome = match frame.header.ty {
6349 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6350 Ok(probe) if probe.op == "route.bind" => {
6351 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6352 Ok(ModuleControlResponse::RouteBindAck {}) => {
6353 RouteBindRelayOutcome::Accepted
6354 }
6355 Ok(other) => {
6356 let message =
6357 format!("route.bind response carried unexpected body: {other:?}");
6358 secondary_error = Some(control_error_frame(
6359 &frame,
6360 "invalid_control_body",
6361 message.clone(),
6362 )?);
6363 RouteBindRelayOutcome::ModuleGone(message)
6364 }
6365 Err(err) => {
6366 let message = format!("malformed route.bind response body: {err}");
6367 secondary_error = Some(control_error_frame(
6368 &frame,
6369 "invalid_control_body",
6370 message.clone(),
6371 )?);
6372 RouteBindRelayOutcome::ModuleGone(message)
6373 }
6374 }
6375 }
6376 Ok(probe) => {
6377 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6378 {
6379 Ok(response) => ModuleControlRpcOutcome::Response(response),
6380 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6381 "malformed {} response body: {err}",
6382 probe.op
6383 )),
6384 };
6385 let completion = self
6386 .forwarding
6387 .complete_module_control_rpc(
6388 connection_id,
6389 frame.header.corr,
6390 Some(&probe.op),
6391 outcome,
6392 )
6393 .map_err(RouterError::Forwarding)?;
6394 if !self.observe_module_control_completion(completion) {
6395 debug!(
6396 connection_id = connection_id.get(),
6397 corr = frame.header.corr,
6398 op = %probe.op,
6399 "dropping late or unknown module-control RPC response"
6400 );
6401 }
6402 return Ok(Vec::new());
6403 }
6404 Err(err) => {
6405 if let Some(expected_op) = self
6406 .forwarding
6407 .pending_module_control_op(connection_id, frame.header.corr)
6408 .map_err(RouterError::Forwarding)?
6409 {
6410 let completion = self
6411 .forwarding
6412 .complete_module_control_rpc(
6413 connection_id,
6414 frame.header.corr,
6415 None,
6416 ModuleControlRpcOutcome::MalformedResponse(format!(
6417 "malformed {expected_op} response body: {err}"
6418 )),
6419 )
6420 .map_err(RouterError::Forwarding)?;
6421 if !self.observe_module_control_completion(completion) {
6422 debug!(
6423 connection_id = connection_id.get(),
6424 corr = frame.header.corr,
6425 "dropping late malformed module-control RPC response"
6426 );
6427 }
6428 return Ok(Vec::new());
6429 }
6430 let message = format!("malformed route.bind response body: {err}");
6431 secondary_error = Some(control_error_frame(
6432 &frame,
6433 "invalid_control_body",
6434 message.clone(),
6435 )?);
6436 RouteBindRelayOutcome::ModuleGone(message)
6437 }
6438 },
6439 FrameType::Error => {
6440 if self
6441 .forwarding
6442 .pending_module_control_op(connection_id, frame.header.corr)
6443 .map_err(RouterError::Forwarding)?
6444 .is_some()
6445 {
6446 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6447 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6448 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6449 "malformed module-control ERROR body: {err}"
6450 )),
6451 };
6452 let completion = self
6453 .forwarding
6454 .complete_module_control_rpc(
6455 connection_id,
6456 frame.header.corr,
6457 None,
6458 outcome,
6459 )
6460 .map_err(RouterError::Forwarding)?;
6461 if !self.observe_module_control_completion(completion) {
6462 debug!(
6463 connection_id = connection_id.get(),
6464 corr = frame.header.corr,
6465 "dropping late or unknown module-control RPC error"
6466 );
6467 }
6468 return Ok(Vec::new());
6469 }
6470 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6471 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6472 Err(err) => {
6473 let message = format!("malformed route.bind ERROR body: {err}");
6474 secondary_error = Some(control_error_frame(
6475 &frame,
6476 "invalid_control_body",
6477 message.clone(),
6478 )?);
6479 RouteBindRelayOutcome::ModuleGone(message)
6480 }
6481 }
6482 }
6483 ty => {
6484 return Ok(vec![control_error_frame(
6485 &frame,
6486 "unsupported_control_frame",
6487 format!("unsupported module channel-0 frame {ty:?}"),
6488 )?])
6489 }
6490 };
6491
6492 let settled =
6493 self.forwarding
6494 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6495 let completion = match settled {
6496 Ok(completion) => completion,
6497 Err(err) => {
6498 self.refuse_to_end_module_connection_for_a_client(
6499 connection_id,
6500 frame.header.corr,
6501 err,
6502 )?;
6503 return Ok(secondary_error.into_iter().collect());
6504 }
6505 };
6506 if let Some(target) = completion.abandoned.as_ref() {
6507 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6508 }
6509 if !completion.settled {
6510 debug!(
6511 connection_id = connection_id.get(),
6512 corr = frame.header.corr,
6513 frame_type = ?frame.header.ty,
6514 "dropping late or unknown route.bind relay response"
6515 );
6516 }
6517 Ok(secondary_error.into_iter().collect())
6518 }
6519
6520 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6521 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6522 self.cleanup_connection_with_end_reason(
6526 connection_id,
6527 RegistrationEndReason::ExplicitGoodbye,
6528 )
6529 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6530 Ok(Vec::new())
6531 }
6532}
6533
6534impl Default for ControlHandler {
6535 fn default() -> Self {
6536 Self::new(Arc::new(Registry::default()))
6537 }
6538}
6539
6540impl crate::supervise::SwapPromotionObserver for ControlHandler {
6541 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6542 self.apply_registration_capabilities(registration);
6543 }
6544}
6545
6546fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6547 CapabilityRequirementStatus {
6548 consumer: status.consumer,
6549 capability: status.capability,
6550 need: match status.need {
6551 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6552 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6553 },
6554 verdict: status.verdict.as_str().to_string(),
6555 episode_seq: status.episode_seq,
6556 config_satisfiable: status.config_satisfiable,
6557 runtime_available: status.runtime_available,
6558 detail: status.detail,
6559 }
6560}
6561
6562fn append_capability_problem_detail(
6563 detail: Option<String>,
6564 capability_detail: Option<String>,
6565) -> Option<String> {
6566 match (detail, capability_detail) {
6567 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6568 (Some(detail), None) => Some(detail),
6569 (None, Some(capability_detail)) => Some(capability_detail),
6570 (None, None) => None,
6571 }
6572}
6573
6574fn subc_ops() -> Vec<String> {
6575 SUBC_CONTROL_OPS
6576 .iter()
6577 .map(|op| (*op).to_string())
6578 .collect()
6579}
6580
6581fn module_subc_ops() -> Vec<String> {
6582 SUBC_CONTROL_OPS
6583 .iter()
6584 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6585 .map(|op| (*op).to_string())
6586 .collect()
6587}
6588
6589#[cfg(test)]
6590fn module_baseline_control_ops() -> Vec<String> {
6591 MODULE_BASELINE_CONTROL_OPS
6592 .iter()
6593 .map(|op| (*op).to_string())
6594 .collect()
6595}
6596
6597fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6598 let mut seen = HashSet::new();
6599 let mut effective = Vec::new();
6600 for op in MODULE_BASELINE_CONTROL_OPS {
6601 if seen.insert((*op).to_string()) {
6602 effective.push((*op).to_string());
6603 }
6604 }
6605 for op in declared.unwrap_or_default() {
6606 if seen.insert(op.clone()) {
6607 effective.push(op);
6608 }
6609 }
6610 effective
6611}
6612
6613fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6614 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6615}
6616
6617fn target_module_id(target: &RouteTarget) -> &str {
6618 match target {
6619 RouteTarget::ToolProvider { module_id }
6620 | RouteTarget::ManagementSurface { module_id }
6621 | RouteTarget::InternalService { module_id, .. } => module_id,
6622 }
6623}
6624
6625fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6626 roles.iter().any(|role| match (target, role) {
6627 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6628 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6629 (
6630 RouteTarget::InternalService { service_id, .. },
6631 ProviderRole::InternalService {
6632 service_id: provided,
6633 ..
6634 },
6635 ) => service_id == provided,
6636 _ => false,
6637 })
6638}
6639
6640fn is_routable_role(role: &ProviderRole) -> bool {
6641 matches!(
6642 role,
6643 ProviderRole::ToolProvider { .. }
6644 | ProviderRole::ManagementSurface { .. }
6645 | ProviderRole::InternalService { .. }
6646 )
6647}
6648
6649#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6650enum ControlRequestBodyError {
6651 UnknownOp,
6652 InvalidBody,
6653}
6654
6655#[derive(Debug, Deserialize)]
6656struct ControlOpProbe {
6657 op: String,
6658}
6659
6660const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6663
6664fn is_known_module_push_op(body: &[u8]) -> bool {
6665 serde_json::from_slice::<ControlOpProbe>(body)
6666 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6667 .unwrap_or(false)
6668}
6669
6670fn is_known_module_request_op(body: &[u8]) -> bool {
6671 serde_json::from_slice::<ControlOpProbe>(body)
6672 .map(|probe| is_module_to_subc_op(&probe.op))
6673 .unwrap_or(false)
6674}
6675
6676fn is_module_to_subc_op(op: &str) -> bool {
6677 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6678}
6679
6680fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6681 debug!(
6682 op = %op,
6683 connection_id = connection_id.get(),
6684 corr,
6685 "control dispatch"
6686 );
6687}
6688
6689fn log_slow_control_dispatch(
6690 dispatch_started_at: Option<StdInstant>,
6691 op: &'static str,
6692 connection_id: ConnectionId,
6693 corr: u64,
6694) {
6695 let Some(dispatch_started_at) = dispatch_started_at else {
6696 return;
6697 };
6698 let elapsed = dispatch_started_at.elapsed();
6699 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6700 warn!(
6701 op = %op,
6702 connection_id = connection_id.get(),
6703 corr,
6704 elapsed_ms = elapsed.as_millis() as u64,
6705 "slow control dispatch"
6706 );
6707 }
6708}
6709
6710#[cfg(test)]
6712type TestCallerLookup =
6713 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
6714
6715fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
6717 match request {
6718 ClientControlRequest::SupervisorRestart { module_id, .. } => {
6719 Some(PrivilegedOperation::Restart { module_id })
6720 }
6721 ClientControlRequest::SupervisorSwap { module_id, .. } => {
6722 Some(PrivilegedOperation::Swap { module_id })
6723 }
6724 ClientControlRequest::SupervisorReload { module_id } => {
6725 Some(PrivilegedOperation::Reload { module_id })
6726 }
6727 ClientControlRequest::SupervisorRescan { preview } => {
6728 if *preview {
6729 None
6730 } else {
6731 Some(PrivilegedOperation::Rescan)
6732 }
6733 }
6734 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
6735 Some(PrivilegedOperation::SetEnabled {
6736 module_id,
6737 enabled: *enabled,
6738 })
6739 }
6740 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
6741 Some(PrivilegedOperation::ReleaseReserved { module_id })
6742 }
6743 ClientControlRequest::ServerDescribe {}
6744 | ClientControlRequest::CatalogList { .. }
6745 | ClientControlRequest::RouteOpen { .. }
6746 | ClientControlRequest::RoutePoll { .. }
6747 | ClientControlRequest::SupervisorList {}
6748 | ClientControlRequest::SupervisorHealth {}
6749 | ClientControlRequest::SupervisorHealthProbe { .. }
6750 | ClientControlRequest::SupervisorRoutes { .. }
6751 | ClientControlRequest::SupervisorProvenance { .. }
6752 | ClientControlRequest::SupervisorStderrTail { .. }
6753 | ClientControlRequest::SupervisorTerminals { .. }
6754 | ClientControlRequest::SupervisorSpawnSnapshot {}
6755 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
6756 }
6757}
6758
6759fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6760 match request {
6761 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6762 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6763 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6764 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6765 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6766 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6767 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6768 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6769 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6770 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6771 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6772 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6773 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6774 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6775 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6776 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6777 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6778 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6779 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6780 }
6781}
6782
6783fn principal_label(principal: &Principal) -> String {
6784 match principal {
6785 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6786 Principal::Direct => "direct".to_string(),
6787 other => format!("{other:?}"),
6788 }
6789}
6790
6791fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6792 match request {
6793 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6794 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6795 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6796 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6797 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6798 }
6799}
6800
6801fn parse_client_control_request(
6802 body: &[u8],
6803) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6804 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6805 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6806 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6807 ControlRequestBodyError::InvalidBody
6808 }
6809 Ok(_) => ControlRequestBodyError::UnknownOp,
6810 Err(_) => ControlRequestBodyError::InvalidBody,
6811 };
6812 (err, classification)
6813 })
6814}
6815
6816fn parse_module_control_request_from_module(
6817 body: &[u8],
6818) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6819 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6820 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6821 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6822 Ok(_) => ControlRequestBodyError::UnknownOp,
6823 Err(_) => ControlRequestBodyError::InvalidBody,
6824 };
6825 (err, classification)
6826 })
6827}
6828
6829#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6830enum ProviderRoleKind {
6831 ToolProvider,
6832 PipelineStage,
6833 ManagementSurface,
6834 InternalService,
6835}
6836
6837fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6838 match role {
6839 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6840 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6841 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6842 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6843 }
6844}
6845
6846fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6847 roles.iter().map(provider_role_kind).collect()
6848}
6849
6850const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6853
6854#[derive(Debug, Default, PartialEq, Eq)]
6856struct ScopeOutcomeCounts {
6857 created: usize,
6858 replaced: usize,
6859 updated: usize,
6860 unchanged: usize,
6861 refused: usize,
6862}
6863
6864impl ScopeOutcomeCounts {
6865 fn of(results: &[ScopeRecordResult]) -> Self {
6866 let mut counts = Self::default();
6867 for result in results {
6868 let slot = match result.outcome {
6869 ScopeRecordOutcome::Created => &mut counts.created,
6870 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6871 ScopeRecordOutcome::Updated => &mut counts.updated,
6872 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6873 ScopeRecordOutcome::Refused => &mut counts.refused,
6874 };
6875 *slot += 1;
6876 }
6877 counts
6878 }
6879}
6880
6881#[cfg(test)]
6882mod scope_outcome_count_tests {
6883 use super::*;
6884
6885 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6886 ScopeRecordResult {
6887 scope_ref: "r".to_string(),
6888 scope_epoch: 1,
6889 outcome,
6890 code: None,
6891 message: None,
6892 version: None,
6893 parent_state: None,
6894 }
6895 }
6896
6897 #[test]
6900 fn every_outcome_is_counted_in_its_own_field() {
6901 let results = [
6902 result(ScopeRecordOutcome::Created),
6903 result(ScopeRecordOutcome::Created),
6904 result(ScopeRecordOutcome::Replaced),
6905 result(ScopeRecordOutcome::Updated),
6906 result(ScopeRecordOutcome::Unchanged),
6907 result(ScopeRecordOutcome::Refused),
6908 result(ScopeRecordOutcome::Refused),
6909 result(ScopeRecordOutcome::Refused),
6910 ];
6911 assert_eq!(
6912 ScopeOutcomeCounts::of(&results),
6913 ScopeOutcomeCounts {
6914 created: 2,
6915 replaced: 1,
6916 updated: 1,
6917 unchanged: 1,
6918 refused: 3,
6919 }
6920 );
6921 }
6922}
6923
6924fn capability_census_trigger(
6928 old: Option<&CapabilityDeclarations>,
6929 new: Option<&CapabilityDeclarations>,
6930) -> bool {
6931 let old_provides = old
6932 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6933 .unwrap_or_default();
6934 let old_denies = old
6935 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6936 .unwrap_or_default();
6937 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6938 provides: Vec::new(),
6939 requires: Vec::new(),
6940 must_never_reach: Vec::new(),
6941 });
6942
6943 new.provides
6944 .iter()
6945 .any(|capability| !old_provides.contains(capability))
6946 || new
6947 .must_never_reach
6948 .iter()
6949 .any(|capability| !old_denies.contains(capability))
6950}
6951
6952fn denied_capability<'a>(
6955 opening_manifest: &'a ModuleManifest,
6956 target_manifest: &ModuleManifest,
6957) -> Option<&'a str> {
6958 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6959 let target_capabilities = target_manifest.capabilities.as_ref()?;
6960 opening_capabilities
6961 .must_never_reach
6962 .iter()
6963 .find(|denied| {
6964 target_capabilities
6965 .provides
6966 .iter()
6967 .any(|provided| provided == *denied)
6968 })
6969 .map(String::as_str)
6970}
6971
6972fn catalog_update_frozen_field_message(
6973 registered: &ModuleManifest,
6974 provides: &[ProviderRole],
6975) -> Option<String> {
6976 let old_has_provides = !registered.provides.is_empty();
6977 let new_has_provides = !provides.is_empty();
6978 if old_has_provides != new_has_provides {
6979 return Some(format!(
6980 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6981 registered.module_id
6982 ));
6983 }
6984
6985 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6986 return Some(format!(
6987 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6988 registered.module_id
6989 ));
6990 }
6991
6992 let registered_concurrency = manifest_concurrency(registered);
6993 let mut candidate = registered.clone();
6994 candidate.provides = provides.to_vec();
6995 let candidate_concurrency = manifest_concurrency(&candidate);
6996 if candidate_concurrency != registered_concurrency {
6997 return Some(format!(
6998 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6999 registered.module_id, registered_concurrency, candidate_concurrency
7000 ));
7001 }
7002
7003 None
7006}
7007
7008fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7009 manifest.provides.iter().any(is_routable_role)
7010}
7011
7012fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7018 manifest
7019 .provides
7020 .iter()
7021 .find_map(|provider| match provider {
7022 ProviderRole::ToolProvider { concurrency, .. }
7023 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7024 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7025 })
7026 .unwrap_or(Concurrency::ModuleManaged)
7027}
7028
7029fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7036 let has_management_surface = manifest
7037 .provides
7038 .iter()
7039 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7040 if !has_management_surface {
7041 return false;
7042 }
7043 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7044 return false;
7045 };
7046 let Some(provides) = raw
7047 .get("manifest")
7048 .and_then(|manifest| manifest.get("provides"))
7049 .and_then(serde_json::Value::as_array)
7050 else {
7051 return false;
7052 };
7053 provides.iter().any(|role| {
7058 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7059 && role.get("concurrency").is_none()
7060 })
7061}
7062
7063fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7064 if peer_version != PROTOCOL_VERSION {
7065 return Err(format!(
7066 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7067 ));
7068 }
7069 Ok(PROTOCOL_VERSION)
7070}
7071
7072fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7073 Frame::build_with_version(
7074 response_version(frame),
7075 FrameType::Pong,
7076 frame.header.flags,
7077 0,
7078 0,
7079 frame.header.corr,
7080 Vec::new(),
7081 )
7082 .map_err(RouterError::FrameBuild)
7083}
7084
7085pub(crate) fn control_error_frame(
7086 frame: &Frame,
7087 code: &'static str,
7088 message: impl Into<String>,
7089) -> Result<Frame, RouterError> {
7090 control_error_body_frame(
7091 frame,
7092 ErrorBody {
7093 code: code.to_string(),
7094 message: message.into(),
7095 detail: None,
7096 },
7097 )
7098}
7099
7100fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7101 let body = serde_json::to_vec(&error).map_err(|err| {
7102 RouterError::backend(
7103 0,
7104 frame.header.corr,
7105 format!("failed to encode control ERROR: {err}"),
7106 )
7107 })?;
7108
7109 Frame::build_with_version(
7110 response_version(frame),
7111 FrameType::Error,
7112 control_flags(),
7113 0,
7114 0,
7115 frame.header.corr,
7116 body,
7117 )
7118 .map_err(RouterError::FrameBuild)
7119}
7120
7121fn control_response_body_frame<T: Serialize>(
7122 frame: &Frame,
7123 reply: &T,
7124 label: &'static str,
7125) -> Result<Frame, RouterError> {
7126 let body = serde_json::to_vec(reply).map_err(|err| {
7127 RouterError::backend(
7128 0,
7129 frame.header.corr,
7130 format!("failed to encode {label}: {err}"),
7131 )
7132 })?;
7133
7134 Frame::build_with_version(
7135 response_version(frame),
7136 FrameType::Response,
7137 control_flags(),
7138 0,
7139 0,
7140 frame.header.corr,
7141 body,
7142 )
7143 .map_err(RouterError::FrameBuild)
7144}
7145
7146fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7176 match err {
7177 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7178 ForwardingError::NoModuleConnection => "target_unavailable",
7179 ForwardingError::ModuleReloading { .. } => "module_reloading",
7180 ForwardingError::ClientRouteChannelExhausted { .. }
7181 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7182 ForwardingError::StaleModuleEndpoint
7183 | ForwardingError::UnknownReservation { .. }
7184 | ForwardingError::ConnectionClosing { .. }
7185 | ForwardingError::ClientEgressClosed { .. }
7186 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7187 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7190 ForwardingError::RelayCorrelationExhausted
7191 | ForwardingError::RouteOpenBuild(_)
7192 | ForwardingError::Poisoned => "forwarding_error",
7193 }
7194}
7195
7196fn response_version(frame: &Frame) -> u8 {
7197 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7198 frame.header.ver
7199 } else {
7200 PROTOCOL_VERSION
7201 }
7202}
7203
7204fn control_flags() -> Flags {
7205 Flags::new(false, Priority::Passive, false)
7206}
7207
7208fn send_goodbye_target_best_effort(
7213 counters: &DaemonCounters,
7214 target: &GoodbyeTarget,
7215 context: &'static str,
7216) {
7217 let Ok(frame) = Frame::build_with_version(
7218 target.negotiated_ver,
7219 FrameType::Goodbye,
7220 control_flags(),
7221 target.channel,
7222 target.epoch,
7223 0,
7224 Vec::new(),
7225 ) else {
7226 return;
7227 };
7228 crate::forwarding::send_module_route_goodbye(
7229 counters,
7230 &target.sink,
7231 frame,
7232 target.module_id.as_deref(),
7233 context,
7234 );
7235}
7236
7237pub(crate) fn send_route_control_pushes(
7238 forwarding: &ForwardingTable,
7239 routes: Vec<EndpointRoute>,
7240 push: ClientControlPush,
7241) {
7242 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7243 for route in routes {
7244 let target = route.goodbye_target;
7245 if let Some((existing, channels)) = targets
7246 .iter_mut()
7247 .find(|(existing, _)| existing.connection_id == target.connection_id)
7248 {
7249 debug_assert_eq!(
7250 existing.negotiated_ver, target.negotiated_ver,
7251 "one connection cannot negotiate multiple frame versions"
7252 );
7253 if !channels.contains(&target.channel) {
7254 channels.push(target.channel);
7255 }
7256 continue;
7257 }
7258 let channel = target.channel;
7259 targets.push((target, vec![channel]));
7260 }
7261 for (target, mut channels) in targets {
7262 channels.sort_unstable();
7263 let mut push = push.clone();
7264 match &mut push {
7265 ClientControlPush::RouteClosing {
7266 channels: covered, ..
7267 }
7268 | ClientControlPush::RouteClosed {
7269 channels: covered, ..
7270 } => *covered = channels,
7271 }
7272 let body = match serde_json::to_vec(&push) {
7273 Ok(body) => body,
7274 Err(err) => {
7275 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7276 continue;
7277 }
7278 };
7279 let frame = match Frame::build_with_version(
7280 target.negotiated_ver,
7281 FrameType::Push,
7282 control_flags(),
7283 0,
7284 0,
7285 0,
7286 body.clone(),
7287 ) {
7288 Ok(frame) => frame,
7289 Err(err) => {
7290 warn!(
7291 route_channel = target.channel,
7292 error = %err,
7293 "failed to build route lifecycle control PUSH frame"
7294 );
7295 continue;
7296 }
7297 };
7298 if let Err(err) = target.sink.try_send(frame) {
7299 if target.close_on_delivery_failure() {
7300 warn!(
7301 target_connection_id = target.connection_id.get(),
7302 route_channel = target.channel,
7303 error = %err,
7304 "route lifecycle control PUSH was not delivered to client; closing target connection"
7305 );
7306 let _ = forwarding.escalate_client_delivery_failure(
7307 target.connection_id,
7308 target.channel,
7309 target.epoch,
7310 CloseReason::new(
7311 "route_lifecycle_push_delivery_failed",
7312 format!(
7313 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7314 target.channel
7315 ),
7316 ),
7317 crate::forwarding::UndeliveredFrame {
7318 module_id: target.module_id.as_deref(),
7319 sink: &target.sink,
7320 },
7321 );
7322 }
7323 }
7324 }
7325}
7326
7327#[cfg(test)]
7328mod tests {
7329 mod activity_read {
7330 include!("activity_read_tests.rs");
7331 }
7332
7333 mod lifecycle_activity {
7334 include!("control_lifecycle_tests.rs");
7335 }
7336 #[cfg(unix)]
7337 #[tokio::test]
7338 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7339 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7340 let path = dir.join("subc.jsonc");
7341 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7342 "program":"/bin/sleep","args":["60"],"protocol":"none",
7343 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7344 }}}).to_string()).unwrap();
7345 let mut configured = crate::daemon_config::load(&path)
7346 .unwrap()
7347 .unwrap()
7348 .modules
7349 .pop()
7350 .unwrap();
7351 let registry = std::sync::Arc::new(crate::Registry::default());
7352 let handle = crate::SupervisorHandle::new();
7353 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7354 .with_handle(handle.clone());
7355 let module = supervisor
7356 .supervise_configured_with_health(
7357 configured.module_spec(),
7358 true,
7359 configured.health.clone(),
7360 None,
7361 configured.restart,
7362 )
7363 .unwrap();
7364 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7365 let before = module.status().unwrap().pid;
7366 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7367 configured.health.cadence = std::time::Duration::from_secs(3600);
7368 let health_only = handler
7369 .reconcile_supervised_modules(
7370 &supervisor,
7371 vec![configured.clone()],
7372 false,
7373 Principal::Direct,
7374 )
7375 .await
7376 .unwrap();
7377 assert!(
7378 health_only.changed_pending_reload.is_empty(),
7379 "health policy is already applied live"
7380 );
7381 assert_eq!(module.status().unwrap().pid, before);
7382 assert_eq!(
7383 module.configuration().unwrap().1.http,
7384 configured.health.http
7385 );
7386 configured.args = vec!["61".into()];
7387 let launch = handler
7388 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7389 .await
7390 .unwrap();
7391 assert_eq!(launch.changed_pending_reload, ["stock"]);
7392 assert_eq!(
7393 module.status().unwrap().pid,
7394 before,
7395 "a launch edit is stored until reload"
7396 );
7397 module.drain().await.unwrap();
7398 }
7399 use cortexkit_test_support::ScratchDir;
7400 use std::{
7401 collections::BTreeMap,
7402 fmt,
7403 path::PathBuf,
7404 sync::{Arc, Mutex},
7405 time::Duration,
7406 };
7407
7408 use serde_json::{json, Value};
7409 use subc_protocol::{
7410 manifest::{
7411 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7412 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7413 },
7414 session::HealthStatus,
7415 FrameType,
7416 };
7417
7418 use super::*;
7419 use crate::{
7420 forwarding::{DataRoute, DataRouteState},
7421 registry::ChannelState,
7422 router::FrameSink,
7423 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7424 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7425 RouteCtx, Router,
7426 };
7427 use tokio::{
7428 sync::mpsc,
7429 time::{sleep, Instant},
7430 };
7431 use tracing::{
7432 field::{Field, Visit},
7433 Event, Subscriber,
7434 };
7435 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7436
7437 fn fake_aft_stub_path() -> PathBuf {
7456 let mut path = std::env::current_exe().expect("current_exe available in tests");
7457 path.pop(); path.pop(); path.push(if cfg!(windows) {
7460 "fake-aft-stub.exe"
7461 } else {
7462 "fake-aft-stub"
7463 });
7464 assert!(
7465 path.exists(),
7466 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7467 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7468 path.display()
7469 );
7470 path
7471 }
7472
7473 fn client_retries(code: &str) -> bool {
7478 subc_protocol::error_codes::is_retryable_route_open(code)
7479 }
7480
7481 #[test]
7492 fn retryability_of_forwarding_codes_matches_the_failure() {
7493 let transient = [
7496 ForwardingError::NoModuleConnection,
7497 ForwardingError::ModuleReloading {
7498 module_id: "m".into(),
7499 },
7500 ForwardingError::StaleModuleEndpoint,
7501 ForwardingError::UnknownReservation {
7502 client_channel: 1,
7503 module_channel: 1,
7504 },
7505 ForwardingError::ConnectionClosing {
7506 connection_id: ConnectionId::new(1),
7507 },
7508 ForwardingError::ClientEgressClosed {
7509 connection_id: ConnectionId::new(1),
7510 },
7511 ForwardingError::ModuleEgressUnavailable {
7512 connection_id: ConnectionId::new(1),
7513 },
7514 ];
7515 for err in transient {
7516 let code = forwarding_error_code(&err);
7517 assert!(
7518 client_retries(code),
7519 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7520 );
7521 }
7522
7523 let permanent = [
7528 ForwardingError::ConnectionRoleConflict {
7529 connection_id: ConnectionId::new(1),
7530 },
7531 ForwardingError::ClientRouteChannelExhausted {
7532 connection_id: ConnectionId::new(1),
7533 },
7534 ForwardingError::ModuleRouteChannelExhausted {
7535 endpoint: ModuleEndpointId {
7536 connection_id: ConnectionId::new(1),
7537 generation: 1,
7538 },
7539 },
7540 ForwardingError::RelayCorrelationExhausted,
7541 ForwardingError::RouteOpenBuild("x".into()),
7542 ForwardingError::Poisoned,
7543 ];
7544 for err in permanent {
7545 let code = forwarding_error_code(&err);
7546 assert!(
7547 !client_retries(code),
7548 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7549 );
7550 }
7551 }
7552
7553 #[tokio::test]
7581 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7582 let handler = ControlHandler::default();
7583 let frame =
7584 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7585
7586 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7590 assert_eq!(
7591 stamped,
7592 Principal::Direct,
7593 "a caller that proved nothing must not be stamped as a supervised module"
7594 );
7595
7596 let forged = handler
7601 .route_open_principal(
7602 &frame,
7603 Some(ConsumerIdentity {
7604 module_id: "aft".to_string(),
7605 launch_nonce: "not-a-real-nonce".to_string(),
7606 }),
7607 )
7608 .unwrap();
7609 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7610 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7611 }
7612
7613 #[test]
7630 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7631 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7632 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7633 let ClientControlRequest::RouteOpen {
7634 consumer_identity, ..
7635 } = parsed
7636 else {
7637 panic!("route.open body must parse as RouteOpen");
7638 };
7639 assert_eq!(
7640 consumer_identity,
7641 Some(ConsumerIdentity {
7642 module_id: "aft".to_string(),
7643 launch_nonce: "n".to_string(),
7644 }),
7645 "the wire field name must reach the value route_open_principal reads"
7646 );
7647 }
7648
7649 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7650 ModuleManifest::builder(module_id, "0.1.0")
7651 .protocol_ver(protocol_ver)
7652 .provides(vec![ProviderRole::ToolProvider {
7653 tools: vec![Tool {
7654 name: "read".to_string(),
7655 description: None,
7656 execution_mode: ExecutionMode::Pure,
7657 schema: json!({"type": "object"}),
7658 }],
7659 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7660 concurrency: Concurrency::ModuleManaged,
7661 emits_push: true,
7662 sub_supervises: true,
7663 }])
7664 .build()
7665 }
7666
7667 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7668 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7669 }
7670
7671 fn hello_frame_with_control_ops(
7672 module_id: &str,
7673 protocol_ver: u8,
7674 corr: u64,
7675 control_ops: Option<Vec<String>>,
7676 ) -> Frame {
7677 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7678 }
7679
7680 fn hello_frame_with_nonce(
7681 module_id: &str,
7682 protocol_ver: u8,
7683 corr: u64,
7684 launch_nonce: Option<&str>,
7685 ) -> Frame {
7686 hello_frame_full(
7687 module_id,
7688 protocol_ver,
7689 corr,
7690 None,
7691 launch_nonce.map(ToOwned::to_owned),
7692 )
7693 }
7694
7695 fn hello_frame_full(
7696 module_id: &str,
7697 protocol_ver: u8,
7698 corr: u64,
7699 control_ops: Option<Vec<String>>,
7700 launch_nonce: Option<String>,
7701 ) -> Frame {
7702 let body = serde_json::to_vec(&ModuleHelloBody {
7703 manifest: manifest(module_id, protocol_ver),
7704 protocol_ver,
7705 control_ops,
7706 launch_nonce,
7707 })
7708 .unwrap();
7709 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7710 }
7711
7712 fn non_routable_hello_frame_with_control_ops(
7713 module_id: &str,
7714 corr: u64,
7715 control_ops: Option<Vec<String>>,
7716 ) -> Frame {
7717 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7718 manifest.provides.clear();
7719 let body = serde_json::to_vec(&ModuleHelloBody {
7720 manifest,
7721 protocol_ver: PROTOCOL_VERSION,
7722 control_ops,
7723 launch_nonce: None,
7724 })
7725 .unwrap();
7726 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7727 }
7728
7729 fn capability_grammar_hello_frame(
7730 capabilities: Value,
7731 runtime_computed: Option<Value>,
7732 corr: u64,
7733 ) -> Frame {
7734 let mut body = serde_json::to_value(ModuleHelloBody {
7735 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7736 protocol_ver: PROTOCOL_VERSION,
7737 control_ops: None,
7738 launch_nonce: None,
7739 })
7740 .expect("HELLO body serializes");
7741 body["manifest"]["capabilities"] = capabilities;
7742 if let Some(runtime_computed) = runtime_computed {
7743 body["runtime_computed"] = runtime_computed;
7744 }
7745 Frame::build(
7746 FrameType::Hello,
7747 control_flags(),
7748 0,
7749 0,
7750 corr,
7751 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7752 )
7753 .expect("HELLO frame builds")
7754 }
7755
7756 fn channel_request(channel: u16, corr: u64) -> Frame {
7757 Frame::build(
7758 FrameType::Request,
7759 Flags::new(true, Priority::Interactive, false),
7760 channel,
7761 0,
7762 corr,
7763 b"opaque".to_vec(),
7764 )
7765 .unwrap()
7766 }
7767
7768 fn route_ctx(
7769 connection_id: ConnectionId,
7770 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7771 let (tx, rx) = mpsc::channel(8);
7772 (
7773 RouteCtx {
7774 connection_id,
7775 egress: FrameSink::new(tx),
7776 },
7777 rx,
7778 )
7779 }
7780
7781 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7782 serde_json::from_slice(&frame.body).unwrap()
7783 }
7784
7785 async fn hello_via_sink(
7790 handler: &ControlHandler,
7791 ctx: &RouteCtx,
7792 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7793 hello: Frame,
7794 ) -> Frame {
7795 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7796 assert!(
7797 replies.is_empty(),
7798 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7799 );
7800 let ack = rx
7801 .try_recv()
7802 .expect("HELLO_ACK is queued on the module sink")
7803 .frame;
7804 assert_eq!(ack.header.ty, FrameType::HelloAck);
7805 ack
7806 }
7807
7808 fn parse_error(frame: &Frame) -> Value {
7809 serde_json::from_slice(&frame.body).unwrap()
7810 }
7811
7812 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7813 serde_json::from_slice(&frame.body).unwrap()
7814 }
7815
7816 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7817 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7818 route_channel,
7819 route_epoch: 0,
7820 kind,
7821 })
7822 .unwrap();
7823 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7824 }
7825
7826 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7827 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7828 module_id: module_id.to_string(),
7829 })
7830 .unwrap();
7831 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7832 }
7833
7834 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7835 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7836 }
7837
7838 fn route_open_frame_with_consumer_capabilities(
7839 corr: u64,
7840 module_id: &str,
7841 project_root: ScratchDir,
7842 consumer_capabilities: Option<Vec<String>>,
7843 ) -> Frame {
7844 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7845 target: RouteTarget::ToolProvider {
7846 module_id: module_id.to_string(),
7847 },
7848 identity: BindIdentity::new(
7849 project_root.path().to_path_buf(),
7850 "unit".to_string(),
7851 "session".to_string(),
7852 ),
7853 consumer_identity: None,
7854 consumer_capabilities,
7855 role_versions: None,
7856 admission_facts: None,
7857 scope: None,
7858 })
7859 .unwrap();
7860 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7861 }
7862
7863 fn route_open_frame_with_role_versions(
7864 corr: u64,
7865 module_id: &str,
7866 project_root: ScratchDir,
7867 role_versions: Option<BTreeMap<String, String>>,
7868 ) -> Frame {
7869 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7870 target: RouteTarget::ToolProvider {
7871 module_id: module_id.to_string(),
7872 },
7873 identity: BindIdentity::new(
7874 project_root.path().to_path_buf(),
7875 "unit".to_string(),
7876 format!("session-{corr}"),
7877 ),
7878 consumer_identity: None,
7879 consumer_capabilities: None,
7880 role_versions,
7881 admission_facts: None,
7882 scope: None,
7883 })
7884 .unwrap();
7885 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7886 }
7887
7888 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7889 entries
7890 .iter()
7891 .map(|(role, version)| (role.to_string(), version.to_string()))
7892 .collect()
7893 }
7894
7895 fn route_open_frame_with_admission_facts(
7896 corr: u64,
7897 module_id: &str,
7898 project_root: ScratchDir,
7899 consumer_identity: Option<subc_control::ConsumerIdentity>,
7900 facts: Option<Value>,
7901 ) -> Frame {
7902 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7903 target: RouteTarget::ToolProvider {
7904 module_id: module_id.to_string(),
7905 },
7906 identity: BindIdentity::new(
7907 project_root.path().to_path_buf(),
7908 "unit".to_string(),
7909 format!("session-{corr}"),
7910 ),
7911 consumer_identity,
7912 consumer_capabilities: None,
7913 role_versions: None,
7914 admission_facts: facts,
7915 scope: None,
7916 })
7917 .unwrap();
7918 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7919 }
7920
7921 #[derive(Clone, Default)]
7922 struct EventCapture {
7923 events: Arc<Mutex<Vec<CapturedEvent>>>,
7924 }
7925
7926 #[derive(Clone, Debug)]
7927 struct CapturedEvent {
7928 target: String,
7929 level: tracing::Level,
7930 fields: BTreeMap<String, String>,
7931 }
7932
7933 impl EventCapture {
7934 fn events(&self) -> Vec<CapturedEvent> {
7935 self.events.lock().unwrap().clone()
7936 }
7937 }
7938
7939 impl<S> Layer<S> for EventCapture
7940 where
7941 S: Subscriber,
7942 {
7943 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7944 let mut visitor = EventFieldVisitor::default();
7945 event.record(&mut visitor);
7946 self.events.lock().unwrap().push(CapturedEvent {
7947 target: event.metadata().target().to_string(),
7948 level: *event.metadata().level(),
7949 fields: visitor.fields,
7950 });
7951 }
7952 }
7953
7954 #[derive(Default)]
7955 struct EventFieldVisitor {
7956 fields: BTreeMap<String, String>,
7957 }
7958
7959 impl Visit for EventFieldVisitor {
7960 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7961 self.fields
7962 .insert(field.name().to_string(), format!("{value:?}"));
7963 }
7964 }
7965
7966 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7967 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7968 status,
7969 detail: Some("warming".to_string()),
7970 metrics: Some(json!({"queue_depth": 3})),
7971 })
7972 .unwrap();
7973 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7974 }
7975
7976 fn route_bind_ack(corr: u64) -> Frame {
7977 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7978 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7979 }
7980
7981 fn unique_project_root(label: &str) -> ScratchDir {
7982 ScratchDir::new(label)
7983 }
7984
7985 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7986 match parse_route_poll(frame) {
7987 ClientControlResponse::RoutePoll {
7988 status: None,
7989 live: Some(live),
7990 ..
7991 } => assert_eq!(live, expected_live),
7992 other => panic!("unexpected route.poll response: {other:?}"),
7993 }
7994 }
7995
7996 fn bind_liveness_route(
7997 registry: &Registry,
7998 forwarding: &ForwardingTable,
7999 module_id: &str,
8000 ) -> (RouteCtx, u16, u32) {
8001 let module_connection = ConnectionId::new(101);
8002 let client_connection = ConnectionId::new(202);
8003 let registration = registry
8004 .register_with_control_ops(
8005 manifest(module_id, PROTOCOL_VERSION),
8006 PROTOCOL_VERSION,
8007 module_connection,
8008 module_baseline_control_ops(),
8009 )
8010 .unwrap();
8011 let (module_tx, _module_rx) = mpsc::channel(8);
8012 let endpoint = forwarding
8013 .register_module_connection(
8014 module_connection,
8015 module_id.to_string(),
8016 PROTOCOL_VERSION,
8017 manifest_concurrency(®istration.manifest),
8018 FrameSink::new(module_tx),
8019 )
8020 .unwrap();
8021 let (client_ctx, _client_rx) = route_ctx(client_connection);
8022 let pending = forwarding
8023 .begin_route_bind_relay_for_test(
8024 client_connection,
8025 client_ctx.egress.clone(),
8026 1,
8027 module_id,
8028 )
8029 .unwrap();
8030 assert_eq!(pending.endpoint, endpoint);
8031 let route_channel = pending.client_channel;
8032 let route_epoch = pending.client_epoch;
8033 forwarding
8034 .complete_pending_relay(
8035 module_connection,
8036 pending.corr,
8037 RouteBindRelayOutcome::Accepted,
8038 )
8039 .unwrap();
8040 (client_ctx, route_channel, route_epoch)
8041 }
8042
8043 struct FakeProcessLiveness {
8044 live: Option<bool>,
8045 }
8046
8047 impl ModuleProcessLiveness for FakeProcessLiveness {
8048 fn process_live(&self, _module_id: &str) -> Option<bool> {
8049 self.live
8050 }
8051 }
8052
8053 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8054 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8055 {
8056 let registry = Arc::new(Registry::default());
8057 let supervisor_handle = SupervisorHandle::new();
8058 let supervisor = Supervisor::new_for_test(
8059 Arc::clone(®istry),
8060 RestartPolicy::new(1, Duration::from_millis(10)),
8061 )
8062 .with_handle(supervisor_handle.clone());
8063 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8064 let module = supervisor
8065 .spawn(ModuleSpec {
8066 module_id: "stderr-tail-wire".to_string(),
8067 program: fake_aft_stub_path(),
8068 args: Vec::new(),
8069 env: vec![
8070 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8071 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8072 ],
8073 reserved: false,
8074 reserved_prefixes: Vec::new(),
8075 protocol: ModuleProtocol::Subc,
8076 overlap: Default::default(),
8077 })
8078 .unwrap();
8079
8080 let deadline = Instant::now() + Duration::from_secs(5);
8081 loop {
8082 let tail = module.stderr_tail(None, None);
8083 if tail
8084 .entries
8085 .iter()
8086 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8087 && tail.entries.iter().any(|entry| {
8088 matches!(
8089 entry,
8090 TailEntry::Line {
8091 truncated: true,
8092 ..
8093 }
8094 )
8095 })
8096 {
8097 break;
8098 }
8099 assert!(
8100 Instant::now() < deadline,
8101 "module did not produce a truncated line and restart boundary: {tail:?}"
8102 );
8103 sleep(Duration::from_millis(10)).await;
8104 }
8105
8106 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8107 let request = ClientControlRequest::SupervisorStderrTail {
8108 module_id: "stderr-tail-wire".to_string(),
8109 max_lines: None,
8110 max_bytes: None,
8111 };
8112 let frame = Frame::build(
8113 FrameType::Request,
8114 control_flags(),
8115 0,
8116 0,
8117 1,
8118 serde_json::to_vec(&request).unwrap(),
8119 )
8120 .unwrap();
8121 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8122 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8123 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8124 serde_json::from_slice(&responses[0].body).unwrap()
8125 else {
8126 panic!("expected supervisor.stderr_tail response");
8127 };
8128
8129 assert!(
8130 tail.entries
8131 .iter()
8132 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8133 "the control response lost the restart boundary"
8134 );
8135 let Some(StderrTailEntry::Line {
8136 text,
8137 truncated,
8138 at_ms,
8139 }) = tail.entries.iter().find(|entry| {
8140 matches!(
8141 entry,
8142 StderrTailEntry::Line {
8143 truncated: true,
8144 ..
8145 }
8146 )
8147 })
8148 else {
8149 panic!("the control response lost the truncated line");
8150 };
8151 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8152 assert!(*truncated);
8153 assert!(
8154 at_ms.is_some(),
8155 "the control response lost the line's capture time"
8156 );
8157 }
8158
8159 #[tokio::test(flavor = "current_thread")]
8164 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8165 let dir = ScratchDir::new("terminals-off-worker");
8166 let journal_path = dir.join("terminals.jsonl");
8167 let registry = Arc::new(Registry::default());
8168 let supervisor_handle = SupervisorHandle::new();
8169 let supervisor =
8170 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8171 .with_handle(supervisor_handle.clone())
8172 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8173 let module = supervisor
8174 .spawn(ModuleSpec {
8175 module_id: "terminal-off-worker".to_string(),
8176 program: fake_aft_stub_path(),
8177 args: Vec::new(),
8178 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8179 reserved: false,
8180 reserved_prefixes: Vec::new(),
8181 protocol: ModuleProtocol::Subc,
8182 overlap: Default::default(),
8183 })
8184 .unwrap();
8185 let deadline = Instant::now() + Duration::from_secs(5);
8186 while module.terminal_history().entries.len() != 2 {
8187 assert!(Instant::now() < deadline, "module did not record two exits");
8188 sleep(Duration::from_millis(10)).await;
8189 }
8190
8191 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8192 let handler =
8193 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8194 let frame = Frame::build(
8195 FrameType::Request,
8196 control_flags(),
8197 0,
8198 0,
8199 1,
8200 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8201 module_id: "terminal-off-worker".to_string(),
8202 })
8203 .unwrap(),
8204 )
8205 .unwrap();
8206 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8207 let spawned_at = std::time::Instant::now();
8208 let read = tokio::spawn({
8209 let handler = Arc::clone(&handler);
8210 async move { handler.handle_control_frame(&ctx, frame).await }
8211 });
8212 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8215 .await
8216 .unwrap()
8217 .expect("the history read reached its pause");
8218 let elapsed = spawned_at.elapsed();
8219 assert!(
8220 elapsed < Duration::from_secs(2) && !read.is_finished(),
8221 "this task could not run while the history read was paused \
8222 (resumed after {elapsed:?}, read finished: {})",
8223 read.is_finished()
8224 );
8225
8226 drop(release);
8227 let responses = read.await.unwrap().unwrap();
8228 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8229 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8230 panic!("expected supervisor.terminals response");
8231 };
8232 assert_eq!(terminals.entries.len(), 2);
8233 assert_eq!(terminals.journal_skipped_lines, 0);
8234 assert_eq!(terminals.journal_read_errors, 0);
8235 }
8236
8237 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8238 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8239 let registry = Arc::new(Registry::default());
8240 let supervisor_handle = SupervisorHandle::new();
8241 let supervisor =
8242 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8243 .with_handle(supervisor_handle.clone());
8244 let module = supervisor
8245 .spawn(ModuleSpec {
8246 module_id: "terminal-golden".to_string(),
8247 program: fake_aft_stub_path(),
8248 args: Vec::new(),
8249 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8250 reserved: false,
8251 reserved_prefixes: Vec::new(),
8252 protocol: ModuleProtocol::Subc,
8253 overlap: Default::default(),
8254 })
8255 .unwrap();
8256
8257 let deadline = Instant::now() + Duration::from_secs(5);
8258 while module.terminal_history().entries.len() != 2 {
8259 assert!(
8260 Instant::now() < deadline,
8261 "module did not retain two terminal exits: {:?}",
8262 module.terminal_history()
8263 );
8264 sleep(Duration::from_millis(10)).await;
8265 }
8266
8267 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8268 let request = ClientControlRequest::SupervisorTerminals {
8269 module_id: "terminal-golden".to_string(),
8270 };
8271 let frame = Frame::build(
8272 FrameType::Request,
8273 control_flags(),
8274 0,
8275 0,
8276 1,
8277 serde_json::to_vec(&request).unwrap(),
8278 )
8279 .unwrap();
8280 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8281 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8282 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8283 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8284 panic!("expected supervisor.terminals response");
8285 };
8286 assert_eq!(terminals.entries.len(), 2);
8287 assert_eq!(terminals.dropped, 0);
8288
8289 let mut rendered = serde_json::to_value(response).unwrap();
8290 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8293 for (index, entry) in rendered["entries"]
8294 .as_array_mut()
8295 .expect("terminal response entries array")
8296 .iter_mut()
8297 .enumerate()
8298 {
8299 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8300 }
8301
8302 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8303 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8304 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8305 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8306 std::fs::write(&golden_path, &serialized).unwrap();
8307 }
8308 let expected: Value =
8309 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8310 assert_eq!(rendered, expected);
8311 }
8312
8313 #[test]
8314 fn hello_registers_manifest_and_returns_ack() {
8315 let registry = Arc::new(Registry::default());
8316 let handler = ControlHandler::new(Arc::clone(®istry));
8317 let conn = ConnectionId::new(1);
8318
8319 let responses = handler
8320 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8321 .unwrap();
8322
8323 assert_eq!(responses.len(), 1);
8324 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8325 assert_eq!(responses[0].header.channel, 0);
8326 assert_eq!(responses[0].header.corr, 7);
8327 let ack = parse_ack(&responses[0]);
8328 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8329 assert!(ack
8330 .subc_capabilities
8331 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8332 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8333 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8334 assert!(ack
8335 .subc_ops
8336 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8337 assert!(ack
8338 .subc_ops
8339 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8340
8341 let registration = registry.get_module("aft").unwrap().unwrap();
8342 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8343 assert_eq!(registration.state, ChannelState::Active);
8344 assert_eq!(registration.connection_id, conn);
8345 assert_eq!(registration.control_ops, module_baseline_control_ops());
8346 }
8347
8348 #[test]
8349 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8350 let invalid_identifiers = [
8351 ("case_change", "credentials-Provider/v1"),
8352 ("leading_zero", "credentials-provider/v01"),
8353 ("trailing_hyphen", "credentials-provider-/v1"),
8354 ("consecutive_hyphens", "credentials--provider/v1"),
8355 ("uppercase", "Credentials-provider/v1"),
8356 ("missing_v", "credentials-provider/1"),
8357 ("whitespace", "credentials provider/v1"),
8358 ("zero_version", "credentials-provider/v0"),
8359 ("out_of_range_version", "credentials-provider/v4294967296"),
8360 (
8361 "overlength_name",
8362 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8363 ),
8364 ];
8365 let mut cases = invalid_identifiers
8366 .into_iter()
8367 .map(|(name, identifier)| {
8368 (
8369 format!("identifier_{name}"),
8370 "capabilities.provides[0]".to_string(),
8371 identifier.to_string(),
8372 json!({ "provides": [identifier] }),
8373 None,
8374 )
8375 })
8376 .collect::<Vec<_>>();
8377 cases.extend([
8378 (
8379 "unknown_need".to_string(),
8380 "capabilities.requires[0].need".to_string(),
8381 "deferred".to_string(),
8382 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8383 None,
8384 ),
8385 (
8386 "duplicate_provides".to_string(),
8387 "capabilities.provides[1]".to_string(),
8388 "credentials-provider/v1".to_string(),
8389 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8390 None,
8391 ),
8392 (
8393 "duplicate_must_never_reach".to_string(),
8394 "capabilities.must_never_reach[1]".to_string(),
8395 "credentials-provider/v1".to_string(),
8396 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8397 None,
8398 ),
8399 (
8400 "duplicate_requires_same_need".to_string(),
8401 "capabilities.requires[1]".to_string(),
8402 "credentials-provider/v1".to_string(),
8403 json!({ "requires": [
8404 { "capability": "credentials-provider/v1", "need": "required" },
8405 { "capability": "credentials-provider/v1", "need": "required" }
8406 ] }),
8407 None,
8408 ),
8409 (
8410 "duplicate_requires_conflicting_need".to_string(),
8411 "capabilities.requires[1]".to_string(),
8412 "credentials-provider/v1".to_string(),
8413 json!({ "requires": [
8414 { "capability": "credentials-provider/v1", "need": "required" },
8415 { "capability": "credentials-provider/v1", "need": "optional" }
8416 ] }),
8417 None,
8418 ),
8419 (
8420 "capabilities_root_pointer".to_string(),
8421 "runtime_computed[0]".to_string(),
8422 "/capabilities".to_string(),
8423 json!({}),
8424 Some(json!(["/capabilities"])),
8425 ),
8426 (
8427 "capabilities_descendant_pointer".to_string(),
8428 "runtime_computed[0]".to_string(),
8429 "/capabilities/provides".to_string(),
8430 json!({}),
8431 Some(json!(["/capabilities/provides"])),
8432 ),
8433 (
8434 "malformed_pointer_without_leading_slash".to_string(),
8435 "runtime_computed[0]".to_string(),
8436 "capabilities".to_string(),
8437 json!({}),
8438 Some(json!(["capabilities"])),
8439 ),
8440 (
8441 "malformed_pointer_escape".to_string(),
8442 "runtime_computed[0]".to_string(),
8443 "/roles/~2/tools".to_string(),
8444 json!({}),
8445 Some(json!(["/roles/~2/tools"])),
8446 ),
8447 (
8448 "unknown_capabilities_field".to_string(),
8449 "capabilities.future".to_string(),
8450 "<array>".to_string(),
8451 json!({ "future": [] }),
8452 None,
8453 ),
8454 ]);
8455
8456 for (index, (name, field, value, capabilities, runtime_computed)) in
8457 cases.into_iter().enumerate()
8458 {
8459 let registry = Arc::new(Registry::default());
8460 let handler = ControlHandler::new(Arc::clone(®istry));
8461 let response = handler
8462 .handle_control(
8463 ConnectionId::new((index + 1) as u64),
8464 capability_grammar_hello_frame(
8465 capabilities,
8466 runtime_computed,
8467 index as u64 + 1,
8468 ),
8469 )
8470 .expect("invalid HELLO returns a refusal");
8471
8472 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8473 let error = parse_error(&response[0]);
8474 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8475 let message = error["message"]
8476 .as_str()
8477 .expect("error message is a string");
8478 assert!(
8479 message.contains(&field),
8480 "{name}: field missing from {message}"
8481 );
8482 assert!(
8483 message.contains(&value),
8484 "{name}: value missing from {message}"
8485 );
8486 assert_eq!(
8487 registry
8488 .active_registration_count()
8489 .expect("registry reads"),
8490 0,
8491 "{name}: refused HELLO must not create a catalog entry"
8492 );
8493 }
8494 }
8495
8496 #[test]
8497 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8498 let registry = Arc::new(Registry::default());
8499 let handler = ControlHandler::new(Arc::clone(®istry));
8500 let capabilities = json!({
8501 "provides": ["credentials-provider/v1"],
8502 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8503 "must_never_reach": ["federation-transport/v1"]
8504 });
8505 let response = handler
8506 .handle_control(
8507 ConnectionId::new(99),
8508 capability_grammar_hello_frame(
8509 capabilities.clone(),
8510 Some(json!(["/roles/0/tools"])),
8511 99,
8512 ),
8513 )
8514 .expect("valid HELLO registers");
8515 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8516
8517 let request = Frame::build(
8518 FrameType::Request,
8519 control_flags(),
8520 0,
8521 0,
8522 100,
8523 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8524 .expect("catalog request serializes"),
8525 )
8526 .expect("catalog request frame builds");
8527 let response = handler
8528 .handle_catalog_list(request, None)
8529 .expect("catalog list succeeds");
8530 let ClientControlResponse::CatalogList { modules, .. } =
8531 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8532 else {
8533 panic!("catalog request must return catalog.list");
8534 };
8535 assert_eq!(modules.len(), 1);
8536 assert_eq!(
8537 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8538 capabilities
8539 );
8540 }
8541
8542 #[test]
8543 fn catalog_list_mirrors_management_operation_description() {
8544 let registry = Arc::new(Registry::default());
8545 let handler = ControlHandler::new(Arc::clone(®istry));
8546 let description = "List managed records and return their identifiers and metadata.";
8547 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8548 manifest.provides = vec![ProviderRole::ManagementSurface {
8549 operations: vec![ManagementOperation {
8550 name: "records.list".to_string(),
8551 kind: ManagementOperationKind::Query,
8552 description: Some(description.to_string()),
8553 }],
8554 config_schema: json!({"type": "object"}),
8555 observability: vec![ObservabilitySurface {
8556 name: "records.stats".to_string(),
8557 kind: ObservabilityKind::Snapshot,
8558 }],
8559 identity_scope: vec![IdentityScope::Project],
8560 concurrency: Concurrency::ModuleManaged,
8561 }];
8562 registry
8563 .register_with_control_ops(
8564 manifest,
8565 PROTOCOL_VERSION,
8566 ConnectionId::new(99),
8567 Vec::new(),
8568 )
8569 .expect("described management manifest registers");
8570
8571 let request = Frame::build(
8572 FrameType::Request,
8573 control_flags(),
8574 0,
8575 0,
8576 100,
8577 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8578 .expect("catalog request serializes"),
8579 )
8580 .expect("catalog request frame builds");
8581 let response = handler
8582 .handle_catalog_list(request, None)
8583 .expect("catalog list succeeds");
8584 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8585 assert_eq!(
8586 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8587 "catalog.list must preserve the declared operation description verbatim"
8588 );
8589 }
8590
8591 #[test]
8592 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8593 let registry = Arc::new(Registry::default());
8594 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8595 [("vault".to_string(), true), ("squatter".to_string(), true)],
8596 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8597 );
8598 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8599 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8600 provides: vec!["credentials-provider/v1".to_string()],
8601 requires: Vec::new(),
8602 must_never_reach: Vec::new(),
8603 });
8604 let frame = Frame::build(
8605 FrameType::Hello,
8606 control_flags(),
8607 0,
8608 0,
8609 77,
8610 serde_json::to_vec(&ModuleHelloBody {
8611 manifest: squatter,
8612 protocol_ver: PROTOCOL_VERSION,
8613 control_ops: None,
8614 launch_nonce: None,
8615 })
8616 .expect("HELLO serializes"),
8617 )
8618 .expect("HELLO frame builds");
8619 let response = handler
8620 .handle_control(ConnectionId::new(77), frame)
8621 .expect("reserved claim receives a typed refusal");
8622 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8623 assert_eq!(
8624 registry
8625 .active_registration_count()
8626 .expect("registry reads"),
8627 0,
8628 "a reserved capability refusal must not leave a catalog entry"
8629 );
8630 }
8631
8632 #[test]
8633 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8634 for settlement in ["timeout", "inconclusive", "drop"] {
8635 let breakers = RouteBindBreakers::default();
8636 let RouteBindAdmission::Admitted {
8637 guard: mut old,
8638 probe: false,
8639 } = breakers.admit("prov")
8640 else {
8641 panic!("ordinary relay admitted")
8642 };
8643 let RouteBindAdmission::Admitted {
8644 guard: mut opener, ..
8645 } = breakers.admit("prov")
8646 else {
8647 panic!("second relay admitted")
8648 };
8649 assert!(
8650 !opener
8651 .record_timeout(1, Duration::ZERO)
8652 .unwrap()
8653 .reopened_after_probe
8654 );
8655 let RouteBindAdmission::Admitted {
8656 guard: mut probe,
8657 probe: true,
8658 } = breakers.admit("prov")
8659 else {
8660 panic!("one cooldown probe admitted")
8661 };
8662 match settlement {
8663 "timeout" => assert!(
8664 !old.record_timeout(1, Duration::ZERO)
8665 .unwrap()
8666 .reopened_after_probe
8667 ),
8668 "inconclusive" => old.record_inconclusive(),
8669 "drop" => drop(old),
8670 _ => unreachable!(),
8671 }
8672 assert!(
8673 matches!(
8674 breakers.admit("prov"),
8675 RouteBindAdmission::Refused {
8676 probe_in_flight: true,
8677 ..
8678 }
8679 ),
8680 "{settlement} of a pre-open relay cannot release the real probe"
8681 );
8682 assert!(
8683 probe
8684 .record_timeout(1, Duration::ZERO)
8685 .unwrap()
8686 .reopened_after_probe
8687 );
8688 assert!(matches!(
8689 breakers.admit("prov"),
8690 RouteBindAdmission::Admitted { probe: true, .. }
8691 ));
8692 }
8693 let breakers = RouteBindBreakers::default();
8694 let admit = || match breakers.admit("prov") {
8695 RouteBindAdmission::Admitted { guard, .. } => guard,
8696 _ => panic!("relay admitted"),
8697 };
8698 admit().record_timeout(1, Duration::ZERO);
8699 let mut old_probe = admit();
8700 breakers.reset_for_new_module_connection("prov");
8701 admit().record_timeout(1, Duration::ZERO);
8702 let _new_probe = admit();
8703 old_probe.record_inconclusive();
8704 assert!(matches!(
8705 breakers.admit("prov"),
8706 RouteBindAdmission::Refused {
8707 probe_in_flight: true,
8708 ..
8709 }
8710 ));
8711 }
8712
8713 #[tokio::test]
8714 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8715 for candidate in [false, true] {
8716 let registry = Arc::new(Registry::default());
8717 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8718 [("vault".to_string(), true), ("squatter".to_string(), true)],
8719 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8720 );
8721 let conn = ConnectionId::new(77);
8722 let (ctx, mut rx) = route_ctx(conn);
8723 let initial = capability_manifest("squatter", &[], &[]);
8724 if candidate {
8725 registry
8726 .register_candidate_with_control_ops(
8727 initial.clone(),
8728 PROTOCOL_VERSION,
8729 conn,
8730 module_baseline_control_ops(),
8731 )
8732 .unwrap();
8733 handler
8734 .forwarding
8735 .register_candidate_module_connection(
8736 conn,
8737 "squatter".to_string(),
8738 PROTOCOL_VERSION,
8739 manifest_concurrency(&initial),
8740 ctx.egress.clone(),
8741 )
8742 .unwrap();
8743 } else {
8744 hello_via_sink(
8745 &handler,
8746 &ctx,
8747 &mut rx,
8748 hello_frame_with_manifest(initial.clone(), 1),
8749 )
8750 .await;
8751 }
8752 let response = handler
8753 .handle_control_frame(
8754 &ctx,
8755 catalog_update_with_capabilities_frame(
8756 2,
8757 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8758 .capabilities
8759 .unwrap(),
8760 ),
8761 )
8762 .await
8763 .unwrap();
8764 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8765 assert_eq!(
8766 registry
8767 .get_module_by_connection(conn)
8768 .unwrap()
8769 .unwrap()
8770 .manifest,
8771 initial
8772 );
8773 }
8774 }
8775
8776 #[test]
8777 fn server_describe_surfaces_required_capability_verdict_fields() {
8778 let registry = Arc::new(Registry::default());
8779 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8780 [
8781 ("consumer".to_string(), true),
8782 ("provider".to_string(), false),
8783 ],
8784 BTreeMap::new(),
8785 );
8786 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8787 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8788 provides: Vec::new(),
8789 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8790 capability: "credentials-provider/v1".to_string(),
8791 need: subc_protocol::manifest::CapabilityNeed::Required,
8792 }],
8793 must_never_reach: Vec::new(),
8794 });
8795 let hello = Frame::build(
8796 FrameType::Hello,
8797 control_flags(),
8798 0,
8799 0,
8800 78,
8801 serde_json::to_vec(&ModuleHelloBody {
8802 manifest: consumer,
8803 protocol_ver: PROTOCOL_VERSION,
8804 control_ops: None,
8805 launch_nonce: None,
8806 })
8807 .expect("HELLO serializes"),
8808 )
8809 .expect("HELLO frame builds");
8810 handler
8811 .handle_control(ConnectionId::new(78), hello)
8812 .expect("consumer registers");
8813 let describe = Frame::build(
8814 FrameType::Request,
8815 control_flags(),
8816 0,
8817 0,
8818 79,
8819 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8820 .expect("request serializes"),
8821 )
8822 .expect("describe frame builds");
8823 let response = handler
8824 .handle_server_describe(describe)
8825 .expect("server.describe succeeds");
8826 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8827 let requirement = &rendered["capability_requirements"][0];
8828 assert_eq!(requirement["consumer"], "consumer");
8829 assert_eq!(requirement["verdict"], "never_provided");
8830 assert_eq!(requirement["episode_seq"], 1);
8831 assert_eq!(requirement["config_satisfiable"], false);
8832 assert_eq!(requirement["runtime_available"], false);
8833 assert!(requirement["detail"]
8834 .as_str()
8835 .expect("detail string")
8836 .contains("credentials-provider/v1"));
8837 }
8838
8839 #[test]
8840 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8841 let registry = Arc::new(Registry::default());
8842 let handler = ControlHandler::new(Arc::clone(®istry));
8843 let hello = handler
8844 .handle_control(
8845 ConnectionId::new(101),
8846 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8847 )
8848 .expect("legacy HELLO registers");
8849 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8850
8851 let request = Frame::build(
8852 FrameType::Request,
8853 control_flags(),
8854 0,
8855 0,
8856 102,
8857 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8858 .expect("catalog request serializes"),
8859 )
8860 .expect("catalog request frame builds");
8861 let response = handler
8862 .handle_catalog_list(request, None)
8863 .expect("catalog list succeeds");
8864 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8865 assert!(
8866 body["modules"][0].get("capabilities").is_none(),
8867 "legacy manifest must retain an absent capabilities field on catalog.list"
8868 );
8869 }
8870
8871 #[test]
8872 fn hello_ack_omits_storage_when_no_storage_config() {
8873 let registry = Arc::new(Registry::default());
8874 let handler = ControlHandler::new(Arc::clone(®istry));
8875 let responses = handler
8876 .handle_control(
8877 ConnectionId::new(1),
8878 hello_frame("aft", PROTOCOL_VERSION, 7),
8879 )
8880 .unwrap();
8881 let ack = parse_ack(&responses[0]);
8882 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8883 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8884 }
8885
8886 #[tokio::test]
8887 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8888 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8889 let registry = Arc::new(Registry::default());
8890 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
8891 let responses = handler
8892 .handle_control(
8893 ConnectionId::new(1),
8894 hello_frame("aft", PROTOCOL_VERSION, 7),
8895 )
8896 .unwrap();
8897 let ack = parse_ack(&responses[0]);
8898 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8899
8900 let described = handler
8901 .handle_control_frame(
8902 &route_ctx(ConnectionId::new(2)).0,
8903 Frame::build(
8904 FrameType::Request,
8905 control_flags(),
8906 0,
8907 0,
8908 9,
8909 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8910 )
8911 .unwrap(),
8912 )
8913 .await
8914 .unwrap();
8915 let ClientControlResponse::ServerDescribe { machine_id, .. } =
8916 serde_json::from_slice(&described[0].body).unwrap()
8917 else {
8918 panic!("server.describe answered with another shape");
8919 };
8920 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8921 }
8922
8923 #[test]
8924 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8925 let registry = Arc::new(Registry::default());
8928 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
8929 crate::daemon_config::StorageConfig::Sqlite {
8930 data_home: std::path::PathBuf::from("/data"),
8931 },
8932 ));
8933
8934 let responses = handler
8935 .handle_control(
8936 ConnectionId::new(1),
8937 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8938 )
8939 .unwrap();
8940 let ack = parse_ack(&responses[0]);
8941 assert_eq!(
8942 ack.storage,
8943 Some(serde_json::json!({
8944 "module_id": "alfonso-routing",
8945 "storage_namespace": "default",
8946 "isolation": { "kind": "module" },
8947 "backend": {
8948 "backend": "sqlite",
8949 "path": "/data/cortexkit/alfonso-routing/store.db"
8950 }
8951 })),
8952 "the delivered descriptor is the module's own sqlite store path"
8953 );
8954 }
8955
8956 #[test]
8957 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8958 let registry = Arc::new(Registry::default());
8959 let handler = ControlHandler::new(Arc::clone(®istry));
8960 let conn = ConnectionId::new(1);
8961 let responses = handler
8962 .handle_control(
8963 conn,
8964 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8965 )
8966 .unwrap();
8967 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8968 let registration = registry.get_module("aft").unwrap().unwrap();
8969 assert_eq!(registration.control_ops, module_baseline_control_ops());
8970
8971 let frame =
8972 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8973 assert!(handler
8974 .guard_module_control_op(&frame, "aft", "route.bind")
8975 .unwrap()
8976 .is_none());
8977 let error = handler
8978 .guard_module_control_op(&frame, "aft", "test.synthetic")
8979 .unwrap()
8980 .expect("synthetic ungranted op should be rejected");
8981 assert_eq!(error.header.ty, FrameType::Error);
8982 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8983 }
8984
8985 #[test]
8986 fn hello_control_ops_some_adds_optional_grants() {
8987 let registry = Arc::new(Registry::default());
8988 let handler = ControlHandler::new(Arc::clone(®istry));
8989 handler
8990 .handle_control(
8991 ConnectionId::new(1),
8992 hello_frame_with_control_ops(
8993 "aft",
8994 PROTOCOL_VERSION,
8995 7,
8996 Some(vec![
8997 "future.synthetic".to_string(),
8998 "route.bind".to_string(),
8999 ]),
9000 ),
9001 )
9002 .unwrap();
9003 let registration = registry.get_module("aft").unwrap().unwrap();
9004 assert_eq!(
9005 registration.control_ops,
9006 vec![
9007 "route.bind".to_string(),
9008 "route.status".to_string(),
9009 "future.synthetic".to_string(),
9010 ]
9011 );
9012 let frame =
9013 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9014 assert!(handler
9015 .guard_module_control_op(&frame, "aft", "future.synthetic")
9016 .unwrap()
9017 .is_none());
9018 }
9019
9020 #[tokio::test]
9021 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9022 let registry = Arc::new(Registry::default());
9023 let forwarding = Arc::new(ForwardingTable::default());
9024 let handler =
9025 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9026 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9027 hello_via_sink(
9028 &handler,
9029 &module_ctx,
9030 &mut module_rx,
9031 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9032 )
9033 .await;
9034
9035 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9036 let responses = handler
9037 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9038 .await
9039 .unwrap();
9040 assert_eq!(responses.len(), 1);
9041 assert_eq!(responses[0].header.ty, FrameType::Error);
9042 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9043 assert!(module_rx.try_recv().is_err());
9044 }
9045
9046 #[tokio::test]
9047 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9048 let registry = Arc::new(Registry::default());
9049 let forwarding = Arc::new(ForwardingTable::default());
9050 let handler =
9051 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9052 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9053 hello_via_sink(
9054 &handler,
9055 &module_ctx,
9056 &mut module_rx,
9057 hello_frame_with_control_ops(
9058 "aft",
9059 PROTOCOL_VERSION,
9060 7,
9061 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9062 ),
9063 )
9064 .await;
9065
9066 let project_root = unique_project_root("demux");
9067 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9068 let route_handler = handler.clone();
9069 let route_task = tokio::spawn(async move {
9070 route_handler
9071 .handle_control_frame(
9072 &route_client_ctx,
9073 route_open_frame(100, "aft", project_root),
9074 )
9075 .await
9076 .unwrap()
9077 });
9078 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9079 .await
9080 .unwrap()
9081 .unwrap();
9082 assert!(matches!(
9083 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9084 ModuleControlRequest::RouteBind { .. }
9085 ));
9086
9087 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9088 let health_handler = handler.clone();
9089 let health_task = tokio::spawn(async move {
9090 health_handler
9091 .handle_control_frame(
9092 &health_client_ctx,
9093 supervisor_health_probe_frame(101, "aft"),
9094 )
9095 .await
9096 .unwrap()
9097 });
9098 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9099 .await
9100 .unwrap()
9101 .unwrap();
9102 assert_eq!(
9103 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9104 ModuleControlRequest::HealthCheck {}
9105 );
9106
9107 handler
9108 .handle_control_frame(
9109 &module_ctx,
9110 health_response(health_frame.header.corr, HealthStatus::Degraded),
9111 )
9112 .await
9113 .unwrap();
9114 let health_response = health_task.await.unwrap();
9115 assert_eq!(health_response.len(), 1);
9116 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9117 ClientControlResponse::SupervisorHealthProbe {
9118 module_id,
9119 status,
9120 detail,
9121 metrics,
9122 } => {
9123 assert_eq!(module_id, "aft");
9124 assert_eq!(status, HealthStatus::Degraded);
9125 assert_eq!(detail.as_deref(), Some("warming"));
9126 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9127 }
9128 other => panic!("unexpected health response: {other:?}"),
9129 }
9130
9131 handler
9132 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9133 .await
9134 .unwrap();
9135 let route_response = route_task.await.unwrap();
9136 assert!(route_response.is_empty());
9137 let published = route_client_rx.recv().await.unwrap();
9138 assert!(matches!(
9139 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9140 ClientControlResponse::RouteOpen { .. }
9141 ));
9142 }
9143
9144 async fn relay_route_open(
9149 handler: &ControlHandler,
9150 client_connection: ConnectionId,
9151 client_egress: &FrameSink,
9152 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9153 corr: u64,
9154 module_id: &str,
9155 project_root_label: &str,
9156 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9157 let ctx = RouteCtx {
9158 connection_id: client_connection,
9159 egress: client_egress.clone(),
9160 };
9161 let handler = handler.clone();
9162 let project_root = unique_project_root(project_root_label);
9163 let module_id = module_id.to_string();
9164 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9165 let task = tokio::spawn(async move {
9166 let _guard = tracing::dispatcher::set_default(&dispatch);
9167 handler
9168 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9169 .await
9170 .unwrap()
9171 });
9172 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9173 .await
9174 .expect("module receives the relayed route.bind")
9175 .expect("module egress is open");
9176 (task, bind.frame)
9177 }
9178
9179 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9180 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9181 ModuleControlRequest::RouteBind {
9182 route_channel,
9183 epoch,
9184 ..
9185 } => (route_channel, epoch),
9186 other => panic!("expected a route.bind request, got {other:?}"),
9187 }
9188 }
9189
9190 fn published_route(frame: &Frame) -> (u16, u32) {
9191 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9192 ClientControlResponse::RouteOpen {
9193 route_channel,
9194 route_epoch,
9195 } => (route_channel, route_epoch),
9196 other => panic!("expected a route.open response, got {other:?}"),
9197 }
9198 }
9199
9200 #[tokio::test]
9220 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9221 let registry = Arc::new(Registry::default());
9222 let forwarding = Arc::new(ForwardingTable::default());
9223 let handler =
9224 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9225
9226 let module_connection = ConnectionId::new(30);
9227 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9228 hello_via_sink(
9229 &handler,
9230 &module_ctx,
9231 &mut module_rx,
9232 hello_frame("aft", PROTOCOL_VERSION, 7),
9233 )
9234 .await;
9235
9236 let dying_client = ConnectionId::new(31);
9237 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9238
9239 let (first_task, first_bind) = relay_route_open(
9242 &handler,
9243 dying_client,
9244 &dying_ctx.egress,
9245 &mut module_rx,
9246 100,
9247 "aft",
9248 "closing-first",
9249 )
9250 .await;
9251 handler
9252 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9253 .await
9254 .unwrap();
9255 assert!(first_task.await.unwrap().is_empty());
9256 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9257
9258 let (second_task, second_bind) = relay_route_open(
9260 &handler,
9261 dying_client,
9262 &dying_ctx.egress,
9263 &mut module_rx,
9264 101,
9265 "aft",
9266 "closing-second",
9267 )
9268 .await;
9269 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9270
9271 assert!(forwarding
9274 .escalate_client_delivery_failure(
9275 dying_client,
9276 first_channel,
9277 first_epoch,
9278 CloseReason::new(
9279 "module_to_client_delivery_failed",
9280 "client egress refused a module frame",
9281 ),
9282 crate::forwarding::UndeliveredFrame {
9283 module_id: None,
9284 sink: &dying_ctx.egress,
9285 },
9286 )
9287 .unwrap());
9288 assert!(!dying_ctx.egress.is_closed());
9289
9290 let ack = handler
9292 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9293 .await;
9294 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9295 if module_loop_error.is_some() {
9296 handler.cleanup_connection(module_connection).unwrap();
9300 }
9301 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9305 .await
9306 .ok()
9307 .flatten();
9308
9309 assert!(
9311 registry
9312 .get_module_by_connection(module_connection)
9313 .unwrap()
9314 .is_some(),
9315 "one client's closing connection ended the shared module connection: \
9316 {module_loop_error:?}"
9317 );
9318 let cotenant = ConnectionId::new(32);
9320 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9321 let (cotenant_task, cotenant_bind) = relay_route_open(
9322 &handler,
9323 cotenant,
9324 &cotenant_ctx.egress,
9325 &mut module_rx,
9326 102,
9327 "aft",
9328 "closing-cotenant",
9329 )
9330 .await;
9331 handler
9332 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9333 .await
9334 .unwrap();
9335 assert!(cotenant_task.await.unwrap().is_empty());
9336 let (cotenant_channel, cotenant_epoch) =
9337 published_route(&cotenant_rx.recv().await.unwrap());
9338 assert!(matches!(
9339 forwarding
9340 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9341 .unwrap(),
9342 DataRoute::Client(DataRouteState::Bound(_))
9343 ));
9344
9345 let goodbye = post_ack_module_frame
9348 .expect("module receives a GOODBYE for the abandoned route channel");
9349 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9350 assert_eq!(goodbye.header.channel, abandoned_channel);
9351 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9352
9353 assert!(dying_rx.try_recv().is_err());
9357 let second_response = second_task.await.unwrap();
9358 assert_eq!(second_response.len(), 1);
9359 assert_eq!(
9360 parse_error(&second_response[0])["code"],
9361 "target_unavailable"
9362 );
9363 }
9364
9365 async fn catalog_snapshot(
9368 handler: &ControlHandler,
9369 client_ctx: &RouteCtx,
9370 corr: u64,
9371 ) -> (u64, Vec<(String, bool)>) {
9372 let body =
9373 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9374 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9375 let reply = handler
9376 .handle_control_frame(client_ctx, request)
9377 .await
9378 .unwrap()
9379 .pop()
9380 .unwrap();
9381 let ClientControlResponse::CatalogList {
9382 generation,
9383 modules,
9384 ..
9385 } = serde_json::from_slice(&reply.body).unwrap()
9386 else {
9387 panic!("not a catalog.list reply");
9388 };
9389 (
9390 generation,
9391 modules
9392 .into_iter()
9393 .map(|entry| (entry.module_id, entry.ready))
9394 .collect(),
9395 )
9396 }
9397
9398 fn hello_held_before_forwarding() -> (
9404 ControlHandler,
9405 Arc<std::sync::Barrier>,
9406 std::thread::JoinHandle<Vec<Frame>>,
9407 RouteCtx,
9408 mpsc::Receiver<crate::router::OutboundFrame>,
9409 ) {
9410 let hold = Arc::new(std::sync::Barrier::new(2));
9411 let handler = ControlHandler::with_forwarding(
9412 Arc::new(Registry::default()),
9413 Arc::new(ForwardingTable::default()),
9414 )
9415 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9416 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9417 let hello = {
9418 let handler = handler.clone();
9419 let module_ctx = module_ctx.clone();
9420 std::thread::spawn(move || {
9421 handler
9422 .handle_hello(
9423 module_ctx.connection_id,
9424 Some(module_ctx.egress.clone()),
9425 hello_frame("aft", PROTOCOL_VERSION, 7),
9426 )
9427 .unwrap()
9428 })
9429 };
9430 (handler, hold, hello, module_ctx, module_rx)
9431 }
9432
9433 #[tokio::test]
9442 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9443 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9444 let client = ConnectionId::new(41);
9445 let (client_ctx, mut client_rx) = route_ctx(client);
9446
9447 hold.wait();
9448 assert!(handler.registry.get_module("aft").unwrap().is_some());
9450 assert!(!handler
9451 .forwarding
9452 .has_live_module_connection("aft")
9453 .unwrap());
9454 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9455 assert!(
9456 listed.is_empty(),
9457 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9458 );
9459 let refused = handler
9460 .handle_control_frame(
9461 &client_ctx,
9462 route_open_frame(301, "aft", unique_project_root("listed-window")),
9463 )
9464 .await
9465 .unwrap();
9466 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9467 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9468 let duplicate = handler
9469 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9470 .await
9471 .unwrap();
9472 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9473
9474 hold.wait();
9475 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9476 let ack = module_rx.try_recv().unwrap().frame;
9477 assert_eq!(ack.header.ty, FrameType::HelloAck);
9478 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9479 assert_eq!(listed, vec![("aft".to_string(), true)]);
9480
9481 let (open_task, bind) = relay_route_open(
9482 &handler,
9483 client,
9484 &client_ctx.egress,
9485 &mut module_rx,
9486 303,
9487 "aft",
9488 "listed-after",
9489 )
9490 .await;
9491 assert!(handler
9492 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9493 .await
9494 .unwrap()
9495 .is_empty());
9496 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9497 published_route(&client_rx.try_recv().unwrap().frame);
9498 }
9499
9500 #[tokio::test]
9507 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9508 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9509 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9510 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9511
9512 hold.wait();
9513 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9514 assert!(listed.is_empty());
9515 assert_eq!(
9516 during, before,
9517 "the generation moved while the module was not listed"
9518 );
9519
9520 hold.wait();
9521 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9522 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9523 assert_eq!(listed, vec![("aft".to_string(), true)]);
9524 assert_eq!(
9525 after,
9526 before + 1,
9527 "the module appeared without exactly one generation bump"
9528 );
9529 }
9530
9531 #[tokio::test]
9536 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
9537 let registry = Arc::new(Registry::default());
9538 let handler = ControlHandler::with_forwarding(
9539 Arc::clone(®istry),
9540 Arc::new(ForwardingTable::default()),
9541 );
9542 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9543 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9544 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9545 drop(module_rx);
9546
9547 let replies = handler
9548 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9549 .await
9550 .unwrap();
9551 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
9552 assert!(registry.get_module("aft").unwrap().is_none());
9553 assert!(registry
9554 .get_module_by_connection(module_ctx.connection_id)
9555 .unwrap()
9556 .is_none());
9557 assert_eq!(
9558 catalog_snapshot(&handler, &client_ctx, 301).await,
9559 (before, Vec::new())
9560 );
9561 }
9562
9563 async fn route_open_held_after_bind_send() -> (
9567 ControlHandler,
9568 Arc<tokio::sync::Barrier>,
9569 RouteCtx,
9570 mpsc::Receiver<crate::router::OutboundFrame>,
9571 ) {
9572 let hold = Arc::new(tokio::sync::Barrier::new(2));
9573 let handler = ControlHandler::with_forwarding(
9574 Arc::new(Registry::default()),
9575 Arc::new(ForwardingTable::default()),
9576 )
9577 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9578 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9579 hello_via_sink(
9580 &handler,
9581 &module_ctx,
9582 &mut module_rx,
9583 hello_frame("aft", PROTOCOL_VERSION, 7),
9584 )
9585 .await;
9586 (handler, hold, module_ctx, module_rx)
9587 }
9588
9589 #[tokio::test]
9596 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9597 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9598 let client = ConnectionId::new(41);
9599 let (client_ctx, mut client_rx) = route_ctx(client);
9600
9601 let (open_task, bind) = relay_route_open(
9602 &handler,
9603 client,
9604 &client_ctx.egress,
9605 &mut module_rx,
9606 200,
9607 "aft",
9608 "acked-before-mark",
9609 )
9610 .await;
9611 let (module_channel, module_epoch) = route_bind_channel(&bind);
9612 hold.wait().await;
9614
9615 assert!(handler
9617 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9618 .await
9619 .unwrap()
9620 .is_empty());
9621 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9622
9623 hold.wait().await;
9624 assert!(
9625 open_task.await.unwrap().is_empty(),
9626 "route.open answers accepted"
9627 );
9628
9629 match module_rx.try_recv() {
9632 Err(mpsc::error::TryRecvError::Empty) => {}
9633 Ok(outbound) => panic!(
9634 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9635 outbound.frame.header.ty,
9636 outbound.frame.header.channel,
9637 outbound.frame.header.epoch
9638 ),
9639 Err(err) => panic!("module egress closed: {err}"),
9640 }
9641
9642 let router = Router::with_control_handler(Arc::new(handler.clone()));
9645 let request = Frame::build(
9646 FrameType::Request,
9647 Flags::new(true, Priority::Interactive, false),
9648 client_channel,
9649 client_epoch,
9650 201,
9651 b"first request".to_vec(),
9652 )
9653 .unwrap();
9654 router
9655 .route_for_connection(&client_ctx, request)
9656 .await
9657 .unwrap();
9658 let delivered = module_rx.try_recv().unwrap().frame;
9659 assert_eq!(delivered.header.ty, FrameType::Request);
9660 assert_eq!(
9661 (delivered.header.channel, delivered.header.epoch),
9662 (module_channel, module_epoch)
9663 );
9664 assert_eq!(delivered.body, b"first request");
9665 }
9666
9667 #[tokio::test]
9673 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9674 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9675 let client = ConnectionId::new(42);
9676 let (client_ctx, _client_rx) = route_ctx(client);
9677
9678 let (open_task, bind) = relay_route_open(
9679 &handler,
9680 client,
9681 &client_ctx.egress,
9682 &mut module_rx,
9683 300,
9684 "aft",
9685 "released-before-mark",
9686 )
9687 .await;
9688 let (module_channel, module_epoch) = route_bind_channel(&bind);
9689 hold.wait().await;
9690
9691 handler.cleanup_connection(client).unwrap();
9692 assert!(
9693 module_rx.try_recv().is_err(),
9694 "client teardown must not GOODBYE a bind route.open is still queueing"
9695 );
9696
9697 hold.wait().await;
9698 let response = open_task.await.unwrap();
9699 assert_eq!(response.len(), 1);
9700 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9701
9702 let goodbye = module_rx
9703 .try_recv()
9704 .expect("module is told to drop the binding the daemon abandoned")
9705 .frame;
9706 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9707 assert_eq!(
9708 (goodbye.header.channel, goodbye.header.epoch),
9709 (module_channel, module_epoch)
9710 );
9711 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9712 }
9713
9714 #[tokio::test]
9719 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
9720 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9721 let client = ConnectionId::new(43);
9722 let (client_ctx, client_rx) = route_ctx(client);
9723
9724 let (open_task, bind) = relay_route_open(
9725 &handler,
9726 client,
9727 &client_ctx.egress,
9728 &mut module_rx,
9729 400,
9730 "aft",
9731 "gone-client-before-mark",
9732 )
9733 .await;
9734 let (module_channel, module_epoch) = route_bind_channel(&bind);
9735 hold.wait().await;
9736
9737 drop(client_rx);
9738 assert!(client_ctx.egress.is_closed());
9739 handler
9740 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9741 .await
9742 .unwrap();
9743
9744 hold.wait().await;
9745 let response = open_task.await.unwrap();
9746 assert_eq!(response.len(), 1);
9747 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9748
9749 let goodbye = module_rx
9750 .try_recv()
9751 .expect("module is told to drop the binding it acked")
9752 .frame;
9753 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9754 assert_eq!(
9755 (goodbye.header.channel, goodbye.header.epoch),
9756 (module_channel, module_epoch)
9757 );
9758 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9759 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
9760 }
9761
9762 #[test]
9769 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9770 let handler = ControlHandler::default();
9771 let module_connection = ConnectionId::new(30);
9772 let client_connection = ConnectionId::new(31);
9773
9774 handler
9775 .refuse_to_end_module_connection_for_a_client(
9776 module_connection,
9777 77,
9778 ForwardingError::ConnectionClosing {
9779 connection_id: client_connection,
9780 },
9781 )
9782 .expect("a closing client must never end the module connection");
9783
9784 assert!(matches!(
9785 handler.refuse_to_end_module_connection_for_a_client(
9786 module_connection,
9787 78,
9788 ForwardingError::ConnectionClosing {
9789 connection_id: module_connection,
9790 },
9791 ),
9792 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9793 connection_id
9794 })) if connection_id == module_connection
9795 ));
9796 assert!(matches!(
9797 handler.refuse_to_end_module_connection_for_a_client(
9798 module_connection,
9799 79,
9800 ForwardingError::Poisoned,
9801 ),
9802 Err(RouterError::Forwarding(ForwardingError::Poisoned))
9803 ));
9804 assert!(matches!(
9805 handler.refuse_to_end_module_connection_for_a_client(
9806 module_connection,
9807 80,
9808 ForwardingError::StaleModuleEndpoint,
9809 ),
9810 Err(RouterError::Forwarding(
9811 ForwardingError::StaleModuleEndpoint
9812 ))
9813 ));
9814 }
9815
9816 #[tokio::test]
9829 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9830 let registry = Arc::new(Registry::default());
9831 let forwarding = Arc::new(ForwardingTable::default());
9832 let supervisor = SupervisorHandle::new();
9833 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9834 let handler =
9835 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9836 .with_supervisor(supervisor);
9837
9838 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9839 hello_via_sink(
9840 &handler,
9841 &target_ctx,
9842 &mut target_rx,
9843 hello_frame("target", PROTOCOL_VERSION, 1),
9844 )
9845 .await;
9846
9847 let wrong_nonce = handler
9851 .handle_control_frame(
9852 &route_ctx(ConnectionId::new(91)).0,
9853 route_open_frame_with_admission_facts(
9854 20,
9855 "target",
9856 unique_project_root("admission-facts"),
9857 Some(subc_control::ConsumerIdentity {
9858 module_id: "fed".to_string(),
9859 launch_nonce: "not-the-real-nonce".to_string(),
9860 }),
9861 None,
9862 ),
9863 )
9864 .await
9865 .unwrap();
9866 assert_eq!(
9867 parse_error(&wrong_nonce[0])["code"],
9868 "bad_consumer_identity",
9869 "a mismatched launch nonce must be refused, not stamped Reserved"
9870 );
9871
9872 let never_spawned = handler
9876 .handle_control_frame(
9877 &route_ctx(ConnectionId::new(92)).0,
9878 route_open_frame_with_admission_facts(
9879 21,
9880 "target",
9881 unique_project_root("admission-facts"),
9882 Some(subc_control::ConsumerIdentity {
9883 module_id: "never-spawned".to_string(),
9884 launch_nonce: "any-nonce".to_string(),
9885 }),
9886 None,
9887 ),
9888 )
9889 .await
9890 .unwrap();
9891 assert_eq!(
9892 parse_error(&never_spawned[0])["code"],
9893 "bad_consumer_identity",
9894 "an unspawned module_id must be refused rather than accepted for lack of a record"
9895 );
9896 }
9897
9898 #[tokio::test]
9921 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
9922 let registry = Arc::new(Registry::default());
9923 let forwarding = Arc::new(ForwardingTable::default());
9924 let supervisor = SupervisorHandle::new();
9925 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9926 let handler =
9927 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9928 .with_supervisor(supervisor);
9929
9930 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
9931 hello_via_sink(
9932 &handler,
9933 &target_ctx,
9934 &mut target_rx,
9935 hello_frame("target", PROTOCOL_VERSION, 1),
9936 )
9937 .await;
9938
9939 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
9940 let route_handler = handler.clone();
9941 let route_task = tokio::spawn(async move {
9942 route_handler
9943 .handle_control_frame(
9944 &client_ctx,
9945 route_open_frame_with_admission_facts(
9946 30,
9947 "target",
9948 unique_project_root("admission-facts"),
9949 Some(subc_control::ConsumerIdentity {
9950 module_id: "fed".to_string(),
9951 launch_nonce: "fed-nonce".to_string(),
9952 }),
9953 None,
9954 ),
9955 )
9956 .await
9957 .unwrap()
9958 });
9959
9960 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9967 .await
9968 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9969 .expect("module control channel closed before route.bind");
9970 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9971 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9972 panic!("expected route.bind")
9973 };
9974 assert_eq!(
9975 principal,
9976 Some(Principal::Reserved {
9977 module_id: "fed".to_string()
9978 }),
9979 "a correctly attested consumer must be stamped Reserved for its own id"
9980 );
9981
9982 handler
9983 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9984 .await
9985 .unwrap();
9986 assert!(route_task.await.unwrap().is_empty());
9987 assert!(
9988 matches!(
9989 serde_json::from_slice::<ClientControlResponse>(
9990 &client_rx.recv().await.unwrap().body
9991 )
9992 .unwrap(),
9993 ClientControlResponse::RouteOpen { .. }
9994 ),
9995 "the route must actually open, not merely avoid an error"
9996 );
9997 }
9998
9999 #[tokio::test(start_paused = true)]
10000 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10001 let registry = Arc::new(Registry::default());
10002 let forwarding = Arc::new(ForwardingTable::default());
10003 let supervisor = SupervisorHandle::new();
10004 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10005 let handler =
10006 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10007 .with_supervisor(supervisor);
10008
10009 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10010 hello_via_sink(
10011 &handler,
10012 &target_ctx,
10013 &mut target_rx,
10014 hello_frame("target", PROTOCOL_VERSION, 1),
10015 )
10016 .await;
10017
10018 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10019 let direct_handler = handler.clone();
10020 let direct_open = tokio::spawn(async move {
10021 direct_handler
10022 .handle_control_frame(
10023 &direct_ctx,
10024 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10025 )
10026 .await
10027 .unwrap()
10028 });
10029 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10030 .await
10031 .expect("no direct route.bind within 5s")
10032 .expect("target control channel closed before direct route.bind");
10033 handler
10034 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10035 .await
10036 .unwrap();
10037 assert!(direct_open.await.unwrap().is_empty());
10038 let _ = direct_rx.recv().await.unwrap();
10039
10040 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10041 let reserved_handler = handler.clone();
10042 let reserved_open = tokio::spawn(async move {
10043 reserved_handler
10044 .handle_control_frame(
10045 &reserved_ctx,
10046 route_open_frame_with_admission_facts(
10047 3,
10048 "target",
10049 unique_project_root("admission-facts"),
10050 Some(ConsumerIdentity {
10051 module_id: "fed".to_string(),
10052 launch_nonce: "fed-nonce".to_string(),
10053 }),
10054 None,
10055 ),
10056 )
10057 .await
10058 .unwrap()
10059 });
10060 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10061 .await
10062 .expect("no reserved route.bind within 5s")
10063 .expect("target control channel closed before reserved route.bind");
10064 handler
10065 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10066 .await
10067 .unwrap();
10068 assert!(reserved_open.await.unwrap().is_empty());
10069 let _ = reserved_rx.recv().await.unwrap();
10070
10071 forwarding
10072 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10073 .unwrap();
10074 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10075 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10076 module_id: Some("target".to_string()),
10077 })
10078 .unwrap();
10079 let census_frame =
10080 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10081 let response = handler
10082 .handle_control_frame(&census_ctx, census_frame)
10083 .await
10084 .unwrap()
10085 .pop()
10086 .unwrap();
10087 let actual: Value = serde_json::from_slice(&response.body).unwrap();
10088 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10089 assert!(matches!(
10090 decoded,
10091 ClientControlResponse::SupervisorRoutes { .. }
10092 ));
10093 let routes = actual["modules"][0]["routes"].as_array().unwrap();
10094 assert_eq!(routes.len(), 2);
10095 assert!(routes.iter().all(|route| route["draining"] == true));
10096 assert!(
10099 routes.iter().all(|route| route["drain_reason"] == "reload"),
10100 "draining routes must name the drain's reason: {routes:?}"
10101 );
10102 assert!(routes.iter().any(|route| {
10103 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10104 }));
10105 assert!(routes.iter().any(|route| {
10106 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10107 }));
10108
10109 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10110 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10111 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10112 std::fs::write(
10113 &golden_path,
10114 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10115 )
10116 .unwrap();
10117 }
10118 let expected: Value =
10119 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10120 assert_eq!(actual, expected);
10121 }
10122
10123 async fn query_live_roots(
10124 handler: &ControlHandler,
10125 module_ctx: &RouteCtx,
10126 ) -> ModuleControlResponseToModule {
10127 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10128 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10129 let response = handler
10130 .handle_control_frame(module_ctx, frame)
10131 .await
10132 .unwrap()
10133 .pop()
10134 .unwrap();
10135 serde_json::from_slice(&response.body).unwrap()
10136 }
10137
10138 #[tokio::test(start_paused = true)]
10139 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10140 let registry = Arc::new(Registry::default());
10141 let forwarding = Arc::new(ForwardingTable::default());
10142 let handler = ControlHandler::with_forwarding(registry, forwarding);
10143 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10144 hello_via_sink(
10145 &handler,
10146 &target_ctx,
10147 &mut target_rx,
10148 hello_frame("target", PROTOCOL_VERSION, 1),
10149 )
10150 .await;
10151 let root = unique_project_root("live-roots-known");
10152 let path = ProjectRootId::from_path_allowing_missing(root.path())
10153 .unwrap()
10154 .as_path()
10155 .to_path_buf();
10156 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10157 let open_handler = handler.clone();
10158 let opened = tokio::spawn(async move {
10159 open_handler
10160 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10161 .await
10162 .unwrap()
10163 });
10164 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10165 .await
10166 .unwrap()
10167 .unwrap();
10168 handler
10169 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10170 .await
10171 .unwrap();
10172 assert!(opened.await.unwrap().is_empty());
10173 let _ = client_rx.recv().await.unwrap();
10174
10175 let root = unique_project_root("live-roots-pending");
10176 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10177 .unwrap()
10178 .as_path()
10179 .to_path_buf();
10180 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10181 let open_handler = handler.clone();
10182 let pending = tokio::spawn(async move {
10183 open_handler
10184 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10185 .await
10186 .unwrap()
10187 });
10188 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10189 .await
10190 .unwrap()
10191 .unwrap();
10192 let actual = query_live_roots(&handler, &target_ctx).await;
10193 let ModuleControlResponseToModule::LiveRoots {
10194 roots,
10195 unknown_root_bindings,
10196 total_bindings,
10197 } = actual
10198 else {
10199 panic!("expected live roots")
10200 };
10201 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10202 assert_eq!(unknown_root_bindings, 0);
10203 assert_eq!(
10204 roots.len(),
10205 2,
10206 "root-known arm must retain each canonical root"
10207 );
10208 assert_eq!(
10209 total_bindings,
10210 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10211 );
10212 let counts = roots
10213 .iter()
10214 .map(|root| (root.project_root.clone(), root.bound, root.pending))
10215 .collect::<Vec<_>>();
10216 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10217 expected.sort_by(|a, b| a.0.cmp(&b.0));
10218 assert_eq!(
10219 counts, expected,
10220 "roots must sort by path and count pending separately"
10221 );
10222 handler
10223 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10224 .await
10225 .unwrap();
10226 assert!(pending.await.unwrap().is_empty());
10227 }
10228
10229 #[tokio::test(start_paused = true)]
10230 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10231 let forwarding = Arc::new(ForwardingTable::default());
10232 let handler =
10233 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10234 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10235 hello_via_sink(
10236 &handler,
10237 &target_ctx,
10238 &mut target_rx,
10239 hello_frame("target", PROTOCOL_VERSION, 1),
10240 )
10241 .await;
10242 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10243 let pending = forwarding
10244 .begin_route_bind_relay_for_test(
10245 client_ctx.connection_id,
10246 client_ctx.egress.clone(),
10247 2,
10248 "target",
10249 )
10250 .unwrap();
10251 forwarding
10252 .complete_pending_relay(
10253 target_ctx.connection_id,
10254 pending.corr,
10255 RouteBindRelayOutcome::Accepted,
10256 )
10257 .unwrap();
10258 let actual = query_live_roots(&handler, &target_ctx).await;
10259 let ModuleControlResponseToModule::LiveRoots {
10260 roots,
10261 unknown_root_bindings,
10262 total_bindings,
10263 } = actual
10264 else {
10265 panic!("expected live roots")
10266 };
10267 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10268 assert_eq!(
10269 unknown_root_bindings, 1,
10270 "unknown-root arm must not read as no bindings"
10271 );
10272 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10273 assert_eq!(
10274 total_bindings,
10275 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10276 );
10277 }
10278
10279 #[tokio::test(start_paused = true)]
10287 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10288 let forwarding = Arc::new(ForwardingTable::default());
10289 let handler =
10290 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10291 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10292 let replies = handler
10293 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10294 .await
10295 .unwrap();
10296 let queued_by_hello = module_rx.len();
10297
10298 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10299 let open_handler = handler.clone();
10300 let open = tokio::spawn(async move {
10301 open_handler
10302 .handle_control_frame(
10303 &client_ctx,
10304 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10305 )
10306 .await
10307 .unwrap()
10308 });
10309 let mut spins = 0;
10311 while module_rx.len() == queued_by_hello {
10312 spins += 1;
10313 assert!(spins < 10_000, "route.open never queued a route.bind");
10314 tokio::task::yield_now().await;
10315 }
10316
10317 for reply in replies {
10319 module_ctx.egress.send(reply).await.unwrap();
10320 }
10321
10322 let first = module_rx.recv().await.unwrap().frame;
10323 assert_eq!(
10324 first.header.ty,
10325 FrameType::HelloAck,
10326 "the first frame a registering module reads must be its HELLO_ACK"
10327 );
10328 assert_eq!(first.header.corr, 7);
10329 let second = module_rx.recv().await.unwrap().frame;
10330 assert_eq!(second.header.ty, FrameType::Request);
10331 assert!(
10332 matches!(
10333 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10334 ModuleControlRequest::RouteBind { .. }
10335 ),
10336 "the route.bind follows the ack"
10337 );
10338 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10339
10340 handler
10341 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10342 .await
10343 .unwrap();
10344 assert!(open.await.unwrap().is_empty());
10345 let _ = client_rx.recv().await.unwrap();
10346 }
10347
10348 #[tokio::test(start_paused = true)]
10349 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10350 let handler = ControlHandler::with_forwarding(
10351 Arc::new(Registry::default()),
10352 Arc::new(ForwardingTable::default()),
10353 );
10354 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10355 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10356 hello_via_sink(
10357 &handler,
10358 &first_ctx,
10359 &mut first_rx,
10360 hello_frame("first", PROTOCOL_VERSION, 1),
10361 )
10362 .await;
10363 hello_via_sink(
10364 &handler,
10365 &second_ctx,
10366 &mut second_rx,
10367 hello_frame("second", PROTOCOL_VERSION, 2),
10368 )
10369 .await;
10370 let root = unique_project_root("second-only");
10371 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10372 let cloned = handler.clone();
10373 let open = tokio::spawn(async move {
10374 cloned
10375 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10376 .await
10377 .unwrap()
10378 });
10379 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10380 .await
10381 .unwrap()
10382 .unwrap();
10383 let first = query_live_roots(&handler, &first_ctx).await;
10384 let second = query_live_roots(&handler, &second_ctx).await;
10385 assert!(
10386 matches!(
10387 first,
10388 ModuleControlResponseToModule::LiveRoots {
10389 total_bindings: 0,
10390 ..
10391 }
10392 ),
10393 "cross-module scope must not expose another module's roots"
10394 );
10395 assert!(
10396 matches!(
10397 second,
10398 ModuleControlResponseToModule::LiveRoots {
10399 total_bindings: 1,
10400 ..
10401 }
10402 ),
10403 "second module must see its pending route"
10404 );
10405 handler
10406 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10407 .await
10408 .unwrap();
10409 assert!(open.await.unwrap().is_empty());
10410 }
10411
10412 #[tokio::test(start_paused = true)]
10413 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10414 let handler = ControlHandler::with_forwarding(
10415 Arc::new(Registry::default()),
10416 Arc::new(ForwardingTable::default()),
10417 );
10418 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10419 hello_via_sink(
10420 &handler,
10421 &target_ctx,
10422 &mut target_rx,
10423 hello_frame("target", PROTOCOL_VERSION, 1),
10424 )
10425 .await;
10426 let actual = query_live_roots(&handler, &target_ctx).await;
10427 let ModuleControlResponseToModule::LiveRoots {
10428 roots,
10429 unknown_root_bindings,
10430 total_bindings,
10431 } = actual
10432 else {
10433 panic!("expected live roots")
10434 };
10435 assert!(roots.is_empty());
10436 assert_eq!(unknown_root_bindings, 0);
10437 assert_eq!(total_bindings, 0);
10438 assert_eq!(
10439 total_bindings,
10440 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10441 );
10442 }
10443
10444 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10451 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10452 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10453 let text = std::fs::read_to_string(&path)
10454 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10455 let vectors: Vec<(String, Value)> = text
10456 .lines()
10457 .filter(|line| !line.trim().is_empty())
10458 .map(|line| {
10459 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10460 let id = entry["corpus_id"]
10461 .as_str()
10462 .expect("every vector carries a corpus_id")
10463 .to_string();
10464 (id, entry["package"].clone())
10465 })
10466 .collect();
10467 assert_eq!(
10471 vectors.len(),
10472 3,
10473 "vendored fed corpus changed size; re-sync from subc-federation"
10474 );
10475
10476 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10491 let richest = vectors
10492 .iter()
10493 .filter_map(|(_, package)| package.as_object())
10494 .map(|object| {
10495 object
10496 .keys()
10497 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10498 .count()
10499 })
10500 .max()
10501 .unwrap_or(0);
10502 assert!(
10503 richest >= 2,
10504 "vendored corpus no longer carries a package with unmodelled fields, \
10505 so the relay test can no longer distinguish a verbatim relay from a lossy one"
10506 );
10507
10508 vectors
10509 }
10510
10511 #[tokio::test]
10519 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10520 for (corpus_id, package) in fed_admission_facts_vectors() {
10521 let registry = Arc::new(Registry::default());
10522 let forwarding = Arc::new(ForwardingTable::default());
10523 let supervisor = SupervisorHandle::new();
10524 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10525 let handler =
10526 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10527 .with_supervisor(supervisor)
10528 .with_admission_facts_config(
10529 Some("fed".to_string()),
10530 Some(vec!["target".to_string()]),
10531 );
10532
10533 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10534 hello_via_sink(
10535 &handler,
10536 &target_ctx,
10537 &mut target_rx,
10538 hello_frame("target", PROTOCOL_VERSION, 1),
10539 )
10540 .await;
10541
10542 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10543 let route_handler = handler.clone();
10544 let expected = package.clone();
10545 let route_task = tokio::spawn(async move {
10546 route_handler
10547 .handle_control_frame(
10548 &client_ctx,
10549 route_open_frame_with_admission_facts(
10550 20,
10551 "target",
10552 unique_project_root("admission-facts"),
10553 Some(subc_control::ConsumerIdentity {
10554 module_id: "fed".to_string(),
10555 launch_nonce: "fed-nonce".to_string(),
10556 }),
10557 Some(package),
10558 ),
10559 )
10560 .await
10561 .unwrap()
10562 });
10563
10564 let bind_frame = target_rx.recv().await.unwrap();
10565 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10566 let ModuleControlRequest::RouteBind {
10567 admission_facts, ..
10568 } = bind
10569 else {
10570 panic!("{corpus_id}: expected route.bind")
10571 };
10572 assert_eq!(
10573 admission_facts,
10574 Some(expected),
10575 "{corpus_id}: relay must not add, drop or reshape any field"
10576 );
10577
10578 handler
10579 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10580 .await
10581 .unwrap();
10582 route_task.await.unwrap();
10583 }
10584 }
10585
10586 #[tokio::test]
10587 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10588 let registry = Arc::new(Registry::default());
10589 let forwarding = Arc::new(ForwardingTable::default());
10590 let supervisor = SupervisorHandle::new();
10591 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10592 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10593 let handler =
10594 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10595 .with_supervisor(supervisor)
10596 .with_admission_facts_config(
10597 Some("fed".to_string()),
10598 Some(vec!["target".to_string()]),
10599 );
10600
10601 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10602 hello_via_sink(
10603 &handler,
10604 &target_ctx,
10605 &mut target_rx,
10606 hello_frame("target", PROTOCOL_VERSION, 1),
10607 )
10608 .await;
10609 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10610 hello_via_sink(
10611 &handler,
10612 &other_ctx,
10613 &mut other_rx,
10614 hello_frame("other", PROTOCOL_VERSION, 2),
10615 )
10616 .await;
10617
10618 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10619 let expected_facts = facts.clone();
10620 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10621 let route_handler = handler.clone();
10622 let route_task = tokio::spawn(async move {
10623 route_handler
10624 .handle_control_frame(
10625 &client_ctx,
10626 route_open_frame_with_admission_facts(
10627 10,
10628 "target",
10629 unique_project_root("admission-facts"),
10630 Some(subc_control::ConsumerIdentity {
10631 module_id: "fed".to_string(),
10632 launch_nonce: "fed-nonce".to_string(),
10633 }),
10634 Some(facts.clone()),
10635 ),
10636 )
10637 .await
10638 .unwrap()
10639 });
10640 let bind_frame = target_rx.recv().await.unwrap();
10641 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10642 let ModuleControlRequest::RouteBind {
10643 admission_facts, ..
10644 } = bind
10645 else {
10646 panic!("expected route.bind")
10647 };
10648 assert_eq!(admission_facts, Some(expected_facts));
10649 handler
10650 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10651 .await
10652 .unwrap();
10653 assert!(route_task.await.unwrap().is_empty());
10654 assert!(matches!(
10655 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10656 .unwrap(),
10657 ClientControlResponse::RouteOpen { .. }
10658 ));
10659
10660 let direct = handler
10661 .handle_control_frame(
10662 &route_ctx(ConnectionId::new(73)).0,
10663 route_open_frame_with_admission_facts(
10664 11,
10665 "target",
10666 unique_project_root("admission-facts"),
10667 None,
10668 Some(json!({"x": 1})),
10669 ),
10670 )
10671 .await
10672 .unwrap();
10673 assert_eq!(
10674 parse_error(&direct[0])["code"],
10675 "admission_facts_not_permitted"
10676 );
10677
10678 let different_reserved = handler
10679 .handle_control_frame(
10680 &route_ctx(ConnectionId::new(77)).0,
10681 route_open_frame_with_admission_facts(
10682 15,
10683 "target",
10684 unique_project_root("admission-facts"),
10685 Some(subc_control::ConsumerIdentity {
10686 module_id: "other".to_string(),
10687 launch_nonce: "other-nonce".to_string(),
10688 }),
10689 Some(json!({"x": 1})),
10690 ),
10691 )
10692 .await
10693 .unwrap();
10694 assert_eq!(
10695 parse_error(&different_reserved[0])["code"],
10696 "admission_facts_not_permitted"
10697 );
10698
10699 let other_target = handler
10700 .handle_control_frame(
10701 &route_ctx(ConnectionId::new(74)).0,
10702 route_open_frame_with_admission_facts(
10703 12,
10704 "other",
10705 unique_project_root("admission-facts"),
10706 Some(subc_control::ConsumerIdentity {
10707 module_id: "fed".to_string(),
10708 launch_nonce: "fed-nonce".to_string(),
10709 }),
10710 Some(json!({"x": 1})),
10711 ),
10712 )
10713 .await
10714 .unwrap();
10715 assert_eq!(
10716 parse_error(&other_target[0])["code"],
10717 "admission_facts_target_not_allowed"
10718 );
10719
10720 let nonexistent = handler
10721 .handle_control_frame(
10722 &route_ctx(ConnectionId::new(75)).0,
10723 route_open_frame_with_admission_facts(
10724 13,
10725 "missing",
10726 unique_project_root("admission-facts"),
10727 None,
10728 Some(json!({"x": 1})),
10729 ),
10730 )
10731 .await
10732 .unwrap();
10733 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
10734
10735 let described = handler
10736 .handle_control_frame(
10737 &route_ctx(ConnectionId::new(76)).0,
10738 Frame::build(
10739 FrameType::Request,
10740 control_flags(),
10741 0,
10742 0,
10743 14,
10744 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10745 )
10746 .unwrap(),
10747 )
10748 .await
10749 .unwrap();
10750 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10751 serde_json::from_slice(&described[0].body).unwrap()
10752 else {
10753 panic!("expected server.describe response")
10754 };
10755 assert!(capabilities
10756 .iter()
10757 .any(|cap| cap == "admission_facts_relay_v1"));
10758 }
10759
10760 #[tokio::test]
10761 async fn admission_facts_without_configured_carrier_are_rejected() {
10762 let registry = Arc::new(Registry::default());
10763 let forwarding = Arc::new(ForwardingTable::default());
10764 let handler = ControlHandler::with_forwarding(registry, forwarding);
10765 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10766 hello_via_sink(
10767 &handler,
10768 &target_ctx,
10769 &mut target_rx,
10770 hello_frame("target", PROTOCOL_VERSION, 1),
10771 )
10772 .await;
10773
10774 let responses = handler
10775 .handle_control_frame(
10776 &route_ctx(ConnectionId::new(79)).0,
10777 route_open_frame_with_admission_facts(
10778 16,
10779 "target",
10780 unique_project_root("admission-facts"),
10781 None,
10782 Some(json!({"x": 1})),
10783 ),
10784 )
10785 .await
10786 .unwrap();
10787 assert_eq!(
10788 parse_error(&responses[0])["code"],
10789 "admission_facts_not_permitted"
10790 );
10791 }
10792
10793 #[tokio::test]
10794 async fn route_open_relays_consumer_capabilities_verbatim() {
10795 let registry = Arc::new(Registry::default());
10796 let forwarding = Arc::new(ForwardingTable::default());
10797 let handler =
10798 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10799 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10800 hello_via_sink(
10801 &handler,
10802 &module_ctx,
10803 &mut module_rx,
10804 hello_frame("aft", PROTOCOL_VERSION, 7),
10805 )
10806 .await;
10807
10808 let expected = vec!["elicitation".to_string(), "roots".to_string()];
10809 let expected_for_request = expected.clone();
10810 let project_root = unique_project_root("consumer-capabilities-present");
10811 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10812 let route_handler = handler.clone();
10813 let route_task = tokio::spawn(async move {
10814 route_handler
10815 .handle_control_frame(
10816 &client_ctx,
10817 route_open_frame_with_consumer_capabilities(
10818 401,
10819 "aft",
10820 project_root,
10821 Some(expected_for_request),
10822 ),
10823 )
10824 .await
10825 .unwrap()
10826 });
10827 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10828 .await
10829 .unwrap()
10830 .unwrap();
10831 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10832 let ModuleControlRequest::RouteBind {
10833 consumer_capabilities,
10834 ..
10835 } = bind
10836 else {
10837 panic!("expected route.bind request, got {bind:?}");
10838 };
10839 assert_eq!(consumer_capabilities, Some(expected.clone()));
10840
10841 handler
10842 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10843 .await
10844 .unwrap();
10845 let route_response = route_task.await.unwrap();
10846 assert!(route_response.is_empty());
10847 let published = client_rx.recv().await.unwrap();
10848 assert!(matches!(
10849 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10850 ClientControlResponse::RouteOpen { .. }
10851 ));
10852 }
10853
10854 #[tokio::test]
10855 async fn route_open_without_consumer_capabilities_relays_none() {
10856 let registry = Arc::new(Registry::default());
10857 let forwarding = Arc::new(ForwardingTable::default());
10858 let handler =
10859 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10860 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10861 hello_via_sink(
10862 &handler,
10863 &module_ctx,
10864 &mut module_rx,
10865 hello_frame("aft", PROTOCOL_VERSION, 7),
10866 )
10867 .await;
10868
10869 let project_root = unique_project_root("consumer-capabilities-absent");
10870 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10871 let route_handler = handler.clone();
10872 let route_task = tokio::spawn(async move {
10873 route_handler
10874 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
10875 .await
10876 .unwrap()
10877 });
10878 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10879 .await
10880 .unwrap()
10881 .unwrap();
10882 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10883 let ModuleControlRequest::RouteBind {
10884 consumer_capabilities,
10885 ..
10886 } = bind
10887 else {
10888 panic!("expected route.bind request, got {bind:?}");
10889 };
10890 assert_eq!(consumer_capabilities, None);
10891
10892 handler
10893 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10894 .await
10895 .unwrap();
10896 let route_response = route_task.await.unwrap();
10897 assert!(route_response.is_empty());
10898 let published = client_rx.recv().await.unwrap();
10899 assert!(matches!(
10900 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10901 ClientControlResponse::RouteOpen { .. }
10902 ));
10903 }
10904
10905 async fn bind_role_versions_for(
10909 sent: Option<BTreeMap<String, String>>,
10910 connection: u64,
10911 ) -> Option<BTreeMap<String, String>> {
10912 let registry = Arc::new(Registry::default());
10913 let forwarding = Arc::new(ForwardingTable::default());
10914 let handler =
10915 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10916 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
10917 hello_via_sink(
10918 &handler,
10919 &module_ctx,
10920 &mut module_rx,
10921 hello_frame("aft", PROTOCOL_VERSION, 7),
10922 )
10923 .await;
10924 let project_root = unique_project_root("role-versions");
10925 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
10926 let route_handler = handler.clone();
10927 let route_task = tokio::spawn(async move {
10928 route_handler
10929 .handle_control_frame(
10930 &client_ctx,
10931 route_open_frame_with_role_versions(403, "aft", project_root, sent),
10932 )
10933 .await
10934 .unwrap()
10935 });
10936 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10937 .await
10938 .expect("a well-formed route.open reaches the module as a bind")
10939 .unwrap();
10940 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10941 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
10942 panic!("expected route.bind request, got {bind:?}");
10943 };
10944 handler
10945 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10946 .await
10947 .unwrap();
10948 assert!(route_task.await.unwrap().is_empty());
10949 let published = client_rx.recv().await.unwrap();
10950 assert!(matches!(
10951 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10952 ClientControlResponse::RouteOpen { .. }
10953 ));
10954 role_versions
10955 }
10956
10957 #[tokio::test]
10958 async fn route_open_relays_role_versions_verbatim() {
10959 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10960 assert_eq!(
10961 bind_role_versions_for(Some(sent.clone()), 141).await,
10962 Some(sent)
10963 );
10964 }
10965
10966 #[tokio::test]
10969 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10970 assert_eq!(bind_role_versions_for(None, 143).await, None);
10971 assert_eq!(
10972 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10973 None
10974 );
10975 }
10976
10977 #[tokio::test]
10978 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10979 let registry = Arc::new(Registry::default());
10980 let forwarding = Arc::new(ForwardingTable::default());
10981 let handler =
10982 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10983 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10984 hello_via_sink(
10985 &handler,
10986 &module_ctx,
10987 &mut module_rx,
10988 hello_frame("aft", PROTOCOL_VERSION, 7),
10989 )
10990 .await;
10991
10992 let nine: BTreeMap<String, String> = (0..9)
10993 .map(|index| (format!("role-{index}"), "v1".to_string()))
10994 .collect();
10995 for (label, malformed) in [
10996 (
10997 "invalid role name",
10998 role_versions(&[("Tool_Provider", "v1")]),
10999 ),
11000 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11001 ("nine entries", nine),
11002 ] {
11003 let responses = tokio::time::timeout(
11006 Duration::from_secs(1),
11007 handler.handle_control_frame(
11008 &route_ctx(ConnectionId::new(148)).0,
11009 route_open_frame_with_role_versions(
11010 404,
11011 "aft",
11012 unique_project_root("role-versions-malformed"),
11013 Some(malformed),
11014 ),
11015 ),
11016 )
11017 .await
11018 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11019 .unwrap();
11020 assert_eq!(responses.len(), 1, "{label}");
11021 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11022 let error = parse_error(&responses[0]);
11023 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11024 assert_eq!(
11025 error["detail"]["field"], "role_versions",
11026 "{label}: {error}"
11027 );
11028 assert!(
11029 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11030 "{label}: a malformed declaration is terminal"
11031 );
11032 assert!(
11033 module_rx.try_recv().is_err(),
11034 "{label}: the module must never see a bind"
11035 );
11036 }
11037 }
11038
11039 #[tokio::test]
11043 async fn route_role_versions_capability_is_advertised() {
11044 let handler = ControlHandler::new(Arc::new(Registry::default()));
11045 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11046 let ack = hello_via_sink(
11047 &handler,
11048 &ctx,
11049 &mut rx,
11050 hello_frame("m", PROTOCOL_VERSION, 1),
11051 )
11052 .await;
11053 let ack = parse_ack(&ack);
11054 assert!(
11055 ack.subc_capabilities
11056 .iter()
11057 .any(|c| c == "route-role-versions/v1"),
11058 "{:?}",
11059 ack.subc_capabilities
11060 );
11061
11062 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11063 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11064 let reply = handler
11065 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11066 .await
11067 .unwrap()
11068 .pop()
11069 .unwrap();
11070 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11071 serde_json::from_slice(&reply.body).unwrap()
11072 else {
11073 panic!("not a server.describe reply");
11074 };
11075 assert!(
11076 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11077 "{capabilities:?}"
11078 );
11079 }
11080
11081 #[tokio::test]
11085 async fn catalog_events_capability_is_advertised() {
11086 let handler = ControlHandler::new(Arc::new(Registry::default()));
11087 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11088 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11089 let reply = handler
11090 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11091 .await
11092 .unwrap()
11093 .pop()
11094 .unwrap();
11095 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11096 serde_json::from_slice(&reply.body).unwrap()
11097 else {
11098 panic!("not a server.describe reply");
11099 };
11100 assert!(
11101 capabilities
11102 .iter()
11103 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11104 "{capabilities:?}"
11105 );
11106 }
11107
11108 #[tokio::test]
11109 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11110 let handler = ControlHandler::new(Arc::new(Registry::default()));
11111 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11112 let ack = hello_via_sink(
11113 &handler,
11114 &module,
11115 &mut module_rx,
11116 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11117 )
11118 .await;
11119 let ack = parse_ack(&ack);
11120 assert!(
11121 ack.subc_capabilities
11122 .iter()
11123 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11124 "{:?}",
11125 ack.subc_capabilities
11126 );
11127
11128 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11129 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11130 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11131 let reply = handler
11132 .handle_control_frame(&client, frame)
11133 .await
11134 .unwrap()
11135 .pop()
11136 .unwrap();
11137 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11138 serde_json::from_slice(&reply.body).unwrap()
11139 else {
11140 panic!("not a server.describe reply");
11141 };
11142 assert!(
11143 capabilities
11144 .iter()
11145 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11146 "{capabilities:?}"
11147 );
11148 }
11149
11150 #[tokio::test]
11151 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11152 let registry = Arc::new(Registry::default());
11153 let forwarding = Arc::new(ForwardingTable::default());
11154 let handler =
11155 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11156 .with_health_probe_timeout(Duration::from_secs(5));
11157 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11158 hello_via_sink(
11159 &handler,
11160 &module_ctx,
11161 &mut module_rx,
11162 non_routable_hello_frame_with_control_ops(
11163 "mcp",
11164 300,
11165 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11166 ),
11167 )
11168 .await;
11169 assert!(registry
11170 .get_module("mcp")
11171 .unwrap()
11172 .unwrap()
11173 .manifest
11174 .provides
11175 .is_empty());
11176
11177 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11178 let route_response = handler
11179 .handle_control_frame(
11180 &route_client_ctx,
11181 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11182 )
11183 .await
11184 .unwrap();
11185 assert_eq!(route_response[0].header.ty, FrameType::Error);
11186 assert_eq!(
11187 parse_error(&route_response[0])["code"],
11188 "target_unavailable"
11189 );
11190 assert!(parse_error(&route_response[0])["message"]
11191 .as_str()
11192 .unwrap()
11193 .contains("does not provide the requested target"));
11194 assert!(module_rx.try_recv().is_err());
11195
11196 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11197 let health_handler = handler.clone();
11198 let health_task = tokio::spawn(async move {
11199 health_handler
11200 .handle_control_frame(
11201 &health_client_ctx,
11202 supervisor_health_probe_frame(302, "mcp"),
11203 )
11204 .await
11205 .unwrap()
11206 });
11207 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11208 .await
11209 .unwrap()
11210 .unwrap();
11211 assert_eq!(
11212 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11213 ModuleControlRequest::HealthCheck {}
11214 );
11215 handler
11216 .handle_control_frame(
11217 &module_ctx,
11218 health_response(health_frame.header.corr, HealthStatus::Ok),
11219 )
11220 .await
11221 .unwrap();
11222 let health_response = health_task.await.unwrap();
11223 assert_eq!(health_response[0].header.ty, FrameType::Response);
11224 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11225 ClientControlResponse::SupervisorHealthProbe {
11226 module_id, status, ..
11227 } => {
11228 assert_eq!(module_id, "mcp");
11229 assert_eq!(status, HealthStatus::Ok);
11230 }
11231 other => panic!("unexpected health response: {other:?}"),
11232 }
11233
11234 forwarding
11239 .cleanup_connection(module_ctx.connection_id)
11240 .unwrap();
11241 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11242 let cleanup_response = tokio::time::timeout(
11243 Duration::from_millis(200),
11244 handler.handle_control_frame(
11245 &cleanup_probe_ctx,
11246 supervisor_health_probe_frame(303, "mcp"),
11247 ),
11248 )
11249 .await
11250 .expect("probe should fail immediately when the control lane is gone")
11251 .unwrap();
11252 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11253 assert_eq!(
11254 parse_error(&cleanup_response[0])["code"],
11255 "target_unavailable"
11256 );
11257 assert!(parse_error(&cleanup_response[0])["message"]
11258 .as_str()
11259 .unwrap()
11260 .contains("no module connection"));
11261
11262 handler
11263 .cleanup_connection(module_ctx.connection_id)
11264 .unwrap();
11265 }
11266
11267 #[tokio::test]
11268 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11269 let registry = Arc::new(Registry::default());
11270 let supervisor_handle = SupervisorHandle::new();
11271 let supervisor =
11272 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11273 .with_handle(supervisor_handle.clone())
11274 .with_connection_file_path(
11275 std::env::temp_dir()
11276 .join(format!("subc-route-open-warming-{}", std::process::id())),
11277 );
11278 let module = supervisor
11279 .supervise_configured(
11280 ModuleSpec {
11281 module_id: "warming".to_string(),
11282 program: fake_aft_stub_path(),
11283 args: Vec::new(),
11284 env: Vec::new(),
11285 reserved: false,
11286 reserved_prefixes: Vec::new(),
11287 protocol: ModuleProtocol::Subc,
11288 overlap: Default::default(),
11289 },
11290 true,
11291 )
11292 .unwrap();
11293 assert_eq!(module.state().unwrap(), ModuleState::Running);
11294
11295 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11296 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11297 let response = handler
11298 .handle_control_frame(
11299 &ctx,
11300 route_open_frame(304, "warming", unique_project_root("warming")),
11301 )
11302 .await
11303 .unwrap();
11304 module.stop().await.unwrap();
11305
11306 assert_eq!(response[0].header.ty, FrameType::Error);
11307 let error = parse_error(&response[0]);
11308 assert_eq!(error["code"], "module_warming");
11309 assert!(error["message"]
11310 .as_str()
11311 .unwrap()
11312 .contains("state=running, enabled=true, live=false"));
11313 }
11314
11315 #[test]
11316 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11317 let handler = ControlHandler::new(Arc::new(Registry::default()));
11318 let capture = EventCapture::default();
11319 let _subscriber =
11320 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11321 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11322 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11323 let pending = (0..limit).collect::<Vec<_>>();
11324 let response = handler
11325 .route_open_capacity_refusal(
11326 &ctx,
11327 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11328 "busy",
11329 pending.len(),
11330 limit,
11331 )
11332 .unwrap();
11333 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11334 let event = capture
11335 .events()
11336 .into_iter()
11337 .find(|event| {
11338 event.target == "control"
11339 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11340 })
11341 .expect("connection admission refusal event");
11342 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11343 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11344 }
11345
11346 #[test]
11347 fn route_open_target_cap_logs_admission_reason_and_capacity() {
11348 let handler = ControlHandler::new(Arc::new(Registry::default()));
11349 let capture = EventCapture::default();
11350 let _subscriber =
11351 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11352 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11353 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11354 let guards = (0..limit)
11355 .map(|_| {
11356 handler
11357 .route_bind_concurrency
11358 .try_admit("busy", limit)
11359 .unwrap()
11360 })
11361 .collect::<Vec<_>>();
11362 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11363 Err(in_flight) => in_flight,
11364 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11365 };
11366 let response = handler
11367 .route_open_target_capacity_refusal(
11368 &ctx,
11369 &route_open_frame(397, "busy", unique_project_root("target-cap")),
11370 "busy",
11371 in_flight,
11372 )
11373 .unwrap();
11374 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11375 let event = capture
11376 .events()
11377 .into_iter()
11378 .find(|event| {
11379 event.target == "control"
11380 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11381 })
11382 .expect("target admission refusal event");
11383 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11384 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11385 drop(guards);
11386 }
11387
11388 #[tokio::test(flavor = "current_thread")]
11389 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11390 let handler = ControlHandler::new(Arc::new(Registry::default()));
11391 let capture = EventCapture::default();
11392 let _subscriber =
11393 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11394 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11395 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11396
11397 let _ = handler
11398 .handle_client_control_request(
11399 &ctx,
11400 frame,
11401 ClientControlRequest::SupervisorSetEnabled {
11402 module_id: "broca".to_string(),
11403 enabled: false,
11404 },
11405 )
11406 .await
11407 .unwrap();
11408
11409 let events = capture
11410 .events()
11411 .into_iter()
11412 .filter(|event| {
11413 event.target == "control"
11414 && event.fields.get("message").map(String::as_str)
11415 == Some("supervisor request received")
11416 })
11417 .collect::<Vec<_>>();
11418 assert_eq!(events.len(), 1, "one supervisor request log line");
11419 let event = &events[0];
11420 assert_eq!(event.level, tracing::Level::INFO);
11421 assert_eq!(
11422 event.fields.get("op"),
11423 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11424 );
11425 assert_eq!(
11426 event.fields.get("module_id"),
11427 Some(&"\"broca\"".to_string())
11428 );
11429 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11430 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11431 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11432 }
11433
11434 #[tokio::test(flavor = "current_thread")]
11435 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11436 let registry = Arc::new(Registry::default());
11437 let module_connection = ConnectionId::new(303);
11438 registry
11439 .register_with_control_ops(
11440 manifest("aft", PROTOCOL_VERSION),
11441 PROTOCOL_VERSION,
11442 module_connection,
11443 module_baseline_control_ops(),
11444 )
11445 .unwrap();
11446 let handler = ControlHandler::new(registry);
11447 let capture = EventCapture::default();
11448 let _subscriber =
11449 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11450 let (ctx, _rx) = route_ctx(module_connection);
11451 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11452
11453 let _ = handler
11454 .handle_client_control_request(
11455 &ctx,
11456 frame,
11457 ClientControlRequest::SupervisorSetEnabled {
11458 module_id: "broca".to_string(),
11459 enabled: false,
11460 },
11461 )
11462 .await
11463 .unwrap();
11464
11465 let callers = capture
11466 .events()
11467 .into_iter()
11468 .filter(|event| {
11469 event.fields.get("message").map(String::as_str)
11470 == Some("supervisor request received")
11471 })
11472 .map(|event| event.fields.get("caller").cloned())
11473 .collect::<Vec<_>>();
11474 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
11475 }
11476
11477 #[tokio::test(flavor = "current_thread")]
11478 async fn supervisor_list_does_not_log_request_received() {
11479 let handler = ControlHandler::new(Arc::new(Registry::default()));
11480 let capture = EventCapture::default();
11481 let _subscriber =
11482 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11483 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
11484 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
11485
11486 handler
11487 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
11488 .await
11489 .unwrap();
11490
11491 assert!(capture.events().into_iter().all(|event| {
11492 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11493 }));
11494 }
11495
11496 #[tokio::test(flavor = "current_thread")]
11497 async fn supervisor_rescan_preview_does_not_log_request_received() {
11498 let handler = ControlHandler::new(Arc::new(Registry::default()));
11499 let capture = EventCapture::default();
11500 let _subscriber =
11501 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11502 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
11503 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
11504
11505 handler
11506 .handle_client_control_request(
11507 &ctx,
11508 frame,
11509 ClientControlRequest::SupervisorRescan { preview: true },
11510 )
11511 .await
11512 .unwrap();
11513
11514 assert!(capture.events().into_iter().all(|event| {
11515 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11516 }));
11517 }
11518
11519 struct ApproveOperator;
11520 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
11521 fn prompt(
11522 &self,
11523 _: &str,
11524 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
11525 ) -> crate::operator_confirm::ProviderResult {
11526 crate::operator_confirm::ProviderResult::Approved
11527 }
11528 }
11529
11530 #[tokio::test]
11531 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
11532 assert_eq!(
11533 ControlHandler::default().privileged_operator_prompt,
11534 PrivilegedOperatorPrompt::Log
11535 );
11536 for (initial, changed, running) in [
11537 ("log", "enforce", PrivilegedOperatorPrompt::Log),
11538 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
11539 ] {
11540 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
11541 let path = dir.join("subc.jsonc");
11542 std::fs::write(
11543 &path,
11544 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
11545 )
11546 .unwrap();
11547 let config = crate::daemon_config::load(&path).unwrap().unwrap();
11548 let registry = Arc::new(Registry::default());
11549 let handle = SupervisorHandle::new();
11550 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
11551 .with_handle(handle.clone());
11552 let handler = ControlHandler::new(registry)
11553 .with_supervisor(handle)
11554 .with_privileged_operator_prompt(config.privileged_operator_prompt)
11555 .with_supervisor_rescan(supervisor, &path, config.port);
11556 handler.forwarding.operator_confirms().configure(
11557 Arc::new(ApproveOperator),
11558 crate::operator_confirm::OperatorLimits::default(),
11559 );
11560 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
11561
11562 for (disk_mode, needs_restart) in [
11563 (Some(initial), false),
11564 (Some(changed), true),
11565 (Some(changed), true),
11566 (None, running == PrivilegedOperatorPrompt::Enforce),
11567 (Some(initial), false),
11568 ] {
11569 let mut doc = json!({"version":1});
11570 if let Some(mode) = disk_mode {
11571 doc["privileged_operator_prompt"] = json!(mode);
11572 }
11573 std::fs::write(&path, doc.to_string()).unwrap();
11574 for preview in [true, false] {
11575 let request = ClientControlRequest::SupervisorRescan { preview };
11576 let frame = Frame::build(
11577 FrameType::Request,
11578 control_flags(),
11579 0,
11580 0,
11581 1,
11582 serde_json::to_vec(&request).unwrap(),
11583 )
11584 .unwrap();
11585 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11586 assert_eq!(frames.len(), 1);
11587 let ClientControlResponse::SupervisorRescan { result } =
11588 serde_json::from_slice(&frames[0].body).unwrap()
11589 else {
11590 panic!("expected a rescan response");
11591 };
11592 let expected = if needs_restart {
11593 vec!["privileged_operator_prompt"]
11594 } else {
11595 Vec::new()
11596 };
11597 assert_eq!(result.restart_required, expected);
11598 assert_eq!(handler.privileged_operator_prompt, running);
11599 assert_eq!(
11600 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
11601 running
11602 );
11603 }
11604 }
11605 }
11606 }
11607
11608 #[tokio::test]
11609 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
11610 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
11611 let path = dir.join("subc.jsonc");
11612 let registry = Arc::new(Registry::default());
11613 let handle = SupervisorHandle::new();
11614 let supervisor =
11615 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
11616 let handler = ControlHandler::new(registry)
11617 .with_supervisor(handle)
11618 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
11619 .with_supervisor_rescan(supervisor, &path, None);
11620 handler.forwarding.operator_confirms().configure(
11621 Arc::new(ApproveOperator),
11622 crate::operator_confirm::OperatorLimits::default(),
11623 );
11624 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
11625 for preview in [true, false] {
11626 std::fs::write(
11627 &path,
11628 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
11629 )
11630 .unwrap();
11631 let request = ClientControlRequest::SupervisorRescan { preview };
11632 let frame = Frame::build(
11633 FrameType::Request,
11634 control_flags(),
11635 0,
11636 0,
11637 1,
11638 serde_json::to_vec(&request).unwrap(),
11639 )
11640 .unwrap();
11641 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11642 assert_eq!(frames.len(), 1);
11643 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
11644 assert_eq!(
11645 handler.privileged_operator_prompt,
11646 PrivilegedOperatorPrompt::Enforce
11647 );
11648 }
11649 }
11650
11651 #[tokio::test]
11656 async fn route_open_refusal_names_the_check_that_refused() {
11657 let handler = ControlHandler::new(Arc::new(Registry::default()));
11658 let capture = EventCapture::default();
11659 let _subscriber =
11660 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11661 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11662 let response = handler
11663 .handle_control_frame(
11664 &ctx,
11665 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
11666 )
11667 .await
11668 .unwrap();
11669
11670 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11671 let event = capture
11672 .events()
11673 .into_iter()
11674 .find(|event| {
11675 event.target == "control"
11676 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11677 })
11678 .expect("route.open refusal event");
11679 assert_eq!(
11680 event.fields.get("reason"),
11681 Some(&"\"not_registered\"".to_string())
11682 );
11683 }
11684
11685 #[tokio::test]
11686 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
11687 let registry = Arc::new(Registry::default());
11688 let supervisor_handle = SupervisorHandle::new();
11689 let supervisor =
11690 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11691 .with_handle(supervisor_handle.clone())
11692 .with_connection_file_path(std::env::temp_dir().join(format!(
11693 "subc-route-open-refusal-info-{}",
11694 std::process::id()
11695 )));
11696 let module = supervisor
11697 .supervise_configured(
11698 ModuleSpec {
11699 module_id: "warming".to_string(),
11700 program: fake_aft_stub_path(),
11701 args: Vec::new(),
11702 env: Vec::new(),
11703 reserved: false,
11704 reserved_prefixes: Vec::new(),
11705 protocol: ModuleProtocol::Subc,
11706 overlap: Default::default(),
11707 },
11708 true,
11709 )
11710 .unwrap();
11711 assert_eq!(module.state().unwrap(), ModuleState::Running);
11712
11713 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11714 assert!(handler
11715 .counters()
11716 .snapshot()
11717 .get("route_open_refused_by_code")
11718 .is_none());
11719 let capture = EventCapture::default();
11720 let _subscriber =
11721 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11722 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
11723 let response = handler
11724 .handle_control_frame(
11725 &ctx,
11726 route_open_frame(394, "warming", unique_project_root("refusal-info")),
11727 )
11728 .await
11729 .unwrap();
11730 module.stop().await.unwrap();
11731
11732 assert_eq!(parse_error(&response[0])["code"], "module_warming");
11733 let event = capture
11734 .events()
11735 .into_iter()
11736 .find(|event| {
11737 event.target == "control"
11738 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
11739 })
11740 .expect("route.open refusal event");
11741 assert_eq!(
11742 event.fields.get("module_id"),
11743 Some(&"\"warming\"".to_string())
11744 );
11745 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
11746 assert_eq!(
11747 event.fields.get("reason"),
11748 Some(&"\"supervised_not_registered\"".to_string())
11749 );
11750 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
11751 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
11752 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
11753 assert_eq!(
11754 handler.counters().snapshot()["route_open_refused_by_code"],
11755 json!({ "module_warming": 1 })
11756 );
11757 }
11758
11759 const OUTAGE_START: &str = "route.open refusing module: not serving";
11760 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
11761
11762 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
11763 capture
11764 .events()
11765 .into_iter()
11766 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
11767 .collect()
11768 }
11769
11770 fn supervise_stub(
11771 registry: &Arc<Registry>,
11772 module_id: &str,
11773 enabled: bool,
11774 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
11775 let supervisor_handle = SupervisorHandle::new();
11776 let supervisor =
11777 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
11778 .with_handle(supervisor_handle.clone())
11779 .with_connection_file_path(std::env::temp_dir().join(format!(
11780 "subc-route-outage-{module_id}-{}",
11781 std::process::id()
11782 )));
11783 let module = supervisor
11784 .supervise_configured(
11785 ModuleSpec {
11786 module_id: module_id.to_string(),
11787 program: fake_aft_stub_path(),
11788 args: Vec::new(),
11789 env: Vec::new(),
11790 reserved: false,
11791 reserved_prefixes: Vec::new(),
11792 protocol: ModuleProtocol::Subc,
11793 overlap: Default::default(),
11794 },
11795 enabled,
11796 )
11797 .unwrap();
11798 (supervisor_handle, module)
11799 }
11800
11801 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
11802 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
11803 module_id: module_id.to_string(),
11804 drain_timeout_ms: Some(50),
11805 })
11806 .unwrap();
11807 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11808 }
11809
11810 #[test]
11814 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
11815 let registry = Arc::new(Registry::default());
11816 let forwarding = Arc::new(ForwardingTable::default());
11817 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11818 let second = ControlHandler::with_forwarding(registry, forwarding);
11819 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
11820 }
11821
11822 #[tokio::test(flavor = "current_thread")]
11826 async fn route_open_unknown_module_refusals_add_no_outage_state() {
11827 let handler = ControlHandler::new(Arc::new(Registry::default()));
11828 let capture = EventCapture::default();
11829 let _subscriber =
11830 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11831 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
11832 for corr in 0..8 {
11833 let response = handler
11834 .handle_control_frame(
11835 &ctx,
11836 route_open_frame(
11837 380 + corr,
11838 &format!("nobody-{corr}"),
11839 unique_project_root("outage-unknown"),
11840 ),
11841 )
11842 .await
11843 .unwrap();
11844 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11845 }
11846
11847 assert_eq!(handler.route_outages.tracked_module_count(), 0);
11848 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
11849 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
11850 }
11851
11852 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11857 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
11858 let registry = Arc::new(Registry::default());
11859 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
11860 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11861 let capture = EventCapture::default();
11862 let _subscriber =
11863 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11864 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
11865 handler.route_outages.record_accepted("outage-restart");
11868
11869 let response = handler
11870 .handle_control_frame(
11871 &ctx,
11872 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
11873 )
11874 .await
11875 .unwrap();
11876 assert_eq!(response[0].header.ty, FrameType::Error);
11877 let starts = outage_lines(&capture, OUTAGE_START);
11878 assert_eq!(starts.len(), 1, "{starts:?}");
11879 assert_eq!(starts[0].level, tracing::Level::WARN);
11880 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
11881 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
11882 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
11883 handler.route_outages.record_accepted("outage-restart");
11884 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
11885
11886 let restart = handler
11887 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
11888 .await
11889 .unwrap();
11890 assert_eq!(
11891 restart[0].header.ty,
11892 FrameType::Response,
11893 "{:?}",
11894 parse_error(&restart[0])
11895 );
11896 handler
11897 .handle_control_frame(
11898 &ctx,
11899 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
11900 )
11901 .await
11902 .unwrap();
11903 module.stop().await.unwrap();
11904
11905 let starts = outage_lines(&capture, OUTAGE_START);
11906 assert_eq!(starts.len(), 2, "{starts:?}");
11907 assert_eq!(starts[1].level, tracing::Level::INFO);
11908 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
11909 }
11910
11911 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11915 async fn failed_operator_restart_leaves_no_operator_mark() {
11916 let registry = Arc::new(Registry::default());
11917 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
11918 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11919 let capture = EventCapture::default();
11920 let _subscriber =
11921 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11922 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
11923 handler.route_outages.record_accepted("outage-disabled");
11924
11925 let restart = handler
11926 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
11927 .await
11928 .unwrap();
11929 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
11930 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
11931
11932 handler
11933 .handle_control_frame(
11934 &ctx,
11935 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
11936 )
11937 .await
11938 .unwrap();
11939 let starts = outage_lines(&capture, OUTAGE_START);
11940 assert_eq!(starts.len(), 1, "{starts:?}");
11941 assert_eq!(starts[0].level, tracing::Level::WARN);
11942 }
11943
11944 #[tokio::test(flavor = "current_thread")]
11945 async fn route_open_unknown_module_escapes_target_module_id() {
11946 let handler = ControlHandler::new(Arc::new(Registry::default()));
11947 let capture = EventCapture::default();
11948 let _subscriber =
11949 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11950 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
11951 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11952 let response = handler
11953 .handle_control_frame(
11954 &ctx,
11955 route_open_frame(
11956 395,
11957 hostile_module_id,
11958 unique_project_root("hostile-target-module-id"),
11959 ),
11960 )
11961 .await
11962 .unwrap();
11963
11964 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11965 let event = capture
11966 .events()
11967 .into_iter()
11968 .find(|event| {
11969 event.target == "control"
11970 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11971 })
11972 .expect("route.open unknown-module refusal event");
11973 let logged = event.fields.get("module_id").expect("module_id field");
11974 assert!(!logged.bytes().any(|byte| byte < 0x20));
11975 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11976 }
11977
11978 #[tokio::test(flavor = "current_thread")]
11979 async fn route_open_module_rejection_uses_daemon_counter_key() {
11980 let registry = Arc::new(Registry::default());
11981 let forwarding = Arc::new(ForwardingTable::default());
11982 let handler =
11983 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11984 let module_connection = ConnectionId::new(95);
11985 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11986 hello_via_sink(
11987 &handler,
11988 &module_ctx,
11989 &mut module_rx,
11990 hello_frame("aft", PROTOCOL_VERSION, 395),
11991 )
11992 .await;
11993
11994 let client_connection = ConnectionId::new(96);
11995 let (client_ctx, _client_rx) = route_ctx(client_connection);
11996 let capture = EventCapture::default();
11997 let _subscriber =
11998 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11999 let (route_task, bind) = relay_route_open(
12000 &handler,
12001 client_connection,
12002 &client_ctx.egress,
12003 &mut module_rx,
12004 396,
12005 "aft",
12006 "hostile-module-code",
12007 )
12008 .await;
12009 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12010 let rejection = Frame::build(
12011 FrameType::Error,
12012 control_flags(),
12013 0,
12014 0,
12015 bind.header.corr,
12016 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12017 )
12018 .unwrap();
12019 handler
12020 .handle_control_frame(&module_ctx, rejection)
12021 .await
12022 .unwrap();
12023
12024 let response = route_task.await.unwrap();
12025 assert_eq!(parse_error(&response[0])["code"], hostile_code);
12026 let counters = handler.counters().snapshot();
12027 assert_eq!(
12028 counters["route_open_refused_by_code"],
12029 json!({ "module_rejected": 1 })
12030 );
12031 assert!(counters["route_open_refused_by_code"]
12032 .get(hostile_code)
12033 .is_none());
12034
12035 let event = capture
12036 .events()
12037 .into_iter()
12038 .find(|event| {
12039 event.target == "control"
12040 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12041 })
12042 .expect("route.open module-rejection refusal event");
12043 let logged = event.fields.get("module_code").expect("module_code field");
12044 assert!(!logged.bytes().any(|byte| byte < 0x20));
12045 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12046 }
12047
12048 #[tokio::test]
12049 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12050 let registry = Arc::new(Registry::default());
12051 let supervisor_handle = SupervisorHandle::new();
12052 let missing_program = std::env::temp_dir().join(format!(
12053 "subc-route-open-missing-program-{}",
12054 std::process::id()
12055 ));
12056 let supervisor =
12057 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12058 .with_handle(supervisor_handle.clone());
12059 let module = supervisor
12060 .supervise_configured(
12061 ModuleSpec {
12062 module_id: "failed".to_string(),
12063 program: missing_program,
12064 args: Vec::new(),
12065 env: Vec::new(),
12066 reserved: false,
12067 reserved_prefixes: Vec::new(),
12068 protocol: ModuleProtocol::Subc,
12069 overlap: Default::default(),
12070 },
12071 true,
12072 )
12073 .unwrap();
12074 assert_eq!(module.state().unwrap(), ModuleState::Failed);
12075
12076 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12077 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12078 let response = handler
12079 .handle_control_frame(
12080 &ctx,
12081 route_open_frame(305, "failed", unique_project_root("failed")),
12082 )
12083 .await
12084 .unwrap();
12085
12086 assert_eq!(response[0].header.ty, FrameType::Error);
12087 let error = parse_error(&response[0]);
12088 assert_eq!(error["code"], "target_unavailable");
12089 assert!(error["message"]
12090 .as_str()
12091 .unwrap()
12092 .contains("state=failed, enabled=true, live=false"));
12093 }
12094
12095 #[tokio::test]
12096 async fn route_open_role_mismatch_remains_target_unavailable() {
12097 let registry = Arc::new(Registry::default());
12098 let handler = ControlHandler::new(Arc::clone(®istry));
12099 handler
12100 .handle_control(
12101 ConnectionId::new(41),
12102 non_routable_hello_frame_with_control_ops("health-only", 306, None),
12103 )
12104 .unwrap();
12105
12106 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12107 let response = handler
12108 .handle_control_frame(
12109 &ctx,
12110 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12111 )
12112 .await
12113 .unwrap();
12114
12115 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12116 assert!(parse_error(&response[0])["message"]
12117 .as_str()
12118 .unwrap()
12119 .contains("does not provide the requested target"));
12120 }
12121
12122 #[tokio::test]
12123 async fn route_open_inactive_registration_remains_target_unavailable() {
12124 let registry = Arc::new(Registry::default());
12125 let handler = ControlHandler::new(Arc::clone(®istry));
12126 handler
12127 .handle_control(
12128 ConnectionId::new(43),
12129 hello_frame("inactive", PROTOCOL_VERSION, 308),
12130 )
12131 .unwrap();
12132 assert!(registry
12133 .set_module_state_for_test("inactive", ChannelState::Closed)
12134 .unwrap());
12135
12136 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12137 let response = handler
12138 .handle_control_frame(
12139 &ctx,
12140 route_open_frame(309, "inactive", unique_project_root("inactive")),
12141 )
12142 .await
12143 .unwrap();
12144
12145 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12146 assert!(parse_error(&response[0])["message"]
12147 .as_str()
12148 .unwrap()
12149 .contains("is not active"));
12150 }
12151
12152 #[tokio::test]
12153 async fn late_health_reply_is_recorded_through_the_module_response_path() {
12154 let registry = Arc::new(Registry::default());
12155 let forwarding = Arc::new(ForwardingTable::default());
12156 let supervisor_handle = SupervisorHandle::new();
12157 let supervisor =
12158 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
12159 .with_forwarding(Arc::clone(&forwarding))
12160 .with_handle(supervisor_handle.clone());
12161 let module = supervisor
12162 .supervise_configured(
12163 crate::ModuleSpec {
12164 module_id: "late-health-response".to_string(),
12165 program: PathBuf::from("disabled-module"),
12166 args: Vec::new(),
12167 env: Vec::new(),
12168 reserved: false,
12169 reserved_prefixes: Vec::new(),
12170 protocol: ModuleProtocol::Subc,
12171 overlap: Default::default(),
12172 },
12173 false,
12174 )
12175 .unwrap();
12176 let handler =
12177 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12178 .with_supervisor(supervisor_handle);
12179 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12180 handler
12181 .handle_control_frame(
12182 &module_ctx,
12183 hello_frame_with_control_ops(
12184 "late-health-response",
12185 PROTOCOL_VERSION,
12186 7,
12187 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12188 ),
12189 )
12190 .await
12191 .unwrap();
12192 let probe_started_at = Instant::now() - Duration::from_millis(80);
12193 let pending = forwarding
12194 .begin_health_probe_rpc_for(
12195 "late-health-response",
12196 MODULE_CONTROL_OP_HEALTH_CHECK,
12197 probe_started_at,
12198 Instant::now() - Duration::from_millis(1),
12199 )
12200 .unwrap();
12201 assert!(forwarding
12202 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12203 .unwrap());
12204
12205 let responses = handler
12206 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12207 .await
12208 .unwrap();
12209
12210 assert!(responses.is_empty());
12211 let health = module.status().unwrap().health;
12212 assert_eq!(health.late_answer_count, 1);
12213 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12214 }
12215
12216 #[tokio::test]
12217 async fn health_probe_timeout_and_module_death_are_typed() {
12218 let registry = Arc::new(Registry::default());
12219 let forwarding = Arc::new(ForwardingTable::default());
12220 let handler =
12221 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12222 .with_health_probe_timeout(Duration::from_millis(50));
12223 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12224 hello_via_sink(
12225 &handler,
12226 &module_ctx,
12227 &mut module_rx,
12228 hello_frame_with_control_ops(
12229 "aft",
12230 PROTOCOL_VERSION,
12231 7,
12232 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12233 ),
12234 )
12235 .await;
12236
12237 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12238 let responses = handler
12239 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12240 .await
12241 .unwrap();
12242 assert_eq!(responses[0].header.ty, FrameType::Error);
12243 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12244 let _ = module_rx.try_recv();
12245
12246 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12247 let health_handler = handler.clone();
12248 let death_task = tokio::spawn(async move {
12249 health_handler
12250 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12251 .await
12252 .unwrap()
12253 });
12254 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12255 .await
12256 .unwrap()
12257 .unwrap();
12258 handler
12259 .cleanup_connection(module_ctx.connection_id)
12260 .unwrap();
12261 let responses = death_task.await.unwrap();
12262 assert_eq!(responses[0].header.ty, FrameType::Error);
12263 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12264 }
12265
12266 #[test]
12267 fn hello_requires_exact_protocol_version() {
12268 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12269 let registry = Arc::new(Registry::default());
12270 let handler = ControlHandler::new(Arc::clone(®istry));
12271 let responses = handler
12272 .handle_control(
12273 ConnectionId::new(connection),
12274 hello_frame("aft", offered, 9),
12275 )
12276 .unwrap();
12277
12278 assert_eq!(responses.len(), 1);
12279 assert_eq!(responses[0].header.ty, FrameType::Error);
12280 let error = parse_error(&responses[0]);
12281 assert_eq!(error["code"], "version_unsupported");
12282 assert!(registry.get_module("aft").unwrap().is_none());
12283 assert_eq!(registry.active_registration_count().unwrap(), 0);
12284 }
12285 }
12286
12287 #[test]
12288 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12289 let registry = Arc::new(Registry::default());
12290 let forwarding = Arc::new(ForwardingTable::default());
12291 let handler =
12292 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12293 let module_connection = ConnectionId::new(301);
12294 let registration = registry
12295 .register_with_control_ops(
12296 manifest("aft-push", PROTOCOL_VERSION),
12297 PROTOCOL_VERSION,
12298 module_connection,
12299 module_baseline_control_ops(),
12300 )
12301 .unwrap();
12302 let (module_tx, _module_rx) = mpsc::channel(8);
12303 let endpoint = forwarding
12304 .register_module_connection(
12305 module_connection,
12306 "aft-push".to_string(),
12307 PROTOCOL_VERSION,
12308 manifest_concurrency(®istration.manifest),
12309 FrameSink::new(module_tx),
12310 )
12311 .unwrap();
12312
12313 let unknown = Frame::build(
12315 FrameType::Push,
12316 control_flags(),
12317 0,
12318 0,
12319 5,
12320 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12321 )
12322 .unwrap();
12323 let out = handler.handle_status_update(endpoint, unknown).unwrap();
12324 assert!(
12325 out.is_empty(),
12326 "unknown push op must be ignored, got {out:?}"
12327 );
12328
12329 let malformed = Frame::build(
12331 FrameType::Push,
12332 control_flags(),
12333 0,
12334 0,
12335 6,
12336 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
12337 )
12338 .unwrap();
12339 let out = handler.handle_status_update(endpoint, malformed).unwrap();
12340 assert_eq!(out.len(), 1);
12341 assert_eq!(out[0].header.ty, FrameType::Error);
12342 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
12343 }
12344
12345 #[test]
12346 fn hello_rejected_when_connection_already_owns_client_routes() {
12347 let registry = Arc::new(Registry::default());
12348 let forwarding = Arc::new(ForwardingTable::default());
12349 let handler =
12350 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12351 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
12353 let client_connection = ConnectionId::new(202);
12354
12355 let responses = handler
12358 .handle_control(
12359 client_connection,
12360 hello_frame("aft-second", PROTOCOL_VERSION, 9),
12361 )
12362 .unwrap();
12363 assert_eq!(responses[0].header.ty, FrameType::Error);
12364 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12365 assert!(registry.get_module("aft-second").unwrap().is_none());
12366 }
12367
12368 #[tokio::test]
12369 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12370 let registry = Arc::new(Registry::default());
12371 let forwarding = Arc::new(ForwardingTable::default());
12372 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12373 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12374 hello_via_sink(
12375 &handler,
12376 &module_ctx,
12377 &mut module_rx,
12378 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12379 )
12380 .await;
12381 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12382 let pending = forwarding
12383 .begin_route_bind_relay_for_test(
12384 client_ctx.connection_id,
12385 client_ctx.egress.clone(),
12386 2,
12387 "alpha",
12388 )
12389 .unwrap();
12390 forwarding
12391 .complete_pending_relay(
12392 module_ctx.connection_id,
12393 pending.corr,
12394 RouteBindRelayOutcome::Accepted,
12395 )
12396 .unwrap();
12397 client_rx.try_recv().unwrap();
12398 for module_id in ["beta", "alpha"] {
12399 let replies = handler
12400 .handle_control_frame(
12401 &module_ctx,
12402 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12403 )
12404 .await
12405 .unwrap();
12406 assert_eq!(replies.len(), 1, "second HELLO must be refused");
12407 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12408 }
12409 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12410 assert!(registry.get_module("beta").unwrap().is_none());
12411 assert!(matches!(
12412 forwarding
12413 .lookup_data_route(
12414 client_ctx.connection_id,
12415 pending.client_channel,
12416 pending.client_epoch,
12417 )
12418 .unwrap(),
12419 DataRoute::Client(DataRouteState::Bound(_))
12420 ));
12421 assert!(handler
12422 .hello_launch_nonces
12423 .lock()
12424 .unwrap()
12425 .presented(module_ctx.connection_id, Some("alpha-nonce")));
12426 assert!(module_rx.try_recv().is_err());
12427 }
12428
12429 #[test]
12430 fn reserved_module_hello_requires_matching_launch_nonce() {
12431 let registry = Arc::new(Registry::default());
12432 let supervisor = SupervisorHandle::new();
12433 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12436 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12437
12438 let no_nonce = handler
12440 .handle_control(
12441 ConnectionId::new(1),
12442 hello_frame("vault", PROTOCOL_VERSION, 1),
12443 )
12444 .unwrap();
12445 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
12446 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
12447 assert!(registry.get_module("vault").unwrap().is_none());
12448
12449 let wrong = handler
12451 .handle_control(
12452 ConnectionId::new(2),
12453 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
12454 )
12455 .unwrap();
12456 assert_eq!(wrong[0].header.ty, FrameType::Error);
12457 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
12458 assert!(registry.get_module("vault").unwrap().is_none());
12459
12460 let ok = handler
12462 .handle_control(
12463 ConnectionId::new(3),
12464 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
12465 )
12466 .unwrap();
12467 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
12468 assert!(registry.get_module("vault").unwrap().is_some());
12469 }
12470
12471 #[test]
12472 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
12473 let registry = Arc::new(Registry::default());
12474 let supervisor = SupervisorHandle::new();
12475 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12476 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12477 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12478
12479 let squat = handler
12480 .handle_control(
12481 ConnectionId::new(1),
12482 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
12483 )
12484 .unwrap();
12485 assert_eq!(squat[0].header.ty, FrameType::Error);
12486 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
12487 assert!(parse_error(&squat[0])["message"]
12488 .as_str()
12489 .unwrap()
12490 .contains("fed:"));
12491
12492 let accepted_peer = handler
12493 .handle_control(
12494 ConnectionId::new(2),
12495 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
12496 )
12497 .unwrap();
12498 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
12499
12500 let accepted_short = handler
12501 .handle_control(
12502 ConnectionId::new(3),
12503 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
12504 )
12505 .unwrap();
12506 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
12507
12508 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
12509 let response = handler
12510 .handle_control(
12511 ConnectionId::new(conn),
12512 hello_frame(module_id, PROTOCOL_VERSION, conn),
12513 )
12514 .unwrap();
12515 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
12516 }
12517 }
12518
12519 #[test]
12520 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
12521 let registry = Arc::new(Registry::default());
12522 let supervisor = SupervisorHandle::new();
12523 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12524 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12525 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
12526 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12527
12528 let owner_nonce = handler
12529 .handle_control(
12530 ConnectionId::new(1),
12531 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
12532 )
12533 .unwrap();
12534 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
12535 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
12536 assert!(registry.get_module("fed:special").unwrap().is_none());
12537
12538 let exact_nonce = handler
12539 .handle_control(
12540 ConnectionId::new(2),
12541 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
12542 )
12543 .unwrap();
12544 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
12545 assert!(registry.get_module("fed:special").unwrap().is_some());
12546 }
12547
12548 #[test]
12549 fn non_reserved_module_ignores_launch_nonce() {
12550 let registry = Arc::new(Registry::default());
12551 let handler = ControlHandler::new(Arc::clone(®istry));
12554 let no_nonce = handler
12555 .handle_control(
12556 ConnectionId::new(1),
12557 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
12558 )
12559 .unwrap();
12560 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
12561 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
12562
12563 let echoed_nonce = handler
12564 .handle_control(
12565 ConnectionId::new(2),
12566 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
12567 )
12568 .unwrap();
12569 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
12570 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
12571 }
12572
12573 #[test]
12574 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
12575 let handler = ControlHandler::default();
12576 let conn = ConnectionId::new(1);
12577 let malformed = Frame::build(
12578 FrameType::Hello,
12579 control_flags(),
12580 0,
12581 0,
12582 3,
12583 b"{not json".to_vec(),
12584 )
12585 .unwrap();
12586
12587 let error = handler.handle_control(conn, malformed).unwrap();
12588 assert_eq!(error[0].header.ty, FrameType::Error);
12589 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
12590
12591 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12592 let pong = handler.handle_control(conn, ping).unwrap();
12593 assert_eq!(pong[0].header.ty, FrameType::Pong);
12594 assert_eq!(pong[0].header.corr, 4);
12595 }
12596
12597 #[test]
12598 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
12599 let registry = Arc::new(Registry::default());
12600 let handler = ControlHandler::new(Arc::clone(®istry));
12601
12602 handler
12603 .handle_control(
12604 ConnectionId::new(1),
12605 hello_frame("aft", PROTOCOL_VERSION, 1),
12606 )
12607 .unwrap();
12608 let duplicate = handler
12609 .handle_control(
12610 ConnectionId::new(2),
12611 hello_frame("aft", PROTOCOL_VERSION, 2),
12612 )
12613 .unwrap();
12614
12615 assert_eq!(duplicate[0].header.ty, FrameType::Error);
12616 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12617 let registration = registry.get_module("aft").unwrap().unwrap();
12618 assert_eq!(registration.connection_id, ConnectionId::new(1));
12619 }
12620
12621 #[test]
12622 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
12623 let registry = Arc::new(Registry::default());
12624 let forwarding = Arc::new(ForwardingTable::default());
12625 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
12626 let handler =
12627 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12628 .with_process_liveness(process_liveness);
12629 let (ctx, route_channel, route_epoch) =
12630 bind_liveness_route(®istry, &forwarding, "aft-dead");
12631 let responses = handler
12632 .handle_route_poll(
12633 &ctx,
12634 route_poll_frame(41, PollKind::Liveness, route_channel),
12635 route_channel,
12636 route_epoch,
12637 PollKind::Liveness,
12638 )
12639 .unwrap();
12640
12641 assert_eq!(responses.len(), 1);
12642 assert_eq!(responses[0].header.ty, FrameType::Response);
12643 assert_route_poll_liveness(&responses[0], false);
12644 }
12645
12646 #[test]
12647 fn liveness_poll_without_process_source_uses_bound_route() {
12648 let registry = Arc::new(Registry::default());
12649 let forwarding = Arc::new(ForwardingTable::default());
12650 let handler =
12651 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12652 let (ctx, route_channel, route_epoch) =
12653 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
12654 let responses = handler
12655 .handle_route_poll(
12656 &ctx,
12657 route_poll_frame(42, PollKind::Liveness, route_channel),
12658 route_channel,
12659 route_epoch,
12660 PollKind::Liveness,
12661 )
12662 .unwrap();
12663
12664 assert_route_poll_liveness(&responses[0], true);
12665 }
12666
12667 #[test]
12668 fn liveness_poll_untracked_process_source_uses_bound_route() {
12669 let registry = Arc::new(Registry::default());
12670 let forwarding = Arc::new(ForwardingTable::default());
12671 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
12672 let handler =
12673 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12674 .with_process_liveness(process_liveness);
12675 let (ctx, route_channel, route_epoch) =
12676 bind_liveness_route(®istry, &forwarding, "aft-untracked");
12677 let responses = handler
12678 .handle_route_poll(
12679 &ctx,
12680 route_poll_frame(43, PollKind::Liveness, route_channel),
12681 route_channel,
12682 route_epoch,
12683 PollKind::Liveness,
12684 )
12685 .unwrap();
12686
12687 assert_route_poll_liveness(&responses[0], true);
12688 }
12689
12690 #[tokio::test]
12691 async fn unknown_op_returns_unknown_control_op() {
12692 let handler = ControlHandler::default();
12693 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
12694 let request = Frame::build(
12695 FrameType::Request,
12696 control_flags(),
12697 0,
12698 0,
12699 55,
12700 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
12701 )
12702 .unwrap();
12703
12704 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12705
12706 assert_eq!(response.len(), 1);
12707 assert_eq!(response[0].header.ty, FrameType::Error);
12708 assert_eq!(response[0].header.corr, 55);
12709 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
12710 }
12711
12712 #[tokio::test]
12713 async fn supervisor_provenance_rejects_unknown_exact_module() {
12714 let handler = ControlHandler::default();
12715 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
12716 let request = Frame::build(
12717 FrameType::Request,
12718 control_flags(),
12719 0,
12720 0,
12721 57,
12722 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
12723 )
12724 .unwrap();
12725
12726 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12727
12728 assert_eq!(response.len(), 1);
12729 assert_eq!(response[0].header.ty, FrameType::Error);
12730 assert_eq!(response[0].header.corr, 57);
12731 let error = parse_error(&response[0]);
12732 assert_eq!(error["code"], "unknown_module");
12733 assert_eq!(error["message"], "module_id 'missing' is not supervised");
12734 }
12735
12736 #[test]
12737 fn provenance_probe_override_keeps_handler_tests_deterministic() {
12738 let expected = subc_control::RunningImageAgreement::Unavailable {
12739 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12740 };
12741 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
12742 assert_eq!(handler.provenance_probe_override, Some(expected));
12743 }
12744
12745 #[test]
12746 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
12747 let verdict = reload_verdict(
12748 std::path::Path::new("/bin/new"),
12749 Some(std::path::Path::new("/bin/old")),
12750 subc_control::RunningImageAgreement::Unavailable {
12751 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12752 },
12753 );
12754 assert!(matches!(
12755 verdict.path,
12756 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
12757 if configured == std::path::Path::new("/bin/new")
12758 && spawned_from == std::path::Path::new("/bin/old")
12759 ));
12760 }
12761
12762 #[cfg(windows)]
12763 #[test]
12764 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
12765 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
12766 let configured = dir.join("module.exe");
12767 std::fs::write(&configured, b"executable path fixture").unwrap();
12768 let spawned = std::fs::canonicalize(&configured).unwrap();
12769 assert_ne!(
12770 configured, spawned,
12771 "fixture must exercise different path spellings"
12772 );
12773 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
12774 let lower_case = configured.to_string_lossy().to_lowercase();
12775 for spelling in [
12776 &configured,
12777 Path::new(&forward_slashes),
12778 Path::new(&lower_case),
12779 ] {
12780 let verdict = reload_verdict(
12781 spelling,
12782 Some(&spawned),
12783 subc_control::RunningImageAgreement::Unavailable {
12784 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12785 },
12786 );
12787 assert_eq!(
12788 verdict.path,
12789 ReloadPathAgreement::Match,
12790 "configured {spelling:?}, spawned {spawned:?}"
12791 );
12792 }
12793 }
12794
12795 #[cfg(windows)]
12796 #[test]
12797 fn windows_reload_path_keeps_different_existing_program_pending() {
12798 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
12799 let old = dir.join("old.exe");
12800 let new = dir.join("new.exe");
12801 std::fs::write(&old, b"same executable content").unwrap();
12802 std::fs::copy(&old, &new).unwrap();
12803 let spawned = std::fs::canonicalize(&old).unwrap();
12804 let image = subc_control::RunningImageAgreement::Unavailable {
12805 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12806 };
12807 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
12808 assert!(
12809 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
12810 if configured == new && spawned_from == spawned)
12811 );
12812 let missing = dir.join("missing.exe");
12813 assert!(matches!(
12814 reload_verdict(&missing, Some(&spawned), image).path,
12815 ReloadPathAgreement::Mismatch { .. }
12816 ));
12817 }
12818
12819 #[test]
12820 fn reload_verdict_detects_replaced_image_at_same_path() {
12821 let image = subc_control::RunningImageAgreement::Mismatch {
12822 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
12823 digest: "old".into(),
12824 },
12825 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
12826 digest: "new".into(),
12827 },
12828 };
12829 let verdict = reload_verdict(
12830 std::path::Path::new("/bin/same"),
12831 Some(std::path::Path::new("/bin/same")),
12832 image.clone(),
12833 );
12834 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12835 assert_eq!(verdict.image, image);
12836 }
12837
12838 #[test]
12839 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
12840 let image = subc_control::RunningImageAgreement::Unavailable {
12841 reason: subc_control::RunningImageUnavailableReason::NotRunning,
12842 };
12843 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
12844 assert_eq!(
12845 verdict.path,
12846 subc_control::ReloadPathAgreement::Unavailable {
12847 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
12848 }
12849 );
12850 assert_eq!(verdict.image, image);
12851
12852 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
12853 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12854 };
12855 let verdict = reload_verdict(
12856 std::path::Path::new("/bin/same"),
12857 Some(std::path::Path::new("/bin/same")),
12858 unconfirmed.clone(),
12859 );
12860 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12861 assert_eq!(verdict.image, unconfirmed);
12862 }
12863
12864 #[test]
12865 fn reload_verdict_preserves_each_image_unavailability_reason() {
12866 use subc_control::RunningImageUnavailableReason as Reason;
12867
12868 for reason in [
12869 Reason::NotRunning,
12870 Reason::UnsupportedPlatform,
12871 Reason::RunningExecutableUnreadable,
12872 Reason::SpawnedPathUnreadable,
12873 Reason::HashFailed,
12874 Reason::ProcessIdentityUnconfirmed,
12875 Reason::Unknown("future_probe_reason".to_string()),
12876 ] {
12877 let image = subc_control::RunningImageAgreement::Unavailable {
12878 reason: reason.clone(),
12879 };
12880 let verdict = reload_verdict(
12881 std::path::Path::new("/bin/same"),
12882 Some(std::path::Path::new("/bin/same")),
12883 image.clone(),
12884 );
12885 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12886 assert_eq!(verdict.image, image, "{reason:?}");
12887 }
12888 }
12889
12890 #[tokio::test]
12891 async fn malformed_control_bodies_return_invalid_control_body() {
12892 let handler = ControlHandler::default();
12893 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
12894
12895 for (corr, body) in [
12896 (56, br#"{"route_channel":1}"#.as_slice()),
12897 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
12898 (
12899 58,
12900 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
12901 ),
12902 ] {
12903 let request = Frame::build(
12904 FrameType::Request,
12905 control_flags(),
12906 0,
12907 0,
12908 corr,
12909 body.to_vec(),
12910 )
12911 .unwrap();
12912 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12913
12914 assert_eq!(response.len(), 1);
12915 assert_eq!(response[0].header.ty, FrameType::Error);
12916 assert_eq!(response[0].header.corr, corr);
12917 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
12918 }
12919 }
12920
12921 #[tokio::test]
12922 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
12923 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12924 let registry = Arc::new(Registry::default());
12925 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
12926 let router = Router::with_control_handler(Arc::clone(&control));
12927 let connection = router.begin_connection();
12928 let (ctx, mut rx) = route_ctx(connection.id());
12929
12930 router
12931 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
12932 .await
12933 .unwrap();
12934 let response = rx.recv().await.unwrap();
12935 let ack = parse_ack(&response);
12936 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12937 let channel = 1;
12938
12939 let goodbye =
12940 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
12941 router.route_for_connection(&ctx, goodbye).await.unwrap();
12942 assert!(rx.try_recv().is_err());
12943 assert!(registry.get_module("aft").unwrap().is_none());
12944
12945 router
12946 .route_for_connection(&ctx, channel_request(channel, 13))
12947 .await
12948 .unwrap();
12949 let error_frame = rx.recv().await.unwrap();
12950 assert_eq!(error_frame.header.ty, FrameType::Error);
12951 assert_eq!(error_frame.header.channel, channel);
12952 let captured = crate::router::test_log::captured_logs(&logs);
12953 assert_eq!(
12954 captured
12955 .lines()
12956 .filter(|line| {
12957 line.contains("module_id=aft")
12958 && line.contains("reason=explicit_goodbye")
12959 && line.contains("module registration ended")
12960 })
12961 .count(),
12962 1,
12963 "unexpected GOODBYE registry log: {captured}"
12964 );
12965 }
12966
12967 #[tokio::test]
12968 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
12969 let registry = Arc::new(Registry::default());
12970 let handler = ControlHandler::new(registry).with_capability_config(
12971 [("prov".to_string(), true), ("cons".to_string(), true)],
12972 BTreeMap::new(),
12973 );
12974 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
12975 register_capability_manifest(
12976 &handler,
12977 &provider_ctx,
12978 &mut provider_rx,
12979 capability_manifest("prov", &["thing/v1"], &[]),
12980 1,
12981 )
12982 .await;
12983 let mut consumer = capability_manifest("cons", &[], &[]);
12984 consumer.capabilities.as_mut().unwrap().requires.push(
12985 subc_protocol::manifest::CapabilityRequirement {
12986 capability: "thing/v1".to_string(),
12987 need: subc_protocol::manifest::CapabilityNeed::Required,
12988 },
12989 );
12990 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
12991 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
12992 assert_eq!(
12993 handler.capability_evaluator.verdict("cons", "thing/v1"),
12994 Some(CapabilityVerdict::Provided)
12995 );
12996 let (mut client_rx, _) = open_route_for_capability_test(
12997 &handler,
12998 &provider_ctx,
12999 &mut provider_rx,
13000 703,
13001 3,
13002 "prov",
13003 None,
13004 )
13005 .await;
13006 let goodbye =
13007 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13008 handler
13009 .handle_control_frame(&provider_ctx, goodbye)
13010 .await
13011 .unwrap();
13012 assert_eq!(
13013 handler.capability_evaluator.verdict("cons", "thing/v1"),
13014 Some(CapabilityVerdict::NeverProvided)
13015 );
13016 let closed = client_rx
13017 .try_recv()
13018 .expect("GOODBYE pushes route.closed before route GOODBYE");
13019 assert!(
13020 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13021 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13022 );
13023 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13024 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13025 }
13026
13027 struct RouteClosedFixture {
13028 handler: ControlHandler,
13029 #[cfg(unix)]
13030 supervisor: Supervisor,
13031 module_ctx: RouteCtx,
13032 #[cfg(unix)]
13033 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13034 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13035 route: Arc<crate::forwarding::RouteBinding>,
13036 }
13037
13038 async fn route_closed_fixture() -> RouteClosedFixture {
13039 let registry = Arc::new(Registry::default());
13040 let handler = ControlHandler::new(Arc::clone(®istry));
13041 #[cfg(unix)]
13042 let supervisor = Supervisor::new(registry, RestartPolicy::default())
13043 .with_forwarding(Arc::clone(&handler.forwarding));
13044 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13045 register_capability_manifest(
13046 &handler,
13047 &module_ctx,
13048 &mut module_rx,
13049 capability_manifest("prov", &[], &[]),
13050 1,
13051 )
13052 .await;
13053 let (client_rx, bind) = open_route_for_capability_test(
13054 &handler,
13055 &module_ctx,
13056 &mut module_rx,
13057 702,
13058 2,
13059 "prov",
13060 None,
13061 )
13062 .await;
13063 let ModuleControlRequest::RouteBind {
13064 route_channel: channel,
13065 epoch: route_epoch,
13066 ..
13067 } = bind
13068 else {
13069 panic!("expected route.bind");
13070 };
13071 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13072 handler
13073 .forwarding
13074 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13075 .unwrap()
13076 else {
13077 panic!("the acknowledged route must be bound");
13078 };
13079 RouteClosedFixture {
13080 handler,
13081 #[cfg(unix)]
13082 supervisor,
13083 module_ctx,
13084 #[cfg(unix)]
13085 module_rx,
13086 client_rx,
13087 route,
13088 }
13089 }
13090
13091 impl RouteClosedFixture {
13092 #[cfg(unix)]
13093 async fn cut(&mut self) {
13094 let notices = async {
13095 let mut draining = self.module_rx.recv().await.unwrap();
13096 assert!(matches!(
13097 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13098 &draining.body
13099 )
13100 .unwrap(),
13101 subc_protocol::session::ModuleControlCommand::Draining {
13102 reason: RouteCloseReason::Restart,
13103 ..
13104 }
13105 ));
13106 draining.flushed.take().unwrap().send(()).unwrap();
13107 let mut closing = self.client_rx.recv().await.unwrap();
13108 assert!(matches!(
13109 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13110 ClientControlPush::RouteClosing {
13111 reason: RouteCloseReason::Restart,
13112 ..
13113 }
13114 ));
13115 closing.flushed.take().unwrap().send(()).unwrap();
13116 };
13117 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13118 result.unwrap();
13119 }
13120
13121 fn disconnect(&mut self) -> ClientControlPush {
13122 self.handler
13123 .cleanup_connection(self.module_ctx.connection_id)
13124 .unwrap();
13125 let closed = self.client_rx.try_recv().unwrap();
13126 assert_eq!(closed.header.ty, FrameType::Push);
13127 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13128 assert!(matches!(
13129 &push,
13130 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13131 if channels == &[self.route.client_channel]
13132 ));
13133 assert_eq!(
13134 self.client_rx.try_recv().unwrap().header.ty,
13135 FrameType::Goodbye
13136 );
13137 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13138 push
13139 }
13140 }
13141
13142 #[cfg(unix)]
13143 #[tokio::test(start_paused = true)]
13144 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13145 let mut fixture = route_closed_fixture().await;
13146 fixture.cut().await;
13147 assert!(matches!(
13148 fixture.disconnect(),
13149 ClientControlPush::RouteClosed {
13150 reason: RouteCloseReason::Restart,
13151 drained: true,
13152 ..
13153 }
13154 ));
13155 }
13156
13157 #[cfg(unix)]
13158 #[tokio::test(start_paused = true)]
13159 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13160 let mut fixture = route_closed_fixture().await;
13161 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13162 let started = tokio::time::Instant::now();
13163 fixture.cut().await;
13164 assert!(started.elapsed() >= Duration::from_secs(2));
13165 assert_eq!(fixture.route.flow.in_flight(), 1);
13166 assert!(matches!(
13167 fixture.disconnect(),
13168 ClientControlPush::RouteClosed {
13169 reason: RouteCloseReason::Restart,
13170 drained: false,
13171 ..
13172 }
13173 ));
13174 }
13175
13176 #[tokio::test]
13177 async fn module_connection_crash_reports_crash_and_not_drained() {
13178 let mut fixture = route_closed_fixture().await;
13179 assert!(matches!(
13183 fixture.disconnect(),
13184 ClientControlPush::RouteClosed {
13185 reason: RouteCloseReason::Crash,
13186 drained: false,
13187 ..
13188 }
13189 ));
13190 }
13191
13192 #[tokio::test]
13193 async fn dropping_router_connection_releases_registration() {
13194 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13195 let registry = Arc::new(Registry::default());
13196 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13197 let router = Router::with_control_handler(Arc::clone(&control));
13198 let connection = router.begin_connection();
13199 let connection_id = connection.id();
13200 let (ctx, mut rx) = route_ctx(connection_id);
13201
13202 router
13203 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13204 .await
13205 .unwrap();
13206 let response = rx.recv().await.unwrap();
13207 let ack = parse_ack(&response);
13208 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13209 assert!(registry.get_module("aft").unwrap().is_some());
13210
13211 drop(connection);
13212
13213 assert!(registry.get_module("aft").unwrap().is_none());
13214 assert_eq!(registry.active_registration_count().unwrap(), 0);
13215
13216 control
13217 .cleanup_connection(ConnectionId::new(u64::MAX))
13218 .unwrap();
13219 let captured = crate::router::test_log::captured_logs(&logs);
13220 println!("captured registry lifecycle logs:\n{captured}");
13221 let events: Vec<_> = captured
13222 .lines()
13223 .filter(|line| {
13224 line.contains("module registered module_id=aft ")
13225 || line.contains("module registration ended")
13226 })
13227 .collect();
13228 assert_eq!(
13229 events.len(),
13230 2,
13231 "unexpected registry lifecycle logs: {captured}"
13232 );
13233 assert!(events[0].contains("module registered module_id=aft "));
13234 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13235 assert!(events[1].contains(&format!(
13236 "module_id=aft connection_id={}",
13237 connection_id.get()
13238 )));
13239 assert!(events[1].contains("reason=connection_closed"));
13240 assert!(events[1].contains("module registration ended"));
13241 assert_eq!(
13242 captured
13243 .lines()
13244 .filter(|line| line.contains("module registered module_id=aft "))
13245 .count(),
13246 1,
13247 "legacy registration admission line must appear once: {captured}"
13248 );
13249 }
13250
13251 #[tokio::test]
13252 async fn hello_registration_keeps_legacy_module_registered_line_once() {
13253 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13254 let registry = Arc::new(Registry::default());
13255 let control = ControlHandler::new(Arc::clone(®istry));
13256 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13257 hello_via_sink(
13258 &control,
13259 &ctx,
13260 &mut rx,
13261 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13262 )
13263 .await;
13264
13265 let captured = crate::router::test_log::captured_logs(&logs);
13266 let admissions: Vec<_> = captured
13267 .lines()
13268 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13269 .collect();
13270 assert_eq!(
13271 admissions.len(),
13272 1,
13273 "legacy admission line must remain exactly once: {captured}"
13274 );
13275 assert!(admissions[0].contains("routable_provider=true"));
13276 assert!(admissions[0].contains("connection_id=777"));
13277 }
13278
13279 fn capability_manifest(
13280 module_id: &str,
13281 provides: &[&str],
13282 must_never_reach: &[&str],
13283 ) -> ModuleManifest {
13284 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13285 manifest.capabilities = Some(CapabilityDeclarations {
13286 provides: provides
13287 .iter()
13288 .map(|capability| (*capability).to_string())
13289 .collect(),
13290 requires: Vec::new(),
13291 must_never_reach: must_never_reach
13292 .iter()
13293 .map(|capability| (*capability).to_string())
13294 .collect(),
13295 });
13296 manifest
13297 }
13298
13299 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13300 Frame::build(
13301 FrameType::Hello,
13302 control_flags(),
13303 0,
13304 0,
13305 corr,
13306 serde_json::to_vec(&ModuleHelloBody {
13307 protocol_ver: manifest.protocol_ver,
13308 manifest,
13309 control_ops: None,
13310 launch_nonce: None,
13311 })
13312 .expect("capability test HELLO serializes"),
13313 )
13314 .expect("capability test HELLO frame builds")
13315 }
13316
13317 fn catalog_update_with_capabilities_frame(
13318 corr: u64,
13319 capabilities: CapabilityDeclarations,
13320 ) -> Frame {
13321 Frame::build(
13322 FrameType::Request,
13323 control_flags(),
13324 0,
13325 0,
13326 corr,
13327 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13328 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
13329 capabilities: Some(capabilities),
13330 ready: None,
13331 })
13332 .expect("capability catalog.update serializes"),
13333 )
13334 .expect("capability catalog.update frame builds")
13335 }
13336
13337 async fn register_capability_manifest(
13338 handler: &ControlHandler,
13339 ctx: &RouteCtx,
13340 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13341 manifest: ModuleManifest,
13342 corr: u64,
13343 ) {
13344 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
13345 }
13346
13347 async fn open_route_for_capability_test(
13348 handler: &ControlHandler,
13349 target_ctx: &RouteCtx,
13350 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13351 client_connection_id: u64,
13352 corr: u64,
13353 target_module_id: &str,
13354 consumer_identity: Option<ConsumerIdentity>,
13355 ) -> (
13356 mpsc::Receiver<crate::router::OutboundFrame>,
13357 ModuleControlRequest,
13358 ) {
13359 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13360 let route_handler = handler.clone();
13361 let target_module_id = target_module_id.to_string();
13362 let route_task = tokio::spawn(async move {
13363 route_handler
13364 .handle_control_frame(
13365 &client_ctx,
13366 route_open_frame_with_admission_facts(
13367 corr,
13368 &target_module_id,
13369 unique_project_root("admission-facts"),
13370 consumer_identity,
13371 None,
13372 ),
13373 )
13374 .await
13375 .expect("capability test route.open succeeds")
13376 });
13377 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13378 .await
13379 .expect("capability test route.open must reach route.bind")
13380 .expect("target control receiver stays open");
13381 let bind_request: ModuleControlRequest =
13382 serde_json::from_slice(&bind.body).expect("route.bind decodes");
13383 handler
13384 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13385 .await
13386 .expect("capability test route.bind ACK succeeds");
13387 assert!(route_task.await.expect("route.open task joins").is_empty());
13388 let opened = client_rx
13389 .recv()
13390 .await
13391 .expect("successful route.open publishes a response");
13392 assert!(matches!(
13393 serde_json::from_slice::<ClientControlResponse>(&opened.body),
13394 Ok(ClientControlResponse::RouteOpen { .. })
13395 ));
13396 (client_rx, bind_request)
13397 }
13398
13399 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13400 assert_eq!(frame.header.ty, FrameType::Push);
13401 assert_eq!(frame.header.channel, 0);
13402 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13403 .expect("route.closed control push decodes");
13404 let ClientControlPush::RouteClosed { channels, .. } = &push else {
13405 panic!("expected route.closed");
13406 };
13407 assert_eq!(channels.len(), 1, "exactly one violating route closed");
13408 let channels = channels.clone();
13409 assert_eq!(
13410 push,
13411 ClientControlPush::RouteClosed {
13412 module_id: target_module_id.to_string(),
13413 channels,
13414 reason: RouteCloseReason::CapabilityDenied,
13415 drained: false,
13416 abandoned: 0,
13417 excluded_subscriptions: 0,
13418 terminal: Some(false),
13419 }
13420 );
13421 }
13422
13423 #[tokio::test]
13424 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
13425 let registry = Arc::new(Registry::default());
13426 let forwarding = Arc::new(ForwardingTable::default());
13427 let supervisor = SupervisorHandle::new();
13428 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13429 let handler =
13430 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13431 .with_supervisor(supervisor);
13432 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
13433 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
13434 register_capability_manifest(
13435 &handler,
13436 &target_ctx,
13437 &mut target_rx,
13438 capability_manifest("target", &["credentials-provider/v1"], &[]),
13439 1,
13440 )
13441 .await;
13442 register_capability_manifest(
13443 &handler,
13444 &opener_ctx,
13445 &mut opener_rx,
13446 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13447 2,
13448 )
13449 .await;
13450
13451 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
13452 let replies = handler
13453 .handle_control_frame(
13454 &client_ctx,
13455 route_open_frame_with_admission_facts(
13456 3,
13457 "target",
13458 unique_project_root("admission-facts"),
13459 Some(ConsumerIdentity {
13460 module_id: "opener".to_string(),
13461 launch_nonce: "opener-nonce".to_string(),
13462 }),
13463 None,
13464 ),
13465 )
13466 .await
13467 .expect("denied route.open returns a typed frame");
13468 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13469 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13470 assert!(
13471 target_rx.try_recv().is_err(),
13472 "forbidden route.open must not relay route.bind"
13473 );
13474 }
13475
13476 #[tokio::test]
13477 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
13478 let registry = Arc::new(Registry::default());
13479 let forwarding = Arc::new(ForwardingTable::default());
13480 let supervisor = SupervisorHandle::new();
13481 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13482 let handler =
13483 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13484 .with_supervisor(supervisor);
13485 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
13486 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
13487 register_capability_manifest(
13488 &handler,
13489 &target_ctx,
13490 &mut target_rx,
13491 capability_manifest("target", &["credentials-provider/v1"], &[]),
13492 1,
13493 )
13494 .await;
13495 register_capability_manifest(
13496 &handler,
13497 &old_opener_ctx,
13498 &mut old_opener_rx,
13499 capability_manifest("opener", &[], &[]),
13500 2,
13501 )
13502 .await;
13503 let (mut client_rx, _) = open_route_for_capability_test(
13504 &handler,
13505 &target_ctx,
13506 &mut target_rx,
13507 712,
13508 3,
13509 "target",
13510 Some(ConsumerIdentity {
13511 module_id: "opener".to_string(),
13512 launch_nonce: "opener-nonce".to_string(),
13513 }),
13514 )
13515 .await;
13516 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13517
13518 handler
13519 .cleanup_connection(old_opener_ctx.connection_id)
13520 .expect("old opener registration cleans up");
13521 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
13522 register_capability_manifest(
13523 &handler,
13524 &new_opener_ctx,
13525 &mut new_opener_rx,
13526 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13527 4,
13528 )
13529 .await;
13530
13531 assert_capability_denied_push(
13532 client_rx
13533 .try_recv()
13534 .expect("HELLO deny addition must emit route.closed")
13535 .frame,
13536 "target",
13537 );
13538 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13539 assert!(matches!(
13540 target_rx.try_recv(),
13541 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13542 ));
13543 }
13544
13545 #[tokio::test]
13546 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
13547 let registry = Arc::new(Registry::default());
13548 let forwarding = Arc::new(ForwardingTable::default());
13549 let supervisor = SupervisorHandle::new();
13550 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13551 let handler =
13552 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13553 .with_supervisor(supervisor);
13554 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
13555 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
13556 register_capability_manifest(
13557 &handler,
13558 &target_ctx,
13559 &mut target_rx,
13560 capability_manifest("target", &[], &[]),
13561 1,
13562 )
13563 .await;
13564 register_capability_manifest(
13565 &handler,
13566 &opener_ctx,
13567 &mut opener_rx,
13568 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13569 2,
13570 )
13571 .await;
13572 let (mut client_rx, _) = open_route_for_capability_test(
13573 &handler,
13574 &target_ctx,
13575 &mut target_rx,
13576 722,
13577 3,
13578 "target",
13579 Some(ConsumerIdentity {
13580 module_id: "opener".to_string(),
13581 launch_nonce: "opener-nonce".to_string(),
13582 }),
13583 )
13584 .await;
13585 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13586
13587 let replies = handler
13588 .handle_control_frame(
13589 &target_ctx,
13590 catalog_update_with_capabilities_frame(
13591 4,
13592 CapabilityDeclarations {
13593 provides: vec!["credentials-provider/v1".to_string()],
13594 requires: Vec::new(),
13595 must_never_reach: Vec::new(),
13596 },
13597 ),
13598 )
13599 .await
13600 .expect("claim catalog.update succeeds");
13601 assert!(matches!(
13602 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
13603 Ok(ModuleControlResponseToModule::CatalogUpdate {})
13604 ));
13605 assert_capability_denied_push(
13606 client_rx
13607 .try_recv()
13608 .expect("claim addition must emit route.closed")
13609 .frame,
13610 "target",
13611 );
13612 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13613 assert!(matches!(
13614 target_rx.try_recv(),
13615 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13616 ));
13617 }
13618
13619 #[tokio::test]
13620 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
13621 let registry = Arc::new(Registry::default());
13622 let forwarding = Arc::new(ForwardingTable::default());
13623 let supervisor = SupervisorHandle::new();
13624 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13625 let handler =
13626 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13627 .with_supervisor(supervisor);
13628 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
13629 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
13630 register_capability_manifest(
13631 &handler,
13632 &target_ctx,
13633 &mut target_rx,
13634 capability_manifest("target", &["credentials-provider/v1"], &[]),
13635 1,
13636 )
13637 .await;
13638 register_capability_manifest(
13639 &handler,
13640 &opener_ctx,
13641 &mut opener_rx,
13642 capability_manifest("opener", &[], &[]),
13643 2,
13644 )
13645 .await;
13646 let (mut client_rx, _) = open_route_for_capability_test(
13647 &handler,
13648 &target_ctx,
13649 &mut target_rx,
13650 732,
13651 3,
13652 "target",
13653 Some(ConsumerIdentity {
13654 module_id: "opener".to_string(),
13655 launch_nonce: "opener-nonce".to_string(),
13656 }),
13657 )
13658 .await;
13659 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13660
13661 handler
13662 .handle_control_frame(
13663 &target_ctx,
13664 catalog_update_with_capabilities_frame(
13665 4,
13666 CapabilityDeclarations {
13667 provides: Vec::new(),
13668 requires: Vec::new(),
13669 must_never_reach: Vec::new(),
13670 },
13671 ),
13672 )
13673 .await
13674 .expect("claim removal catalog.update succeeds");
13675 assert_eq!(
13676 forwarding.active_binding_count().unwrap(),
13677 1,
13678 "removing an attested target claim must leave the route census unchanged"
13679 );
13680 assert!(
13681 client_rx.try_recv().is_err(),
13682 "claim removal must not emit route.closed capability_denied"
13683 );
13684 assert!(
13685 target_rx.try_recv().is_err(),
13686 "claim removal must not send the target a route GOODBYE"
13687 );
13688 }
13689
13690 #[tokio::test]
13693 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
13694 let registry = Arc::new(Registry::default());
13695 let forwarding = Arc::new(ForwardingTable::default());
13696 let supervisor = SupervisorHandle::new();
13697 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13698 let handler =
13699 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13700 .with_supervisor(supervisor);
13701 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
13702 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
13703 register_capability_manifest(
13704 &handler,
13705 &target_ctx,
13706 &mut target_rx,
13707 capability_manifest("target", &["credentials-provider/v1"], &[]),
13708 1,
13709 )
13710 .await;
13711 register_capability_manifest(
13712 &handler,
13713 &opener_ctx,
13714 &mut opener_rx,
13715 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13716 2,
13717 )
13718 .await;
13719
13720 let (_client_rx, bind) = open_route_for_capability_test(
13721 &handler,
13722 &target_ctx,
13723 &mut target_rx,
13724 742,
13725 3,
13726 "target",
13727 None,
13728 )
13729 .await;
13730 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
13731 panic!("direct scope-honesty route must bind");
13732 };
13733 assert_eq!(principal, Some(Principal::Direct));
13734 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13735 }
13736
13737 #[tokio::test]
13740 async fn must_never_reach_self_route_is_capability_forbidden() {
13741 let registry = Arc::new(Registry::default());
13742 let forwarding = Arc::new(ForwardingTable::default());
13743 let supervisor = SupervisorHandle::new();
13744 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
13745 let handler =
13746 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13747 .with_supervisor(supervisor);
13748 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
13749 register_capability_manifest(
13750 &handler,
13751 &self_ctx,
13752 &mut self_rx,
13753 capability_manifest(
13754 "self-provider",
13755 &["credentials-provider/v1"],
13756 &["credentials-provider/v1"],
13757 ),
13758 1,
13759 )
13760 .await;
13761
13762 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
13763 let replies = handler
13764 .handle_control_frame(
13765 &client_ctx,
13766 route_open_frame_with_admission_facts(
13767 2,
13768 "self-provider",
13769 unique_project_root("admission-facts"),
13770 Some(ConsumerIdentity {
13771 module_id: "self-provider".to_string(),
13772 launch_nonce: "self-nonce".to_string(),
13773 }),
13774 None,
13775 ),
13776 )
13777 .await
13778 .expect("self-route refusal returns a typed frame");
13779 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13780 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13781 assert!(
13782 self_rx.try_recv().is_err(),
13783 "self denial must not relay route.bind"
13784 );
13785 }
13786
13787 #[test]
13788 fn unsupported_channel_zero_frame_returns_error() {
13789 let handler = ControlHandler::default();
13790 let request = Frame::build(
13791 FrameType::Request,
13792 control_flags(),
13793 0,
13794 0,
13795 21,
13796 b"opaque".to_vec(),
13797 )
13798 .unwrap();
13799
13800 let response = handler
13801 .handle_control(ConnectionId::new(1), request)
13802 .unwrap();
13803
13804 assert_eq!(response[0].header.ty, FrameType::Error);
13805 assert_eq!(
13806 parse_error(&response[0])["code"],
13807 "unsupported_control_frame"
13808 );
13809 }
13810
13811 mod swap {
13816 use super::*;
13817
13818 const INCUMBENT: ConnectionId = ConnectionId::new(30);
13819 const CANDIDATE: ConnectionId = ConnectionId::new(40);
13820
13821 struct Swap {
13822 registry: Arc<Registry>,
13823 forwarding: Arc<ForwardingTable>,
13824 handler: ControlHandler,
13825 incumbent_ctx: RouteCtx,
13826 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13827 candidate_ctx: RouteCtx,
13828 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13829 }
13830
13831 async fn swap_with_incumbent() -> Swap {
13832 let registry = Arc::new(Registry::default());
13833 let forwarding = Arc::new(ForwardingTable::default());
13834 let handler =
13835 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13836 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
13837 hello_via_sink(
13838 &handler,
13839 &incumbent_ctx,
13840 &mut incumbent_rx,
13841 hello_frame("aft", PROTOCOL_VERSION, 7),
13842 )
13843 .await;
13844 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
13845 Swap {
13846 registry,
13847 forwarding,
13848 handler,
13849 incumbent_ctx,
13850 incumbent_rx,
13851 candidate_ctx,
13852 candidate_rx,
13853 }
13854 }
13855
13856 fn register_candidate(swap: &Swap, ready: Option<bool>) {
13857 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
13858 candidate_manifest.ready = ready;
13859 let registration = swap
13860 .registry
13861 .register_candidate_with_control_ops(
13862 candidate_manifest,
13863 PROTOCOL_VERSION,
13864 CANDIDATE,
13865 module_baseline_control_ops(),
13866 )
13867 .unwrap();
13868 swap.forwarding
13869 .register_candidate_module_connection(
13870 CANDIDATE,
13871 "aft".to_string(),
13872 PROTOCOL_VERSION,
13873 manifest_concurrency(®istration.manifest),
13874 swap.candidate_ctx.egress.clone(),
13875 )
13876 .unwrap();
13877 }
13878
13879 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
13880 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
13881 swap.registry.promote_candidate("aft").unwrap().unwrap();
13882 cutover.incumbent.unwrap()
13883 }
13884
13885 #[tokio::test]
13894 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
13895 let mut swap = swap_with_incumbent().await;
13896 let hold = Arc::new(std::sync::Barrier::new(2));
13897 let handler = swap
13898 .handler
13899 .clone()
13900 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
13901 handler
13902 .supervisor
13903 .open_swap("aft", "candidate-nonce".to_string());
13904 let hello = {
13905 let handler = handler.clone();
13906 let candidate_ctx = swap.candidate_ctx.clone();
13907 std::thread::spawn(move || {
13908 handler
13909 .handle_hello(
13910 candidate_ctx.connection_id,
13911 Some(candidate_ctx.egress.clone()),
13912 hello_frame_with_nonce(
13913 "aft",
13914 PROTOCOL_VERSION,
13915 8,
13916 Some("candidate-nonce"),
13917 ),
13918 )
13919 .unwrap()
13920 })
13921 };
13922
13923 hold.wait();
13924 assert!(swap
13927 .registry
13928 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
13929 .unwrap()
13930 .is_some());
13931 assert!(swap
13932 .registry
13933 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
13934 .unwrap()
13935 .is_none());
13936 assert!(swap
13937 .forwarding
13938 .module_endpoint_for_connection(CANDIDATE)
13939 .unwrap()
13940 .is_none());
13941
13942 hold.wait();
13943 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
13944 assert_eq!(
13945 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
13946 FrameType::HelloAck
13947 );
13948 let published = swap
13949 .registry
13950 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
13951 .unwrap()
13952 .expect("candidate is published once its HELLO completes");
13953 assert_eq!(published.connection_id, CANDIDATE);
13954 assert!(swap
13955 .forwarding
13956 .module_endpoint_for_connection(CANDIDATE)
13957 .unwrap()
13958 .is_some());
13959 }
13960
13961 fn keyed_total(counters: &Value, key: &str) -> u64 {
13962 counters[key]
13963 .as_object()
13964 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
13965 .unwrap_or(0)
13966 }
13967
13968 #[tokio::test]
13969 async fn replacement_logs_old_end_and_new_admission_once() {
13970 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13971 let mut swap = swap_with_incumbent().await;
13972 swap.handler
13973 .supervisor
13974 .open_swap("aft", "candidate-nonce".to_string());
13975 hello_via_sink(
13976 &swap.handler,
13977 &swap.candidate_ctx,
13978 &mut swap.candidate_rx,
13979 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
13980 )
13981 .await;
13982 cutover(&swap);
13983 swap.handler.cleanup_connection(INCUMBENT).unwrap();
13984
13985 let captured = crate::router::test_log::captured_logs(&logs);
13986 println!("captured replacement registry logs:\n{captured}");
13987 let new_admissions: Vec<_> = captured
13988 .lines()
13989 .filter(|line| {
13990 line.contains("connection_id=40")
13991 && line.contains("swap candidate registered; not routable until cutover")
13992 })
13993 .collect();
13994 assert_eq!(
13995 new_admissions.len(),
13996 1,
13997 "unexpected admission logs: {captured}"
13998 );
13999 assert!(new_admissions[0].contains("module_id=aft"));
14000 assert!(new_admissions[0].contains("connection_id=40"));
14001 assert!(
14002 new_admissions[0].contains("swap candidate registered; not routable until cutover")
14003 );
14004 assert!(new_admissions[0].contains("ready=true"));
14005
14006 let old_admissions: Vec<_> = captured
14007 .lines()
14008 .filter(|line| {
14009 line.contains("module registered module_id=aft ")
14010 && line.contains("connection_id=30")
14011 })
14012 .collect();
14013 assert_eq!(
14014 old_admissions.len(),
14015 1,
14016 "unexpected incumbent admission: {captured}"
14017 );
14018
14019 let old_ends: Vec<_> = captured
14020 .lines()
14021 .filter(|line| {
14022 line.contains("connection_id=30") && line.contains("module registration ended")
14023 })
14024 .collect();
14025 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14026 assert!(old_ends[0].contains("module_id=aft"));
14027 assert!(old_ends[0].contains("connection_id=30"));
14028 assert!(old_ends[0].contains("reason=replaced"));
14029 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14030 assert_eq!(
14031 captured
14032 .lines()
14033 .filter(|line| line.contains("module registration promoted"))
14034 .count(),
14035 1,
14036 "unexpected promotion logs: {captured}"
14037 );
14038 }
14039
14040 #[tokio::test]
14046 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14047 let mut swap = swap_with_incumbent().await;
14048 let handler = swap.handler.clone();
14049
14050 let cotenant = ConnectionId::new(31);
14052 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14053 let (cotenant_task, cotenant_bind) = relay_route_open(
14054 &handler,
14055 cotenant,
14056 &cotenant_ctx.egress,
14057 &mut swap.incumbent_rx,
14058 100,
14059 "aft",
14060 "swap-cotenant",
14061 )
14062 .await;
14063 handler
14064 .handle_control_frame(
14065 &swap.incumbent_ctx,
14066 route_bind_ack(cotenant_bind.header.corr),
14067 )
14068 .await
14069 .unwrap();
14070 assert!(cotenant_task.await.unwrap().is_empty());
14071 let (cotenant_channel, cotenant_epoch) =
14072 published_route(&cotenant_rx.recv().await.unwrap());
14073
14074 let caller = ConnectionId::new(32);
14076 let (caller_ctx, mut caller_rx) = route_ctx(caller);
14077 let (caller_task, caller_bind) = relay_route_open(
14078 &handler,
14079 caller,
14080 &caller_ctx.egress,
14081 &mut swap.incumbent_rx,
14082 101,
14083 "aft",
14084 "swap-caller",
14085 )
14086 .await;
14087 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14088
14089 register_candidate(&swap, None);
14090 cutover(&swap);
14091
14092 let ack = handler
14094 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14095 .await;
14096 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14097 if module_loop_error.is_some() {
14098 handler.cleanup_connection(INCUMBENT).unwrap();
14101 }
14102
14103 assert!(
14105 cotenant_rx.try_recv().is_err(),
14106 "the co-tenant route on the incumbent was torn down by one late ack: \
14107 {module_loop_error:?}"
14108 );
14109 assert!(matches!(
14110 swap.forwarding
14111 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14112 .unwrap(),
14113 DataRoute::Client(DataRouteState::Bound(_))
14114 ));
14115 assert_eq!(module_loop_error, None);
14116 assert!(swap
14117 .registry
14118 .get_module_by_connection(INCUMBENT)
14119 .unwrap()
14120 .is_some());
14121
14122 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14124 .await
14125 .expect("the incumbent is told to drop the abandoned binding")
14126 .unwrap()
14127 .frame;
14128 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14129 assert_eq!(goodbye.header.channel, abandoned_channel);
14130 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14131 assert!(swap.incumbent_rx.try_recv().is_err());
14132
14133 let response = caller_task.await.unwrap();
14135 assert_eq!(response.len(), 1);
14136 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14137 assert!(caller_rx.try_recv().is_err());
14138
14139 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14143 let counters = handler.counters().snapshot();
14144 assert_eq!(
14145 keyed_total(&counters, "route_open_accepted_by_principal"),
14146 1
14147 );
14148 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14149 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14150 }
14151
14152 #[tokio::test]
14156 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14157 let mut swap = swap_with_incumbent().await;
14158 register_candidate(&swap, None);
14159 let incumbent = cutover(&swap);
14160 swap.forwarding
14161 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14162 .unwrap()
14163 .expect("the incumbent is still registered");
14164
14165 let client = ConnectionId::new(33);
14166 let (client_ctx, mut client_rx) = route_ctx(client);
14167 let route_handler = swap.handler.clone();
14168 let open_ctx = RouteCtx {
14169 connection_id: client,
14170 egress: client_ctx.egress.clone(),
14171 };
14172 let mut route_task = tokio::spawn(async move {
14173 route_handler
14174 .handle_control_frame(
14175 &open_ctx,
14176 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14177 )
14178 .await
14179 .unwrap()
14180 });
14181 let bind = tokio::select! {
14182 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14183 response = &mut route_task => {
14184 let response = response.unwrap();
14185 panic!(
14186 "post-cutover route.open was refused instead of relayed to the candidate: {}",
14187 parse_error(&response[0])["code"]
14188 );
14189 }
14190 };
14191 swap.handler
14192 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14193 .await
14194 .unwrap();
14195 assert!(route_task.await.unwrap().is_empty());
14196 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14197 match swap
14198 .forwarding
14199 .lookup_data_route(client, channel, epoch)
14200 .unwrap()
14201 {
14202 DataRoute::Client(DataRouteState::Bound(route)) => {
14203 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14204 }
14205 other => panic!("expected a bound route on the candidate, got {other:?}"),
14206 }
14207 assert!(swap.incumbent_rx.try_recv().is_err());
14208 }
14209
14210 #[tokio::test]
14215 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14216 let swap = swap_with_incumbent().await;
14217 register_candidate(&swap, Some(false));
14218 let update = Frame::build(
14219 FrameType::Request,
14220 control_flags(),
14221 0,
14222 0,
14223 55,
14224 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14225 provides: manifest("aft", PROTOCOL_VERSION).provides,
14226 capabilities: None,
14227 ready: Some(true),
14228 })
14229 .unwrap(),
14230 )
14231 .unwrap();
14232
14233 let replies = swap
14234 .handler
14235 .handle_control_frame(&swap.candidate_ctx, update)
14236 .await
14237 .unwrap();
14238
14239 assert_eq!(replies.len(), 1);
14240 assert_eq!(
14241 replies[0].header.ty,
14242 FrameType::Response,
14243 "candidate catalog.update was refused: {:?}",
14244 serde_json::from_slice::<Value>(&replies[0].body).ok()
14245 );
14246 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14247 assert_eq!(
14248 swap.registry
14249 .get_module("aft")
14250 .unwrap()
14251 .unwrap()
14252 .connection_id,
14253 INCUMBENT
14254 );
14255 }
14256 }
14257
14258 mod swap_admission {
14262 use super::*;
14263
14264 const INCUMBENT_NONCE: &str = "incumbent-nonce";
14265 const CANDIDATE_NONCE: &str = "candidate-nonce";
14266
14267 fn handler_with_incumbent(
14268 module_id: &str,
14269 reserved: bool,
14270 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14271 let registry = Arc::new(Registry::default());
14272 let supervisor = SupervisorHandle::new();
14273 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14274 if reserved {
14275 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14276 }
14277 let handler =
14278 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
14279 let incumbent = handler
14280 .handle_control(
14281 ConnectionId::new(1),
14282 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14283 )
14284 .unwrap();
14285 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14286 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14287 (registry, supervisor, handler)
14288 }
14289
14290 #[test]
14296 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14297 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14298
14299 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14300 let replies = handler
14301 .handle_control(
14302 ConnectionId::new(connection),
14303 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
14304 )
14305 .unwrap();
14306 assert_eq!(replies[0].header.ty, FrameType::Error);
14307 assert_eq!(
14308 parse_error(&replies[0])["code"],
14309 "swap_token_invalid",
14310 "nonce {nonce:?}"
14311 );
14312 }
14313 assert!(registry.get_candidate("aft").unwrap().is_none());
14314 assert_eq!(
14315 registry.get_module("aft").unwrap().unwrap().connection_id,
14316 ConnectionId::new(1)
14317 );
14318
14319 let admitted = handler
14322 .handle_control(
14323 ConnectionId::new(4),
14324 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
14325 )
14326 .unwrap();
14327 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
14328 assert_eq!(
14329 registry
14330 .get_candidate("aft")
14331 .unwrap()
14332 .unwrap()
14333 .connection_id,
14334 ConnectionId::new(4)
14335 );
14336 assert_eq!(
14337 registry.get_module("aft").unwrap().unwrap().connection_id,
14338 ConnectionId::new(1),
14339 "the candidate must not take the active slot"
14340 );
14341 let replayed = handler
14342 .handle_control(
14343 ConnectionId::new(5),
14344 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
14345 )
14346 .unwrap();
14347 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
14348
14349 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
14353 let squatter = handler
14354 .handle_control(
14355 ConnectionId::new(6),
14356 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
14357 )
14358 .unwrap();
14359 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14360 assert!(
14361 registry.get_module("aft").unwrap().is_none(),
14362 "a squatter took the active slot of an id being swapped"
14363 );
14364 }
14365
14366 #[test]
14371 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14372 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14373
14374 let replies = handler
14375 .handle_control(
14376 ConnectionId::new(2),
14377 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14378 )
14379 .unwrap();
14380
14381 assert_eq!(
14382 replies[0].header.ty,
14383 FrameType::HelloAck,
14384 "reserved candidate refused: {:?}",
14385 serde_json::from_slice::<Value>(&replies[0].body).ok()
14386 );
14387 assert_eq!(
14388 registry
14389 .get_candidate("vault")
14390 .unwrap()
14391 .unwrap()
14392 .connection_id,
14393 ConnectionId::new(2)
14394 );
14395 }
14396
14397 #[test]
14400 fn without_an_open_swap_the_ordinary_gates_decide() {
14401 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14402 supervisor.close_swap("vault");
14403
14404 let candidate = handler
14405 .handle_control(
14406 ConnectionId::new(2),
14407 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14408 )
14409 .unwrap();
14410 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14411 let duplicate = handler
14412 .handle_control(
14413 ConnectionId::new(3),
14414 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
14415 )
14416 .unwrap();
14417 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14418 assert!(registry.get_candidate("vault").unwrap().is_none());
14419 }
14420 }
14421
14422 mod scopes {
14426 mod activity {
14427 include!("activity_scope_tests.rs");
14428 }
14429 mod delta {
14430 include!("scope_handler_delta_tests.rs");
14431 }
14432 mod call_modes {
14433 include!("scope_call_mode_tests.rs");
14434 }
14435
14436 use subc_protocol::scope::{
14437 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
14438 ScopeStatus,
14439 };
14440
14441 use super::*;
14442
14443 const OWNER: &str = "prefrontal-core";
14444
14445 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
14446 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
14447 }
14448
14449 async fn call(
14450 handler: &ControlHandler,
14451 ctx: &RouteCtx,
14452 request: &ModuleControlRequestFromModule,
14453 ) -> Frame {
14454 let body = serde_json::to_vec(request).unwrap();
14455 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
14456 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
14457 assert_eq!(replies.len(), 1, "{replies:?}");
14458 replies.pop().unwrap()
14459 }
14460
14461 async fn sync(
14462 handler: &ControlHandler,
14463 ctx: &RouteCtx,
14464 generation: u64,
14465 scopes: Vec<ScopeRecord>,
14466 ) -> Result<ModuleControlResponseToModule, String> {
14467 let reply = call(
14468 handler,
14469 ctx,
14470 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
14471 )
14472 .await;
14473 match reply.header.ty {
14474 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
14475 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
14476 }
14477 }
14478
14479 async fn describe(
14480 handler: &ControlHandler,
14481 ctx: &RouteCtx,
14482 owner: &str,
14483 scope_ref: &str,
14484 ) -> ModuleControlResponseToModule {
14485 let reply = call(
14486 handler,
14487 ctx,
14488 &ModuleControlRequestFromModule::ScopeDescribe {
14489 owner: Principal::Reserved {
14490 module_id: owner.to_string(),
14491 },
14492 scope_ref: scope_ref.to_string(),
14493 },
14494 )
14495 .await;
14496 assert_eq!(
14497 reply.header.ty,
14498 FrameType::Response,
14499 "{:?}",
14500 parse_error(&reply)
14501 );
14502 serde_json::from_slice(&reply.body).unwrap()
14503 }
14504
14505 #[tokio::test]
14506 async fn scope_describe_never_echoes_bind_proof_or_hash() {
14507 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
14508 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
14509 let mut rig = rig().await;
14510 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
14511 .await;
14512 let _bound = rig
14513 .bound(
14514 Some(AFT),
14515 PLEXUS,
14516 Some(
14517 rig.selector("s", Some(1))
14518 .with_bind_proof(Some(PROOF.into())),
14519 ),
14520 )
14521 .await;
14522 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
14523 let output = serde_json::to_string(&reply).unwrap();
14524 assert!(output.contains("\"status\":\"live\""));
14525 assert!(!output.contains(PROOF));
14526 assert!(!output.contains(HASH));
14527 }
14528
14529 async fn module(
14531 handler: &ControlHandler,
14532 connection: u64,
14533 module_id: &str,
14534 nonce: Option<&str>,
14535 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14536 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
14537 hello_via_sink(
14538 handler,
14539 &ctx,
14540 &mut rx,
14541 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
14542 )
14543 .await;
14544 (ctx, rx)
14545 }
14546
14547 #[tokio::test]
14550 async fn a_client_connection_cannot_sync_or_describe() {
14551 let handler = ControlHandler::new(Arc::new(Registry::default()));
14552 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
14553 for request in [
14554 ModuleControlRequestFromModule::ScopeSync {
14555 generation: 1,
14556 scopes: vec![head("s", 1)],
14557 },
14558 ModuleControlRequestFromModule::ScopeDescribe {
14559 owner: Principal::Direct,
14560 scope_ref: "s".to_string(),
14561 },
14562 ] {
14563 let reply = call(&handler, &ctx, &request).await;
14564 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
14565 }
14566 assert!(
14567 !handler
14568 .scopes
14569 .read()
14570 .unwrap()
14571 .describe(
14572 &Principal::Reserved {
14573 module_id: OWNER.to_string()
14574 },
14575 "s"
14576 )
14577 .owner_synced
14578 );
14579 }
14580
14581 #[tokio::test]
14584 async fn a_module_without_a_supervised_launch_cannot_sync() {
14585 let handler = ControlHandler::new(Arc::new(Registry::default()));
14586 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
14587 assert_eq!(
14588 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
14589 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14590 );
14591 }
14592
14593 #[tokio::test]
14594 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
14595 let supervisor = SupervisorHandle::new();
14596 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14597 let handler = ControlHandler::new(Arc::new(Registry::default()))
14598 .with_supervisor(supervisor.clone());
14599 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14600 sync(&handler, &incumbent, 1, vec![head("s", 1)])
14601 .await
14602 .expect("the current launch syncs");
14603
14604 supervisor.open_swap(OWNER, "n2".to_string());
14607 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
14608 assert_eq!(
14609 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14610 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14611 );
14612 sync(&handler, &incumbent, 2, vec![head("s", 1)])
14613 .await
14614 .expect("the serving owner syncs during the swap");
14615
14616 supervisor.close_swap(OWNER);
14619 assert_eq!(
14620 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14621 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14622 );
14623 sync(&handler, &incumbent, 3, vec![head("s", 1)])
14624 .await
14625 .expect("the serving owner syncs after the rollback");
14626 handler.cleanup_connection(candidate.connection_id).unwrap();
14627
14628 supervisor.open_swap(OWNER, "n3".to_string());
14633 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
14634 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
14635 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
14636 .await
14637 .expect("the promoted launch takes authority");
14638 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14639 panic!("unexpected reply {reply:?}");
14640 };
14641 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
14642 assert_eq!(
14643 sync(&handler, &incumbent, 4, Vec::new()).await,
14644 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14645 );
14646 }
14647
14648 #[tokio::test]
14651 async fn closing_the_authority_connection_frees_sync_authority() {
14652 let supervisor = SupervisorHandle::new();
14653 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14654 let handler = ControlHandler::new(Arc::new(Registry::default()))
14655 .with_supervisor(supervisor.clone());
14656 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14657 sync(&handler, &first, 10, vec![head("s", 1)])
14658 .await
14659 .unwrap();
14660 handler.cleanup_connection(first.connection_id).unwrap();
14661
14662 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
14663 sync(&handler, &second, 1, vec![head("s", 1)])
14664 .await
14665 .expect("the next connection takes the released authority");
14666 }
14667
14668 #[tokio::test]
14669 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
14670 let supervisor = SupervisorHandle::new();
14671 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14672 let handler =
14673 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
14674 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
14675 sync(&handler, &first, 10, vec![head("s", 1)])
14676 .await
14677 .unwrap();
14678 handler
14679 .handle_control_frame(
14680 &first,
14681 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
14682 )
14683 .await
14684 .unwrap();
14685 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
14686 sync(&handler, &second, 1, vec![head("s", 1)])
14687 .await
14688 .expect("GOODBYE releases authority even if the old socket remains open");
14689 }
14690
14691 #[tokio::test]
14692 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
14693 let registry = Arc::new(Registry::default());
14694 let supervisor_handle = SupervisorHandle::new();
14695 let supervisor =
14696 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14697 .with_handle(supervisor_handle.clone())
14698 .with_daemon_incarnation("incarnation-7".to_string());
14699 supervisor
14702 .supervise_configured(
14703 ModuleSpec {
14704 module_id: OWNER.to_string(),
14705 program: PathBuf::from("/nonexistent/prefrontal-core"),
14706 args: Vec::new(),
14707 env: Vec::new(),
14708 reserved: false,
14709 reserved_prefixes: Vec::new(),
14710 protocol: ModuleProtocol::Subc,
14711 overlap: Default::default(),
14712 },
14713 false,
14714 )
14715 .unwrap();
14716 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
14717 let handler =
14718 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14719 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
14720
14721 let ModuleControlResponseToModule::ScopeDescribe {
14723 status,
14724 daemon_incarnation,
14725 owner_synced,
14726 owner_configured,
14727 scope,
14728 ..
14729 } = describe(&handler, &reader, OWNER, "s").await
14730 else {
14731 panic!("not a describe reply");
14732 };
14733 assert_eq!(status, ScopeStatus::NotLive);
14734 assert_eq!(daemon_incarnation, "incarnation-7");
14735 assert!(!owner_synced);
14736 assert!(owner_configured);
14737 assert!(scope.is_none());
14738
14739 let ModuleControlResponseToModule::ScopeDescribe {
14742 status,
14743 owner_configured,
14744 ..
14745 } = describe(&handler, &reader, "ghost", "s").await
14746 else {
14747 panic!("not a describe reply");
14748 };
14749 assert_eq!(status, ScopeStatus::NotLive);
14750 assert!(!owner_configured);
14751
14752 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
14754 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
14755 let ModuleControlResponseToModule::ScopeDescribe {
14756 status,
14757 scope_epoch,
14758 owner_synced,
14759 scope,
14760 ..
14761 } = describe(&handler, &reader, OWNER, "s").await
14762 else {
14763 panic!("not a describe reply");
14764 };
14765 assert_eq!(status, ScopeStatus::Live);
14766 assert_eq!(scope_epoch, Some(4));
14767 assert!(owner_synced);
14768 let stamp = scope.expect("a live scope carries its stamp");
14769 assert!(
14770 stamp.owner_authorized,
14771 "prefrontal-core is the default authority"
14772 );
14773 assert_eq!(stamp.kind, ScopeKind::Head);
14774 }
14775
14776 #[tokio::test]
14777 async fn scope_authority_owners_decides_owner_authorized() {
14778 let supervisor = SupervisorHandle::new();
14779 supervisor.set_spawn_nonce("broca", "b1".to_string());
14780 let handler = ControlHandler::new(Arc::new(Registry::default()))
14781 .with_supervisor(supervisor)
14782 .with_scope_authority_owners(vec!["broca".to_string()]);
14783 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
14784 let mut gated = head("s", 1);
14785 gated.attributes.agent_id = Some("agent".to_string());
14786 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
14787 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
14788 describe(&handler, &broca, "broca", "s").await
14789 else {
14790 panic!("not a describe reply");
14791 };
14792 assert!(scope.unwrap().owner_authorized);
14793 }
14794
14795 #[tokio::test]
14799 async fn scope_ops_and_the_scopes_capability_are_advertised() {
14800 let handler = ControlHandler::new(Arc::new(Registry::default()));
14801 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
14802 let ack = hello_via_sink(
14803 &handler,
14804 &ctx,
14805 &mut rx,
14806 hello_frame("m", PROTOCOL_VERSION, 1),
14807 )
14808 .await;
14809 let ack = parse_ack(&ack);
14810 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
14811 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
14812 }
14813 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
14814
14815 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
14816 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14817 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14818 let reply = handler
14819 .handle_control_frame(&client, frame)
14820 .await
14821 .unwrap()
14822 .pop()
14823 .unwrap();
14824 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14825 serde_json::from_slice(&reply.body).unwrap()
14826 else {
14827 panic!("not a server.describe reply");
14828 };
14829 assert!(
14830 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
14831 "{capabilities:?}"
14832 );
14833 }
14834
14835 const PLEXUS: &str = "plexus";
14838 const OTHER: &str = "other";
14839 const AFT: &str = "aft";
14840 const BROCA: &str = "broca";
14841 const MAGIC: &str = "magic-context";
14842
14843 fn nonce(module_id: &str) -> String {
14844 format!("nonce-{module_id}")
14845 }
14846
14847 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14848 let (tx, rx) = mpsc::channel(64);
14849 (
14850 RouteCtx {
14851 connection_id: ConnectionId::new(connection),
14852 egress: FrameSink::new(tx),
14853 },
14854 rx,
14855 )
14856 }
14857
14858 struct Rig {
14862 handler: ControlHandler,
14863 forwarding: Arc<ForwardingTable>,
14864 owner: RouteCtx,
14865 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14866 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
14867 generation: u64,
14868 next_connection: u64,
14869 _supervisor: Supervisor,
14870 }
14871
14872 async fn rig() -> Rig {
14873 rig_with_flow_support(true).await
14874 }
14875
14876 async fn rig_with_flow_support(flow_support: bool) -> Rig {
14877 rig_with_scope_capabilities(flow_support, false).await
14878 }
14879
14880 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
14881 let mut provides = Vec::new();
14882 if flow_support {
14883 provides.push("flow-scopes/v1");
14884 }
14885 if run_support {
14886 provides.push("agent-run-scopes/v1");
14887 }
14888 rig_with_target_capabilities(&provides).await
14889 }
14890
14891 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
14892 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
14893 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14894 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14897 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
14898 Frame::build(
14899 FrameType::Hello,
14900 control_flags(),
14901 0,
14902 0,
14903 connection,
14904 serde_json::to_vec(&body).unwrap(),
14905 )
14906 .unwrap()
14907 }
14908
14909 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
14910 let registry = Arc::new(Registry::default());
14911 let forwarding = Arc::new(ForwardingTable::default());
14912 let supervisor_handle = SupervisorHandle::new();
14913 let supervisor =
14914 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14915 .with_handle(supervisor_handle.clone());
14916 supervisor
14917 .supervise_configured(
14918 ModuleSpec {
14919 module_id: OWNER.to_string(),
14920 program: PathBuf::from("/nonexistent/prefrontal-core"),
14921 args: Vec::new(),
14922 env: Vec::new(),
14923 reserved: false,
14924 reserved_prefixes: Vec::new(),
14925 protocol: ModuleProtocol::Subc,
14926 overlap: Default::default(),
14927 },
14928 false,
14929 )
14930 .unwrap();
14931 for module_id in [OWNER, AFT, BROCA, MAGIC] {
14932 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
14933 }
14934 let handler =
14935 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14936 .with_supervisor(supervisor_handle);
14937 let (owner, mut owner_rx) = wide_ctx(1);
14938 hello_via_sink(
14939 &handler,
14940 &owner,
14941 &mut owner_rx,
14942 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
14943 )
14944 .await;
14945 let mut modules = BTreeMap::new();
14946 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
14947 let (ctx, mut rx) = wide_ctx(connection);
14948 let hello = hello_with_capabilities(module_id, connection, provides);
14949 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
14950 modules.insert(module_id.to_string(), (ctx, rx));
14951 }
14952 Rig {
14953 handler,
14954 forwarding,
14955 owner,
14956 _owner_rx: owner_rx,
14957 modules,
14958 generation: 0,
14959 next_connection: 100,
14960 _supervisor: supervisor,
14961 }
14962 }
14963
14964 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
14965 ScopeCarrier::new(Principal::Reserved {
14966 module_id: module_id.to_string(),
14967 })
14968 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
14969 }
14970
14971 fn session(scope_epoch: u64) -> ScopeRecord {
14974 let mut record = head("s", scope_epoch);
14975 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
14976 record.attributes.agent_id = Some("agent-1".to_string());
14977 record.attributes.delegates = true;
14978 record
14979 }
14980
14981 impl Rig {
14982 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
14983 self.generation += 1;
14984 sync(&self.handler, &self.owner, self.generation, scopes)
14985 .await
14986 .expect("the owner's sync is accepted");
14987 }
14988
14989 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
14990 ScopeSelector::new(
14991 Principal::Reserved {
14992 module_id: OWNER.to_string(),
14993 },
14994 scope_ref.to_string(),
14995 scope_epoch,
14996 )
14997 }
14998
14999 fn open_frame(
15000 &mut self,
15001 opener: Option<&str>,
15002 target: &str,
15003 scope: Option<ScopeSelector>,
15004 ) -> (
15005 RouteCtx,
15006 mpsc::Receiver<crate::router::OutboundFrame>,
15007 Frame,
15008 ) {
15009 self.next_connection += 1;
15010 let (ctx, rx) = wide_ctx(self.next_connection);
15011 let root = unique_project_root("scoped-open");
15012 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15013 target: RouteTarget::ToolProvider {
15014 module_id: target.to_string(),
15015 },
15016 identity: BindIdentity::new(
15017 root.path().to_path_buf(),
15018 "unit".to_string(),
15019 "session".to_string(),
15020 ),
15021 consumer_identity: opener.map(|module_id| ConsumerIdentity {
15022 module_id: module_id.to_string(),
15023 launch_nonce: nonce(module_id),
15024 }),
15025 consumer_capabilities: None,
15026 role_versions: None,
15027 admission_facts: None,
15028 scope,
15029 })
15030 .unwrap();
15031 let frame = Frame::build(
15032 FrameType::Request,
15033 control_flags(),
15034 0,
15035 0,
15036 self.next_connection,
15037 body,
15038 )
15039 .unwrap();
15040 (ctx, rx, frame)
15041 }
15042
15043 async fn refused(
15045 &mut self,
15046 opener: Option<&str>,
15047 target: &str,
15048 scope: Option<ScopeSelector>,
15049 ) -> String {
15050 self.refusal_body(opener, target, scope).await["code"]
15051 .as_str()
15052 .unwrap()
15053 .to_string()
15054 }
15055
15056 async fn refusal_body(
15057 &mut self,
15058 opener: Option<&str>,
15059 target: &str,
15060 scope: Option<ScopeSelector>,
15061 ) -> Value {
15062 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15063 let replies = tokio::time::timeout(
15064 Duration::from_secs(2),
15065 self.handler.handle_control_frame(&ctx, frame),
15066 )
15067 .await
15068 .expect("the open must be refused before waiting for a bind ack")
15069 .unwrap();
15070 assert_eq!(replies.len(), 1, "{replies:?}");
15071 assert_eq!(replies[0].header.ty, FrameType::Error);
15072 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15073 assert!(
15074 module_rx.try_recv().is_err(),
15075 "a refused open relays nothing"
15076 );
15077 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15078 parse_error(&replies[0])
15079 }
15080
15081 async fn relayed(
15083 &mut self,
15084 opener: Option<&str>,
15085 target: &str,
15086 scope: Option<ScopeSelector>,
15087 ) -> Relayed {
15088 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15089 let handler = self.handler.clone();
15090 let task_ctx = ctx.clone();
15091 let task = tokio::spawn(async move {
15092 handler
15093 .handle_control_frame(&task_ctx, frame)
15094 .await
15095 .unwrap()
15096 });
15097 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15098 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15099 .await
15100 .expect("the target receives the relayed route.bind")
15101 .unwrap()
15102 .frame;
15103 Relayed {
15104 target: target.to_string(),
15105 client: ctx,
15106 client_rx: rx,
15107 task,
15108 bind,
15109 }
15110 }
15111
15112 async fn ack(&self, relayed: &Relayed) {
15113 let (module, _) = &self.modules[&relayed.target];
15114 self.handler
15115 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15116 .await
15117 .unwrap();
15118 }
15119
15120 async fn bound(
15122 &mut self,
15123 opener: Option<&str>,
15124 target: &str,
15125 scope: Option<ScopeSelector>,
15126 ) -> Bound {
15127 let relayed = self.relayed(opener, target, scope).await;
15128 self.ack(&relayed).await;
15129 let Relayed {
15130 target,
15131 client,
15132 mut client_rx,
15133 task,
15134 bind,
15135 } = relayed;
15136 assert!(
15137 task.await.unwrap().is_empty(),
15138 "the open is answered by commit"
15139 );
15140 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15141 Bound {
15142 target,
15143 client,
15144 client_rx,
15145 channel,
15146 epoch,
15147 bind,
15148 }
15149 }
15150
15151 fn live(&self, route: &Bound) -> bool {
15152 matches!(
15153 self.forwarding
15154 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15155 .unwrap(),
15156 DataRoute::Client(DataRouteState::Bound(_))
15157 )
15158 }
15159 }
15160
15161 struct Relayed {
15162 target: String,
15163 client: RouteCtx,
15164 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15165 task: tokio::task::JoinHandle<Vec<Frame>>,
15166 bind: Frame,
15167 }
15168
15169 struct Bound {
15170 target: String,
15171 client: RouteCtx,
15172 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15173 channel: u16,
15174 epoch: u32,
15175 bind: Frame,
15176 }
15177
15178 impl Bound {
15179 fn closed_reason(&mut self) -> RouteCloseReason {
15182 let mut reason = None;
15183 let mut goodbye = false;
15184 while let Ok(outbound) = self.client_rx.try_recv() {
15185 let frame = outbound.frame;
15186 match frame.header.ty {
15187 FrameType::Goodbye => {
15188 assert_eq!(
15189 (frame.header.channel, frame.header.epoch),
15190 (self.channel, self.epoch)
15191 );
15192 goodbye = true;
15193 }
15194 FrameType::Push => {
15195 let ClientControlPush::RouteClosed {
15196 reason: r,
15197 module_id,
15198 ..
15199 } = serde_json::from_slice(&frame.body).unwrap()
15200 else {
15201 panic!("unexpected push");
15202 };
15203 assert_eq!(module_id, self.target);
15204 reason = Some(r);
15205 }
15206 other => panic!("unexpected frame {other:?}"),
15207 }
15208 }
15209 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15210 reason.expect("the client is told why the route closed")
15211 }
15212
15213 fn untouched(&mut self) -> bool {
15214 self.client_rx.try_recv().is_err()
15215 }
15216
15217 fn stamp(&self) -> Option<ScopeStamp> {
15218 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15219 ModuleControlRequest::RouteBind { scope, .. } => scope,
15220 other => panic!("expected a route.bind, got {other:?}"),
15221 }
15222 }
15223 }
15224
15225 #[tokio::test]
15226 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15227 ) {
15228 let mut rig = rig().await;
15229 let mut record = session(1);
15230 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15231 record.child_owners = vec![Principal::Reserved {
15232 module_id: MAGIC.to_string(),
15233 }];
15234 rig.sync(vec![record]).await;
15235 let scope = || Some(rig_selector("s", Some(1)));
15236
15237 rig.bound(Some(OWNER), PLEXUS, scope()).await;
15240 rig.bound(Some(AFT), OTHER, scope()).await;
15241 rig.bound(Some(BROCA), PLEXUS, scope()).await;
15242
15243 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15247 assert_eq!(
15248 rig.refused(opener, target, scope()).await,
15249 error_codes::SCOPE_NOT_CARRIER,
15250 "{opener:?} -> {target}"
15251 );
15252 }
15253 }
15254
15255 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15256 ScopeSelector::new(
15257 Principal::Reserved {
15258 module_id: OWNER.to_string(),
15259 },
15260 scope_ref.to_string(),
15261 scope_epoch,
15262 )
15263 }
15264
15265 #[tokio::test]
15266 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
15267 let mut rig = rig().await;
15268 rig.sync(vec![session(1)]).await;
15269 for opener in [OWNER, AFT] {
15270 assert_eq!(
15271 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
15272 .await,
15273 error_codes::SCOPE_EPOCH_REQUIRED,
15274 "{opener}"
15275 );
15276 }
15277 }
15278
15279 #[tokio::test]
15280 async fn admission_separates_not_synced_not_live_and_ended() {
15281 let mut rig = rig().await;
15282 let code = rig
15284 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15285 .await;
15286 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
15287 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
15288
15289 let ghost = ScopeSelector::new(
15291 Principal::Reserved {
15292 module_id: "ghost".to_string(),
15293 },
15294 "s".to_string(),
15295 Some(1),
15296 );
15297 assert_eq!(
15298 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
15299 error_codes::SCOPE_NOT_LIVE
15300 );
15301
15302 rig.sync(vec![session(2)]).await;
15303 assert_eq!(
15304 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
15305 .await,
15306 error_codes::SCOPE_NOT_LIVE
15307 );
15308 for epoch in [1, 3] {
15309 assert_eq!(
15310 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
15311 .await,
15312 error_codes::SCOPE_ENDED,
15313 "epoch {epoch}"
15314 );
15315 }
15316 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
15318 .await;
15319 }
15320
15321 #[tokio::test]
15322 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
15323 let mut rig = rig().await;
15324 rig.sync(vec![session(1)]).await;
15325 let route = rig
15326 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15327 .await;
15328 let stamp = route.stamp().expect("a scoped bind carries the stamp");
15329 assert_eq!(stamp.scope_ref, "s");
15330 assert_eq!(stamp.scope_epoch, 1);
15331 assert_eq!(stamp.kind, ScopeKind::Head);
15332 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
15333 assert!(stamp.attributes.delegates);
15334 assert!(stamp.owner_authorized);
15335 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15336 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
15337
15338 let (broca, mut broca_rx) = wide_ctx(50);
15341 hello_via_sink(
15342 &rig.handler,
15343 &broca,
15344 &mut broca_rx,
15345 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
15346 )
15347 .await;
15348 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
15349 .await
15350 .unwrap();
15351 let own = ScopeSelector::new(
15352 Principal::Reserved {
15353 module_id: BROCA.to_string(),
15354 },
15355 "b".to_string(),
15356 Some(1),
15357 );
15358 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
15359 assert!(!route.stamp().unwrap().owner_authorized);
15360 }
15361
15362 #[tokio::test]
15363 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
15364 let mut rig = rig().await;
15365 let mut record = head("s", 1);
15366 record.carriers = vec![carrier(AFT, None)];
15367 let flow_id = "Flow:run-7/step_2!~";
15368 record.attributes.flow_id = Some(flow_id.to_string());
15369 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
15370 .await
15371 .unwrap();
15372 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15373 panic!("not a sync reply");
15374 };
15375 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
15376 let route = rig
15377 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15378 .await;
15379 assert!(rig.live(&route), "the stamped bind committed");
15380 let stamp = route.stamp().expect("a flow scope carries a stamp");
15381 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
15382 assert_eq!(stamp.attributes.agent_id, None);
15383 assert!(!stamp.attributes.delegates);
15384 assert!(stamp.owner_authorized);
15385 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15386 assert_eq!(unscoped.stamp(), None);
15387 }
15388
15389 #[tokio::test]
15390 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
15391 let mut rig = rig_with_flow_support(false).await;
15392 let mut record = session(1);
15393 record.attributes.flow_id = Some("flow:7".to_string());
15394 rig.sync(vec![record]).await;
15395 for opener in [OWNER, AFT] {
15396 let body = rig
15397 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15398 .await;
15399 assert_eq!(body["code"], "target_flow_unsupported");
15400 let message = body["message"].as_str().unwrap();
15401 for required in [PLEXUS, "flow-scopes/v1"] {
15402 assert!(message.contains(required), "{message}");
15403 }
15404 }
15405 }
15406
15407 #[tokio::test]
15408 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
15409 let mut rig = rig_with_flow_support(true).await;
15410 let mut record = session(1);
15411 record.attributes.flow_id = Some("flow:7".to_string());
15412 rig.sync(vec![record]).await;
15413 for opener in [OWNER, AFT] {
15414 let route = rig
15415 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15416 .await;
15417 assert!(rig.live(&route));
15418 assert_eq!(
15419 route.stamp().unwrap().attributes.flow_id.as_deref(),
15420 Some("flow:7")
15421 );
15422 }
15423 }
15424
15425 #[tokio::test]
15426 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
15427 use std::future::Future;
15428
15429 let mut rig = rig_with_flow_support(true).await;
15430 let mut record = session(1);
15431 record.attributes.flow_id = Some("flow:7".to_string());
15432 rig.sync(vec![record]).await;
15433 let (client, mut client_rx, frame) =
15434 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
15435 for _ in 0..64 {
15438 client.egress.try_send(route_bind_ack(1)).unwrap();
15439 }
15440 let handler = rig.handler.clone();
15441 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15442 std::future::poll_fn(|cx| {
15443 assert!(open.as_mut().poll(cx).is_pending());
15444 std::task::Poll::Ready(())
15445 })
15446 .await;
15447 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15448
15449 let old_connection = rig.modules[PLEXUS].0.connection_id;
15450 rig.handler.cleanup_connection(old_connection).unwrap();
15451 let (replacement, mut replacement_rx) = wide_ctx(200);
15452 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
15453 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15454 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15455 let hello = Frame::build(
15456 FrameType::Hello,
15457 control_flags(),
15458 0,
15459 0,
15460 200,
15461 serde_json::to_vec(&body).unwrap(),
15462 )
15463 .unwrap();
15464 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
15465
15466 client_rx.try_recv().unwrap();
15467 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15468 .await
15469 .expect("the replacement is refused without waiting for a bind ack")
15470 .unwrap();
15471 assert_eq!(replies.len(), 1);
15472 let body = parse_error(&replies[0]);
15473 assert_eq!(body["code"], "target_flow_unsupported");
15474 for required in [PLEXUS, "flow-scopes/v1"] {
15475 assert!(body["message"].as_str().unwrap().contains(required));
15476 }
15477 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
15478 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15479 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15480 assert!(rig
15481 .handler
15482 .registry
15483 .get_module_by_connection(replacement.connection_id)
15484 .unwrap()
15485 .is_some());
15486 }
15487
15488 #[tokio::test]
15489 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
15490 use std::future::Future;
15491 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15492 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15493 const NEXT_HASH: &str =
15494 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15495 let mut rig = rig().await;
15496 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15497 .await;
15498 let (client, mut rx, frame) = rig.open_frame(
15499 Some(AFT),
15500 PLEXUS,
15501 Some(
15502 rig.selector("s", Some(1))
15503 .with_bind_proof(Some(PROOF.into())),
15504 ),
15505 );
15506 for _ in 0..64 {
15509 client.egress.try_send(route_bind_ack(1)).unwrap();
15510 }
15511 let handler = rig.handler.clone();
15512 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15513 std::future::poll_fn(|cx| {
15514 assert!(open.as_mut().poll(cx).is_pending());
15515 std::task::Poll::Ready(())
15516 })
15517 .await;
15518 rig.sync(vec![
15519 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
15520 ])
15521 .await;
15522 rx.try_recv().unwrap();
15523 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15524 .await
15525 .unwrap()
15526 .unwrap();
15527 assert_eq!(
15528 parse_error(&replies[0])["code"],
15529 error_codes::SCOPE_BIND_PROOF_MISMATCH
15530 );
15531 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
15532 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15533 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15534 }
15535
15536 #[tokio::test]
15537 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
15538 {
15539 let mut rig = rig_with_flow_support(false).await;
15540 rig.sync(vec![session(1)]).await;
15541 let route = rig
15542 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15543 .await;
15544 assert!(rig.live(&route));
15545 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
15546 let mut record = session(1);
15547 record.attributes.flow_id = Some("flow:7".to_string());
15548 rig.sync(vec![record]).await;
15549 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
15550 assert!(rig.live(&unscoped));
15551 assert_eq!(unscoped.stamp(), None);
15552 }
15553
15554 #[tokio::test]
15555 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
15556 let mut rig = rig().await;
15557 let mut record = session(1);
15558 record.attributes.flow_id = Some("flow:7".to_string());
15559 rig.sync(vec![record.clone()]).await;
15560 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15561 let mut owner_route = rig
15562 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15563 .await;
15564 let mut carrier_route = rig
15565 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
15566 .await;
15567 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15568 rig.sync(vec![record.clone()]).await;
15569 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15570 assert!(rig.live(&owner_route) && owner_route.untouched());
15571 assert!(rig.live(&carrier_route) && carrier_route.untouched());
15572
15573 record.attributes.flow_id = Some("flow:8".to_string());
15574 rig.sync(vec![record]).await;
15575 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
15576 let before = before.unwrap();
15577 assert_eq!(after.scope_epoch, before.scope_epoch);
15578 assert!(after.version > before.version);
15579 for route in [&mut owner_route, &mut carrier_route] {
15580 assert!(!rig.live(route));
15581 assert_eq!(
15582 route.closed_reason(),
15583 RouteCloseReason::ScopeDelegationChanged
15584 );
15585 }
15586 assert!(rig.live(&unscoped) && unscoped.untouched());
15587 for target in [PLEXUS, OTHER] {
15590 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
15591 let goodbye = module_rx
15592 .try_recv()
15593 .expect("the provider sees the drain")
15594 .frame;
15595 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15596 assert!(module_rx.try_recv().is_err());
15597 }
15598 let rebound = rig
15599 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15600 .await;
15601 assert_eq!(
15602 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
15603 Some("flow:8")
15604 );
15605 }
15606
15607 #[tokio::test]
15612 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
15613 let mut rig = rig().await;
15614 rig.sync(vec![session(1)]).await;
15615 let mut cotenant = rig
15616 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15617 .await;
15618
15619 let mut changed = session(1);
15620 changed.child_owners.push(Principal::Reserved {
15621 module_id: MAGIC.to_string(),
15622 });
15623 let mut ended = None;
15624 for (code, next) in [
15625 (error_codes::SCOPE_CHANGED, vec![changed]),
15626 (error_codes::SCOPE_ENDED, Vec::new()),
15627 ] {
15628 let relayed = rig
15629 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15630 .await;
15631 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
15632 ended = Some(next.is_empty());
15633 rig.sync(next).await;
15634 rig.ack(&relayed).await;
15635 let replies = relayed.task.await.unwrap();
15636 assert_eq!(replies.len(), 1, "{replies:?}");
15637 assert_eq!(parse_error(&replies[0])["code"], code);
15638 assert_eq!(
15639 subc_protocol::error_codes::is_retryable_route_open(code),
15640 code == error_codes::SCOPE_CHANGED
15641 );
15642 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15643 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15645 let mut goodbyes = Vec::new();
15648 while let Ok(outbound) = plexus_rx.try_recv() {
15649 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
15650 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
15651 }
15652 assert!(
15653 goodbyes.contains(&(bind_channel, bind_epoch)),
15654 "{goodbyes:?}"
15655 );
15656 assert!(rig
15657 .handler
15658 .registry
15659 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
15660 .unwrap()
15661 .is_some());
15662 }
15663 assert_eq!(ended, Some(true));
15664 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
15667 }
15668
15669 #[tokio::test]
15672 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
15673 struct Case {
15674 name: &'static str,
15675 change: fn(&mut ScopeRecord),
15676 closed: [Option<RouteCloseReason>; 4],
15679 }
15680 use RouteCloseReason::*;
15681 let cases = [
15682 Case {
15683 name: "a carrier entry removed",
15684 change: |r| {
15685 r.carriers.retain(|c| {
15686 c.principal
15687 != Principal::Reserved {
15688 module_id: AFT.to_string(),
15689 }
15690 })
15691 },
15692 closed: [None, Some(ScopeCarrierRemoved), None, None],
15693 },
15694 Case {
15695 name: "a target removed from a carrier",
15696 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
15697 closed: [None, None, None, Some(ScopeCarrierRemoved)],
15698 },
15699 Case {
15700 name: "a bare carrier narrowed to targets",
15701 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
15702 closed: [None, Some(ScopeCarrierRemoved), None, None],
15703 },
15704 Case {
15705 name: "delegates turned off",
15706 change: |r| r.attributes.delegates = false,
15707 closed: [Some(ScopeDelegationChanged); 4],
15708 },
15709 Case {
15710 name: "agent_id changed",
15711 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
15712 closed: [Some(ScopeDelegationChanged); 4],
15713 },
15714 Case {
15715 name: "a carrier added, child owners changed, the record re-sent",
15716 change: |r| {
15717 r.carriers.push(carrier(MAGIC, None));
15718 r.child_owners.push(Principal::Reserved {
15719 module_id: MAGIC.to_string(),
15720 });
15721 },
15722 closed: [None; 4],
15723 },
15724 Case {
15725 name: "a target added",
15726 change: |r| {
15727 r.carriers[1]
15728 .targets
15729 .as_mut()
15730 .unwrap()
15731 .push("third".to_string())
15732 },
15733 closed: [None; 4],
15734 },
15735 Case {
15736 name: "delegates turned on",
15737 change: |r| r.attributes.delegates = true,
15738 closed: [None; 4],
15739 },
15740 ];
15741 for case in cases {
15742 let mut rig = rig().await;
15743 rig.sync(vec![session(1)]).await;
15744 let scope = || Some(rig_selector("s", Some(1)));
15745 let mut routes = [
15746 rig.bound(Some(OWNER), PLEXUS, scope()).await,
15747 rig.bound(Some(AFT), PLEXUS, scope()).await,
15748 rig.bound(Some(BROCA), PLEXUS, scope()).await,
15749 rig.bound(Some(BROCA), OTHER, scope()).await,
15750 ];
15751 let mut record = session(1);
15752 (case.change)(&mut record);
15753 rig.sync(vec![record]).await;
15754 for (index, expected) in case.closed.iter().enumerate() {
15755 let route = &mut routes[index];
15756 match expected {
15757 Some(reason) => {
15758 assert!(!rig.live(route), "{}: route {index} still live", case.name);
15759 assert_eq!(
15760 route.closed_reason(),
15761 *reason,
15762 "{}: route {index}",
15763 case.name
15764 );
15765 }
15766 None => {
15767 assert!(rig.live(route), "{}: route {index} closed", case.name);
15768 assert!(
15769 route.untouched(),
15770 "{}: route {index} was told something",
15771 case.name
15772 );
15773 }
15774 }
15775 }
15776 }
15777 }
15778
15779 #[tokio::test]
15780 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
15781 for next in [Vec::new(), vec![session(2)]] {
15783 let mut rig = rig().await;
15784 rig.sync(vec![session(1)]).await;
15785 let mut route = rig
15786 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15787 .await;
15788 rig.sync(next).await;
15789 assert!(!rig.live(&route));
15790 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
15791 }
15792
15793 let mut rig = rig().await;
15796 let mut child = session(1);
15797 child.scope_ref = "child".to_string();
15798 child.kind = ScopeKind::Worker;
15799 child.parent = Some(ScopeParent::new(
15800 Principal::Reserved {
15801 module_id: OWNER.to_string(),
15802 },
15803 "s".to_string(),
15804 1,
15805 ));
15806 rig.sync(vec![session(1), child.clone()]).await;
15807 let mut child_route = rig
15808 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
15809 .await;
15810 assert_eq!(
15811 child_route.stamp().unwrap().parent_state,
15812 Some(ParentState::Linked)
15813 );
15814 rig.sync(vec![child]).await;
15815 assert!(!rig.live(&child_route));
15816 assert_eq!(
15817 child_route.closed_reason(),
15818 RouteCloseReason::ScopeParentEnded
15819 );
15820 }
15821
15822 #[tokio::test]
15823 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
15824 ) {
15825 let mut rig = rig().await;
15826 rig.sync(vec![session(1)]).await;
15827 let mut route = rig
15828 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15829 .await;
15830 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15831 rig.sync(vec![session(1)]).await;
15832 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15833 assert!(rig.live(&route) && route.untouched());
15834
15835 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
15839 .forwarding
15840 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15841 .unwrap()
15842 else {
15843 panic!("the route is bound");
15844 };
15845 binding.flow.acquire_tagged(9, false).await.unwrap();
15846 let mut widened = session(1);
15847 widened.carriers.push(carrier(MAGIC, None));
15848 rig.sync(vec![widened]).await;
15849 assert!(rig.live(&route) && route.untouched());
15850 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15851 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
15852 assert_eq!(binding.flow.in_flight(), 1);
15855 binding
15856 .flow
15857 .acquire_tagged(10, false)
15858 .await
15859 .expect("the flow is still open");
15860 }
15861
15862 #[tokio::test]
15865 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
15866 let mut rig = rig().await;
15867 rig.sync(vec![session(1)]).await;
15868 let mut on_incumbent = rig
15869 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15870 .await;
15871
15872 let (candidate, _candidate_rx) = wide_ctx(9);
15875 let registration = rig
15876 .handler
15877 .registry
15878 .register_candidate_with_control_ops(
15879 manifest(PLEXUS, PROTOCOL_VERSION),
15880 PROTOCOL_VERSION,
15881 candidate.connection_id,
15882 module_baseline_control_ops(),
15883 )
15884 .unwrap();
15885 rig.forwarding
15886 .register_candidate_module_connection(
15887 candidate.connection_id,
15888 PLEXUS.to_string(),
15889 PROTOCOL_VERSION,
15890 manifest_concurrency(®istration.manifest),
15891 candidate.egress.clone(),
15892 )
15893 .unwrap();
15894 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
15895 rig.handler
15896 .registry
15897 .promote_candidate(PLEXUS)
15898 .unwrap()
15899 .unwrap();
15900 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
15901
15902 rig.sync(Vec::new()).await;
15903 assert!(!rig.live(&on_incumbent));
15904 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
15905 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15906 let goodbye = incumbent_rx
15907 .try_recv()
15908 .expect("the superseded endpoint is told")
15909 .frame;
15910 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15911 }
15912 }
15913}
15914
15915#[cfg(test)]
15916mod concurrency_default_exposure_tests {
15917 use super::*;
15918
15919 fn hello_body(role_json: &str) -> Vec<u8> {
15920 format!(
15921 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
15922 )
15923 .into_bytes()
15924 }
15925
15926 fn manifest_from(body: &[u8]) -> ModuleManifest {
15927 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
15928 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
15929 .expect("manifest parses")
15930 }
15931
15932 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
15933
15934 #[test]
15935 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
15936 let body = hello_body(&format!(
15937 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
15938 ));
15939 let manifest = manifest_from(&body);
15940 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15943 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
15944 }
15945
15946 #[test]
15947 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
15948 let body = hello_body(&format!(
15949 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
15950 ));
15951 let manifest = manifest_from(&body);
15952 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15953 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
15954 }
15955
15956 #[test]
15957 fn non_management_roles_are_never_reported() {
15958 let body = hello_body(
15959 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
15960 );
15961 let manifest = manifest_from(&body);
15962 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
15963 }
15964}